diff options
319 files changed, 13931 insertions, 5032 deletions
diff --git a/.gitattributes b/.gitattributes index 32583149c2..0accd23848 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,6 +1,6 @@ * whitespace=!indent,trail,space -*.[ch] whitespace=indent,trail,space diff=cpp -*.sh whitespace=indent,trail,space text eol=lf +*.[ch] whitespace=indent,trail,space,incomplete diff=cpp +*.sh whitespace=indent,trail,space,incomplete text eol=lf *.perl text eol=lf diff=perl *.pl text eof=lf diff=perl *.pm text eol=lf diff=perl diff --git a/.github/workflows/check-style.yml b/.github/workflows/check-style.yml index c052a5df23..19a145d4ad 100644 --- a/.github/workflows/check-style.yml +++ b/.github/workflows/check-style.yml @@ -20,7 +20,7 @@ jobs: jobname: ClangFormat runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 diff --git a/.github/workflows/check-whitespace.yml b/.github/workflows/check-whitespace.yml index d0a78fc426..928fd4cfe2 100644 --- a/.github/workflows/check-whitespace.yml +++ b/.github/workflows/check-whitespace.yml @@ -19,7 +19,7 @@ jobs: check-whitespace: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index 01a0437b2f..cfa17d394a 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -38,7 +38,7 @@ jobs: COVERITY_LANGUAGE: cxx COVERITY_PLATFORM: overridden-below steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: install minimal Git for Windows SDK if: contains(matrix.os, 'windows') uses: git-for-windows/setup-git-for-windows-sdk@v1 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 393ea4d1cc..816d5a34c4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -63,7 +63,7 @@ jobs: echo "skip_concurrent=$skip_concurrent" >>$GITHUB_OUTPUT - name: skip if the commit or tree was already tested id: skip-if-redundant - uses: actions/github-script@v7 + uses: actions/github-script@v8 if: steps.check-ref.outputs.enabled == 'yes' with: github-token: ${{secrets.GITHUB_TOKEN}} @@ -112,7 +112,7 @@ jobs: group: windows-build-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: git-for-windows/setup-git-for-windows-sdk@v1 - name: build shell: bash @@ -123,7 +123,7 @@ jobs: - name: zip up tracked files run: git archive -o artifacts/tracked.tar.gz HEAD - name: upload tracked files and build artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: windows-artifacts path: artifacts @@ -140,7 +140,7 @@ jobs: cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - name: download tracked files and build artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v6 with: name: windows-artifacts path: ${{github.workspace}} @@ -157,7 +157,7 @@ jobs: run: ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: failed-tests-windows-${{ matrix.nr }} path: ${{env.FAILED_TEST_ARTIFACTS}} @@ -173,10 +173,10 @@ jobs: group: vs-build-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: git-for-windows/setup-git-for-windows-sdk@v1 - name: initialize vcpkg - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'microsoft/vcpkg' path: 'compat/vcbuild/vcpkg' @@ -208,7 +208,7 @@ jobs: - name: zip up tracked files run: git archive -o artifacts/tracked.tar.gz HEAD - name: upload tracked files and build artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: vs-artifacts path: artifacts @@ -226,7 +226,7 @@ jobs: steps: - uses: git-for-windows/setup-git-for-windows-sdk@v1 - name: download tracked files and build artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v6 with: name: vs-artifacts path: ${{github.workspace}} @@ -244,7 +244,7 @@ jobs: run: ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: failed-tests-windows-vs-${{ matrix.nr }} path: ${{env.FAILED_TEST_ARTIFACTS}} @@ -258,8 +258,8 @@ jobs: group: windows-meson-build-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 - name: Set up dependencies shell: pwsh run: pip install meson ninja @@ -270,7 +270,7 @@ jobs: shell: pwsh run: meson compile -C build - name: Upload build artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: windows-meson-artifacts path: build @@ -286,13 +286,13 @@ jobs: group: windows-meson-test-${{ matrix.nr }}-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 - name: Set up dependencies shell: pwsh run: pip install meson ninja - name: Download build artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v6 with: name: windows-meson-artifacts path: build @@ -313,16 +313,16 @@ jobs: vector: - jobname: osx-clang cc: clang - pool: macos-13 + pool: macos-14 - jobname: osx-reftable cc: clang - pool: macos-13 + pool: macos-14 - jobname: osx-gcc cc: gcc-13 - pool: macos-13 + pool: macos-14 - jobname: osx-meson cc: clang - pool: macos-13 + pool: macos-14 env: CC: ${{matrix.vector.cc}} CC_PACKAGE: ${{matrix.vector.cc_package}} @@ -331,7 +331,7 @@ jobs: TEST_OUTPUT_DIRECTORY: ${{github.workspace}}/t runs-on: ${{matrix.vector.pool}} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - run: ci/install-dependencies.sh - run: ci/run-build-and-tests.sh - name: print test failures @@ -339,7 +339,7 @@ jobs: run: ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: failed-tests-${{matrix.vector.jobname}} path: ${{env.FAILED_TEST_ARTIFACTS}} @@ -352,7 +352,7 @@ jobs: CI_JOB_IMAGE: ubuntu-latest runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - run: ci/install-dependencies.sh - run: ci/run-build-and-minimal-fuzzers.sh dockerized: @@ -429,7 +429,7 @@ jobs: else apt-get -q update && apt-get -q -y install git fi - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - run: ci/install-dependencies.sh - run: useradd builder --create-home - run: chown -R builder . @@ -439,7 +439,7 @@ jobs: run: sudo --preserve-env --set-home --user=builder ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: failed-tests-${{matrix.vector.jobname}} path: ${{env.FAILED_TEST_ARTIFACTS}} @@ -454,10 +454,25 @@ jobs: group: static-analysis-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - run: ci/install-dependencies.sh - run: ci/run-static-analysis.sh - run: ci/check-directional-formatting.bash + rust-analysis: + needs: ci-config + if: needs.ci-config.outputs.enabled == 'yes' + env: + jobname: RustAnalysis + CI_JOB_IMAGE: ubuntu:rolling + runs-on: ubuntu-latest + container: ubuntu:rolling + concurrency: + group: rust-analysis-${{ github.ref }} + cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} + steps: + - uses: actions/checkout@v4 + - run: ci/install-dependencies.sh + - run: ci/run-rust-checks.sh sparse: needs: ci-config if: needs.ci-config.outputs.enabled == 'yes' @@ -469,7 +484,7 @@ jobs: group: sparse-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Install other dependencies run: ci/install-dependencies.sh - run: make sparse @@ -485,6 +500,6 @@ jobs: CI_JOB_IMAGE: ubuntu-latest runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - run: ci/install-dependencies.sh - run: ci/test-documentation.sh diff --git a/.gitignore b/.gitignore index 78a45cb5be..24635cf2d6 100644 --- a/.gitignore +++ b/.gitignore @@ -79,6 +79,7 @@ /git-grep /git-hash-object /git-help +/git-history /git-hook /git-http-backend /git-http-fetch diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f7d57d1ee9..b419a84e2c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -161,7 +161,7 @@ test:mingw64: - saas-windows-medium-amd64 before_script: - *windows_before_script - - choco install -y git meson ninja + - choco install -y git meson ninja rust-ms - Import-Module $env:ChocolateyInstall\helpers\chocolateyProfile.psm1 - refreshenv @@ -212,6 +212,17 @@ static-analysis: - ./ci/run-static-analysis.sh - ./ci/check-directional-formatting.bash +rust-analysis: + image: ubuntu:rolling + stage: analyze + needs: [ ] + variables: + jobname: RustAnalysis + before_script: + - ./ci/install-dependencies.sh + script: + - ./ci/run-rust-checks.sh + check-whitespace: image: ubuntu:latest stage: analyze diff --git a/Cargo.toml b/Cargo.toml index 45c9b34981..2f51bf5d5f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ name = "gitcore" version = "0.1.0" edition = "2018" +rust-version = "1.49.0" [lib] crate-type = ["staticlib"] diff --git a/Documentation/BreakingChanges.adoc b/Documentation/BreakingChanges.adoc index 90b53abcea..f814450d2f 100644 --- a/Documentation/BreakingChanges.adoc +++ b/Documentation/BreakingChanges.adoc @@ -295,6 +295,26 @@ The command will be removed. + cf. <xmqqa59i45wc.fsf@gitster.g> +* Support for `core.preferSymlinkRefs=true` has been deprecated and will be + removed in Git 3.0. Writing symbolic refs as symbolic links will be phased + out in favor of using plain files using the textual representation of + symbolic refs. ++ +Symbolic references were initially always stored as a symbolic link. This was +changed in 9b143c6e15 (Teach update-ref about a symbolic ref stored in a +textfile., 2005-09-25), where a new textual symref format was introduced to +store those symbolic refs in a plain file. In 9f0bb90d16 +(core.prefersymlinkrefs: use symlinks for .git/HEAD, 2006-05-02), the Git +project switched the default to use the textual symrefs in favor of symbolic +links. ++ +The migration away from symbolic links has happened almost 20 years ago by now, +and there is no known reason why one should prefer them nowadays. Furthermore, +symbolic links are not supported on some platforms. ++ +Note that only the writing side for such symbolic links is deprecated. Reading +such symbolic links is still supported for now. + == Superseded features that will not be deprecated Some features have gained newer replacements that aim to improve the design in diff --git a/Documentation/Makefile b/Documentation/Makefile index a3fbd29744..47208269a2 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -34,6 +34,7 @@ MAN5_TXT += gitformat-bundle.adoc MAN5_TXT += gitformat-chunk.adoc MAN5_TXT += gitformat-commit-graph.adoc MAN5_TXT += gitformat-index.adoc +MAN5_TXT += gitformat-loose.adoc MAN5_TXT += gitformat-pack.adoc MAN5_TXT += gitformat-signature.adoc MAN5_TXT += githooks.adoc @@ -52,6 +53,7 @@ MAN7_TXT += gitcli.adoc MAN7_TXT += gitcore-tutorial.adoc MAN7_TXT += gitcredentials.adoc MAN7_TXT += gitcvs-migration.adoc +MAN7_TXT += gitdatamodel.adoc MAN7_TXT += gitdiffcore.adoc MAN7_TXT += giteveryday.adoc MAN7_TXT += gitfaq.adoc @@ -122,6 +124,7 @@ TECH_DOCS += technical/bundle-uri TECH_DOCS += technical/commit-graph TECH_DOCS += technical/directory-rename-detection TECH_DOCS += technical/hash-function-transition +TECH_DOCS += technical/large-object-promisors TECH_DOCS += technical/long-running-process-protocol TECH_DOCS += technical/multi-pack-index TECH_DOCS += technical/packfile-uri diff --git a/Documentation/MyFirstContribution.adoc b/Documentation/MyFirstContribution.adoc index 02ba8ba5f6..f186dfbc89 100644 --- a/Documentation/MyFirstContribution.adoc +++ b/Documentation/MyFirstContribution.adoc @@ -1153,6 +1153,11 @@ NOTE: When you are sending a real patch, it will go to git@vger.kernel.org - but please don't send your patchset from the tutorial to the real mailing list! For now, you can send it to yourself, to make sure you understand how it will look. +NOTE: After sending your patches, you can confirm that they reached the mailing +list by visiting https://lore.kernel.org/git/. Use the search bar to find your +name or the subject of your patch. If it appears, your email was successfully +delivered. + After you run the command above, you will be presented with an interactive prompt for each patch that's about to go out. This gives you one last chance to edit or quit sending something (but again, don't edit code this way). Once you diff --git a/Documentation/RelNotes/2.51.2.adoc b/Documentation/RelNotes/2.51.2.adoc new file mode 100644 index 0000000000..f0be60333a --- /dev/null +++ b/Documentation/RelNotes/2.51.2.adoc @@ -0,0 +1,45 @@ +Git 2.51.2 Release Notes +======================== + +In addition to fixes for an unfortunate regression introduced in Git +2.51.1 that caused "git diff --quiet -w" to be not so quiet when there +are additions, deletions and conflicts, this maintenance release merges +more fixes/improvements that have landed on the master front, primarily +to make the CI part of the system a bit more robust. + + +Fixes since Git 2.51.1 +---------------------- + + * Recently we attempted to improve "git diff -w --quiet" and friends + to handle cases where patch output would be suppressed, but it + introduced a bug that emits unnecessary output, which has been + corrected. + + * The code to squelch output from "git diff -w --name-status" + etc. for paths that "git diff -w -p" would have stayed silent + leaked output from dry-run patch generation, which has been + corrected. + + * Windows "real-time monitoring" interferes with the execution of + tests and affects negatively in both correctness and performance, + which has been disabled in Gitlab CI. + + * An earlier addition to "git diff --no-index A B" to limit the + output with pathspec after the two directories misbehaved when + these directories were given with a trailing slash, which has been + corrected. + + * The "--short" option of "git status" that meant output for humans + and "-z" option to show NUL delimited output format did not mix + well, and colored some but not all things. The command has been + updated to color all elements consistently in such a case. + + * Unicode width table update. + + * Recent OpenSSH creates the Unix domain socket to communicate with + ssh-agent under $HOME instead of /tmp, which causes our test to + fail doe to overly long pathname in our test environment, which has + been worked around by using "ssh-agent -T". + +Also contains various documentation updates, code cleanups and minor fixups. diff --git a/Documentation/RelNotes/2.52.0.adoc b/Documentation/RelNotes/2.52.0.adoc index ef5f91fcc0..6c0e7d05c0 100644 --- a/Documentation/RelNotes/2.52.0.adoc +++ b/Documentation/RelNotes/2.52.0.adoc @@ -55,6 +55,30 @@ UI, Workflows & Features (e.g. blame.ignorerevsfile) can be marked as optional by prefixing ":(optoinal)" before its value. + * Show 'P'ipe command in "git add -p". + + * "git sparse-checkout" subcommand learned a new "clean" action to + prune otherwise unused working-tree files that are outside the + areas of interest. + + * "git fast-import" is taught to handle signed tags, just like it + recently learned to handle signed commits, in different ways. + + * A new configuration variable commitGraph.changedPaths allows to + turn "--changed-paths" on by default for "git commit-graph". + + * "Symlink symref" has been added to the list of things that will + disappear at Git 3.0 boundary. + + * "git maintenance" command learns the "geometric" strategy where it + avoids doing maintenance tasks that rebuilds everything from + scratch. + + * "git repo structure", a new command. + + * The help text and manual page of "git bisect" command have been + made consistent with each other. + Performance, Internal Implementation, Development Support etc. -------------------------------------------------------------- @@ -102,7 +126,6 @@ Performance, Internal Implementation, Development Support etc. * Adjust to the way newer versions of cURL selectively enable tracing options, so that our tests can continue to work. - (merge 1b5a6bfff3 jk/curl-global-trace-components later to maint). * The clear_alloc_state() API function was not fully clearing the structure for reuse, but since nobody reuses it, replace it with a @@ -131,6 +154,34 @@ Performance, Internal Implementation, Development Support etc. and one for xdiff), roll everything into a single libgit.a archive. This would help later effort to FFI into Rust. + * The beginning of SHA1-SHA256 interoperability work. + + * Build procedure for a few credential helpers (in contrib/) have + been updated. + + * CI improvements to handle the recent Rust integration better. + + * The code in "git repack" machinery has been cleaned up to prepare + for incremental update of midx files. + + * Two slightly different ways to get at "all the packfiles" in API + has been cleaned up. + + * The code to walk revision graph to compute merge base has been + optimized. + + * AI guidelines has been added to our documentation set. + + * Contributed credential helpers (obviously in contrib/) now have "cd + $there && make install" target. + + * The "MyFirstContribution" tutorial tells the reader how to send out + their patches; the section gained a hint to verify the message + reached the mailing list. + + * The "debug" ref-backend was missing a method implementation, which + has been corrected. + Fixes since v2.51 ----------------- @@ -140,11 +191,9 @@ including security updates, are included in this release. * During interactive rebase, using 'drop' on a merge commit lead to an error, which was incorrect. - (merge 4d491ade8f js/rebase-i-allow-drop-on-a-merge later to maint). * "git refs migrate" to migrate the reflog entries from a refs backend to another had a handful of bugs squashed. - (merge 465eff81de ps/reflog-migrate-fixes later to maint). * "git remote rename origin upstream" failed to move origin/HEAD to upstream/HEAD when origin/HEAD is unborn and performed other @@ -157,11 +206,9 @@ including security updates, are included in this release. * "git push" had a code path that led to BUG() but it should have been a die(), as it is a response to a usual but invalid end-user action to attempt pushing an object that does not exist. - (merge dfbfc2221b dl/push-missing-object-error later to maint). * Various bugs about rename handling in "ort" merge strategy have been fixed. - (merge f6ecb603ff en/ort-rename-fixes later to maint). * "git jump" (in contrib/) fails to parse the diff header correctly when a file has a space in its name, which has been corrected. @@ -172,7 +219,6 @@ including security updates, are included in this release. the prefix from the output, and oddballs like "-" (stdin) did not work correctly because of it. Correct the set-up by undoing what the set-up sequence did to cwd and prefix. - (merge e1d3d61a45 jc/diff-no-index-in-subdir later to maint). * Various options to "git diff" that makes comparison ignore certain aspects of the differences (like "space changes are ignored", @@ -180,19 +226,19 @@ including security updates, are included in this release. ignored") did not work well with "--name-only" and friends. (merge b55e6d36eb ly/diff-name-only-with-diff-from-content later to maint). + * The above caused regressions, which has been corrected. + * Documentation for "git rebase" has been updated. (merge 3f7f2b0359 je/doc-rebase later to maint). * The start_delayed_progress() function in the progress eye-candy API did not clear its internal state, making an initial delay value larger than 1 second ineffective, which has been corrected. - (merge 457534d041 js/progress-delay-fix later to maint). * The compatObjectFormat extension is used to hide an incomplete feature that is not yet usable for any purpose other than developing the feature further. Document it as such to discourage its use by mere mortals. - (merge 716d905792 bc/doc-compat-object-format-not-working later to maint). * "git log -L..." compared trees of multiple parents with the tree of the merge result in an unnecessarily inefficient way. @@ -202,7 +248,6 @@ including security updates, are included in this release. repository, especially a partially cloned one, "git fetch" may mistakenly think some objects we do have are missing, which has been corrected. - (merge 8f32a5a6c0 jk/fetch-check-graph-objects-fix later to maint). * "git fetch" can clobber a symref that is dangling when the remote-tracking HEAD is set to auto update, which has been @@ -214,20 +259,16 @@ including security updates, are included in this release. * Manual page for "gitk" is updated with the current maintainer's name. - (merge bcb20dda83 js/doc-gitk-history later to maint). * Update the instructions for using GGG in the MyFirstContribution document to say that a GitHub PR could be made against `git/git` instead of `gitgitgadget/git`. - (merge 37001cdbc4 ds/doc-ggg-pr-fork-clarify later to maint). * Makefile tried to run multiple "cargo build" which would not work very well; serialize their execution to work around this problem. - (merge 0eeacde50e da/cargo-serialize later to maint). * "git repack --path-walk" lost objects in some corner cases, which has been corrected. - (merge 93afe9b060 ds/path-walk-repack-fix later to maint). * "git ls-files <pathspec>..." should not necessarily have to expand the index fully if a sparsified directory is excluded by the @@ -238,15 +279,12 @@ including security updates, are included in this release. * Windows "real-time monitoring" interferes with the execution of tests and affects negatively in both correctness and performance, which has been disabled in Gitlab CI. - (merge 608cf5b793 ps/gitlab-ci-disable-windows-monitoring later to maint). * A broken or malicious "git fetch" can say that it has the same object for many many times, and the upload-pack serving it can exhaust memory storing them redundantly, which has been corrected. - (merge 88a2dc68c8 ps/upload-pack-oom-protection later to maint). * A corner case bug in "git log -L..." has been corrected. - (merge e3106998ff sg/line-log-boundary-fixes later to maint). * "git rev-parse --short" and friends failed to disambiguate two objects with object names that share common prefix longer than 32 @@ -256,7 +294,6 @@ including security updates, are included in this release. * Some among "git add -p" and friends ignored color.diff and/or color.ui configuration variables, which is an old regression, which has been corrected. - (merge 1092cd6435 jk/add-i-color later to maint). * "git subtree" (in contrib/) did not work correctly when splitting squashed subtrees, which has been improved. @@ -272,7 +309,6 @@ including security updates, are included in this release. * "git rebase -i" failed to clean-up the commit log message when the command commits the final one in a chain of "fixup" commands, which has been corrected. - (merge 82a0a73e15 pw/rebase-i-cleanup-fix later to maint). * There are double frees and leaks around setup_revisions() API used in "git stash show", which has been fixed, and setup_revisions() @@ -283,7 +319,6 @@ including security updates, are included in this release. * Deal more gracefully with directory / file conflicts when the files backend is used for ref storage, by failing only the ones that are involved in the conflict while allowing others. - (merge 948b2ab0d8 kn/refs-files-case-insensitive later to maint). * "git last-modified" operating in non-recursive mode used to trigger a BUG(), which has been corrected. @@ -296,16 +331,13 @@ including security updates, are included in this release. * The "do you still use it?" message given by a command that is deeply deprecated and allow us to suggest alternatives has been updated. - (merge 54a60e5b38 kh/you-still-use-whatchanged-fix later to maint). * Clang-format update to let our control macros be formatted the way we had them traditionally, e.g., "for_each_string_list_item()" without space before the parentheses. - (merge 3721541d35 jt/clang-format-foreach-wo-space-before-parenthesis later to maint). * A few places where a size_t value was cast to curl_off_t without checking has been updated to use the existing helper function. - (merge ecc5749578 js/curl-off-t-fixes later to maint). * "git reflog write" did not honor the configured user.name/email which has been corrected. @@ -317,7 +349,6 @@ including security updates, are included in this release. environment, but Ubuntu replaced with "sudo" with an implementation that lacks the feature. Work this around by reinstalling the original version. - (merge fddb484255 ps/ci-avoid-broken-sudo-on-ubuntu later to maint). * The reftable backend learned to sanity check its on-disk data more carefully. @@ -344,35 +375,69 @@ including security updates, are included in this release. output with pathspec after the two directories misbehaved when these directories were given with a trailing slash, which has been corrected. - (merge c0bec06cfe jk/diff-no-index-with-pathspec-fix later to maint). + + * The "--short" option of "git status" that meant output for humans + and "-z" option to show NUL delimited output format did not mix + well, and colored some but not all things. The command has been + updated to color all elements consistently in such a case. + + * Unicode width table update. + + * GPG signing test set-up has been broken for a year, which has been + corrected. + (merge 516bf45749 jc/t1016-setup-fix later to maint). + + * Recent OpenSSH creates the Unix domain socket to communicate with + ssh-agent under $HOME instead of /tmp, which causes our test to + fail doe to overly long pathname in our test environment, which has + been worked around by using "ssh-agent -T". + + * strbuf_split*() to split a string into multiple strbufs is often a + wrong API to use. A few uses of it have been removed by + simplifying the code. + (merge 2ab72a16d9 ob/gpg-interface-cleanup later to maint). + + * "git shortlog" knows "--committer" and "--author" options, which + the command line completion (in contrib/) did not handle well, + which has been corrected. + (merge c568fa8e1c kf/log-shortlog-completion-fix later to maint). + + * "git bisect" command did not react correctly to "git bisect help" + and "git bisect unknown", which has been corrected. + (merge 2bb3a012f3 rz/bisect-help-unknown later to maint). + + * The 'q'(uit) command in "git add -p" has been improved to quit + without doing any meaningless work before leaving, and giving EOF + (typically control-D) to the prompt is made to behave the same way. + + * The wildmatch code had a corner case bug that mistakenly makes + "foo**/bar" match with "foobar", which has been corrected. + (merge 1940a02dc1 jk/match-pathname-fix later to maint). + + * Tests did not set up GNUPGHOME correctly, which is fixed but some + flaky tests are exposed in t1016, which needs to be addressed + before this topic can move forward. + (merge 6cd8369ef3 tz/test-prepare-gnupghome later to maint). + + * The patterns used in the .gitignore files use backslash in the way + documented for fnmatch(3); document as such to reduce confusion. + (merge 8a6d158a1d jk/doc-backslash-in-exclude later to maint). + + * The version of macos image used in GitHub CI has been updated to + macos-14, as the macos-13 that we have been using got deprecated. + (merge 73b9cdb7c4 jc/ci-use-macos-14 later to maint). * Other code cleanup, docfix, build fix, etc. - (merge 823d537fa7 kh/doc-git-log-markup-fix later to maint). - (merge cf7efa4f33 rj/t6137-cygwin-fix later to maint). (merge 529a60a885 ua/t1517-short-help-tests later to maint). (merge 22d421fed9 ac/deglobal-fmt-merge-log-config later to maint). - (merge 741f36c7d9 kr/clone-synopsis-fix later to maint). (merge a60203a015 dk/t7005-editor-updates later to maint). - (merge 7d4a5fef7d ds/doc-count-objects-fix later to maint). (merge 16684b6fae ps/reftable-libgit2-cleanup later to maint). - (merge f38786baa7 ja/asciidoc-doctor-verbatim-fixes later to maint). - (merge 374579c6d4 kh/doc-interpret-trailers-markup-fix later to maint). - (merge 44dce6541c kh/doc-config-typofix later to maint). - (merge 785628b173 js/doc-sending-patch-via-thunderbird later to maint). (merge e5c27bd3d8 je/doc-add later to maint). (merge 13296ac909 ps/object-store-midx-dedup-info later to maint). - (merge 2f4bf83ffc km/alias-doc-markup-fix later to maint). - (merge b0d97aac19 kh/doc-markup-fixes later to maint). (merge f9a6705d9a tc/t0450-harden later to maint). - (merge c25651aefd ds/midx-write-fixes later to maint). - (merge 069c15d256 rs/object-name-extend-abbrev-len-update later to maint). - (merge bf5c224537 mm/worktree-doc-typofix later to maint). - (merge 31397bc4f7 kh/doc-fast-import-markup-fix later to maint). - (merge ac7096723b jc/doc-includeif-hasconfig-remote-url-fix later to maint). - (merge fafc9b08b8 ag/doc-sendmail-gmail-example-update later to maint). (merge a66fc22bf9 rs/get-oid-with-flags-cleanup later to maint). - (merge e1d062e8ba ps/odb-clean-stale-wrappers later to maint). - (merge fdd21ba116 mh/doc-credential-url-prefix later to maint). - (merge 1c573a3451 en/doc-merge-tree-describe-merge-base later to maint). - (merge 84a6bf7965 ja/doc-markup-attached-paragraph-fix later to maint). - (merge 399694384b kh/doc-patch-id-markup-fix later to maint). + (merge 15b8abde07 js/mingw-includes-cleanup later to maint). + (merge 2cebca0582 tb/cat-file-objectmode-update later to maint). + (merge 8f487db07a kh/doc-patch-id-1 later to maint). + (merge f711f37b05 eb/t1016-hash-transition-fix later to maint). + (merge 85333aa1af jk/test-delete-gpgsig-leakfix later to maint). diff --git a/Documentation/SubmittingPatches b/Documentation/SubmittingPatches index d620bd93bd..e270ccbe85 100644 --- a/Documentation/SubmittingPatches +++ b/Documentation/SubmittingPatches @@ -446,6 +446,34 @@ highlighted above. Only capitalize the very first letter of the trailer, i.e. favor "Signed-off-by" over "Signed-Off-By" and "Acked-by:" over "Acked-By". +[[ai]] +=== Use of Artificial Intelligence (AI) + +The Developer's Certificate of Origin requires contributors to certify +that they know the origin of their contributions to the project and +that they have the right to submit it under the project's license. +It's not yet clear that this can be legally satisfied when submitting +significant amount of content that has been generated by AI tools. + +Another issue with AI generated content is that AIs still often +hallucinate or just produce bad code, commit messages, documentation +or output, even when you point out their mistakes. + +To avoid these issues, we will reject anything that looks AI +generated, that sounds overly formal or bloated, that looks like AI +slop, that looks good on the surface but makes no sense, or that +senders don’t understand or cannot explain. + +We strongly recommend using AI tools carefully and responsibly. + +Contributors would often benefit more from AI by using it to guide and +help them step by step towards producing a solution by themselves +rather than by asking for a full solution that they would then mostly +copy-paste. They can also use AI to help with debugging, or with +checking for obvious mistakes, things that can be improved, things +that don’t match our style, guidelines or our feedback, before sending +it to us. + [[git-tools]] === Generate your patch using Git tools out of your commits. diff --git a/Documentation/config/commitgraph.adoc b/Documentation/config/commitgraph.adoc index 7f8c9d6638..70a56c53d2 100644 --- a/Documentation/config/commitgraph.adoc +++ b/Documentation/config/commitgraph.adoc @@ -8,6 +8,17 @@ commitGraph.maxNewFilters:: Specifies the default value for the `--max-new-filters` option of `git commit-graph write` (c.f., linkgit:git-commit-graph[1]). +commitGraph.changedPaths:: + If true, then `git commit-graph write` will compute and write + changed-path Bloom filters by default, equivalent to passing + `--changed-paths`. If false or unset, changed-paths Bloom filters will + be written during `git commit-graph write` only if the filters already + exist in the current commit-graph file. This matches the default + behavior of `git commit-graph write` without any `--[no-]changed-paths` + option. To rewrite a commit-graph file without any filters, use the + `--no-changed-paths` option. Command-line option `--[no-]changed-paths` + always takes precedence over this configuration. Defaults to unset. + commitGraph.readChangedPaths:: Deprecated. Equivalent to commitGraph.changedPathsVersion=-1 if true, and commitGraph.changedPathsVersion=0 if false. (If commitGraph.changedPathVersion diff --git a/Documentation/config/core.adoc b/Documentation/config/core.adoc index 08739bb9d4..01202da7cd 100644 --- a/Documentation/config/core.adoc +++ b/Documentation/config/core.adoc @@ -75,8 +75,8 @@ The built-in file system monitor is currently available only on a limited set of supported platforms. Currently, this includes Windows and MacOS. + - Otherwise, this variable contains the pathname of the "fsmonitor" - hook command. +Otherwise, this variable contains the pathname of the "fsmonitor" +hook command. + This hook command is used to identify all files that may have changed since the requested date/time. This information is used to speed up @@ -290,6 +290,9 @@ core.preferSymlinkRefs:: and other symbolic reference files, use symbolic links. This is sometimes needed to work with old scripts that expect HEAD to be a symbolic link. ++ +This configuration is deprecated and will be removed in Git 3.0. Symbolic refs +will always be written as textual symrefs. core.alternateRefsCommand:: When advertising tips of available history from an alternate, use the shell to @@ -626,6 +629,8 @@ core.whitespace:: part of the line terminator, i.e. with it, `trailing-space` does not trigger if the character before such a carriage-return is not a whitespace (not enabled by default). +* `incomplete-line` treats the last line of a file that is missing the + newline at the end as an error (not enabled by default). * `tabwidth=<n>` tells how many character positions a tab occupies; this is relevant for `indent-with-non-tab` and when Git fixes `tab-in-indent` errors. The default tab width is 8. Allowed values are 1 to 63. diff --git a/Documentation/config/extensions.adoc b/Documentation/config/extensions.adoc index 532456644b..e33040fff5 100644 --- a/Documentation/config/extensions.adoc +++ b/Documentation/config/extensions.adoc @@ -73,6 +73,12 @@ relativeWorktrees::: repaired with either the `--relative-paths` option or with the `worktree.useRelativePaths` config set to `true`. +submoduleEncoding::: + If enabled, submodule gitdir paths are encoded to avoid filesystem + conflicts due to nested gitdirs, case insensitivity or other issues + When enabled, the submodule.<name>.gitdir config is always set for + all submodulesand is the single point of authority for gitdir paths. + worktreeConfig::: If enabled, then worktrees will load config settings from the `$GIT_DIR/config.worktree` file in addition to the diff --git a/Documentation/config/maintenance.adoc b/Documentation/config/maintenance.adoc index 2f71934218..d0c38f03fa 100644 --- a/Documentation/config/maintenance.adoc +++ b/Documentation/config/maintenance.adoc @@ -16,19 +16,36 @@ detach. maintenance.strategy:: This string config option provides a way to specify one of a few - recommended schedules for background maintenance. This only affects - which tasks are run during `git maintenance run --schedule=X` - commands, provided no `--task=<task>` arguments are provided. - Further, if a `maintenance.<task>.schedule` config value is set, - then that value is used instead of the one provided by - `maintenance.strategy`. The possible strategy strings are: + recommended strategies for repository maintenance. This affects + which tasks are run during `git maintenance run`, provided no + `--task=<task>` arguments are provided. This setting impacts manual + maintenance, auto-maintenance as well as scheduled maintenance. The + tasks that run may be different depending on the maintenance type. + -* `none`: This default setting implies no tasks are run at any schedule. +The maintenance strategy can be further tweaked by setting +`maintenance.<task>.enabled` and `maintenance.<task>.schedule`. If set, these +values are used instead of the defaults provided by `maintenance.strategy`. ++ +The possible strategies are: ++ +* `none`: This strategy implies no tasks are run at all. This is the default + strategy for scheduled maintenance. +* `gc`: This strategy runs the `gc` task. This is the default strategy for + manual maintenance. +* `geometric`: This strategy performs geometric repacking of packfiles and + keeps auxiliary data structures up-to-date. The strategy expires data in the + reflog and removes worktrees that cannot be located anymore. When the + geometric repacking strategy would decide to do an all-into-one repack, then + the strategy generates a cruft pack for all unreachable objects. Objects that + are already part of a cruft pack will be expired. ++ +This repacking strategy is a full replacement for the `gc` strategy and is +recommended for large repositories. * `incremental`: This setting optimizes for performing small maintenance activities that do not delete any data. This does not schedule the `gc` task, but runs the `prefetch` and `commit-graph` tasks hourly, the `loose-objects` and `incremental-repack` tasks daily, and the `pack-refs` - task weekly. + task weekly. Manual repository maintenance uses the `gc` task. maintenance.<task>.enabled:: This boolean config option controls whether the maintenance task @@ -75,6 +92,22 @@ maintenance.incremental-repack.auto:: number of pack-files not in the multi-pack-index is at least the value of `maintenance.incremental-repack.auto`. The default value is 10. +maintenance.geometric-repack.auto:: + This integer config option controls how often the `geometric-repack` + task should be run as part of `git maintenance run --auto`. If zero, + then the `geometric-repack` task will not run with the `--auto` + option. A negative value will force the task to run every time. + Otherwise, a positive value implies the command should run either when + there are packfiles that need to be merged together to retain the + geometric progression, or when there are at least this many loose + objects that would be written into a new packfile. The default value is + 100. + +maintenance.geometric-repack.splitFactor:: + This integer config option controls the factor used for the geometric + sequence. See the `--geometric=` option in linkgit:git-repack[1] for + more details. Defaults to `2`. + maintenance.reflog-expire.auto:: This integer config option controls how often the `reflog-expire` task should be run as part of `git maintenance run --auto`. If zero, then diff --git a/Documentation/config/replay.adoc b/Documentation/config/replay.adoc new file mode 100644 index 0000000000..7d549d2f0e --- /dev/null +++ b/Documentation/config/replay.adoc @@ -0,0 +1,11 @@ +replay.refAction:: + Specifies the default mode for handling reference updates in + `git replay`. The value can be: ++ +-- + * `update`: Update refs directly using an atomic transaction (default behavior). + * `print`: Output update-ref commands for pipeline use. +-- ++ +This setting can be overridden with the `--ref-action` command-line option. +When not configured, `git replay` defaults to `update` mode. diff --git a/Documentation/config/stash.adoc b/Documentation/config/stash.adoc index 7fc32027f7..a1197ffd7d 100644 --- a/Documentation/config/stash.adoc +++ b/Documentation/config/stash.adoc @@ -11,6 +11,10 @@ endif::git-stash[] behave as if `--index` was supplied. Defaults to false. ifndef::git-stash[] See the descriptions in linkgit:git-stash[1]. ++ +This also affects invocations of linkgit:git-stash[1] via `--autostash` from +commands like linkgit:git-merge[1], linkgit:git-rebase[1], and +linkgit:git-pull[1]. endif::git-stash[] `stash.showIncludeUntracked`:: diff --git a/Documentation/config/submodule.adoc b/Documentation/config/submodule.adoc index 0672d99117..ddaadc3dc5 100644 --- a/Documentation/config/submodule.adoc +++ b/Documentation/config/submodule.adoc @@ -52,6 +52,11 @@ submodule.<name>.active:: submodule.active config option. See linkgit:gitsubmodules[7] for details. +submodule.<name>.gitdir:: + This option sets the gitdir path for submodule <name>, allowing users to + override the default path. Only works when `extensions.submoduleEncoding` + is enabled, otherwise does nothing. See linkgit:git-config[1] for details. + submodule.active:: A repeated field which contains a pathspec used to match against a submodule's path to determine if the submodule is of interest to git diff --git a/Documentation/diff-algorithm-option.adoc b/Documentation/diff-algorithm-option.adoc new file mode 100644 index 0000000000..8e3a0b63d7 --- /dev/null +++ b/Documentation/diff-algorithm-option.adoc @@ -0,0 +1,20 @@ +`--diff-algorithm=(patience|minimal|histogram|myers)`:: + Choose a diff algorithm. The variants are as follows: ++ +-- + `default`;; + `myers`;; + The basic greedy diff algorithm. Currently, this is the default. + `minimal`;; + Spend extra time to make sure the smallest possible diff is + produced. + `patience`;; + Use "patience diff" algorithm when generating patches. + `histogram`;; + This algorithm extends the patience algorithm to "support + low-occurrence common elements". +-- ++ +For instance, if you configured the `diff.algorithm` variable to a +non-default value and want to use the default one, then you +have to use `--diff-algorithm=default` option. diff --git a/Documentation/diff-options.adoc b/Documentation/diff-options.adoc index ae31520f7f..9cdad6f72a 100644 --- a/Documentation/diff-options.adoc +++ b/Documentation/diff-options.adoc @@ -197,26 +197,7 @@ and starts with _<text>_, this algorithm attempts to prevent it from appearing as a deletion or addition in the output. It uses the "patience diff" algorithm internally. -`--diff-algorithm=(patience|minimal|histogram|myers)`:: - Choose a diff algorithm. The variants are as follows: -+ --- - `default`;; - `myers`;; - The basic greedy diff algorithm. Currently, this is the default. - `minimal`;; - Spend extra time to make sure the smallest possible diff is - produced. - `patience`;; - Use "patience diff" algorithm when generating patches. - `histogram`;; - This algorithm extends the patience algorithm to "support - low-occurrence common elements". --- -+ -For instance, if you configured the `diff.algorithm` variable to a -non-default value and want to use the default one, then you -have to use `--diff-algorithm=default` option. +include::diff-algorithm-option.adoc[] `--stat[=<width>[,<name-width>[,<count>]]]`:: Generate a diffstat. By default, as much space as necessary diff --git a/Documentation/fsck-msgids.adoc b/Documentation/fsck-msgids.adoc index 81f11ba125..acac9683af 100644 --- a/Documentation/fsck-msgids.adoc +++ b/Documentation/fsck-msgids.adoc @@ -10,6 +10,12 @@ `badFilemode`:: (INFO) A tree contains a bad filemode entry. +`badGpgsig`:: + (ERROR) A tag contains a bad (truncated) signature (e.g., `gpgsig`) header. + +`badHeaderContinuation`:: + (ERROR) A continuation header (such as for `gpgsig`) is unexpectedly truncated. + `badName`:: (ERROR) An author/committer name is empty. diff --git a/Documentation/git-add.adoc b/Documentation/git-add.adoc index 3116a2cac5..6192daeb03 100644 --- a/Documentation/git-add.adoc +++ b/Documentation/git-add.adoc @@ -349,6 +349,7 @@ patch:: s - split the current hunk into smaller hunks e - manually edit the current hunk p - print the current hunk + P - print the current hunk using the pager ? - print help + After deciding the fate for all hunks, if there is any hunk diff --git a/Documentation/git-bisect.adoc b/Documentation/git-bisect.adoc index 58dbb74a15..b0078dda0e 100644 --- a/Documentation/git-bisect.adoc +++ b/Documentation/git-bisect.adoc @@ -9,26 +9,22 @@ git-bisect - Use binary search to find the commit that introduced a bug SYNOPSIS -------- [verse] -'git bisect' <subcommand> <options> +'git bisect' start [--term-(bad|new)=<term-new> --term-(good|old)=<term-old>] + [--no-checkout] [--first-parent] [<bad> [<good>...]] [--] [<pathspec>...] +'git bisect' (bad|new|<term-new>) [<rev>] +'git bisect' (good|old|<term-old>) [<rev>...] +'git bisect' terms [--term-(good|old) | --term-(bad|new)] +'git bisect' skip [(<rev>|<range>)...] +'git bisect' next +'git bisect' reset [<commit>] +'git bisect' (visualize|view) +'git bisect' replay <logfile> +'git bisect' log +'git bisect' run <cmd> [<arg>...] +'git bisect' help DESCRIPTION ----------- -The command takes various subcommands, and different options depending -on the subcommand: - - git bisect start [--term-(bad|new)=<term-new> --term-(good|old)=<term-old>] - [--no-checkout] [--first-parent] [<bad> [<good>...]] [--] [<pathspec>...] - git bisect (bad|new|<term-new>) [<rev>] - git bisect (good|old|<term-old>) [<rev>...] - git bisect terms [--term-(good|old) | --term-(bad|new)] - git bisect skip [(<rev>|<range>)...] - git bisect reset [<commit>] - git bisect (visualize|view) - git bisect replay <logfile> - git bisect log - git bisect run <cmd> [<arg>...] - git bisect help - This command uses a binary search algorithm to find which commit in your project's history introduced a bug. You use it by first telling it a "bad" commit that is known to contain the bug, and a "good" @@ -295,6 +291,19 @@ $ git bisect skip v2.5 v2.5..v2.6 This tells the bisect process that the commits between `v2.5` and `v2.6` (inclusive) should be skipped. +Bisect next +~~~~~~~~~~~ + +Normally, after marking a revision as good or bad, Git automatically +computes and checks out the next revision to test. However, if you need to +explicitly request the next bisection step, you can use: + +------------ +$ git bisect next +------------ + +You might use this to resume the bisection process after interrupting it +by checking out a different revision. Cutting down bisection by giving more parameters to bisect start ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/Documentation/git-blame.adoc b/Documentation/git-blame.adoc index e438d28625..adcbb6f5dc 100644 --- a/Documentation/git-blame.adoc +++ b/Documentation/git-blame.adoc @@ -85,6 +85,8 @@ include::blame-options.adoc[] Ignore whitespace when comparing the parent's version and the child's to find where the lines came from. +include::diff-algorithm-option.adoc[] + --abbrev=<n>:: Instead of using the default 7+1 hexadecimal digits as the abbreviated object name, use <m>+1 digits, where <m> is at diff --git a/Documentation/git-checkout.adoc b/Documentation/git-checkout.adoc index 431185ca0b..6f281b298e 100644 --- a/Documentation/git-checkout.adoc +++ b/Documentation/git-checkout.adoc @@ -61,7 +61,7 @@ uncommitted changes. `git checkout -B <branch> [<start-point>]`:: The same as `-b`, except that if the branch already exists it - resets `_<branch>_` to the start point instead of failing. + resets _<branch>_ to the start point instead of failing. `git checkout --detach [<branch>]`:: `git checkout [--detach] <commit>`:: @@ -155,7 +155,7 @@ of it"). `-B <new-branch>`:: The same as `-b`, except that if the branch already exists it - resets `_<branch>_` to the start point instead of failing. + resets _<branch>_ to the start point instead of failing. `-t`:: `--track[=(direct|inherit)]`:: diff --git a/Documentation/git-commit-graph.adoc b/Documentation/git-commit-graph.adoc index e9558173c0..6d19026035 100644 --- a/Documentation/git-commit-graph.adoc +++ b/Documentation/git-commit-graph.adoc @@ -71,7 +71,7 @@ take a while on large repositories. It provides significant performance gains for getting history of a directory or a file with `git log -- <path>`. If this option is given, future commit-graph writes will automatically assume that this option was intended. Use `--no-changed-paths` to stop storing this -data. +data. `--changed-paths` is implied by config `commitGraph.changedPaths=true`. + With the `--max-new-filters=<n>` option, generate at most `n` new Bloom filters (if `--changed-paths` is specified). If `n` is `-1`, no limit is diff --git a/Documentation/git-config.adoc b/Documentation/git-config.adoc index 36d2845152..cc054fa7e1 100644 --- a/Documentation/git-config.adoc +++ b/Documentation/git-config.adoc @@ -117,15 +117,15 @@ OPTIONS --comment <message>:: Append a comment at the end of new or modified lines. - - If _<message>_ begins with one or more whitespaces followed - by "#", it is used as-is. If it begins with "#", a space is - prepended before it is used. Otherwise, a string " # " (a - space followed by a hash followed by a space) is prepended - to it. And the resulting string is placed immediately after - the value defined for the variable. The _<message>_ must - not contain linefeed characters (no multi-line comments are - permitted). ++ +If _<message>_ begins with one or more whitespaces followed +by "#", it is used as-is. If it begins with "#", a space is +prepended before it is used. Otherwise, a string " # " (a +space followed by a hash followed by a space) is prepended +to it. And the resulting string is placed immediately after +the value defined for the variable. The _<message>_ must +not contain linefeed characters (no multi-line comments are +permitted). --all:: With `get`, return all values for a multi-valued key. diff --git a/Documentation/git-fast-import.adoc b/Documentation/git-fast-import.adoc index 85ed7a7270..b74179a6c8 100644 --- a/Documentation/git-fast-import.adoc +++ b/Documentation/git-fast-import.adoc @@ -66,6 +66,11 @@ fast-import stream! This option is enabled automatically for remote-helpers that use the `import` capability, as they are already trusted to run their own code. +--signed-tags=(verbatim|warn-verbatim|warn-strip|strip|abort):: + Specify how to handle signed tags. Behaves in the same way + as the same option in linkgit:git-fast-export[1], except that + default is 'verbatim' (instead of 'abort'). + --signed-commits=(verbatim|warn-verbatim|warn-strip|strip|abort):: Specify how to handle signed commits. Behaves in the same way as the same option in linkgit:git-fast-export[1], except that diff --git a/Documentation/git-history.adoc b/Documentation/git-history.adoc new file mode 100644 index 0000000000..3d6b2665f8 --- /dev/null +++ b/Documentation/git-history.adoc @@ -0,0 +1,111 @@ +git-history(1) +============== + +NAME +---- +git-history - EXPERIMENTAL: Rewrite history of the current branch + +SYNOPSIS +-------- +[synopsis] +git history reword <commit> +git history split <commit> [--] [<pathspec>...] + +DESCRIPTION +----------- + +Rewrite history by rearranging or modifying specific commits in the +history. + +THIS COMMAND IS EXPERIMENTAL. THE BEHAVIOR MAY CHANGE. + +This command is similar to linkgit:git-rebase[1] and uses the same +underlying machinery. You should use rebases if you want to reapply a range of +commits onto a different base, or interactive rebases if you want to edit a +range of commits. + +Note that this command does not (yet) work with histories that contain +merges. You should use linkgit:git-rebase[1] with the `--rebase-merges` +flag instead. + +COMMANDS +-------- + +Several commands are available to rewrite history in different ways: + +`reword <commit>`:: + Rewrite the commit message of the specified commit. All the other + details of this commit remain unchanged. This command will spawn an + editor with the current message of that commit. + +`split <commit> [--] [<pathspec>...]`:: + Interactively split up <commit> into two commits by choosing + hunks introduced by it that will be moved into the new split-out + commit. These hunks will then be written into a new commit that + becomes the parent of the previous commit. The original commit + stays intact, except that its parent will be the newly split-out + commit. ++ +The commit message of the new commit will be asked for by launching the +configured editor. Authorship of the commit will be the same as for the +original commit. ++ +If passed, _<pathspec>_ can be used to limit which changes shall be split out +of the original commit. Files not matching any of the pathspecs will remain +part of the original commit. For more details, see the 'pathspec' entry in +linkgit:gitglossary[7]. ++ +It is invalid to select either all or no hunks, as that would lead to +one of the commits becoming empty. + +CONFIGURATION +------------- + +include::includes/cmd-config-section-all.adoc[] + +include::config/sequencer.adoc[] + +EXAMPLES +-------- + +Split a commit +~~~~~~~~~~~~~~ + +---------- +$ git log --stat --oneline +3f81232 (HEAD -> main) original + bar | 1 + + foo | 1 + + 2 files changed, 2 insertions(+) + +$ git history split HEAD +diff --git a/bar b/bar +new file mode 100644 +index 0000000..5716ca5 +--- /dev/null ++++ b/bar +@@ -0,0 +1 @@ ++bar +(1/1) Stage addition [y,n,q,a,d,e,p,?]? y + +diff --git a/foo b/foo +new file mode 100644 +index 0000000..257cc56 +--- /dev/null ++++ b/foo +@@ -0,0 +1 @@ ++foo +(1/1) Stage addition [y,n,q,a,d,e,p,?]? n + +$ git log --stat --oneline +7cebe64 (HEAD -> main) original + foo | 1 + + 1 file changed, 1 insertion(+) +d1582f3 split-out commit + bar | 1 + + 1 file changed, 1 insertion(+) +---------- + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/git-maintenance.adoc b/Documentation/git-maintenance.adoc index 540b5cf68b..37939510d4 100644 --- a/Documentation/git-maintenance.adoc +++ b/Documentation/git-maintenance.adoc @@ -12,6 +12,7 @@ SYNOPSIS 'git maintenance' run [<options>] 'git maintenance' start [--scheduler=<scheduler>] 'git maintenance' (stop|register|unregister) [<options>] +'git maintenance' is-needed [<options>] DESCRIPTION @@ -84,6 +85,16 @@ The `unregister` subcommand will report an error if the current repository is not already registered. Use the `--force` option to return success even when the current repository is not registered. +is-needed:: + Check whether maintenance needs to be run without actually running it. + Exits with a 0 status code if maintenance needs to be run, 1 otherwise. + Ideally used with the '--auto' flag. ++ +If one or more `--task` options are specified, then those tasks are checked +in that order. Otherwise, the tasks are determined by which +`maintenance.<task>.enabled` config options are true. By default, only +`maintenance.gc.enabled` is true. + TASKS ----- @@ -183,6 +194,8 @@ OPTIONS in the `gc.auto` config setting, or when the number of pack-files exceeds the `gc.autoPackLimit` config setting. Not compatible with the `--schedule` option. + When combined with the `is-needed` subcommand, check if the required + thresholds are met without actually running maintenance. --schedule:: When combined with the `run` subcommand, run maintenance tasks diff --git a/Documentation/git-patch-id.adoc b/Documentation/git-patch-id.adoc index 45da0f27ac..92a1af36a2 100644 --- a/Documentation/git-patch-id.adoc +++ b/Documentation/git-patch-id.adoc @@ -7,8 +7,8 @@ git-patch-id - Compute unique ID for a patch SYNOPSIS -------- -[verse] -'git patch-id' [--stable | --unstable | --verbatim] +[synopsis] +git patch-id [--stable | --unstable | --verbatim] DESCRIPTION ----------- @@ -21,7 +21,7 @@ the same time also reasonably unique, i.e., two patches that have the same The main usecase for this command is to look for likely duplicate commits. -When dealing with 'git diff-tree' output, it takes advantage of +When dealing with `git diff-tree` output, it takes advantage of the fact that the patch is prefixed with the object name of the commit, and outputs two 40-byte hexadecimal strings. The first string is the patch ID, and the second string is the commit ID. @@ -30,35 +30,35 @@ This can be used to make a mapping from patch ID to commit ID. OPTIONS ------- ---verbatim:: +`--verbatim`:: Calculate the patch-id of the input as it is given, do not strip any whitespace. + -This is the default if patchid.verbatim is true. +This is the default if `patchid.verbatim` is `true`. ---stable:: +`--stable`:: Use a "stable" sum of hashes as the patch ID. With this option: + -- - Reordering file diffs that make up a patch does not affect the ID. In particular, two patches produced by comparing the same two trees - with two different settings for "-O<orderfile>" result in the same + with two different settings for `-O<orderfile>` result in the same patch ID signature, thereby allowing the computed result to be used as a key to index some meta-information about the change between the two trees; - Result is different from the value produced by git 1.9 and older - or produced when an "unstable" hash (see --unstable below) is + or produced when an "unstable" hash (see `--unstable` below) is configured - even when used on a diff output taken without any use - of "-O<orderfile>", thereby making existing databases storing such + of `-O<orderfile>`, thereby making existing databases storing such "unstable" or historical patch-ids unusable. - All whitespace within the patch is ignored and does not affect the id. -- + -This is the default if patchid.stable is set to true. +This is the default if `patchid.stable` is set to `true`. ---unstable:: +`--unstable`:: Use an "unstable" hash as the patch ID. With this option, the result produced is compatible with the patch-id value produced by git 1.9 and older and whitespace is ignored. Users with pre-existing diff --git a/Documentation/git-pull.adoc b/Documentation/git-pull.adoc index 48e924a10a..cd3bbc90e3 100644 --- a/Documentation/git-pull.adoc +++ b/Documentation/git-pull.adoc @@ -15,68 +15,54 @@ SYNOPSIS DESCRIPTION ----------- -Incorporates changes from a remote repository into the current branch. -If the current branch is behind the remote, then by default it will -fast-forward the current branch to match the remote. If the current -branch and the remote have diverged, the user needs to specify how to -reconcile the divergent branches with `--rebase` or `--no-rebase` (or -the corresponding configuration option in `pull.rebase`). - -More precisely, `git pull` runs `git fetch` with the given parameters -and then depending on configuration options or command line flags, -will call either `git rebase` or `git merge` to reconcile diverging -branches. - -<repository> should be the name of a remote repository as -passed to linkgit:git-fetch[1]. <refspec> can name an -arbitrary remote ref (for example, the name of a tag) or even -a collection of refs with corresponding remote-tracking branches -(e.g., refs/heads/{asterisk}:refs/remotes/origin/{asterisk}), -but usually it is the name of a branch in the remote repository. - -Default values for <repository> and <branch> are read from the -"remote" and "merge" configuration for the current branch -as set by linkgit:git-branch[1] `--track`. - -Assume the following history exists and the current branch is -"`master`": +Integrate changes from a remote repository into the current branch. ------------- - A---B---C master on origin - / - D---E---F---G master - ^ - origin/master in your repository ------------- +First, `git pull` runs `git fetch` with the same arguments +(excluding merge options) to fetch remote branch(es). +Then it decides which remote branch to integrate: if you run `git pull` +with no arguments this defaults to the <<UPSTREAM-BRANCHES,upstream>> +for the current branch. +Then it integrates that branch into the current branch. -Then "`git pull`" will fetch and replay the changes from the remote -`master` branch since it diverged from the local `master` (i.e., `E`) -until its current commit (`C`) on top of `master` and record the -result in a new commit along with the names of the two parent commits -and a log message from the user describing the changes. - ------------- - A---B---C origin/master - / \ - D---E---F---G---H master ------------- +There are 4 main options for integrating the remote branch: -See linkgit:git-merge[1] for details, including how conflicts -are presented and handled. +1. `git pull --ff-only` will only do "fast-forward" updates: it + fails if your local branch has diverged from the remote branch. + This is the default. +2. `git pull --rebase` runs `git rebase` +3. `git pull --no-rebase` runs `git merge`. +4. `git pull --squash` runs `git merge --squash` -In Git 1.7.0 or later, to cancel a conflicting merge, use -`git reset --merge`. *Warning*: In older versions of Git, running 'git pull' -with uncommitted changes is discouraged: while possible, it leaves you -in a state that may be hard to back out of in the case of a conflict. +You can also set the configuration options `pull.rebase`, `pull.squash`, +or `pull.ff` with your preferred behaviour. -If any of the remote changes overlap with local uncommitted changes, -the merge will be automatically canceled and the work tree untouched. -It is generally best to get any local changes in working order before -pulling or stash them away with linkgit:git-stash[1]. +If there's a merge conflict during the merge or rebase that you don't +want to handle, you can safely abort it with `git merge --abort` or `git +--rebase abort`. OPTIONS ------- +<repository>:: + The "remote" repository to pull from. This can be either + a URL (see the section <<URLS,GIT URLS>> below) or the name + of a remote (see the section <<REMOTES,REMOTES>> below). ++ +Defaults to the configured upstream for the current branch, or `origin`. +See <<UPSTREAM-BRANCHES,UPSTREAM BRANCHES>> below for more on how to +configure upstreams. + +<refspec>:: + Which branch or other reference(s) to fetch and integrate into the + current branch, for example `main` in `git pull origin main`. + Defaults to the configured upstream for the current branch. ++ +This can be a branch, tag, or other collection of reference(s). +See <<fetch-refspec,<refspec>>> below under "Options related to fetching" +for the full syntax, and <<DEFAULT-BEHAVIOUR,DEFAULT BEHAVIOUR>> below +for how `git pull` uses this argument to determine which remote branch +to integrate. + -q:: --quiet:: This is passed to both underlying git-fetch to squelch reporting of @@ -145,6 +131,7 @@ include::urls-remotes.adoc[] include::merge-strategies.adoc[] +[[DEFAULT-BEHAVIOUR]] DEFAULT BEHAVIOUR ----------------- diff --git a/Documentation/git-rebase.adoc b/Documentation/git-rebase.adoc index 005caf6164..4d2fe4be6e 100644 --- a/Documentation/git-rebase.adoc +++ b/Documentation/git-rebase.adoc @@ -487,9 +487,16 @@ See also INCOMPATIBLE OPTIONS below. Add a `Signed-off-by` trailer to all the rebased commits. Note that if `--interactive` is given then only commits marked to be picked, edited or reworded will have the trailer added. -+ + See also INCOMPATIBLE OPTIONS below. +--trailer=<trailer>:: + Append the given trailer line(s) to every rebased commit + message, processed via linkgit:git-interpret-trailers[1]. + When this option is present *rebase automatically implies* + `--force-rebase` so that fast‑forwarded commits are also + rewritten. + -i:: --interactive:: Make a list of the commits which are about to be rebased. Let the diff --git a/Documentation/git-replay.adoc b/Documentation/git-replay.adoc index 0b12bf8aa4..dcb26e8a8e 100644 --- a/Documentation/git-replay.adoc +++ b/Documentation/git-replay.adoc @@ -9,15 +9,16 @@ git-replay - EXPERIMENTAL: Replay commits on a new base, works with bare repos t SYNOPSIS -------- [verse] -(EXPERIMENTAL!) 'git replay' ([--contained] --onto <newbase> | --advance <branch>) <revision-range>... +(EXPERIMENTAL!) 'git replay' ([--contained] --onto <newbase> | --advance <branch>) [--ref-action[=<mode>]] <revision-range>... DESCRIPTION ----------- Takes ranges of commits and replays them onto a new location. Leaves -the working tree and the index untouched, and updates no references. -The output of this command is meant to be used as input to -`git update-ref --stdin`, which would update the relevant branches +the working tree and the index untouched. By default, updates the +relevant references using an atomic transaction (all refs update or +none). Use `--ref-action=print` to avoid automatic ref updates and +instead get update commands that can be piped to `git update-ref --stdin` (see the OUTPUT section below). THIS COMMAND IS EXPERIMENTAL. THE BEHAVIOR MAY CHANGE. @@ -29,18 +30,29 @@ OPTIONS Starting point at which to create the new commits. May be any valid commit, and not just an existing branch name. + -When `--onto` is specified, the update-ref command(s) in the output will -update the branch(es) in the revision range to point at the new -commits, similar to the way how `git rebase --update-refs` updates -multiple branches in the affected range. +When `--onto` is specified, the branch(es) in the revision range will be +updated to point at the new commits, similar to the way `git rebase --update-refs` +updates multiple branches in the affected range. --advance <branch>:: Starting point at which to create the new commits; must be a branch name. + -When `--advance` is specified, the update-ref command(s) in the output -will update the branch passed as an argument to `--advance` to point at -the new commits (in other words, this mimics a cherry-pick operation). +The history is replayed on top of the <branch> and <branch> is updated to +point at the tip of the resulting history. This is different from `--onto`, +which uses the target only as a starting point without updating it. + +--ref-action[=<mode>]:: + Control how references are updated. The mode can be: ++ +-- + * `update` (default): Update refs directly using an atomic transaction. + All refs are updated or none are (all-or-nothing behavior). + * `print`: Output update-ref commands for pipeline use. This is the + traditional behavior where output can be piped to `git update-ref --stdin`. +-- ++ +The default mode can be configured via the `replay.refAction` configuration variable. <revision-range>:: Range of commits to replay. More than one <revision-range> can @@ -54,8 +66,11 @@ include::rev-list-options.adoc[] OUTPUT ------ -When there are no conflicts, the output of this command is usable as -input to `git update-ref --stdin`. It is of the form: +By default, or with `--ref-action=update`, this command produces no output on +success, as refs are updated directly using an atomic transaction. + +When using `--ref-action=print`, the output is usable as input to +`git update-ref --stdin`. It is of the form: update refs/heads/branch1 ${NEW_branch1_HASH} ${OLD_branch1_HASH} update refs/heads/branch2 ${NEW_branch2_HASH} ${OLD_branch2_HASH} @@ -81,6 +96,14 @@ To simply rebase `mybranch` onto `target`: ------------ $ git replay --onto target origin/main..mybranch +------------ + +The refs are updated atomically and no output is produced on success. + +To see what would be updated without actually updating: + +------------ +$ git replay --ref-action=print --onto target origin/main..mybranch update refs/heads/mybranch ${NEW_mybranch_HASH} ${OLD_mybranch_HASH} ------------ @@ -88,33 +111,29 @@ To cherry-pick the commits from mybranch onto target: ------------ $ git replay --advance target origin/main..mybranch -update refs/heads/target ${NEW_target_HASH} ${OLD_target_HASH} ------------ Note that the first two examples replay the exact same commits and on top of the exact same new base, they only differ in that the first -provides instructions to make mybranch point at the new commits and -the second provides instructions to make target point at them. +updates mybranch to point at the new commits and the second updates +target to point at them. What if you have a stack of branches, one depending upon another, and you'd really like to rebase the whole set? ------------ $ git replay --contained --onto origin/main origin/main..tipbranch -update refs/heads/branch1 ${NEW_branch1_HASH} ${OLD_branch1_HASH} -update refs/heads/branch2 ${NEW_branch2_HASH} ${OLD_branch2_HASH} -update refs/heads/tipbranch ${NEW_tipbranch_HASH} ${OLD_tipbranch_HASH} ------------ +All three branches (`branch1`, `branch2`, and `tipbranch`) are updated +atomically. + When calling `git replay`, one does not need to specify a range of commits to replay using the syntax `A..B`; any range expression will do: ------------ $ git replay --onto origin/main ^base branch1 branch2 branch3 -update refs/heads/branch1 ${NEW_branch1_HASH} ${OLD_branch1_HASH} -update refs/heads/branch2 ${NEW_branch2_HASH} ${OLD_branch2_HASH} -update refs/heads/branch3 ${NEW_branch3_HASH} ${OLD_branch3_HASH} ------------ This will simultaneously rebase `branch1`, `branch2`, and `branch3`, diff --git a/Documentation/git-repo.adoc b/Documentation/git-repo.adoc index 209afd1b61..70f0a6d2e4 100644 --- a/Documentation/git-repo.adoc +++ b/Documentation/git-repo.adoc @@ -8,7 +8,8 @@ git-repo - Retrieve information about the repository SYNOPSIS -------- [synopsis] -git repo info [--format=(keyvalue|nul)] [-z] [<key>...] +git repo info [--format=(keyvalue|nul)] [-z] [--all | <key>...] +git repo structure [--format=(table|keyvalue|nul)] DESCRIPTION ----------- @@ -18,13 +19,13 @@ THIS COMMAND IS EXPERIMENTAL. THE BEHAVIOR MAY CHANGE. COMMANDS -------- -`info [--format=(keyvalue|nul)] [-z] [<key>...]`:: +`info [--format=(keyvalue|nul)] [-z] [--all | <key>...]`:: Retrieve metadata-related information about the current repository. Only the requested data will be returned based on their keys (see "INFO KEYS" section below). + The values are returned in the same order in which their respective keys were -requested. +requested. The `--all` flag requests the values for all the available keys. + The output format can be chosen through the flag `--format`. Two formats are supported: @@ -43,6 +44,35 @@ supported: + `-z` is an alias for `--format=nul`. +`structure [--format=(table|keyvalue|nul)]`:: + Retrieve statistics about the current repository structure. The + following kinds of information are reported: ++ +* Reference counts categorized by type +* Reachable object counts categorized by type + ++ +The output format can be chosen through the flag `--format`. Three formats are +supported: ++ +`table`::: + Outputs repository stats in a human-friendly table. This format may + change and is not intended for machine parsing. This is the default + format. + +`keyvalue`::: + Each line of output contains a key-value pair for a repository stat. + The '=' character is used to delimit between the key and the value. + Values containing "unusual" characters are quoted as explained for the + configuration variable `core.quotePath` (see linkgit:git-config[1]). + +`nul`::: + Similar to `keyvalue`, but uses a NUL character to delimit between + key-value pairs instead of a newline. Also uses a newline character as + the delimiter between the key and value instead of '='. Unlike the + `keyvalue` format, values containing "unusual" characters are never + quoted. + INFO KEYS --------- In order to obtain a set of values from `git repo info`, you should provide diff --git a/Documentation/git-rev-parse.adoc b/Documentation/git-rev-parse.adoc index cc32b4b4f0..5398691f3f 100644 --- a/Documentation/git-rev-parse.adoc +++ b/Documentation/git-rev-parse.adoc @@ -174,13 +174,13 @@ for another option. Allow oids to be input from any object format that the current repository supports. - - Specifying "sha1" translates if necessary and returns a sha1 oid. - - Specifying "sha256" translates if necessary and returns a sha256 oid. - - Specifying "storage" translates if necessary and returns an oid in - encoded in the storage hash algorithm. ++ +Specifying "sha1" translates if necessary and returns a sha1 oid. ++ +Specifying "sha256" translates if necessary and returns a sha256 oid. ++ +Specifying "storage" translates if necessary and returns an oid in +encoded in the storage hash algorithm. Options for Objects ~~~~~~~~~~~~~~~~~~~ @@ -324,11 +324,12 @@ The following options are unaffected by `--path-format`: path of the current directory relative to the top-level directory. ---show-object-format[=(storage|input|output)]:: - Show the object format (hash algorithm) used for the repository - for storage inside the `.git` directory, input, or output. For - input, multiple algorithms may be printed, space-separated. - If not specified, the default is "storage". +--show-object-format[=(storage|input|output|compat)]:: + Show the object format (hash algorithm) used for the repository for storage + inside the `.git` directory, input, output, or compatibility. For input, + multiple algorithms may be printed, space-separated. If `compat` is + requested and no compatibility algorithm is enabled, prints an empty line. If + not specified, the default is "storage". --show-ref-format:: Show the reference storage format used for the repository. diff --git a/Documentation/git-shortlog.adoc b/Documentation/git-shortlog.adoc index d8ab38dcc1..aa92800c69 100644 --- a/Documentation/git-shortlog.adoc +++ b/Documentation/git-shortlog.adoc @@ -44,8 +44,8 @@ OPTIONS describe each commit. '<format>' can be any string accepted by the `--format` option of 'git log', such as '* [%h] %s'. (See the "PRETTY FORMATS" section of linkgit:git-log[1].) - - Each pretty-printed commit will be rewrapped before it is shown. ++ +Each pretty-printed commit will be rewrapped before it is shown. --date=<format>:: Show dates formatted according to the given date string. (See diff --git a/Documentation/git-sparse-checkout.adoc b/Documentation/git-sparse-checkout.adoc index 529a8edd9c..0d1618f161 100644 --- a/Documentation/git-sparse-checkout.adoc +++ b/Documentation/git-sparse-checkout.adoc @@ -9,7 +9,7 @@ git-sparse-checkout - Reduce your working tree to a subset of tracked files SYNOPSIS -------- [verse] -'git sparse-checkout' (init | list | set | add | reapply | disable | check-rules) [<options>] +'git sparse-checkout' (init | list | set | add | reapply | disable | check-rules | clean) [<options>] DESCRIPTION @@ -111,6 +111,37 @@ flags, with the same meaning as the flags from the `set` command, in order to change which sparsity mode you are using without needing to also respecify all sparsity paths. +'clean':: + Opportunistically remove files outside of the sparse-checkout + definition. This command requires cone mode to use recursive + directory matches to determine which files should be removed. A + file is considered for removal if it is contained within a tracked + directory that is outside of the sparse-checkout definition. ++ +Some special cases, such as merge conflicts or modified files outside of +the sparse-checkout definition could lead to keeping files that would +otherwise be removed. Resolve conflicts, stage modifications, and use +`git sparse-checkout reapply` in conjunction with `git sparse-checkout +clean` to resolve these cases. ++ +This command can be used to be sure the sparse index works efficiently, +though it does not require enabling the sparse index feature via the +`index.sparse=true` configuration. ++ +To prevent accidental deletion of worktree files, the `clean` subcommand +will not delete any files without the `-f` or `--force` option, unless +the `clean.requireForce` config option is set to `false`. ++ +The `--dry-run` option will list the directories that would be removed +without deleting them. Running in this mode can be helpful to predict the +behavior of the clean comand or to determine which kinds of files are left +in the sparse directories. ++ +The `--verbose` option will list every file within the directories that +are considered for removal. This option is helpful to determine if those +files are actually important or perhaps to explain why the directory is +still present despite the current sparse-checkout. + 'disable':: Disable the `core.sparseCheckout` config setting, and restore the working directory to include all files. @@ -264,34 +295,50 @@ patterns in non-cone mode has a number of shortcomings: inconsistent. * It has edge cases where the "right" behavior is unclear. Two examples: - - First, two users are in a subdirectory, and the first runs - git sparse-checkout set '/toplevel-dir/*.c' - while the second runs - git sparse-checkout set relative-dir - Should those arguments be transliterated into - current/subdirectory/toplevel-dir/*.c - and - current/subdirectory/relative-dir - before inserting into the sparse-checkout file? The user who typed - the first command is probably aware that arguments to set/add are - supposed to be patterns in non-cone mode, and probably would not be - happy with such a transliteration. However, many gitignore-style - patterns are just paths, which might be what the user who typed the - second command was thinking, and they'd be upset if their argument - wasn't transliterated. - - Second, what should bash-completion complete on for set/add commands - for non-cone users? If it suggests paths, is it exacerbating the - problem above? Also, if it suggests paths, what if the user has a - file or directory that begins with either a '!' or '#' or has a '*', - '\', '?', '[', or ']' in its name? And if it suggests paths, will - it complete "/pro" to "/proc" (in the root filesystem) rather than to - "/progress.txt" in the current directory? (Note that users are - likely to want to start paths with a leading '/' in non-cone mode, - for the same reason that .gitignore files often have one.) - Completing on files or directories might give nasty surprises in - all these cases. ++ +First, two users are in a subdirectory, and the first runs ++ +---- +git sparse-checkout set '/toplevel-dir/*.c' +---- ++ +while the second runs ++ +---- +git sparse-checkout set relative-dir +---- ++ +Should those arguments be transliterated into ++ +---- +current/subdirectory/toplevel-dir/*.c +---- ++ +and ++ +---- +current/subdirectory/relative-dir +---- ++ +before inserting into the sparse-checkout file? The user who typed +the first command is probably aware that arguments to set/add are +supposed to be patterns in non-cone mode, and probably would not be +happy with such a transliteration. However, many gitignore-style +patterns are just paths, which might be what the user who typed the +second command was thinking, and they'd be upset if their argument +wasn't transliterated. ++ +Second, what should bash-completion complete on for set/add commands +for non-cone users? If it suggests paths, is it exacerbating the +problem above? Also, if it suggests paths, what if the user has a +file or directory that begins with either a '!' or '#' or has a '*', +'\', '?', '[', or ']' in its name? And if it suggests paths, will +it complete "/pro" to "/proc" (in the root filesystem) rather than to +"/progress.txt" in the current directory? (Note that users are +likely to want to start paths with a leading '/' in non-cone mode, +for the same reason that .gitignore files often have one.) +Completing on files or directories might give nasty surprises in +all these cases. * The excessive flexibility made other extensions essentially impractical. `--sparse-index` is likely impossible in non-cone diff --git a/Documentation/git-tag.adoc b/Documentation/git-tag.adoc index 0f7badc116..cea3202fdb 100644 --- a/Documentation/git-tag.adoc +++ b/Documentation/git-tag.adoc @@ -3,7 +3,7 @@ git-tag(1) NAME ---- -git-tag - Create, list, delete or verify a tag object signed with GPG +git-tag - Create, list, delete or verify tags SYNOPSIS @@ -38,15 +38,17 @@ and `-a`, `-s`, and `-u <key-id>` are absent, `-a` is implied. Otherwise, a tag reference that points directly at the given object (i.e., a lightweight tag) is created. -A GnuPG signed tag object will be created when `-s` or `-u -<key-id>` is used. When `-u <key-id>` is not used, the -committer identity for the current user is used to find the -GnuPG key for signing. The configuration variable `gpg.program` -is used to specify custom GnuPG binary. +A cryptographically signed tag object will be created when `-s` or +`-u <key-id>` is used. The signing backend (GPG, X.509, SSH, etc.) is +controlled by the `gpg.format` configuration variable, defaulting to +OpenPGP. When `-u <key-id>` is not used, the committer identity for +the current user is used to find the key for signing. The +configuration variable `gpg.program` is used to specify a custom +signing binary. Tag objects (created with `-a`, `-s`, or `-u`) are called "annotated" tags; they contain a creation date, the tagger name and e-mail, a -tagging message, and an optional GnuPG signature. Whereas a +tagging message, and an optional cryptographic signature. Whereas a "lightweight" tag is simply a name for an object (usually a commit object). @@ -64,10 +66,12 @@ OPTIONS `-s`:: `--sign`:: - Make a GPG-signed tag, using the default e-mail address's key. - The default behavior of tag GPG-signing is controlled by `tag.gpgSign` - configuration variable if it exists, or disabled otherwise. - See linkgit:git-config[1]. + Make a cryptographically signed tag, using the default signing + key. The signing backend used depends on the `gpg.format` + configuration variable. The default key is determined by the + backend. For GPG, it's based on the committer's email address, + while for SSH it may be a specific key file or agent + identity. See linkgit:git-config[1]. `--no-sign`:: Override `tag.gpgSign` configuration variable that is @@ -75,7 +79,10 @@ OPTIONS `-u <key-id>`:: `--local-user=<key-id>`:: - Make a GPG-signed tag, using the given key. + Make a cryptographically signed tag using the given key. The + format of the <key-id> and the backend used depend on the + `gpg.format` configuration variable. See + linkgit:git-config[1]. `-f`:: `--force`:: @@ -87,7 +94,7 @@ OPTIONS `-v`:: `--verify`:: - Verify the GPG signature of the given tag names. + Verify the cryptographic signature of the given tags. `-n<num>`:: _<num>_ specifies how many lines from the annotation, if any, @@ -235,12 +242,23 @@ it in the repository configuration as follows: ------------------------------------- [user] - signingKey = <gpg-key-id> + signingKey = <key-id> ------------------------------------- +The signing backend can be chosen via the `gpg.format` configuration +variable, which defaults to `openpgp`. See linkgit:git-config[1] +for a list of other supported formats. + +The path to the program used for each signing backend can be specified +with the `gpg.<format>.program` configuration variable. For the +`openpgp` backend, `gpg.program` can be used as a synonym for +`gpg.openpgp.program`. See linkgit:git-config[1] for details. + `pager.tag` is only respected when listing tags, i.e., when `-l` is used or implied. The default is to use a pager. -See linkgit:git-config[1]. + +See linkgit:git-config[1] for more details and other configuration +variables. DISCUSSION ---------- diff --git a/Documentation/git-worktree.adoc b/Documentation/git-worktree.adoc index f272f79783..0f82ec5439 100644 --- a/Documentation/git-worktree.adoc +++ b/Documentation/git-worktree.adoc @@ -79,6 +79,9 @@ with a matching name, treat as equivalent to: $ git worktree add --track -b <branch> <path> <remote>/<branch> ------------ + +For best results it is advised to specify _<path>_ outside of the repository +and existing worktrees - see <<EXAMPLES,EXAMPLES>> ++ If the branch exists in multiple remotes and one of them is named by the `checkout.defaultRemote` configuration variable, we'll use that one for the purposes of disambiguation, even if the _<branch>_ isn't @@ -502,6 +505,7 @@ locked "reason\nwhy is locked" ... ------------ +[[EXAMPLES]] EXAMPLES -------- You are in the middle of a refactoring session and your boss comes in and @@ -522,6 +526,16 @@ $ popd $ git worktree remove ../temp ------------ +Side by side branch checkouts for a repository using multiple worktrees + +------------ +mkdir some-repository +cd some-repository +git clone --bare gitforge@someforge.example.com:some-org/some-repository some-repository.git +git --git-dir=some-repository.git worktree add some-branch +git --git-dir=some-repository.git worktree add another-branch +------------ + CONFIGURATION ------------- diff --git a/Documentation/gitcli.adoc b/Documentation/gitcli.adoc index ef2a0a399d..6815d6bfb7 100644 --- a/Documentation/gitcli.adoc +++ b/Documentation/gitcli.adoc @@ -223,7 +223,7 @@ Options that take a filename allow a prefix `:(optional)`. For example: ---------------------------- git commit -F :(optional)COMMIT_EDITMSG -# if COMMIT_EDITMSG does not exist, equivalent to +# if COMMIT_EDITMSG does not exist, the above is equivalent to git commit ---------------------------- diff --git a/Documentation/gitdatamodel.adoc b/Documentation/gitdatamodel.adoc new file mode 100644 index 0000000000..7d8d707d2d --- /dev/null +++ b/Documentation/gitdatamodel.adoc @@ -0,0 +1,302 @@ +gitdatamodel(7) +=============== + +NAME +---- +gitdatamodel - Git's core data model + +SYNOPSIS +-------- +gitdatamodel + +DESCRIPTION +----------- + +It's not necessary to understand Git's data model to use Git, but it's +very helpful when reading Git's documentation so that you know what it +means when the documentation says "object", "reference" or "index". + +Git's core operations use 4 kinds of data: + +1. <<object,Objects>>: commits, trees, blobs, and tag objects +2. <<references,References>>: branches, tags, + remote-tracking branches, etc +3. <<index,The index>>, also known as the staging area +4. <<reflogs,Reflogs>>: logs of changes to references ("ref log") + +[[object]] +OBJECTS +------- + +All of the commits and files in a Git repository are stored as "Git objects". +Git objects never change after they're created, and every object has an ID, +like `1b61de420a21a2f1aaef93e38ecd0e45e8bc9f0a`. + +This means that if you have an object's ID, you can always recover its +exact contents as long as the object hasn't been deleted. + +Every object has: + +[[object-id]] +1. an *ID* (aka "object name"), which is a cryptographic hash of its + type and contents. + It's fast to look up a Git object using its ID. + This is usually represented in hexadecimal, like + `1b61de420a21a2f1aaef93e38ecd0e45e8bc9f0a`. +2. a *type*. There are 4 types of objects: + <<commit,commits>>, <<tree,trees>>, <<blob,blobs>>, + and <<tag-object,tag objects>>. +3. *contents*. The structure of the contents depends on the type. + +Here's how each type of object is structured: + +[[commit]] +commit:: + A commit contains these required fields + (though there are other optional fields): ++ +1. The full directory structure of all the files in that version of the + repository and each file's contents, stored as the *<<tree,tree>>* ID + of the commit's base directory +2. Its *parent commit ID(s)*. The first commit in a repository has 0 parents, + regular commits have 1 parent, merge commits have 2 or more parents +3. An *author* and the time the commit was authored +4. A *committer* and the time the commit was committed +5. A *commit message* ++ +Here's how an example commit is stored: ++ +---- +tree 1b61de420a21a2f1aaef93e38ecd0e45e8bc9f0a +parent 4ccb6d7b8869a86aae2e84c56523f8705b50c647 +author Maya <maya@example.com> 1759173425 -0400 +committer Maya <maya@example.com> 1759173425 -0400 + +Add README +---- ++ +Like all other objects, commits can never be changed after they're created. +For example, "amending" a commit with `git commit --amend` creates a new +commit with the same parent. ++ +Git does not store the diff for a commit: when you ask Git to show +the commit with linkgit:git-show[1], it calculates the diff from its +parent on the fly. + +[[tree]] +tree:: + A tree is how Git represents a directory. + It can contain files or other trees (which are subdirectories). + It lists, for each item in the tree: ++ +1. The *filename*, for example `hello.py` +2. The *file mode*. These are all of the file modes in Git. + They're only spiritually related to Unix file modes. ++ + - `100644`: regular file (with <<object,object type>> `blob`) + - `100755`: executable file (with type `blob`) + - `120000`: symbolic link (with type `blob`) + - `040000`: directory (with type `tree`) + - `160000`: gitlink, for use with submodules (with type `commit`) + +3. The <<object-id,*object ID*>> with the contents of the file or directory ++ +For example, this is how a tree containing one directory (`src`) and one file +(`README.md`) is stored: ++ +---- +100644 blob 8728a858d9d21a8c78488c8b4e70e531b659141f README.md +040000 tree 89b1d2e0495f66d6929f4ff76ff1bb07fc41947d src +---- + +[[blob]] +blob:: + A blob object contains a file's contents. ++ +When you make a commit, Git stores the full contents of each file that +you changed as a blob. +For example, if you have a commit that changes 2 files in a repository +with 1000 files, that commit will create 2 new blobs, and use the +previous blob ID for the other 998 files. +This means that commits can use relatively little disk space even in a +very large repository. + +[[tag-object]] +tag object:: + Tag objects contain these required fields + (though there are other optional fields): ++ +1. The *ID* of the object it references +2. The *type* of the object it references +3. The *tagger* and tag date +4. A *tag message*, similar to a commit message + +Here's how an example tag object is stored: + +---- +object 750b4ead9c87ceb3ddb7a390e6c7074521797fb3 +type commit +tag v1.0.0 +tagger Maya <maya@example.com> 1759927359 -0400 + +Release version 1.0.0 +---- + +NOTE: All of the examples in this section were generated with +`git cat-file -p <object-id>`. + +[[references]] +REFERENCES +---------- + +References are a way to give a name to a commit. +It's easier to remember "the changes I'm working on are on the `turtle` +branch" than "the changes are in commit bb69721404348e". +Git often uses "ref" as shorthand for "reference". + +References can either refer to: + +1. An object ID, usually a <<commit,commit>> ID +2. Another reference. This is called a "symbolic reference" + +References are stored in a hierarchy, and Git handles references +differently based on where they are in the hierarchy. +Most references are under `refs/`. Here are the main types: + +[[branch]] +branches: `refs/heads/<name>`:: + A branch refers to a commit ID. + That commit is the latest commit on the branch. ++ +To get the history of commits on a branch, Git will start at the commit +ID the branch references, and then look at the commit's parent(s), +the parent's parent, etc. + +[[tag]] +tags: `refs/tags/<name>`:: + A tag refers to a commit ID, tag object ID, or other object ID. + There are two types of tags: + 1. "Annotated tags", which reference a <<tag-object,tag object>> ID + which contains a tag message + 2. "Lightweight tags", which reference a commit, blob, or tree ID + directly ++ +Even though branches and tags both refer to a commit ID, Git +treats them very differently. +Branches are expected to change over time: when you make a commit, Git +will update your <<HEAD,current branch>> to point to the new commit. +Tags are usually not changed after they're created. + +[[HEAD]] +HEAD: `HEAD`:: + `HEAD` is where Git stores your current <<branch,branch>>, + if there is a current branch. `HEAD` can either be: ++ +1. A symbolic reference to your current branch, for example `ref: + refs/heads/main` if your current branch is `main`. +2. A direct reference to a commit ID. In this case there is no current branch. + This is called "detached HEAD state", see the DETACHED HEAD section + of linkgit:git-checkout[1] for more. + +[[remote-tracking-branch]] +remote-tracking branches: `refs/remotes/<remote>/<branch>`:: + A remote-tracking branch refers to a commit ID. + It's how Git stores the last-known state of a branch in a remote + repository. `git fetch` updates remote-tracking branches. When + `git status` says "you're up to date with origin/main", it's looking at + this. ++ +`refs/remotes/<remote>/HEAD` is a symbolic reference to the remote's +default branch. This is the branch that `git clone` checks out by default. + +[[other-refs]] +Other references:: + Git tools may create references anywhere under `refs/`. + For example, linkgit:git-stash[1], linkgit:git-bisect[1], + and linkgit:git-notes[1] all create their own references + in `refs/stash`, `refs/bisect`, etc. + Third-party Git tools may also create their own references. ++ +Git may also create references other than `HEAD` at the base of the +hierarchy, like `ORIG_HEAD`. + +NOTE: Git may delete objects that aren't "reachable" from any reference +or <<reflogs,reflog>>. +An object is "reachable" if we can find it by following tags to whatever +they tag, commits to their parents or trees, and trees to the trees or +blobs that they contain. +For example, if you amend a commit with `git commit --amend`, +there will no longer be a branch that points at the old commit. +The old commit is recorded in the current branch's <<reflogs,reflog>>, +so it is still "reachable", but when the reflog entry expires it may +become unreachable and get deleted. + +the old commit will usually not be reachable, so it may be deleted eventually. +Reachable objects will never be deleted. + +[[index]] +THE INDEX +--------- +The index, also known as the "staging area", is a list of files and +the contents of each file, stored as a <<blob,blob>>. +You can add files to the index or update the contents of a file in the +index with linkgit:git-add[1]. This is called "staging" the file for commit. + +Unlike a <<tree,tree>>, the index is a flat list of files. +When you commit, Git converts the list of files in the index to a +directory <<tree,tree>> and uses that tree in the new <<commit,commit>>. + +Each index entry has 4 fields: + +1. The *file mode*, which must be one of: + - `100644`: regular file (with <<object,object type>> `blob`) + - `100755`: executable file (with type `blob`) + - `120000`: symbolic link (with type `blob`) + - `160000`: gitlink, for use with submodules (with type `commit`) +2. The *<<blob,blob>>* ID of the file, + or (rarely) the *<<commit,commit>>* ID of the submodule +3. The *stage number*, either 0, 1, 2, or 3. This is normally 0, but if + there's a merge conflict there can be multiple versions of the same + filename in the index. +4. The *file path*, for example `src/hello.py` + +It's extremely uncommon to look at the index directly: normally you'd +run `git status` to see a list of changes between the index and <<HEAD,HEAD>>. +But you can use `git ls-files --stage` to see the index. +Here's the output of `git ls-files --stage` in a repository with 2 files: + +---- +100644 8728a858d9d21a8c78488c8b4e70e531b659141f 0 README.md +100644 665c637a360874ce43bf74018768a96d2d4d219a 0 src/hello.py +---- + +[[reflogs]] +REFLOGS +------- + +Every time a branch, remote-tracking branch, or HEAD is updated, Git +updates a log called a "reflog" for that <<references,reference>>. +This means that if you make a mistake and "lose" a commit, you can +generally recover the commit ID by running `git reflog <reference>`. + +A reflog is a list of log entries. Each entry has: + +1. The *commit ID* +2. *Timestamp* when the change was made +3. *Log message*, for example `pull: Fast-forward` + +Reflogs only log changes made in your local repository. +They are not shared with remotes. + +You can view a reflog with `git reflog <reference>`. +For example, here's the reflog for a `main` branch which has changed twice: + +---- +$ git reflog main --date=iso --no-decorate +750b4ea main@{2025-09-29 15:17:05 -0400}: commit: Add README +4ccb6d7 main@{2025-09-29 15:16:48 -0400}: commit (initial): Initial commit +---- + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/gitformat-loose.adoc b/Documentation/gitformat-loose.adoc new file mode 100644 index 0000000000..4850c91669 --- /dev/null +++ b/Documentation/gitformat-loose.adoc @@ -0,0 +1,157 @@ +gitformat-loose(5) +================== + +NAME +---- +gitformat-loose - Git loose object format + + +SYNOPSIS +-------- +[verse] +$GIT_DIR/objects/[0-9a-f][0-9a-f]/* +$GIT_DIR/objects/loose-object-idx +$GIT_DIR/objects/loose-map/map-*.map + +DESCRIPTION +----------- + +Loose objects are how Git stores individual objects, where every object is +written as a separate file. + +Over the lifetime of a repository, objects are usually written as loose objects +initially. Eventually, these loose objects will be compacted into packfiles +via repository maintenance to improve disk space usage and speed up the lookup +of these objects. + +== Loose objects + +Each loose object contains a prefix, followed immediately by the data of the +object. The prefix contains `<type> <size>\0`. `<type>` is one of `blob`, +`tree`, `commit`, or `tag` and `size` is the size of the data (without the +prefix) as a decimal integer expressed in ASCII. + +The entire contents, prefix and data concatenated, is then compressed with zlib +and the compressed data is stored in the file. The object ID of the object is +the SHA-1 or SHA-256 (as appropriate) hash of the uncompressed data. + +The file for the loose object is stored under the `objects` directory, with the +first two hex characters of the object ID being the directory and the remaining +characters being the file name. This is done to shard the data and avoid too +many files being in one directory, since some file systems perform poorly with +many items in a directory. + +As an example, the empty tree contains the data (when uncompressed) `tree 0\0` +and, in a SHA-256 repository, would have the object ID +`6ef19b41225c5369f1c104d45d8d85efa9b057b53b14b4b9b939dd74decc5321` and would be +stored under +`$GIT_DIR/objects/6e/f19b41225c5369f1c104d45d8d85efa9b057b53b14b4b9b939dd74decc5321`. + +Similarly, a blob containing the contents `abc` would have the uncompressed +data of `blob 3\0abc`. + +== Loose object mapping + +When the `compatObjectFormat` option is used, Git needs to store a mapping +between the repository's main algorithm and the compatibility algorithm. There +are two formats for this: the legacy mapping and the modern mapping. + +=== Legacy mapping + +The compatibility mapping is stored in a file called +`$GIT_DIR/objects/loose-object-idx`. The format of this file looks like this: + + # loose-object-idx + (main-name SP compat-name LF)* + +`main-name` refers to hexadecimal object ID of the object in the main +repository format and `compat-name` refers to the same thing, but for the +compatibility format. + +This format is read if it exists but is not written. + +Note that carriage returns are not permitted in this file, regardless of the +host system or configuration. + +=== Modern mapping + +The modern mapping consists of a set of files under `$GIT_DIR/objects/loose` +ending in `.map`. The portion of the filename before the extension is that of +the hash checksum in hex format. + +`git pack-objects` will repack existing entries into one file, removing any +unnecessary objects, such as obsolete shallow entries or loose objects that +have been packed. + +==== Mapping file format + +- A header appears at the beginning and consists of the following: + * A 4-byte mapping signature: `LMAP` + * 4-byte version number: 1 + * 4-byte length of the header section. + * 4-byte number of objects declared in this map file. + * 4-byte number of object formats declared in this map file. + * For each object format: + ** 4-byte format identifier (e.g., `sha1` for SHA-1) + ** 4-byte length in bytes of shortened object names. This is the + shortest possible length needed to make names in the shortened + object name table unambiguous. + ** 8-byte integer, recording where tables relating to this format + are stored in this index file, as an offset from the beginning. + * 8-byte offset to the trailer from the beginning of this file. + * Zero or more additional key/value pairs (4-byte key, 4-byte value), which + may optionally declare one or more chunks. No chunks are currently + defined. Readers must ignore unrecognized keys. +- Zero or more NUL bytes. These are used to improve the alignment of the + 4-byte quantities below. +- Tables for the first object format: + * A sorted table of shortened object names. These are prefixes of the names + of all objects in this file, packed together without offset values to + reduce the cache footprint of the binary search for a specific object name. + * A sorted table of full object names. + * A table of 4-byte metadata values. + * Zero or more chunks. A chunk starts with a four-byte chunk identifier and + a four-byte parameter (which, if unneeded, is all zeros) and an eight-byte + size (not including the identifier, parameter, or size), plus the chunk + data. +- Zero or more NUL bytes. +- Tables for subsequent object formats: + * A sorted table of shortened object names. These are prefixes of the names + of all objects in this file, packed together without offset values to + reduce the cache footprint of the binary search for a specific object name. + * A table of full object names in the order specified by the first object format. + * A table of 4-byte values mapping object name order to the order of the + first object format. For an object in the table of sorted shortened object + names, the value at the corresponding index in this table is the index in + the previous table for that same object. + * Zero or more NUL bytes. +- The trailer consists of the following: + * Hash checksum of all of the above. + +The lower six bits of each metadata table contain a type field indicating the +reason that this object is stored: + +0:: + Reserved. +1:: + This object is stored as a loose object in the repository. +2:: + This object is a shallow entry. The mapping refers to a shallow value + returned by a remote server. +3:: + This object is a submodule entry. The mapping refers to the commit stored + representing a submodule. + +Other data may be stored in this field in the future. Bits that are not used +must be zero. + +All 4-byte numbers are in network order and must be 4-byte aligned in the file, +so the NUL padding may be required in some cases. + +Note that the hash at the end of the file is in whatever the repository's main +algorithm is. In the usual case when there are multiple algorithms, the main +algorithm will be SHA-256 and the compatibility algorithm will be SHA-1. + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/gitformat-pack.adoc b/Documentation/gitformat-pack.adoc index d6ae229be5..1b4db4aa61 100644 --- a/Documentation/gitformat-pack.adoc +++ b/Documentation/gitformat-pack.adoc @@ -32,6 +32,10 @@ In a repository using the traditional SHA-1, pack checksums, index checksums, and object IDs (object names) mentioned below are all computed using SHA-1. Similarly, in SHA-256 repositories, these values are computed using SHA-256. +CRC32 checksums are always computed over the entire packed object, including +the header (n-byte type and length); the base object name or offset, if any; +and the entire compressed object. The CRC32 algorithm used is that of zlib. + == pack-*.pack files have the following format: - A header appears at the beginning and consists of the following: @@ -80,6 +84,16 @@ Valid object types are: Type 5 is reserved for future expansion. Type 0 is invalid. +=== Object encoding + +Unlike loose objects, packed objects do not have a prefix containing the type, +size, and a NUL byte. These are not necessary because they can be determined by +the n-byte type and length that prefixes the data and so they are omitted from +the compressed and deltified data. + +The computation of the object ID still uses this prefix by reconstructing it +from the type and length as needed. + === Size encoding This document uses the following "size encoding" of non-negative @@ -92,6 +106,11 @@ values are more significant. This size encoding should not be confused with the "offset encoding", which is also used in this document. +When encoding the size of an undeltified object in a pack, the size is that of +the uncompressed raw object. For deltified objects, it is the size of the +uncompressed delta. The base object name or offset is not included in the size +computation. + === Deltified representation Conceptually there are only four object types: commit, tree, tag and diff --git a/Documentation/gitignore.adoc b/Documentation/gitignore.adoc index 5e0964ef41..9fccab4ae8 100644 --- a/Documentation/gitignore.adoc +++ b/Documentation/gitignore.adoc @@ -111,6 +111,11 @@ PATTERN FORMAT one of the characters in a range. See fnmatch(3) and the FNM_PATHNAME flag for a more detailed description. + - A backslash ("`\`") can be used to escape any character. E.g., "`\*`" + matches a literal asterisk (and "`\a`" matches "`a`", even though + there is no need for escaping there). As with fnmatch(3), a backslash + at the end of a pattern is an invalid pattern that never matches. + Two consecutive asterisks ("`**`") in patterns matched against full pathname may have special meaning: diff --git a/Documentation/gitprotocol-http.adoc b/Documentation/gitprotocol-http.adoc index d024010414..e2ef7f0459 100644 --- a/Documentation/gitprotocol-http.adoc +++ b/Documentation/gitprotocol-http.adoc @@ -443,7 +443,8 @@ If no "want" objects are received, send an error: TODO: Define error if no "want" lines are requested. If any "want" object is not reachable, send an error: -TODO: Define error if an invalid "want" is requested. +When a Git server receives an invalid or malformed `want` line, it +responds with an error message that includes the offending object name. Create an empty list, `s_common`. diff --git a/Documentation/glossary-content.adoc b/Documentation/glossary-content.adoc index e423e4765b..20ba121314 100644 --- a/Documentation/glossary-content.adoc +++ b/Documentation/glossary-content.adoc @@ -297,8 +297,8 @@ This commit is referred to as a "merge commit", or sometimes just a identified by its <<def_object_name,object name>>. The objects usually live in `$GIT_DIR/objects/`. -[[def_object_identifier]]object identifier (oid):: - Synonym for <<def_object_name,object name>>. +[[def_object_identifier]]object identifier, object ID, oid:: + Synonyms for <<def_object_name,object name>>. [[def_object_name]]object name:: The unique identifier of an <<def_object,object>>. The diff --git a/Documentation/howto/meson.build b/Documentation/howto/meson.build index ece20244af..16b9056f24 100644 --- a/Documentation/howto/meson.build +++ b/Documentation/howto/meson.build @@ -35,7 +35,7 @@ doc_targets += custom_target( output: 'howto-index.html', depends: documentation_deps, install: true, - install_dir: get_option('datadir') / 'doc/git-doc', + install_dir: htmldir, ) foreach howto : howto_sources @@ -57,6 +57,6 @@ foreach howto : howto_sources output: fs.stem(howto_stripped.full_path()) + '.html', depends: documentation_deps, install: true, - install_dir: get_option('datadir') / 'doc/git-doc/howto', + install_dir: htmldir / 'howto', ) endforeach diff --git a/Documentation/meson.build b/Documentation/meson.build index 44f94cdb7b..fd2e8cc02d 100644 --- a/Documentation/meson.build +++ b/Documentation/meson.build @@ -64,6 +64,7 @@ manpages = { 'git-gui.adoc' : 1, 'git-hash-object.adoc' : 1, 'git-help.adoc' : 1, + 'git-history.adoc' : 1, 'git-hook.adoc' : 1, 'git-http-backend.adoc' : 1, 'git-http-fetch.adoc' : 1, @@ -173,6 +174,7 @@ manpages = { 'gitformat-chunk.adoc' : 5, 'gitformat-commit-graph.adoc' : 5, 'gitformat-index.adoc' : 5, + 'gitformat-loose.adoc' : 5, 'gitformat-pack.adoc' : 5, 'gitformat-signature.adoc' : 5, 'githooks.adoc' : 5, @@ -192,6 +194,7 @@ manpages = { 'gitcore-tutorial.adoc' : 7, 'gitcredentials.adoc' : 7, 'gitcvs-migration.adoc' : 7, + 'gitdatamodel.adoc' : 7, 'gitdiffcore.adoc' : 7, 'giteveryday.adoc' : 7, 'gitfaq.adoc' : 7, @@ -411,7 +414,7 @@ foreach manpage, category : manpages input: manpage, output: fs.stem(manpage) + '.html', install: true, - install_dir: get_option('datadir') / 'doc/git-doc', + install_dir: htmldir, ) endif endforeach @@ -422,7 +425,7 @@ if get_option('docs').contains('html') output: 'docinfo.html', copy: true, install: true, - install_dir: get_option('datadir') / 'doc/git-doc', + install_dir: htmldir, ) configure_file( @@ -430,11 +433,11 @@ if get_option('docs').contains('html') output: 'docbook-xsl.css', copy: true, install: true, - install_dir: get_option('datadir') / 'doc/git-doc', + install_dir: htmldir, ) install_symlink('index.html', - install_dir: get_option('datadir') / 'doc/git-doc', + install_dir: htmldir, pointing_to: 'git.html', ) @@ -465,7 +468,7 @@ if get_option('docs').contains('html') input: 'docbook.xsl', output: 'user-manual.html', install: true, - install_dir: get_option('datadir') / 'doc/git-doc', + install_dir: htmldir, ) articles = [ @@ -491,7 +494,7 @@ if get_option('docs').contains('html') output: fs.stem(article) + '.html', depends: documentation_deps, install: true, - install_dir: get_option('datadir') / 'doc/git-doc', + install_dir: htmldir, ) endforeach diff --git a/Documentation/pull-fetch-param.adoc b/Documentation/pull-fetch-param.adoc index d79d2f6065..bb2cf6a462 100644 --- a/Documentation/pull-fetch-param.adoc +++ b/Documentation/pull-fetch-param.adoc @@ -11,6 +11,7 @@ ifndef::git-pull[] (See linkgit:git-config[1]). endif::git-pull[] +[[fetch-refspec]] <refspec>:: Specifies which refs to fetch and which local refs to update. When no <refspec>s appear on the command line, the refs to fetch diff --git a/Documentation/technical/commit-graph.adoc b/Documentation/technical/commit-graph.adoc index 2c26e95e51..a259d1567b 100644 --- a/Documentation/technical/commit-graph.adoc +++ b/Documentation/technical/commit-graph.adoc @@ -39,6 +39,7 @@ A consumer may load the following info for a commit from the graph: Values 1-4 satisfy the requirements of parse_commit_gently(). There are two definitions of generation number: + 1. Corrected committer dates (generation number v2) 2. Topological levels (generation number v1) @@ -158,7 +159,8 @@ number of commits in the full history. By creating a "chain" of commit-graphs, we enable fast writes of new commit data without rewriting the entire commit history -- at least, most of the time. -## File Layout +File Layout +~~~~~~~~~~~ A commit-graph chain uses multiple files, and we use a fixed naming convention to organize these files. Each commit-graph file has a name @@ -170,11 +172,11 @@ hashes for the files in order from "lowest" to "highest". For example, if the `commit-graph-chain` file contains the lines -``` +---- {hash0} {hash1} {hash2} -``` +---- then the commit-graph chain looks like the following diagram: @@ -213,7 +215,8 @@ specifying the hashes of all files in the lower layers. In the above example, `graph-{hash1}.graph` contains `{hash0}` while `graph-{hash2}.graph` contains `{hash0}` and `{hash1}`. -## Merging commit-graph files +Merging commit-graph files +~~~~~~~~~~~~~~~~~~~~~~~~~~ If we only added a new commit-graph file on every write, we would run into a linear search problem through many commit-graph files. Instead, we use a merge @@ -225,6 +228,7 @@ is determined by the merge strategy that the files should collapse to the commits in `graph-{hash1}` should be combined into a new `graph-{hash3}` file. +.... +---------------------+ | | | (new commits) | @@ -250,6 +254,7 @@ file. | | | | +-----------------------+ +.... During this process, the commits to write are combined, sorted and we write the contents to a temporary file, all while holding a `commit-graph-chain.lock` @@ -257,14 +262,15 @@ lock-file. When the file is flushed, we rename it to `graph-{hash3}` according to the computed `{hash3}`. Finally, we write the new chain data to `commit-graph-chain.lock`: -``` +---- {hash3} {hash0} -``` +---- We then close the lock-file. -## Merge Strategy +Merge Strategy +~~~~~~~~~~~~~~ When writing a set of commits that do not exist in the commit-graph stack of height N, we default to creating a new file at level N + 1. We then decide to @@ -289,7 +295,8 @@ The merge strategy values (2 for the size multiple, 64,000 for the maximum number of commits) could be extracted into config settings for full flexibility. -## Handling Mixed Generation Number Chains +Handling Mixed Generation Number Chains +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ With the introduction of generation number v2 and generation data chunk, the following scenario is possible: @@ -318,7 +325,8 @@ have corrected commit dates when written by compatible versions of Git. Thus, rewriting split commit-graph as a single file (`--split=replace`) creates a single layer with corrected commit dates. -## Deleting graph-{hash} files +Deleting graph-\{hash\} files +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ After a new tip file is written, some `graph-{hash}` files may no longer be part of a chain. It is important to remove these files from disk, eventually. @@ -333,7 +341,8 @@ files whose modified times are older than a given expiry window. This window defaults to zero, but can be changed using command-line arguments or a config setting. -## Chains across multiple object directories +Chains across multiple object directories +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In a repo with alternates, we look for the `commit-graph-chain` file starting in the local object directory and then in each alternate. The first file that diff --git a/Documentation/technical/hash-function-transition.adoc b/Documentation/technical/hash-function-transition.adoc index f047fd80ca..2359d7d106 100644 --- a/Documentation/technical/hash-function-transition.adoc +++ b/Documentation/technical/hash-function-transition.adoc @@ -227,9 +227,9 @@ network byte order): ** 4-byte length in bytes of shortened object names. This is the shortest possible length needed to make names in the shortened object name table unambiguous. - ** 4-byte integer, recording where tables relating to this format + ** 8-byte integer, recording where tables relating to this format are stored in this index file, as an offset from the beginning. - * 4-byte offset to the trailer from the beginning of this file. + * 8-byte offset to the trailer from the beginning of this file. * Zero or more additional key/value pairs (4-byte key, 4-byte value). Only one key is supported: 'PSRC'. See the "Loose objects and unreachable objects" section for supported values and how this @@ -260,12 +260,10 @@ network byte order): compressed data to be copied directly from pack to pack during repacking without undetected data corruption. - * A table of 4-byte offset values. For an object in the table of - sorted shortened object names, the value at the corresponding - index in this table indicates where that object can be found in - the pack file. These are usually 31-bit pack file offsets, but - large offsets are encoded as an index into the next table with the - most significant bit set. + * A table of 4-byte offset values. The index of this table in pack order + indicates where that object can be found in the pack file. These are + usually 31-bit pack file offsets, but large offsets are encoded as + an index into the next table with the most significant bit set. * A table of 8-byte offset entries (empty for pack files less than 2 GiB). Pack files are organized with heavily used objects toward @@ -276,10 +274,14 @@ network byte order): up to and not including the table of CRC32 values. - Zero or more NUL bytes. - The trailer consists of the following: - * A copy of the 20-byte SHA-256 checksum at the end of the + * A copy of the full main hash checksum at the end of the corresponding packfile. - * 20-byte SHA-256 checksum of all of the above. + * Full main hash checksum of all of the above. + +The "full main hash" is a full-length hash of the main (not compatibility) +algorithm in the repository. Thus, if the main algorithm is SHA-256, this is +a 32-byte SHA-256 hash and for SHA-1, it's a 20-byte SHA-1 hash. Loose object index ~~~~~~~~~~~~~~~~~~ @@ -427,17 +429,19 @@ ordinary unsigned commit. Signed Tags ~~~~~~~~~~~ -We add a new field "gpgsig-sha256" to the tag object format to allow -signing tags without relying on SHA-1. Its signed payload is the -SHA-256 content of the tag with its gpgsig-sha256 field and "-----BEGIN PGP -SIGNATURE-----" delimited in-body signature removed. - -This means tags can be signed - -1. using SHA-1 only, as in existing signed tag objects -2. using both SHA-1 and SHA-256, by using gpgsig-sha256 and an in-body - signature. -3. using only SHA-256, by only using the gpgsig-sha256 field. +We add new fields "gpgsig" and "gpgsig-sha256" to the tag object format to +allow signing tags in both formats. The in-body signature is used for the +signature in the current hash algorithm and the header is used for the +signature in the other algorithm. Thus, a dual-signature tag will contain both +an in-body signature and a gpgsig-sha256 header for the SHA-1 format of an +object or both an in-body signature and a gpgsig header for the SHA-256 format +of and object. + +The signed payload of the tag is the content of the tag in the current +algorithm with both its gpgsig and gpgsig-sha256 fields and +"-----BEGIN PGP SIGNATURE-----" delimited in-body signature removed. + +This means tags can be signed using one or both algorithms. Mergetag embedding ~~~~~~~~~~~~~~~~~~ diff --git a/Documentation/technical/large-object-promisors.adoc b/Documentation/technical/large-object-promisors.adoc index dea8dafa66..2aa815e023 100644 --- a/Documentation/technical/large-object-promisors.adoc +++ b/Documentation/technical/large-object-promisors.adoc @@ -34,8 +34,8 @@ a new object representation for large blobs as discussed in: https://lore.kernel.org/git/xmqqbkdometi.fsf@gitster.g/ -0) Non goals ------------- +Non goals +--------- - We will not discuss those client side improvements here, as they would require changes in different parts of Git than this effort. @@ -90,8 +90,8 @@ later in this document: even more to host content with larger blobs or more large blobs than currently. -I) Issues with the current situation ------------------------------------- +I Issues with the current situation +----------------------------------- - Some statistics made on GitLab repos have shown that more than 75% of the disk space is used by blobs that are larger than 1MB and @@ -138,8 +138,8 @@ I) Issues with the current situation complaining that these tools require significant effort to set up, learn and use correctly. -II) Main features of the "Large Object Promisors" solution ----------------------------------------------------------- +II Main features of the "Large Object Promisors" solution +--------------------------------------------------------- The main features below should give a rough overview of how the solution may work. Details about needed elements can be found in @@ -166,7 +166,7 @@ format. They should be used along with main remotes that contain the other objects. Note 1 -++++++ +^^^^^^ To clarify, a LOP is a normal promisor remote, except that: @@ -178,7 +178,7 @@ To clarify, a LOP is a normal promisor remote, except that: itself. Note 2 -++++++ +^^^^^^ Git already makes it possible for a main remote to also be a promisor remote storing both regular objects and large blobs for a client that @@ -186,13 +186,13 @@ clones from it with a filter on blob size. But here we explicitly want to avoid that. Rationale -+++++++++ +^^^^^^^^^ LOPs aim to be good at handling large blobs while main remotes are already good at handling other objects. Implementation -++++++++++++++ +^^^^^^^^^^^^^^ Git already has support for multiple promisor remotes, see link:partial-clone.html#using-many-promisor-remotes[the partial clone documentation]. @@ -213,19 +213,19 @@ remote helper (see linkgit:gitremote-helpers[7]) which makes the underlying object storage appear like a remote to Git. Note -++++ +^^^^ A LOP can be a promisor remote accessed using a remote helper by both some clients and the main remote. Rationale -+++++++++ +^^^^^^^^^ This looks like the simplest way to create LOPs that can cheaply handle many large blobs. Implementation -++++++++++++++ +^^^^^^^^^^^^^^ Remote helpers are quite easy to write as shell scripts, but it might be more efficient and maintainable to write them using other languages @@ -247,7 +247,7 @@ The underlying object storage that a LOP uses could also serve as storage for large files handled by Git LFS. Rationale -+++++++++ +^^^^^^^^^ This would simplify the server side if it wants to both use a LOP and act as a Git LFS server. @@ -259,7 +259,7 @@ On the server side, a main remote should have a way to offload to a LOP all its blobs with a size over a configurable threshold. Rationale -+++++++++ +^^^^^^^^^ This makes it easy to set things up and to clean things up. For example, an admin could use this to manually convert a repo not using @@ -268,7 +268,7 @@ some users would sometimes push large blobs, a cron job could use this to regularly make sure the large blobs are moved to the LOP. Implementation -++++++++++++++ +^^^^^^^^^^^^^^ Using something based on `git repack --filter=...` to separate the blobs we want to offload from the other Git objects could be a good @@ -284,13 +284,13 @@ should have ways to prevent oversize blobs to be fetched, and also perhaps pushed, into it. Rationale -+++++++++ +^^^^^^^^^ A main remote containing many oversize blobs would defeat the purpose of LOPs. Implementation -++++++++++++++ +^^^^^^^^^^^^^^ The way to offload to a LOP discussed in 4) above can be used to regularly offload oversize blobs. About preventing oversize blobs from @@ -326,18 +326,18 @@ large blobs directly from the LOP and the server would not need to fetch those blobs from the LOP to be able to serve the client. Note -++++ +^^^^ For fetches instead of clones, a protocol negotiation might not always happen, see the "What about fetches?" FAQ entry below for details. Rationale -+++++++++ +^^^^^^^^^ Security, configurability and efficiency of setting things up. Implementation -++++++++++++++ +^^^^^^^^^^^^^^ A "promisor-remote" protocol v2 capability looks like a good way to implement this. The way the client and server use this capability @@ -356,7 +356,7 @@ the client should be able to offload some large blobs it has fetched, but might not need anymore, to the LOP. Note -++++ +^^^^ It might depend on the context if it should be OK or not for clients to offload large blobs they have created, instead of fetched, directly @@ -367,13 +367,13 @@ This should be discussed and refined when we get closer to implementing this feature. Rationale -+++++++++ +^^^^^^^^^ On the client, the easiest way to deal with unneeded large blobs is to offload them. Implementation -++++++++++++++ +^^^^^^^^^^^^^^ This is very similar to what 4) above is about, except on the client side instead of the server side. So a good solution to 4) could likely @@ -385,8 +385,8 @@ when cloning (see 6) above). Also if the large blobs were fetched from a LOP, it is likely, and can easily be confirmed, that the LOP still has them, so that they can just be removed from the client. -III) Benefits of using LOPs ---------------------------- +III Benefits of using LOPs +-------------------------- Many benefits are related to the issues discussed in "I) Issues with the current situation" above: @@ -406,8 +406,8 @@ the current situation" above: - Reduced storage needs on the client side. -IV) FAQ -------- +IV FAQ +------ What about using multiple LOPs on the server and client side? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -533,7 +533,7 @@ some objects it already knows about but doesn't have because they are on a promisor remote. Regular fetch -+++++++++++++ +^^^^^^^^^^^^^ In a regular fetch, the client will contact the main remote and a protocol negotiation will happen between them. It's a good thing that @@ -551,7 +551,7 @@ new fetch will happen in the same way as the previous clone or fetch, using, or not using, the same LOP(s) as last time. "Backfill" or "lazy" fetch -++++++++++++++++++++++++++ +^^^^^^^^^^^^^^^^^^^^^^^^^^ When there is a backfill fetch, the client doesn't necessarily contact the main remote first. It will try to fetch from its promisor remotes @@ -576,8 +576,8 @@ from the client when it fetches from them. The client could get the token when performing a protocol negotiation with the main remote (see section II.6 above). -V) Future improvements ----------------------- +V Future improvements +--------------------- It is expected that at the beginning using LOPs will be mostly worth it either in a corporate context where the Git version that clients diff --git a/Documentation/technical/meson.build b/Documentation/technical/meson.build index 858af811a7..faff3964a9 100644 --- a/Documentation/technical/meson.build +++ b/Documentation/technical/meson.build @@ -13,6 +13,7 @@ articles = [ 'commit-graph.adoc', 'directory-rename-detection.adoc', 'hash-function-transition.adoc', + 'large-object-promisors.adoc', 'long-running-process-protocol.adoc', 'multi-pack-index.adoc', 'packfile-uri.adoc', @@ -52,7 +53,7 @@ doc_targets += custom_target( output: 'api-index.html', depends: documentation_deps, install: true, - install_dir: get_option('datadir') / 'doc/git-doc/technical', + install_dir: htmldir / 'technical', ) foreach article : api_docs + articles @@ -62,6 +63,6 @@ foreach article : api_docs + articles output: fs.stem(article) + '.html', depends: documentation_deps, install: true, - install_dir: get_option('datadir') / 'doc/git-doc/technical', + install_dir: htmldir / 'technical', ) endforeach diff --git a/Documentation/technical/remembering-renames.adoc b/Documentation/technical/remembering-renames.adoc index 73f41761e2..6155f36c72 100644 --- a/Documentation/technical/remembering-renames.adoc +++ b/Documentation/technical/remembering-renames.adoc @@ -10,32 +10,32 @@ history as an optimization, assuming all merges are automatic and clean Outline: - 0. Assumptions + 1. Assumptions - 1. How rebasing and cherry-picking work + 2. How rebasing and cherry-picking work - 2. Why the renames on MERGE_SIDE1 in any given pick are *always* a + 3. Why the renames on MERGE_SIDE1 in any given pick are *always* a superset of the renames on MERGE_SIDE1 for the next pick. - 3. Why any rename on MERGE_SIDE1 in any given pick is _almost_ always also + 4. Why any rename on MERGE_SIDE1 in any given pick is _almost_ always also a rename on MERGE_SIDE1 for the next pick - 4. A detailed description of the counter-examples to #3. + 5. A detailed description of the counter-examples to #4. - 5. Why the special cases in #4 are still fully reasonable to use to pair + 6. Why the special cases in #5 are still fully reasonable to use to pair up files for three-way content merging in the merge machinery, and why they do not affect the correctness of the merge. - 6. Interaction with skipping of "irrelevant" renames + 7. Interaction with skipping of "irrelevant" renames - 7. Additional items that need to be cached + 8. Additional items that need to be cached - 8. How directory rename detection interacts with the above and why this + 9. How directory rename detection interacts with the above and why this optimization is still safe even if merge.directoryRenames is set to "true". -=== 0. Assumptions === +== 1. Assumptions == There are two assumptions that will hold throughout this document: @@ -44,8 +44,8 @@ There are two assumptions that will hold throughout this document: * All merges are fully automatic -and a third that will hold in sections 2-5 for simplicity, that I'll later -address in section 8: +and a third that will hold in sections 3-6 for simplicity, that I'll later +address in section 9: * No directory renames occur @@ -77,9 +77,9 @@ conflicts that the user needs to resolve), the cache of renames is not stored on disk, and thus is thrown away as soon as the rebase or cherry pick stops for the user to resolve the operation. -The third assumption makes sections 2-5 simpler, and allows people to +The third assumption makes sections 3-6 simpler, and allows people to understand the basics of why this optimization is safe and effective, and -then I can go back and address the specifics in section 8. It is probably +then I can go back and address the specifics in section 9. It is probably also worth noting that if directory renames do occur, then the default of merge.directoryRenames being set to "conflict" means that the operation will stop for users to resolve the conflicts and the cache will be thrown @@ -88,22 +88,26 @@ reason we need to address directory renames specifically, is that some users will have set merge.directoryRenames to "true" to allow the merges to continue to proceed automatically. The optimization is still safe with this config setting, but we have to discuss a few more cases to show why; -this discussion is deferred until section 8. +this discussion is deferred until section 9. -=== 1. How rebasing and cherry-picking work === +== 2. How rebasing and cherry-picking work == Consider the following setup (from the git-rebase manpage): +------------ A---B---C topic / D---E---F---G main +------------ After rebasing or cherry-picking topic onto main, this will appear as: +------------ A'--B'--C' topic / D---E---F---G main +------------ The way the commits A', B', and C' are created is through a series of merges, where rebase or cherry-pick sequentially uses each of the three @@ -111,6 +115,7 @@ A-B-C commits in a special merge operation. Let's label the three commits in the merge operation as MERGE_BASE, MERGE_SIDE1, and MERGE_SIDE2. For this picture, the three commits for each of the three merges would be: +.... To create A': MERGE_BASE: E MERGE_SIDE1: G @@ -125,6 +130,7 @@ To create C': MERGE_BASE: B MERGE_SIDE1: B' MERGE_SIDE2: C +.... Sometimes, folks are surprised that these three-way merges are done. It can be useful in understanding these three-way merges to view them in a @@ -138,8 +144,7 @@ Conceptually the two statements above are the same as a three-way merge of B, B', and C, at least the parts before you decide to record a commit. -=== 2. Why the renames on MERGE_SIDE1 in any given pick are always a === -=== superset of the renames on MERGE_SIDE1 for the next pick. === +== 3. Why the renames on MERGE_SIDE1 in any given pick are always a superset of the renames on MERGE_SIDE1 for the next pick. == The merge machinery uses the filenames it is fed from MERGE_BASE, MERGE_SIDE1, and MERGE_SIDE2. It will only move content to a different @@ -156,6 +161,7 @@ filename under one of three conditions: First, let's remember what commits are involved in the first and second picks of the cherry-pick or rebase sequence: +.... To create A': MERGE_BASE: E MERGE_SIDE1: G @@ -165,6 +171,7 @@ To create B': MERGE_BASE: A MERGE_SIDE1: A' MERGE_SIDE2: B +.... So, in particular, we need to show that the renames between E and G are a superset of those between A and A'. @@ -181,11 +188,11 @@ are a subset of those between E and G. Equivalently, all renames between E and G are a superset of those between A and A'. -=== 3. Why any rename on MERGE_SIDE1 in any given pick is _almost_ === -=== always also a rename on MERGE_SIDE1 for the next pick. === +== 4. Why any rename on MERGE_SIDE1 in any given pick is _almost_ always also a rename on MERGE_SIDE1 for the next pick. == Let's again look at the first two picks: +.... To create A': MERGE_BASE: E MERGE_SIDE1: G @@ -195,17 +202,25 @@ To create B': MERGE_BASE: A MERGE_SIDE1: A' MERGE_SIDE2: B +.... Now let's look at any given rename from MERGE_SIDE1 of the first pick, i.e. any given rename from E to G. Let's use the filenames 'oldfile' and 'newfile' for demonstration purposes. That first pick will function as follows; when the rename is detected, the merge machinery will do a three-way content merge of the following: + +.... E:oldfile G:newfile A:oldfile +.... + and produce a new result: + +.... A':newfile +.... Note above that I've assumed that E->A did not rename oldfile. If that side did rename, then we most likely have a rename/rename(1to2) conflict @@ -254,19 +269,21 @@ were detected as renames, A:oldfile and A':newfile should also be detectable as renames almost always. -=== 4. A detailed description of the counter-examples to #3. === +== 5. A detailed description of the counter-examples to #4. == -We already noted in section 3 that rename/rename(1to1) (i.e. both sides +We already noted in section 4 that rename/rename(1to1) (i.e. both sides renaming a file the same way) was one counter-example. The more interesting bit, though, is why did we need to use the "almost" qualifier when stating that A:oldfile and A':newfile are "almost" always detectable as renames? -Let's repeat an earlier point that section 3 made: +Let's repeat an earlier point that section 4 made: +.... A':newfile was created by applying the changes between E:oldfile and G:newfile to A:oldfile. The changes between E:oldfile and G:newfile were <50% of the size of E:oldfile. +.... If those changes that were <50% of the size of E:oldfile are also <50% of the size of A:oldfile, then A:oldfile and A':newfile will be detectable as @@ -276,18 +293,21 @@ still somehow merge cleanly), then traditional rename detection would not detect A:oldfile and A':newfile as renames. Here's an example where that can happen: + * E:oldfile had 20 lines * G:newfile added 10 new lines at the beginning of the file * A:oldfile kept the first 3 lines of the file, and deleted all the rest + then + +.... => A':newfile would have 13 lines, 3 of which matches those in A:oldfile. -E:oldfile -> G:newfile would be detected as a rename, but A:oldfile and -A':newfile would not be. + E:oldfile -> G:newfile would be detected as a rename, but A:oldfile and + A':newfile would not be. +.... -=== 5. Why the special cases in #4 are still fully reasonable to use to === -=== pair up files for three-way content merging in the merge machinery, === -=== and why they do not affect the correctness of the merge. === +== 6. Why the special cases in #5 are still fully reasonable to use to pair up files for three-way content merging in the merge machinery, and why they do not affect the correctness of the merge. == In the rename/rename(1to1) case, A:newfile and A':newfile are not renames since they use the *same* filename. However, files with the same filename @@ -295,14 +315,14 @@ are obviously fine to pair up for three-way content merging (the merge machinery has never employed break detection). The interesting counter-example case is thus not the rename/rename(1to1) case, but the case where A did not rename oldfile. That was the case that we spent most of -the time discussing in sections 3 and 4. The remainder of this section +the time discussing in sections 4 and 5. The remainder of this section will be devoted to that case as well. So, even if A:oldfile and A':newfile aren't detectable as renames, why is it still reasonable to pair them up for three-way content merging in the merge machinery? There are multiple reasons: - * As noted in sections 3 and 4, the diff between A:oldfile and A':newfile + * As noted in sections 4 and 5, the diff between A:oldfile and A':newfile is *exactly* the same as the diff between E:oldfile and G:newfile. The latter pair were detected as renames, so it seems unlikely to surprise users for us to treat A:oldfile and A':newfile as renames. @@ -394,7 +414,7 @@ cases 1 and 3 seem to provide as good or better behavior with the optimization than without. -=== 6. Interaction with skipping of "irrelevant" renames === +== 7. Interaction with skipping of "irrelevant" renames == Previous optimizations involved skipping rename detection for paths considered to be "irrelevant". See for example the following commits: @@ -421,24 +441,27 @@ detection -- though we can limit it to the paths for which we have not already detected renames. -=== 7. Additional items that need to be cached === +== 8. Additional items that need to be cached == It turns out we have to cache more than just renames; we also cache: +.... A) non-renames (i.e. unpaired deletes) B) counts of renames within directories C) sources that were marked as RELEVANT_LOCATION, but which were downgraded to RELEVANT_NO_MORE D) the toplevel trees involved in the merge +.... These are all stored in struct rename_info, and respectively appear in + * cached_pairs (along side actual renames, just with a value of NULL) * dir_rename_counts * cached_irrelevant * merge_trees -The reason for (A) comes from the irrelevant renames skipping -optimization discussed in section 6. The fact that irrelevant renames +The reason for `(A)` comes from the irrelevant renames skipping +optimization discussed in section 7. The fact that irrelevant renames are skipped means we only get a subset of the potential renames detected and subsequent commits may need to run rename detection on the upstream side on a subset of the remaining renames (to get the @@ -447,23 +470,24 @@ deletes are involved in rename detection too, we don't want to repeatedly check that those paths remain unpaired on the upstream side with every commit we are transplanting. -The reason for (B) is that diffcore_rename_extended() is what +The reason for `(B)` is that diffcore_rename_extended() is what generates the counts of renames by directory which is needed in directory rename detection, and if we don't run diffcore_rename_extended() again then we need to have the output from it, including dir_rename_counts, from the previous run. -The reason for (C) is that merge-ort's tree traversal will again think +The reason for `(C)` is that merge-ort's tree traversal will again think those paths are relevant (marking them as RELEVANT_LOCATION), but the fact that they were downgraded to RELEVANT_NO_MORE means that dir_rename_counts already has the information we need for directory rename detection. (A path which becomes RELEVANT_CONTENT in a subsequent commit will be removed from cached_irrelevant.) -The reason for (D) is that is how we determine whether the remember +The reason for `(D)` is that is how we determine whether the remember renames optimization can be used. In particular, remembering that our sequence of merges looks like: +.... Merge 1: MERGE_BASE: E MERGE_SIDE1: G @@ -475,6 +499,7 @@ sequence of merges looks like: MERGE_SIDE1: A' MERGE_SIDE2: B => Creates B' +.... It is the fact that the trees A and A' appear both in Merge 1 and in Merge 2, with A as a parent of A' that allows this optimization. So @@ -482,12 +507,11 @@ we store the trees to compare with what we are asked to merge next time. -=== 8. How directory rename detection interacts with the above and === -=== why this optimization is still safe even if === -=== merge.directoryRenames is set to "true". === +== 9. How directory rename detection interacts with the above and why this optimization is still safe even if merge.directoryRenames is set to "true". == As noted in the assumptions section: +.... """ ...if directory renames do occur, then the default of merge.directoryRenames being set to "conflict" means that the operation @@ -497,11 +521,13 @@ As noted in the assumptions section: is that some users will have set merge.directoryRenames to "true" to allow the merges to continue to proceed automatically. """ +.... Let's remember that we need to look at how any given pick affects the next one. So let's again use the first two picks from the diagram in section one: +.... First pick does this three-way merge: MERGE_BASE: E MERGE_SIDE1: G @@ -513,6 +539,7 @@ one: MERGE_SIDE1: A' MERGE_SIDE2: B => creates B' +.... Now, directory rename detection exists so that if one side of history renames a directory, and the other side adds a new file to the old @@ -545,7 +572,7 @@ while considering all of these cases: concerned; see the assumptions section). Two interesting sub-notes about these counts: - * If we need to perform rename-detection again on the given side (e.g. + ** If we need to perform rename-detection again on the given side (e.g. some paths are relevant for rename detection that weren't before), then we clear dir_rename_counts and recompute it, making use of cached_pairs. The reason it is important to do this is optimizations @@ -556,7 +583,7 @@ while considering all of these cases: easiest way to "fix up" dir_rename_counts in such cases is to just recompute it. - * If we prune rename/rename(1to1) entries from the cache, then we also + ** If we prune rename/rename(1to1) entries from the cache, then we also need to update dir_rename_counts to decrement the counts for the involved directory and any relevant parent directories (to undo what update_dir_rename_counts() in diffcore-rename.c incremented when the @@ -578,6 +605,7 @@ in order: Case 1: MERGE_SIDE1 renames old dir, MERGE_SIDE2 adds new file to old dir +.... This case looks like this: MERGE_BASE: E, Has olddir/ @@ -595,10 +623,13 @@ Case 1: MERGE_SIDE1 renames old dir, MERGE_SIDE2 adds new file to old dir * MERGE_SIDE1 has cached olddir/newfile -> newdir/newfile Given the cached rename noted above, the second merge can proceed as expected without needing to perform rename detection from A -> A'. +.... Case 2: MERGE_SIDE1 renames old dir, MERGE_SIDE2 renames file into old dir +.... This case looks like this: + MERGE_BASE: E oldfile, olddir/ MERGE_SIDE1: G oldfile, olddir/ -> newdir/ MERGE_SIDE2: A oldfile -> olddir/newfile @@ -617,9 +648,11 @@ Case 2: MERGE_SIDE1 renames old dir, MERGE_SIDE2 renames file into old dir Given the cached rename noted above, the second merge can proceed as expected without needing to perform rename detection from A -> A'. +.... Case 3: MERGE_SIDE1 adds new file to old dir, MERGE_SIDE2 renames old dir +.... This case looks like this: MERGE_BASE: E, Has olddir/ @@ -635,9 +668,11 @@ Case 3: MERGE_SIDE1 adds new file to old dir, MERGE_SIDE2 renames old dir In this case, with the optimization, note that after the first commit there were no renames on MERGE_SIDE1, and any renames on MERGE_SIDE2 are tossed. But the second merge didn't need any renames so this is fine. +.... Case 4: MERGE_SIDE1 renames file into old dir, MERGE_SIDE2 renames old dir +.... This case looks like this: MERGE_BASE: E, Has olddir/ @@ -658,6 +693,7 @@ Case 4: MERGE_SIDE1 renames file into old dir, MERGE_SIDE2 renames old dir Given the cached rename noted above, the second merge can proceed as expected without needing to perform rename detection from A -> A'. +.... Finally, I'll just note here that interactions with the skip-irrelevant-renames optimization means we sometimes don't detect diff --git a/Documentation/technical/sparse-checkout.adoc b/Documentation/technical/sparse-checkout.adoc index 0f750ef3e3..3fa8e53655 100644 --- a/Documentation/technical/sparse-checkout.adoc +++ b/Documentation/technical/sparse-checkout.adoc @@ -14,37 +14,41 @@ Table of contents: * Reference Emails -=== Terminology === +== Terminology == -cone mode: one of two modes for specifying the desired subset of files +*`cone mode`*:: + one of two modes for specifying the desired subset of files in a sparse-checkout. In cone-mode, the user specifies directories (getting both everything under that directory as well as everything in leading directories), while in non-cone mode, the user specifies gitignore-style patterns. Controlled by the --[no-]cone option to sparse-checkout init|set. -SKIP_WORKTREE: When tracked files do not match the sparse specification and +*`SKIP_WORKTREE`*:: + When tracked files do not match the sparse specification and are removed from the working tree, the file in the index is marked with a SKIP_WORKTREE bit. Note that if a tracked file has the SKIP_WORKTREE bit set but the file is later written by the user to the working tree anyway, the SKIP_WORKTREE bit will be cleared at the beginning of any subsequent Git operation. - - Most sparse checkout users are unaware of this implementation - detail, and the term should generally be avoided in user-facing - descriptions and command flags. Unfortunately, prior to the - `sparse-checkout` subcommand this low-level detail was exposed, - and as of time of writing, is still exposed in various places. - -sparse-checkout: a subcommand in git used to reduce the files present in ++ +Most sparse checkout users are unaware of this implementation +detail, and the term should generally be avoided in user-facing +descriptions and command flags. Unfortunately, prior to the +`sparse-checkout` subcommand this low-level detail was exposed, +and as of time of writing, is still exposed in various places. + +*`sparse-checkout`*:: + a subcommand in git used to reduce the files present in the working tree to a subset of all tracked files. Also, the name of the file in the $GIT_DIR/info directory used to track the sparsity patterns corresponding to the user's desired subset. -sparse cone: see cone mode +*`sparse cone`*:: see cone mode -sparse directory: An entry in the index corresponding to a directory, which +*`sparse directory`*:: + An entry in the index corresponding to a directory, which appears in the index instead of all the files under that directory that would normally appear. See also sparse-index. Something that can cause confusion is that the "sparse directory" does NOT match @@ -52,7 +56,8 @@ sparse directory: An entry in the index corresponding to a directory, which working tree. May be renamed in the future (e.g. to "skipped directory"). -sparse index: A special mode for sparse-checkout that also makes the +*`sparse index`*:: + A special mode for sparse-checkout that also makes the index sparse by recording a directory entry in lieu of all the files underneath that directory (thus making that a "skipped directory" which unfortunately has also been called a "sparse @@ -60,7 +65,8 @@ sparse index: A special mode for sparse-checkout that also makes the directories. Controlled by the --[no-]sparse-index option to init|set|reapply. -sparsity patterns: patterns from $GIT_DIR/info/sparse-checkout used to +*`sparsity patterns`*:: + patterns from $GIT_DIR/info/sparse-checkout used to define the set of files of interest. A warning: It is easy to over-use this term (or the shortened "patterns" term), for two reasons: (1) users in cone mode specify directories rather than @@ -70,7 +76,8 @@ sparsity patterns: patterns from $GIT_DIR/info/sparse-checkout used to transiently differ in the working tree or index from the sparsity patterns (see "Sparse specification vs. sparsity patterns"). -sparse specification: The set of paths in the user's area of focus. This +*`sparse specification`*:: + The set of paths in the user's area of focus. This is typically just the tracked files that match the sparsity patterns, but the sparse specification can temporarily differ and include additional files. (See also "Sparse specification @@ -87,12 +94,13 @@ sparse specification: The set of paths in the user's area of focus. This * If working with the index and the working copy, the sparse specification is the union of the paths from above. -vivifying: When a command restores a tracked file to the working tree (and +*`vivifying`*:: + When a command restores a tracked file to the working tree (and hopefully also clears the SKIP_WORKTREE bit in the index for that file), this is referred to as "vivifying" the file. -=== Purpose of sparse-checkouts === +== Purpose of sparse-checkouts == sparse-checkouts exist to allow users to work with a subset of their files. @@ -120,14 +128,12 @@ those usecases, sparse-checkouts can modify different subcommands in over a half dozen different ways. Let's start by considering the high level usecases: - A) Users are _only_ interested in the sparse portion of the repo - - A*) Users are _only_ interested in the sparse portion of the repo - that they have downloaded so far - - B) Users want a sparse working tree, but are working in a larger whole - - C) sparse-checkout is a behind-the-scenes implementation detail allowing +[horizontal] +A):: Users are _only_ interested in the sparse portion of the repo +A*):: Users are _only_ interested in the sparse portion of the repo + that they have downloaded so far +B):: Users want a sparse working tree, but are working in a larger whole +C):: sparse-checkout is a behind-the-scenes implementation detail allowing Git to work with a specially crafted in-house virtual file system; users are actually working with a "full" working tree that is lazily populated, and sparse-checkout helps with the lazy population @@ -136,7 +142,7 @@ usecases: It may be worth explaining each of these in a bit more detail: - (Behavior A) Users are _only_ interested in the sparse portion of the repo +=== (Behavior A) Users are _only_ interested in the sparse portion of the repo These folks might know there are other things in the repository, but don't care. They are uninterested in other parts of the repository, and @@ -163,8 +169,7 @@ side-effects of various other commands (such as the printed diffstat after a merge or pull) can lead to worries about local repository size growing unnecessarily[10]. - (Behavior A*) Users are _only_ interested in the sparse portion of the repo - that they have downloaded so far (a variant on the first usecase) +=== (Behavior A*) Users are _only_ interested in the sparse portion of the repo that they have downloaded so far (a variant on the first usecase) This variant is driven by folks who using partial clones together with sparse checkouts and do disconnected development (so far sounding like a @@ -173,15 +178,14 @@ reason for yet another variant is that downloading even just the blobs through history within their sparse specification may be too much, so they only download some. They would still like operations to succeed without network connectivity, though, so things like `git log -S${SEARCH_TERM} -p` -or `git grep ${SEARCH_TERM} OLDREV ` would need to be prepared to provide +or `git grep ${SEARCH_TERM} OLDREV` would need to be prepared to provide partial results that depend on what happens to have been downloaded. This variant could be viewed as Behavior A with the sparse specification for history querying operations modified from "sparsity patterns" to "sparsity patterns limited to the blobs we have already downloaded". - (Behavior B) Users want a sparse working tree, but are working in a - larger whole +=== (Behavior B) Users want a sparse working tree, but are working in a larger whole Stolee described this usecase this way[11]: @@ -229,8 +233,7 @@ those expensive checks when interacting with the working copy, and may prefer getting "unrelated" results from their history queries over having slow commands. - (Behavior C) sparse-checkout is an implementational detail supporting a - special VFS. +=== (Behavior C) sparse-checkout is an implementational detail supporting a special VFS. This usecase goes slightly against the traditional definition of sparse-checkout in that it actually tries to present a full or dense @@ -255,13 +258,13 @@ will perceive the checkout as dense, and commands should thus behave as if all files are present. -=== Usecases of primary concern === +== Usecases of primary concern == Most of the rest of this document will focus on Behavior A and Behavior B. Some notes about the other two cases and why we are not focusing on them: - (Behavior A*) +=== (Behavior A*) Supporting this usecase is estimated to be difficult and a lot of work. There are no plans to implement it currently, but it may be a potential @@ -275,7 +278,7 @@ valid for this usecase, with the only exception being that it redefines the sparse specification to restrict it to already-downloaded blobs. The hard part is in making commands capable of respecting that modified definition. - (Behavior C) +=== (Behavior C) This usecase violates some of the early sparse-checkout documented assumptions (since files marked as SKIP_WORKTREE will be displayed to users @@ -300,20 +303,20 @@ Behavior C do not assume they are part of the Behavior B camp and propose patches that break things for the real Behavior B folks. -=== Oversimplified mental models === +== Oversimplified mental models == An oversimplification of the differences in the above behaviors is: - Behavior A: Restrict worktree and history operations to sparse specification - Behavior B: Restrict worktree operations to sparse specification; have any - history operations work across all files - Behavior C: Do not restrict either worktree or history operations to the - sparse specification...with the exception of branch checkouts or - switches which avoid writing files that will match the index so - they can later lazily be populated instead. +(Behavior A):: Restrict worktree and history operations to sparse specification +(Behavior B):: Restrict worktree operations to sparse specification; have any + history operations work across all files +(Behavior C):: Do not restrict either worktree or history operations to the + sparse specification...with the exception of branch checkouts or + switches which avoid writing files that will match the index so + they can later lazily be populated instead. -=== Desired behavior === +== Desired behavior == As noted previously, despite the simple idea of just working with a subset of files, there are a range of different behavioral changes that need to be @@ -326,37 +329,38 @@ understanding these differences can be beneficial. * Commands behaving the same regardless of high-level use-case - * commands that only look at files within the sparsity specification + ** commands that only look at files within the sparsity specification - * diff (without --cached or REVISION arguments) - * grep (without --cached or REVISION arguments) - * diff-files + *** diff (without --cached or REVISION arguments) + *** grep (without --cached or REVISION arguments) + *** diff-files - * commands that restore files to the working tree that match sparsity + ** commands that restore files to the working tree that match sparsity patterns, and remove unmodified files that don't match those patterns: - * switch - * checkout (the switch-like half) - * read-tree - * reset --hard + *** switch + *** checkout (the switch-like half) + *** read-tree + *** reset --hard - * commands that write conflicted files to the working tree, but otherwise + ** commands that write conflicted files to the working tree, but otherwise will omit writing files to the working tree that do not match the sparsity patterns: - * merge - * rebase - * cherry-pick - * revert + *** merge + *** rebase + *** cherry-pick + *** revert - * `am` and `apply --cached` should probably be in this section but + *** `am` and `apply --cached` should probably be in this section but are buggy (see the "Known bugs" section below) The behavior for these commands somewhat depends upon the merge strategy being used: - * `ort` behaves as described above - * `octopus` and `resolve` will always vivify any file changed in the merge + + *** `ort` behaves as described above + *** `octopus` and `resolve` will always vivify any file changed in the merge relative to the first parent, which is rather suboptimal. It is also important to note that these commands WILL update the index @@ -372,21 +376,21 @@ understanding these differences can be beneficial. specification and the sparsity patterns (much like the commands in the previous section). - * commands that always ignore sparsity since commits must be full-tree + ** commands that always ignore sparsity since commits must be full-tree - * archive - * bundle - * commit - * format-patch - * fast-export - * fast-import - * commit-tree + *** archive + *** bundle + *** commit + *** format-patch + *** fast-export + *** fast-import + *** commit-tree - * commands that write any modified file to the working tree (conflicted + ** commands that write any modified file to the working tree (conflicted or not, and whether those paths match sparsity patterns or not): - * stash - * apply (without `--index` or `--cached`) + *** stash + *** apply (without `--index` or `--cached`) * Commands that may slightly differ for behavior A vs. behavior B: @@ -394,19 +398,20 @@ understanding these differences can be beneficial. behaviors, but may differ in verbosity and types of warning and error messages. - * commands that make modifications to which files are tracked: - * add - * rm - * mv - * update-index + ** commands that make modifications to which files are tracked: + + *** add + *** rm + *** mv + *** update-index The fact that files can move between the 'tracked' and 'untracked' categories means some commands will have to treat untracked files differently. But if we have to treat untracked files differently, then additional commands may also need changes: - * status - * clean + *** status + *** clean In particular, `status` may need to report any untracked files outside the sparsity specification as an erroneous condition (especially to @@ -420,9 +425,10 @@ understanding these differences can be beneficial. may need to ignore the sparse specification by its nature. Also, its current --[no-]ignore-skip-worktree-entries default is totally bogus. - * commands for manually tweaking paths in both the index and the working tree - * `restore` - * the restore-like half of `checkout` + ** commands for manually tweaking paths in both the index and the working tree + + *** `restore` + *** the restore-like half of `checkout` These commands should be similar to add/rm/mv in that they should only operate on the sparse specification by default, and require a @@ -433,18 +439,19 @@ understanding these differences can be beneficial. * Commands that significantly differ for behavior A vs. behavior B: - * commands that query history - * diff (with --cached or REVISION arguments) - * grep (with --cached or REVISION arguments) - * show (when given commit arguments) - * blame (only matters when one or more -C flags are passed) - * and annotate - * log - * whatchanged (may not exist anymore) - * ls-files - * diff-index - * diff-tree - * ls-tree + ** commands that query history + + *** diff (with --cached or REVISION arguments) + *** grep (with --cached or REVISION arguments) + *** show (when given commit arguments) + *** blame (only matters when one or more -C flags are passed) + **** and annotate + *** log + *** whatchanged (may not exist anymore) + *** ls-files + *** diff-index + *** diff-tree + *** ls-tree Note: for log and whatchanged, revision walking logic is unaffected but displaying of patches is affected by scoping the command to the @@ -458,91 +465,91 @@ understanding these differences can be beneficial. * Commands I don't know how to classify - * range-diff + ** range-diff Is this like `log` or `format-patch`? - * cherry + ** cherry See range-diff * Commands unaffected by sparse-checkouts - * shortlog - * show-branch - * rev-list - * bisect - - * branch - * describe - * fetch - * gc - * init - * maintenance - * notes - * pull (merge & rebase have the necessary changes) - * push - * submodule - * tag - - * config - * filter-branch (works in separate checkout without sparse-checkout setup) - * pack-refs - * prune - * remote - * repack - * replace - - * bugreport - * count-objects - * fsck - * gitweb - * help - * instaweb - * merge-tree (doesn't touch worktree or index, and merges always compute full-tree) - * rerere - * verify-commit - * verify-tag - - * commit-graph - * hash-object - * index-pack - * mktag - * mktree - * multi-pack-index - * pack-objects - * prune-packed - * symbolic-ref - * unpack-objects - * update-ref - * write-tree (operates on index, possibly optimized to use sparse dir entries) - - * for-each-ref - * get-tar-commit-id - * ls-remote - * merge-base (merges are computed full tree, so merge base should be too) - * name-rev - * pack-redundant - * rev-parse - * show-index - * show-ref - * unpack-file - * var - * verify-pack - - * <Everything under 'Interacting with Others' in 'git help --all'> - * <Everything under 'Low-level...Syncing' in 'git help --all'> - * <Everything under 'Low-level...Internal Helpers' in 'git help --all'> - * <Everything under 'External commands' in 'git help --all'> + ** shortlog + ** show-branch + ** rev-list + ** bisect + + ** branch + ** describe + ** fetch + ** gc + ** init + ** maintenance + ** notes + ** pull (merge & rebase have the necessary changes) + ** push + ** submodule + ** tag + + ** config + ** filter-branch (works in separate checkout without sparse-checkout setup) + ** pack-refs + ** prune + ** remote + ** repack + ** replace + + ** bugreport + ** count-objects + ** fsck + ** gitweb + ** help + ** instaweb + ** merge-tree (doesn't touch worktree or index, and merges always compute full-tree) + ** rerere + ** verify-commit + ** verify-tag + + ** commit-graph + ** hash-object + ** index-pack + ** mktag + ** mktree + ** multi-pack-index + ** pack-objects + ** prune-packed + ** symbolic-ref + ** unpack-objects + ** update-ref + ** write-tree (operates on index, possibly optimized to use sparse dir entries) + + ** for-each-ref + ** get-tar-commit-id + ** ls-remote + ** merge-base (merges are computed full tree, so merge base should be too) + ** name-rev + ** pack-redundant + ** rev-parse + ** show-index + ** show-ref + ** unpack-file + ** var + ** verify-pack + + ** <Everything under 'Interacting with Others' in 'git help --all'> + ** <Everything under 'Low-level...Syncing' in 'git help --all'> + ** <Everything under 'Low-level...Internal Helpers' in 'git help --all'> + ** <Everything under 'External commands' in 'git help --all'> * Commands that might be affected, but who cares? - * merge-file - * merge-index - * gitk? + ** merge-file + ** merge-index + ** gitk? -=== Behavior classes === +== Behavior classes == From the above there are a few classes of behavior: @@ -573,18 +580,19 @@ From the above there are a few classes of behavior: Commands in this class generally behave like the "restrict" class, except that: - (1) they will ignore the sparse specification and write files with - conflicts to the working tree (thus temporarily expanding the - sparse specification to include such files.) - (2) they are grouped with commands which move to a new commit, since - they often create a commit and then move to it, even though we - know there are many exceptions to moving to the new commit. (For - example, the user may rebase a commit that becomes empty, or have - a cherry-pick which conflicts, or a user could run `merge - --no-commit`, and we also view `apply --index` kind of like `am - --no-commit`.) As such, these commands can make changes to index - files outside the sparse specification, though they'll mark such - files with SKIP_WORKTREE. + + (1) they will ignore the sparse specification and write files with + conflicts to the working tree (thus temporarily expanding the + sparse specification to include such files.) + (2) they are grouped with commands which move to a new commit, since + they often create a commit and then move to it, even though we + know there are many exceptions to moving to the new commit. (For + example, the user may rebase a commit that becomes empty, or have + a cherry-pick which conflicts, or a user could run `merge + --no-commit`, and we also view `apply --index` kind of like `am + --no-commit`.) As such, these commands can make changes to index + files outside the sparse specification, though they'll mark such + files with SKIP_WORKTREE. * "restrict also specially applied to untracked files" @@ -609,37 +617,39 @@ From the above there are a few classes of behavior: specification. -=== Subcommand-dependent defaults === +== Subcommand-dependent defaults == Note that we have different defaults depending on the command for the desired behavior : * Commands defaulting to "restrict": - * diff-files - * diff (without --cached or REVISION arguments) - * grep (without --cached or REVISION arguments) - * switch - * checkout (the switch-like half) - * reset (<commit>) - - * restore - * checkout (the restore-like half) - * checkout-index - * reset (with pathspec) + + ** diff-files + ** diff (without --cached or REVISION arguments) + ** grep (without --cached or REVISION arguments) + ** switch + ** checkout (the switch-like half) + ** reset (<commit>) + + ** restore + ** checkout (the restore-like half) + ** checkout-index + ** reset (with pathspec) This behavior makes sense; these interact with the working tree. * Commands defaulting to "restrict modulo conflicts": - * merge - * rebase - * cherry-pick - * revert - * am - * apply --index (which is kind of like an `am --no-commit`) + ** merge + ** rebase + ** cherry-pick + ** revert + + ** am + ** apply --index (which is kind of like an `am --no-commit`) - * read-tree (especially with -m or -u; is kind of like a --no-commit merge) - * reset (<tree-ish>, due to similarity to read-tree) + ** read-tree (especially with -m or -u; is kind of like a --no-commit merge) + ** reset (<tree-ish>, due to similarity to read-tree) These also interact with the working tree, but require slightly different behavior either so that (a) conflicts can be resolved or (b) @@ -648,16 +658,17 @@ desired behavior : (See also the "Known bugs" section below regarding `am` and `apply`) * Commands defaulting to "no restrict": - * archive - * bundle - * commit - * format-patch - * fast-export - * fast-import - * commit-tree - * stash - * apply (without `--index`) + ** archive + ** bundle + ** commit + ** format-patch + ** fast-export + ** fast-import + ** commit-tree + + ** stash + ** apply (without `--index`) These have completely different defaults and perhaps deserve the most detailed explanation: @@ -679,53 +690,59 @@ desired behavior : sparse specification then we'll lose changes from the user. * Commands defaulting to "restrict also specially applied to untracked files": - * add - * rm - * mv - * update-index - * status - * clean (?) - - Our original implementation for the first three of these commands was - "no restrict", but it had some severe usability issues: - * `git add <somefile>` if honored and outside the sparse - specification, can result in the file randomly disappearing later - when some subsequent command is run (since various commands - automatically clean up unmodified files outside the sparse - specification). - * `git rm '*.jpg'` could very negatively surprise users if it deletes - files outside the range of the user's interest. - * `git mv` has similar surprises when moving into or out of the cone, - so best to restrict by default - - So, we switched `add` and `rm` to default to "restrict", which made - usability problems much less severe and less frequent, but we still got - complaints because commands like: - git add <file-outside-sparse-specification> - git rm <file-outside-sparse-specification> - would silently do nothing. We should instead print an error in those - cases to get usability right. - - update-index needs to be updated to match, and status and maybe clean - also need to be updated to specially handle untracked paths. - - There may be a difference in here between behavior A and behavior B in - terms of verboseness of errors or additional warnings. + + ** add + ** rm + ** mv + ** update-index + ** status + ** clean (?) + +.... + Our original implementation for the first three of these commands was + "no restrict", but it had some severe usability issues: + + * `git add <somefile>` if honored and outside the sparse + specification, can result in the file randomly disappearing later + when some subsequent command is run (since various commands + automatically clean up unmodified files outside the sparse + specification). + * `git rm '*.jpg'` could very negatively surprise users if it deletes + files outside the range of the user's interest. + * `git mv` has similar surprises when moving into or out of the cone, + so best to restrict by default + + So, we switched `add` and `rm` to default to "restrict", which made + usability problems much less severe and less frequent, but we still got + complaints because commands like: + + git add <file-outside-sparse-specification> + git rm <file-outside-sparse-specification> + + would silently do nothing. We should instead print an error in those + cases to get usability right. + + update-index needs to be updated to match, and status and maybe clean + also need to be updated to specially handle untracked paths. + + There may be a difference in here between behavior A and behavior B in + terms of verboseness of errors or additional warnings. +.... * Commands falling under "restrict or no restrict dependent upon behavior A vs. behavior B" - * diff (with --cached or REVISION arguments) - * grep (with --cached or REVISION arguments) - * show (when given commit arguments) - * blame (only matters when one or more -C flags passed) - * and annotate - * log - * and variants: shortlog, gitk, show-branch, whatchanged, rev-list - * ls-files - * diff-index - * diff-tree - * ls-tree + ** diff (with --cached or REVISION arguments) + ** grep (with --cached or REVISION arguments) + ** show (when given commit arguments) + ** blame (only matters when one or more -C flags passed) + *** and annotate + ** log + *** and variants: shortlog, gitk, show-branch, whatchanged, rev-list + ** ls-files + ** diff-index + ** diff-tree + ** ls-tree For now, we default to behavior B for these, which want a default of "no restrict". @@ -749,7 +766,7 @@ desired behavior : implemented. -=== Sparse specification vs. sparsity patterns === +== Sparse specification vs. sparsity patterns == In a well-behaved situation, the sparse specification is given directly by the $GIT_DIR/info/sparse-checkout file. However, it can transiently @@ -821,45 +838,48 @@ under behavior B index operations are lumped with history and tend to operate full-tree. -=== Implementation Questions === - - * Do the options --scope={sparse,all} sound good to others? Are there better - options? - * Names in use, or appearing in patches, or previously suggested: - * --sparse/--dense - * --ignore-skip-worktree-bits - * --ignore-skip-worktree-entries - * --ignore-sparsity - * --[no-]restrict-to-sparse-paths - * --full-tree/--sparse-tree - * --[no-]restrict - * --scope={sparse,all} - * --focus/--unfocus - * --limit/--unlimited - * Rationale making me lean slightly towards --scope={sparse,all}: - * We want a name that works for many commands, so we need a name that +== Implementation Questions == + + * Do the options --scope={sparse,all} sound good to others? Are there better options? + + ** Names in use, or appearing in patches, or previously suggested: + + *** --sparse/--dense + *** --ignore-skip-worktree-bits + *** --ignore-skip-worktree-entries + *** --ignore-sparsity + *** --[no-]restrict-to-sparse-paths + *** --full-tree/--sparse-tree + *** --[no-]restrict + *** --scope={sparse,all} + *** --focus/--unfocus + *** --limit/--unlimited + + ** Rationale making me lean slightly towards --scope={sparse,all}: + + *** We want a name that works for many commands, so we need a name that does not conflict - * We know that we have more than two possible usecases, so it is best + *** We know that we have more than two possible usecases, so it is best to avoid a flag that appears to be binary. - * --scope={sparse,all} isn't overly long and seems relatively + *** --scope={sparse,all} isn't overly long and seems relatively explanatory - * `--sparse`, as used in add/rm/mv, is totally backwards for + *** `--sparse`, as used in add/rm/mv, is totally backwards for grep/log/etc. Changing the meaning of `--sparse` for these commands would fix the backwardness, but possibly break existing scripts. Using a new name pairing would allow us to treat `--sparse` in these commands as a deprecated alias. - * There is a different `--sparse`/`--dense` pair for commands using + *** There is a different `--sparse`/`--dense` pair for commands using revision machinery, so using that naming might cause confusion - * There is also a `--sparse` in both pack-objects and show-branch, which + *** There is also a `--sparse` in both pack-objects and show-branch, which don't conflict but do suggest that `--sparse` is overloaded - * The name --ignore-skip-worktree-bits is a double negative, is + *** The name --ignore-skip-worktree-bits is a double negative, is quite a mouthful, refers to an implementation detail that many users may not be familiar with, and we'd need a negation for it which would probably be even more ridiculously long. (But we can make --ignore-skip-worktree-bits a deprecated alias for --no-restrict.) - * If a config option is added (sparse.scope?) what should the values and + ** If a config option is added (sparse.scope?) what should the values and description be? "sparse" (behavior A), "worktree-sparse-history-dense" (behavior B), "dense" (behavior C)? There's a risk of confusion, because even for Behaviors A and B we want some commands to be @@ -868,19 +888,20 @@ operate full-tree. the primary difference we are focusing is just the history-querying commands (log/diff/grep). Previous config suggestion here: [13] - * Is `--no-expand` a good alias for ls-files's `--sparse` option? + ** Is `--no-expand` a good alias for ls-files's `--sparse` option? (`--sparse` does not map to either `--scope=sparse` or `--scope=all`, because in non-cone mode it does nothing and in cone-mode it shows the sparse directory entries which are technically outside the sparse specification) - * Under Behavior A: - * Does ls-files' `--no-expand` override the default `--scope=all`, or + ** Under Behavior A: + + *** Does ls-files' `--no-expand` override the default `--scope=all`, or does it need an extra flag? - * Does ls-files' `-t` option imply `--scope=all`? - * Does update-index's `--[no-]skip-worktree` option imply `--scope=all`? + *** Does ls-files' `-t` option imply `--scope=all`? + *** Does update-index's `--[no-]skip-worktree` option imply `--scope=all`? - * sparse-checkout: once behavior A is fully implemented, should we take + ** sparse-checkout: once behavior A is fully implemented, should we take an interim measure to ease people into switching the default? Namely, if folks are not already in a sparse checkout, then require `sparse-checkout init/set` to take a @@ -892,7 +913,7 @@ operate full-tree. is seamless for them. -=== Implementation Goals/Plans === +== Implementation Goals/Plans == * Get buy-in on this document in general. @@ -910,25 +931,26 @@ operate full-tree. request that they not trigger this bug." flag * Flags & Config - * Make `--sparse` in add/rm/mv a deprecated alias for `--scope=all` - * Make `--ignore-skip-worktree-bits` in checkout-index/checkout/restore + + ** Make `--sparse` in add/rm/mv a deprecated alias for `--scope=all` + ** Make `--ignore-skip-worktree-bits` in checkout-index/checkout/restore a deprecated aliases for `--scope=all` - * Create config option (sparse.scope?), tie it to the "Cliff notes" + ** Create config option (sparse.scope?), tie it to the "Cliff notes" overview - * Add --scope=sparse (and --scope=all) flag to each of the history querying + ** Add --scope=sparse (and --scope=all) flag to each of the history querying commands. IMPORTANT: make sure diff machinery changes don't mess with format-patch, fast-export, etc. -=== Known bugs === +== Known bugs == This list used to be a lot longer (see e.g. [1,2,3,4,5,6,7,8,9]), but we've been working on it. -0. Behavior A is not well supported in Git. (Behavior B didn't used to +1. Behavior A is not well supported in Git. (Behavior B didn't used to be either, but was the easier of the two to implement.) -1. am and apply: +2. am and apply: apply, without `--index` or `--cached`, relies on files being present in the working copy, and also writes to them unconditionally. As @@ -948,7 +970,7 @@ been working on it. files and then complain that those vivified files would be overwritten by merge. -2. reset --hard: +3. reset --hard: reset --hard provides confusing error message (works correctly, but misleads the user into believing it didn't): @@ -971,13 +993,13 @@ been working on it. `git reset --hard` DID remove addme from the index and the working tree, contrary to the error message, but in line with how reset --hard should behave. -3. read-tree +4. read-tree `read-tree` doesn't apply the 'SKIP_WORKTREE' bit to *any* of the entries it reads into the index, resulting in all your files suddenly appearing to be "deleted". -4. Checkout, restore: +5. Checkout, restore: These command do not handle path & revision arguments appropriately: @@ -1030,7 +1052,7 @@ been working on it. S tracked H tracked-but-maybe-skipped -5. checkout and restore --staged, continued: +6. checkout and restore --staged, continued: These commands do not correctly scope operations to the sparse specification, and make it worse by not setting important SKIP_WORKTREE @@ -1046,56 +1068,82 @@ been working on it. the sparse specification, but then it will be important to set the SKIP_WORKTREE bits appropriately. -6. Performance issues; see: - https://lore.kernel.org/git/CABPp-BEkJQoKZsQGCYioyga_uoDQ6iBeW+FKr8JhyuuTMK1RDw@mail.gmail.com/ +7. Performance issues; see: + + https://lore.kernel.org/git/CABPp-BEkJQoKZsQGCYioyga_uoDQ6iBeW+FKr8JhyuuTMK1RDw@mail.gmail.com/ -=== Reference Emails === +== Reference Emails == Emails that detail various bugs we've had in sparse-checkout: -[1] (Original descriptions of behavior A & behavior B) - https://lore.kernel.org/git/CABPp-BGJ_Nvi5TmgriD9Bh6eNXE2EDq2f8e8QKXAeYG3BxZafA@mail.gmail.com/ -[2] (Fix stash applications in sparse checkouts; bugs from behavioral differences) - https://lore.kernel.org/git/ccfedc7140dbf63ba26a15f93bd3885180b26517.1606861519.git.gitgitgadget@gmail.com/ -[3] (Present-despite-skipped entries) - https://lore.kernel.org/git/11d46a399d26c913787b704d2b7169cafc28d639.1642175983.git.gitgitgadget@gmail.com/ -[4] (Clone --no-checkout interaction) - https://lore.kernel.org/git/pull.801.v2.git.git.1591324899170.gitgitgadget@gmail.com/ (clone --no-checkout) -[5] (The need for update_sparsity() and avoiding `read-tree -mu HEAD`) - https://lore.kernel.org/git/3a1f084641eb47515b5a41ed4409a36128913309.1585270142.git.gitgitgadget@gmail.com/ -[6] (SKIP_WORKTREE is advisory, not mandatory) - https://lore.kernel.org/git/844306c3e86ef67591cc086decb2b760e7d710a3.1585270142.git.gitgitgadget@gmail.com/ -[7] (`worktree add` should copy sparsity settings from current worktree) - https://lore.kernel.org/git/c51cb3714e7b1d2f8c9370fe87eca9984ff4859f.1644269584.git.gitgitgadget@gmail.com/ -[8] (Avoid negative surprises in add, rm, and mv) - https://lore.kernel.org/git/cover.1617914011.git.matheus.bernardino@usp.br/ - https://lore.kernel.org/git/pull.1018.v4.git.1632497954.gitgitgadget@gmail.com/ -[9] (Move from out-of-cone to in-cone) - https://lore.kernel.org/git/20220630023737.473690-6-shaoxuan.yuan02@gmail.com/ - https://lore.kernel.org/git/20220630023737.473690-4-shaoxuan.yuan02@gmail.com/ -[10] (Unnecessarily downloading objects outside sparse specification) - https://lore.kernel.org/git/CAOLTT8QfwOi9yx_qZZgyGa8iL8kHWutEED7ok_jxwTcYT_hf9Q@mail.gmail.com/ - -[11] (Stolee's comments on high-level usecases) - https://lore.kernel.org/git/1a1e33f6-3514-9afc-0a28-5a6b85bd8014@gmail.com/ +[1] (Original descriptions of behavior A & behavior B): + +https://lore.kernel.org/git/CABPp-BGJ_Nvi5TmgriD9Bh6eNXE2EDq2f8e8QKXAeYG3BxZafA@mail.gmail.com/ + +[2] (Fix stash applications in sparse checkouts; bugs from behavioral differences): + +https://lore.kernel.org/git/ccfedc7140dbf63ba26a15f93bd3885180b26517.1606861519.git.gitgitgadget@gmail.com/ + +[3] (Present-despite-skipped entries): + +https://lore.kernel.org/git/11d46a399d26c913787b704d2b7169cafc28d639.1642175983.git.gitgitgadget@gmail.com/ + +[4] (Clone --no-checkout interaction): + +https://lore.kernel.org/git/pull.801.v2.git.git.1591324899170.gitgitgadget@gmail.com/ (clone --no-checkout) + +[5] (The need for update_sparsity() and avoiding `read-tree -mu HEAD`): + +https://lore.kernel.org/git/3a1f084641eb47515b5a41ed4409a36128913309.1585270142.git.gitgitgadget@gmail.com/ + +[6] (SKIP_WORKTREE is advisory, not mandatory): + +https://lore.kernel.org/git/844306c3e86ef67591cc086decb2b760e7d710a3.1585270142.git.gitgitgadget@gmail.com/ + +[7] (`worktree add` should copy sparsity settings from current worktree): + +https://lore.kernel.org/git/c51cb3714e7b1d2f8c9370fe87eca9984ff4859f.1644269584.git.gitgitgadget@gmail.com/ + +[8] (Avoid negative surprises in add, rm, and mv): + + * https://lore.kernel.org/git/cover.1617914011.git.matheus.bernardino@usp.br/ + * https://lore.kernel.org/git/pull.1018.v4.git.1632497954.gitgitgadget@gmail.com/ + +[9] (Move from out-of-cone to in-cone): + + * https://lore.kernel.org/git/20220630023737.473690-6-shaoxuan.yuan02@gmail.com/ + * https://lore.kernel.org/git/20220630023737.473690-4-shaoxuan.yuan02@gmail.com/ + +[10] (Unnecessarily downloading objects outside sparse specification): + +https://lore.kernel.org/git/CAOLTT8QfwOi9yx_qZZgyGa8iL8kHWutEED7ok_jxwTcYT_hf9Q@mail.gmail.com/ + +[11] (Stolee's comments on high-level usecases): + +https://lore.kernel.org/git/1a1e33f6-3514-9afc-0a28-5a6b85bd8014@gmail.com/ [12] Others commenting on eventually switching default to behavior A: + * https://lore.kernel.org/git/xmqqh719pcoo.fsf@gitster.g/ * https://lore.kernel.org/git/xmqqzgeqw0sy.fsf@gitster.g/ * https://lore.kernel.org/git/a86af661-cf58-a4e5-0214-a67d3a794d7e@github.com/ -[13] Previous config name suggestion and description - * https://lore.kernel.org/git/CABPp-BE6zW0nJSStcVU=_DoDBnPgLqOR8pkTXK3dW11=T01OhA@mail.gmail.com/ +[13] Previous config name suggestion and description: + + https://lore.kernel.org/git/CABPp-BE6zW0nJSStcVU=_DoDBnPgLqOR8pkTXK3dW11=T01OhA@mail.gmail.com/ [14] Tangential issue: switch to cone mode as default sparse specification mechanism: - https://lore.kernel.org/git/a1b68fd6126eb341ef3637bb93fedad4309b36d0.1650594746.git.gitgitgadget@gmail.com/ + +https://lore.kernel.org/git/a1b68fd6126eb341ef3637bb93fedad4309b36d0.1650594746.git.gitgitgadget@gmail.com/ [15] Lengthy email on grep behavior, covering what should be searched: - * https://lore.kernel.org/git/CABPp-BGVO3QdbfE84uF_3QDF0-y2iHHh6G5FAFzNRfeRitkuHw@mail.gmail.com/ + +https://lore.kernel.org/git/CABPp-BGVO3QdbfE84uF_3QDF0-y2iHHh6G5FAFzNRfeRitkuHw@mail.gmail.com/ [16] Email explaining sparsity patterns vs. SKIP_WORKTREE and history operations, search for the parenthetical comment starting "We do not check". - https://lore.kernel.org/git/CABPp-BFsCPPNOZ92JQRJeGyNd0e-TCW-LcLyr0i_+VSQJP+GCg@mail.gmail.com/ + +https://lore.kernel.org/git/CABPp-BFsCPPNOZ92JQRJeGyNd0e-TCW-LcLyr0i_+VSQJP+GCg@mail.gmail.com/ [17] https://lore.kernel.org/git/20220207190320.2960362-1-jonathantanmy@google.com/ diff --git a/Documentation/technical/unambiguous-types.adoc b/Documentation/technical/unambiguous-types.adoc new file mode 100644 index 0000000000..658a5b578e --- /dev/null +++ b/Documentation/technical/unambiguous-types.adoc @@ -0,0 +1,229 @@ += Unambiguous types + +Most of these mappings are obvious, but there are some nuances and gotchas with +Rust FFI (Foreign Function Interface). + +This document defines clear, one-to-one mappings between primitive types in C, +Rust (and possible other languages in the future). Its purpose is to eliminate +ambiguity in type widths, signedness, and binary representation across +platforms and languages. + +For Git, the only header required to use these unambiguous types in C is +`git-compat-util.h`. + +== Boolean types +[cols="1,1", options="header"] +|=== +| C Type | Rust Type +| bool^1^ | bool +|=== + +== Integer types + +In C, `<stdint.h>` (or an equivalent) must be included. + +[cols="1,1", options="header"] +|=== +| C Type | Rust Type +| uint8_t | u8 +| uint16_t | u16 +| uint32_t | u32 +| uint64_t | u64 + +| int8_t | i8 +| int16_t | i16 +| int32_t | i32 +| int64_t | i64 +|=== + +== Floating-point types + +Rust requires IEEE-754 semantics. +In C, that is typically true, but not guaranteed by the standard. + +[cols="1,1", options="header"] +|=== +| C Type | Rust Type +| float^2^ | f32 +| double^2^ | f64 +|=== + +== Size types + +These types represent pointer-sized integers and are typically defined in +`<stddef.h>` or an equivalent header. + +Size types should be used any time pointer arithmetic is performed e.g. +indexing an array, describing the number of elements in memory, etc... + +[cols="1,1", options="header"] +|=== +| C Type | Rust Type +| size_t^3^ | usize +| ptrdiff_t^4^ | isize +|=== + +== Character types + +This is where C and Rust don't have a clean one-to-one mapping. A C `char` is +an 8-bit type that is signless (neither signed nor unsigned) which causes +problems with e.g. `make DEVELOPER=1`. Rust's `char` type is an unsigned 32-bit +integer that is used to describe Unicode code points. Even though a C `char` +is the same width as `u8`, `char` should be converted to u8 where it is +describing bytes in memory. If a C `char` is not describing bytes, then it +should be converted to a more accurate unambiguous type. + +While you could specify `char` in the C code and `u8` in Rust code, it's not as +clear what the appropriate type is, but it would work across the FFI boundary. +However the bigger problem comes from code generation tools like cbindgen and +bindgen. When cbindgen see u8 in Rust it will generate uint8_t on the C side +which will cause differ in signedness warnings/errors. Similaraly if bindgen +see `char` on the C side it will generate `std::ffi::c_char` which has its own +problems. + +=== Notes +^1^ This is only true if stdbool.h (or equivalent) is used. + +^2^ C does not enforce IEEE-754 compatibility, but Rust expects it. If the +platform/arch for C does not follow IEEE-754 then this equivalence does not +hold. Also, it's assumed that `float` is 32 bits and `double` is 64, but +there may be a strange platform/arch where even this isn't true. + +^3^ C also defines uintptr_t, but this should not be used in Git. + +^4^ C also defines ssize_t and intptr_t, but these should not be used in Git. + + +== Problems with std::ffi::c_* types in Rust +TL;DR: They're not guaranteed to match C types for all possible C +compilers/platforms/architectures. + +Only a few of Rust's C FFI types are considered safe and semantically clear to +use: + + +* `c_void` +* `CStr` +* `CString` + +Even then, they should be used sparingly, and only where the semantics match +exactly. + +The std::os::raw::c_* (which is deprecated) directly inherits the problems of +core::ffi, which changes over time and seems to make a best guess at the +correct definition for a given platform/target. This probably isn't a problem +for all platforms that Rust supports currently, but can anyone say that Rust +got it right for all C compilers of all platforms/targets? + +On top of all of that we're targeting an older version of Rust which doesn't +have the latest mappings. + +To give an example: c_long is defined in +footnote:[https://doc.rust-lang.org/1.63.0/src/core/ffi/mod.rs.html#175-189[c_long in 1.63.0]] +footnote:[https://doc.rust-lang.org/1.89.0/src/core/ffi/primitives.rs.html#135-151[c_long in 1.89.0]] + +=== Rust version 1.63.0 + +[source] +---- +mod c_long_definition { + cfg_if! { + if #[cfg(all(target_pointer_width = "64", not(windows)))] { + pub type c_long = i64; + pub type NonZero_c_long = crate::num::NonZeroI64; + pub type c_ulong = u64; + pub type NonZero_c_ulong = crate::num::NonZeroU64; + } else { + // The minimal size of `long` in the C standard is 32 bits + pub type c_long = i32; + pub type NonZero_c_long = crate::num::NonZeroI32; + pub type c_ulong = u32; + pub type NonZero_c_ulong = crate::num::NonZeroU32; + } + } +} +---- + +=== Rust version 1.89.0 + +[source] +---- +mod c_long_definition { + crate::cfg_select! { + any( + all(target_pointer_width = "64", not(windows)), + // wasm32 Linux ABI uses 64-bit long + all(target_arch = "wasm32", target_os = "linux") + ) => { + pub(super) type c_long = i64; + pub(super) type c_ulong = u64; + } + _ => { + // The minimal size of `long` in the C standard is 32 bits + pub(super) type c_long = i32; + pub(super) type c_ulong = u32; + } + } +} +---- + +Even for the cases where C types are correctly mapped to Rust types via +std::ffi::c_* there are still problems. Let's take c_char for example. On some +platforms it's u8 on others it's i8. + +=== Subtraction underflow in debug mode + +The following code will panic in debug on platforms that define c_char as u8, +but won't if it's an i8. + +[source] +---- +let mut x: std::ffi::c_char = 0; +x -= 1; +---- + +=== Inconsistent shift behavior + +`x` will be 0xC0 for platforms that use i8, but will be 0x40 where it's u8. + +[source] +---- +let mut x: std::ffi::c_char = 0x80; +x >>= 1; +---- + +=== Equality fails to compile on some platforms + +The following will not compile on platforms that define c_char as i8, but will +if it's u8. You can cast x e.g. `assert_eq!(x as u8, b'a');`, but then you get +a warning on platforms that use u8 and a clean compilation where i8 is used. + +[source] +---- +let mut x: std::ffi::c_char = 0x61; +assert_eq!(x, b'a'); +---- + +== Enum types +Rust enum types should not be used as FFI types. Rust enum types are more like +C union types than C enum's. For something like: + +[source] +---- +#[repr(C, u8)] +enum Fruit { + Apple, + Banana, + Cherry, +} +---- + +It's easy enough to make sure the Rust enum matches what C would expect, but a +more complex type like. + +[source] +---- +enum HashResult { + SHA1([u8; 20]), + SHA256([u8; 32]), +} +---- + +The Rust compiler has to add a discriminant to the enum to distinguish between +the variants. The width, location, and values for that discriminant is up to +the Rust compiler and is not ABI stable. diff --git a/GIT-VERSION-GEN b/GIT-VERSION-GEN index b16db85e77..4929570f2c 100755 --- a/GIT-VERSION-GEN +++ b/GIT-VERSION-GEN @@ -1,6 +1,6 @@ #!/bin/sh -DEF_VER=v2.51.GIT +DEF_VER=v2.52.0-rc1 LF=' ' @@ -927,10 +927,17 @@ export PYTHON_PATH TEST_SHELL_PATH = $(SHELL_PATH) LIB_FILE = libgit.a + ifdef DEBUG -RUST_LIB = target/debug/libgitcore.a +RUST_TARGET_DIR = target/debug else -RUST_LIB = target/release/libgitcore.a +RUST_TARGET_DIR = target/release +endif + +ifeq ($(uname_S),Windows) +RUST_LIB = $(RUST_TARGET_DIR)/gitcore.lib +else +RUST_LIB = $(RUST_TARGET_DIR)/libgitcore.a endif GITLIBS = common-main.o $(LIB_FILE) @@ -1260,7 +1267,14 @@ LIB_OBJS += reftable/table.o LIB_OBJS += reftable/tree.o LIB_OBJS += reftable/writer.o LIB_OBJS += remote.o +LIB_OBJS += repack.o +LIB_OBJS += repack-cruft.o +LIB_OBJS += repack-filtered.o +LIB_OBJS += repack-geometry.o +LIB_OBJS += repack-midx.o +LIB_OBJS += repack-promisor.o LIB_OBJS += replace-object.o +LIB_OBJS += replay.o LIB_OBJS += repo-settings.o LIB_OBJS += repository.o LIB_OBJS += rerere.o @@ -1394,6 +1408,7 @@ BUILTIN_OBJS += builtin/get-tar-commit-id.o BUILTIN_OBJS += builtin/grep.o BUILTIN_OBJS += builtin/hash-object.o BUILTIN_OBJS += builtin/help.o +BUILTIN_OBJS += builtin/history.o BUILTIN_OBJS += builtin/hook.o BUILTIN_OBJS += builtin/index-pack.o BUILTIN_OBJS += builtin/init-db.o @@ -1521,7 +1536,10 @@ CLAR_TEST_OBJS += $(UNIT_TEST_DIR)/unit-test.o UNIT_TEST_OBJS += $(UNIT_TEST_DIR)/test-lib.o +RUST_SOURCES += src/csum_file.rs +RUST_SOURCES += src/hash.rs RUST_SOURCES += src/lib.rs +RUST_SOURCES += src/loose.rs RUST_SOURCES += src/varint.rs GIT-VERSION-FILE: FORCE @@ -1556,6 +1574,9 @@ ALL_LDFLAGS = $(LDFLAGS) $(LDFLAGS_APPEND) ifdef WITH_RUST BASIC_CFLAGS += -DWITH_RUST GITLIBS += $(RUST_LIB) +ifeq ($(uname_S),Windows) +EXTLIBS += -luserenv +endif endif ifdef SANITIZE @@ -2947,7 +2968,7 @@ scalar$X: scalar.o GIT-LDFLAGS $(GITLIBS) $(LIB_FILE): $(LIB_OBJS) $(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^ -$(RUST_LIB): Cargo.toml $(RUST_SOURCES) +$(RUST_LIB): Cargo.toml $(RUST_SOURCES) $(XDIFF_LIB) $(LIB_FILE) $(REFTABLE_LIB) $(QUIET_CARGO)cargo build $(CARGO_ARGS) .PHONY: rust diff --git a/add-interactive.c b/add-interactive.c index 68fc09547d..05d2e7eefe 100644 --- a/add-interactive.c +++ b/add-interactive.c @@ -3,7 +3,6 @@ #include "git-compat-util.h" #include "add-interactive.h" #include "color.h" -#include "config.h" #include "diffcore.h" #include "gettext.h" #include "hash.h" @@ -20,119 +19,18 @@ #include "prompt.h" #include "tree.h" -static void init_color(struct repository *r, enum git_colorbool use_color, - const char *section_and_slot, char *dst, - const char *default_color) -{ - char *key = xstrfmt("color.%s", section_and_slot); - const char *value; - - if (!want_color(use_color)) - dst[0] = '\0'; - else if (repo_config_get_value(r, key, &value) || - color_parse(value, dst)) - strlcpy(dst, default_color, COLOR_MAXLEN); - - free(key); -} - -static enum git_colorbool check_color_config(struct repository *r, const char *var) -{ - const char *value; - enum git_colorbool ret; - - if (repo_config_get_value(r, var, &value)) - ret = GIT_COLOR_UNKNOWN; - else - ret = git_config_colorbool(var, value); - - /* - * Do not rely on want_color() to fall back to color.ui for us. It uses - * the value parsed by git_color_config(), which may not have been - * called by the main command. - */ - if (ret == GIT_COLOR_UNKNOWN && - !repo_config_get_value(r, "color.ui", &value)) - ret = git_config_colorbool("color.ui", value); - - return ret; -} - void init_add_i_state(struct add_i_state *s, struct repository *r, - struct add_p_opt *add_p_opt) + struct interactive_options *opts) { s->r = r; - s->context = -1; - s->interhunkcontext = -1; - - s->use_color_interactive = check_color_config(r, "color.interactive"); - - init_color(r, s->use_color_interactive, "interactive.header", - s->header_color, GIT_COLOR_BOLD); - init_color(r, s->use_color_interactive, "interactive.help", - s->help_color, GIT_COLOR_BOLD_RED); - init_color(r, s->use_color_interactive, "interactive.prompt", - s->prompt_color, GIT_COLOR_BOLD_BLUE); - init_color(r, s->use_color_interactive, "interactive.error", - s->error_color, GIT_COLOR_BOLD_RED); - strlcpy(s->reset_color_interactive, - want_color(s->use_color_interactive) ? GIT_COLOR_RESET : "", COLOR_MAXLEN); - - s->use_color_diff = check_color_config(r, "color.diff"); - - init_color(r, s->use_color_diff, "diff.frag", s->fraginfo_color, - diff_get_color(s->use_color_diff, DIFF_FRAGINFO)); - init_color(r, s->use_color_diff, "diff.context", s->context_color, - "fall back"); - if (!strcmp(s->context_color, "fall back")) - init_color(r, s->use_color_diff, "diff.plain", - s->context_color, - diff_get_color(s->use_color_diff, DIFF_CONTEXT)); - init_color(r, s->use_color_diff, "diff.old", s->file_old_color, - diff_get_color(s->use_color_diff, DIFF_FILE_OLD)); - init_color(r, s->use_color_diff, "diff.new", s->file_new_color, - diff_get_color(s->use_color_diff, DIFF_FILE_NEW)); - strlcpy(s->reset_color_diff, - want_color(s->use_color_diff) ? GIT_COLOR_RESET : "", COLOR_MAXLEN); - - FREE_AND_NULL(s->interactive_diff_filter); - repo_config_get_string(r, "interactive.difffilter", - &s->interactive_diff_filter); - - FREE_AND_NULL(s->interactive_diff_algorithm); - repo_config_get_string(r, "diff.algorithm", - &s->interactive_diff_algorithm); - - if (!repo_config_get_int(r, "diff.context", &s->context)) - if (s->context < 0) - die(_("%s cannot be negative"), "diff.context"); - if (!repo_config_get_int(r, "diff.interHunkContext", &s->interhunkcontext)) - if (s->interhunkcontext < 0) - die(_("%s cannot be negative"), "diff.interHunkContext"); - - repo_config_get_bool(r, "interactive.singlekey", &s->use_single_key); - if (s->use_single_key) - setbuf(stdin, NULL); - - if (add_p_opt->context != -1) { - if (add_p_opt->context < 0) - die(_("%s cannot be negative"), "--unified"); - s->context = add_p_opt->context; - } - if (add_p_opt->interhunkcontext != -1) { - if (add_p_opt->interhunkcontext < 0) - die(_("%s cannot be negative"), "--inter-hunk-context"); - s->interhunkcontext = add_p_opt->interhunkcontext; - } + interactive_config_init(&s->cfg, r, opts); } void clear_add_i_state(struct add_i_state *s) { - FREE_AND_NULL(s->interactive_diff_filter); - FREE_AND_NULL(s->interactive_diff_algorithm); + interactive_config_clear(&s->cfg); memset(s, 0, sizeof(*s)); - s->use_color_interactive = GIT_COLOR_UNKNOWN; - s->use_color_diff = GIT_COLOR_UNKNOWN; + interactive_config_clear(&s->cfg); } /* @@ -286,7 +184,7 @@ static void list(struct add_i_state *s, struct string_list *list, int *selected, return; if (opts->header) - color_fprintf_ln(stdout, s->header_color, + color_fprintf_ln(stdout, s->cfg.header_color, "%s", opts->header); for (i = 0; i < list->nr; i++) { @@ -354,7 +252,7 @@ static ssize_t list_and_choose(struct add_i_state *s, list(s, &items->items, items->selected, &opts->list_opts); - color_fprintf(stdout, s->prompt_color, "%s", opts->prompt); + color_fprintf(stdout, s->cfg.prompt_color, "%s", opts->prompt); fputs(singleton ? "> " : ">> ", stdout); fflush(stdout); @@ -432,7 +330,7 @@ static ssize_t list_and_choose(struct add_i_state *s, if (from < 0 || from >= items->items.nr || (singleton && from + 1 != to)) { - color_fprintf_ln(stderr, s->error_color, + color_fprintf_ln(stderr, s->cfg.error_color, _("Huh (%s)?"), p); break; } else if (singleton) { @@ -992,7 +890,7 @@ static int run_patch(struct add_i_state *s, const struct pathspec *ps, free(files->items.items[i].string); } else if (item->index.unmerged || item->worktree.unmerged) { - color_fprintf_ln(stderr, s->error_color, + color_fprintf_ln(stderr, s->cfg.error_color, _("ignoring unmerged: %s"), files->items.items[i].string); free(item); @@ -1014,9 +912,9 @@ static int run_patch(struct add_i_state *s, const struct pathspec *ps, opts->prompt = N_("Patch update"); count = list_and_choose(s, files, opts); if (count > 0) { - struct add_p_opt add_p_opt = { - .context = s->context, - .interhunkcontext = s->interhunkcontext, + struct interactive_options opts = { + .context = s->cfg.context, + .interhunkcontext = s->cfg.interhunkcontext, }; struct strvec args = STRVEC_INIT; struct pathspec ps_selected = { 0 }; @@ -1028,7 +926,7 @@ static int run_patch(struct add_i_state *s, const struct pathspec *ps, parse_pathspec(&ps_selected, PATHSPEC_ALL_MAGIC & ~PATHSPEC_LITERAL, PATHSPEC_LITERAL_PATH, "", args.v); - res = run_add_p(s->r, ADD_P_ADD, &add_p_opt, NULL, &ps_selected); + res = run_add_p(s->r, ADD_P_ADD, &opts, NULL, &ps_selected); strvec_clear(&args); clear_pathspec(&ps_selected); } @@ -1064,10 +962,10 @@ static int run_diff(struct add_i_state *s, const struct pathspec *ps, struct child_process cmd = CHILD_PROCESS_INIT; strvec_pushl(&cmd.args, "git", "diff", "-p", "--cached", NULL); - if (s->context != -1) - strvec_pushf(&cmd.args, "--unified=%i", s->context); - if (s->interhunkcontext != -1) - strvec_pushf(&cmd.args, "--inter-hunk-context=%i", s->interhunkcontext); + if (s->cfg.context != -1) + strvec_pushf(&cmd.args, "--unified=%i", s->cfg.context); + if (s->cfg.interhunkcontext != -1) + strvec_pushf(&cmd.args, "--inter-hunk-context=%i", s->cfg.interhunkcontext); strvec_pushl(&cmd.args, oid_to_hex(!is_initial ? &oid : s->r->hash_algo->empty_tree), "--", NULL); for (i = 0; i < files->items.nr; i++) @@ -1085,17 +983,17 @@ static int run_help(struct add_i_state *s, const struct pathspec *ps UNUSED, struct prefix_item_list *files UNUSED, struct list_and_choose_options *opts UNUSED) { - color_fprintf_ln(stdout, s->help_color, "status - %s", + color_fprintf_ln(stdout, s->cfg.help_color, "status - %s", _("show paths with changes")); - color_fprintf_ln(stdout, s->help_color, "update - %s", + color_fprintf_ln(stdout, s->cfg.help_color, "update - %s", _("add working tree state to the staged set of changes")); - color_fprintf_ln(stdout, s->help_color, "revert - %s", + color_fprintf_ln(stdout, s->cfg.help_color, "revert - %s", _("revert staged set of changes back to the HEAD version")); - color_fprintf_ln(stdout, s->help_color, "patch - %s", + color_fprintf_ln(stdout, s->cfg.help_color, "patch - %s", _("pick hunks and update selectively")); - color_fprintf_ln(stdout, s->help_color, "diff - %s", + color_fprintf_ln(stdout, s->cfg.help_color, "diff - %s", _("view diff between HEAD and index")); - color_fprintf_ln(stdout, s->help_color, "add untracked - %s", + color_fprintf_ln(stdout, s->cfg.help_color, "add untracked - %s", _("add contents of untracked files to the staged set of changes")); return 0; @@ -1103,21 +1001,21 @@ static int run_help(struct add_i_state *s, const struct pathspec *ps UNUSED, static void choose_prompt_help(struct add_i_state *s) { - color_fprintf_ln(stdout, s->help_color, "%s", + color_fprintf_ln(stdout, s->cfg.help_color, "%s", _("Prompt help:")); - color_fprintf_ln(stdout, s->help_color, "1 - %s", + color_fprintf_ln(stdout, s->cfg.help_color, "1 - %s", _("select a single item")); - color_fprintf_ln(stdout, s->help_color, "3-5 - %s", + color_fprintf_ln(stdout, s->cfg.help_color, "3-5 - %s", _("select a range of items")); - color_fprintf_ln(stdout, s->help_color, "2-3,6-9 - %s", + color_fprintf_ln(stdout, s->cfg.help_color, "2-3,6-9 - %s", _("select multiple ranges")); - color_fprintf_ln(stdout, s->help_color, "foo - %s", + color_fprintf_ln(stdout, s->cfg.help_color, "foo - %s", _("select item based on unique prefix")); - color_fprintf_ln(stdout, s->help_color, "-... - %s", + color_fprintf_ln(stdout, s->cfg.help_color, "-... - %s", _("unselect specified items")); - color_fprintf_ln(stdout, s->help_color, "* - %s", + color_fprintf_ln(stdout, s->cfg.help_color, "* - %s", _("choose all items")); - color_fprintf_ln(stdout, s->help_color, " - %s", + color_fprintf_ln(stdout, s->cfg.help_color, " - %s", _("(empty) finish selecting")); } @@ -1152,7 +1050,7 @@ static void print_command_item(int i, int selected UNUSED, static void command_prompt_help(struct add_i_state *s) { - const char *help_color = s->help_color; + const char *help_color = s->cfg.help_color; color_fprintf_ln(stdout, help_color, "%s", _("Prompt help:")); color_fprintf_ln(stdout, help_color, "1 - %s", _("select a numbered item")); @@ -1163,7 +1061,7 @@ static void command_prompt_help(struct add_i_state *s) } int run_add_i(struct repository *r, const struct pathspec *ps, - struct add_p_opt *add_p_opt) + struct interactive_options *interactive_opts) { struct add_i_state s = { NULL }; struct print_command_item_data data = { "[", "]" }; @@ -1206,15 +1104,15 @@ int run_add_i(struct repository *r, const struct pathspec *ps, ->util = util; } - init_add_i_state(&s, r, add_p_opt); + init_add_i_state(&s, r, interactive_opts); /* * When color was asked for, use the prompt color for * highlighting, otherwise use square brackets. */ - if (want_color(s.use_color_interactive)) { - data.color = s.prompt_color; - data.reset = s.reset_color_interactive; + if (want_color(s.cfg.use_color_interactive)) { + data.color = s.cfg.prompt_color; + data.reset = s.cfg.reset_color_interactive; } print_file_item_data.color = data.color; print_file_item_data.reset = data.reset; diff --git a/add-interactive.h b/add-interactive.h index da49502b76..eefa2edc7c 100644 --- a/add-interactive.h +++ b/add-interactive.h @@ -1,55 +1,21 @@ #ifndef ADD_INTERACTIVE_H #define ADD_INTERACTIVE_H -#include "color.h" +#include "add-patch.h" -struct add_p_opt { - int context; - int interhunkcontext; -}; - -#define ADD_P_OPT_INIT { .context = -1, .interhunkcontext = -1 } +struct pathspec; +struct repository; struct add_i_state { struct repository *r; - enum git_colorbool use_color_interactive; - enum git_colorbool use_color_diff; - char header_color[COLOR_MAXLEN]; - char help_color[COLOR_MAXLEN]; - char prompt_color[COLOR_MAXLEN]; - char error_color[COLOR_MAXLEN]; - char reset_color_interactive[COLOR_MAXLEN]; - - char fraginfo_color[COLOR_MAXLEN]; - char context_color[COLOR_MAXLEN]; - char file_old_color[COLOR_MAXLEN]; - char file_new_color[COLOR_MAXLEN]; - char reset_color_diff[COLOR_MAXLEN]; - - int use_single_key; - char *interactive_diff_filter, *interactive_diff_algorithm; - int context, interhunkcontext; + struct interactive_config cfg; }; void init_add_i_state(struct add_i_state *s, struct repository *r, - struct add_p_opt *add_p_opt); + struct interactive_options *opts); void clear_add_i_state(struct add_i_state *s); -struct repository; -struct pathspec; int run_add_i(struct repository *r, const struct pathspec *ps, - struct add_p_opt *add_p_opt); - -enum add_p_mode { - ADD_P_ADD, - ADD_P_STASH, - ADD_P_RESET, - ADD_P_CHECKOUT, - ADD_P_WORKTREE, -}; - -int run_add_p(struct repository *r, enum add_p_mode mode, - struct add_p_opt *o, const char *revision, - const struct pathspec *ps); + struct interactive_options *opts); #endif diff --git a/add-patch.c b/add-patch.c index 9402dc71bc..e689932448 100644 --- a/add-patch.c +++ b/add-patch.c @@ -2,11 +2,15 @@ #define DISABLE_SIGN_COMPARE_WARNINGS #include "git-compat-util.h" -#include "add-interactive.h" +#include "add-patch.h" #include "advice.h" +#include "commit.h" +#include "config.h" +#include "diff.h" #include "editor.h" #include "environment.h" #include "gettext.h" +#include "hex.h" #include "object-name.h" #include "pager.h" #include "read-cache-ll.h" @@ -45,7 +49,7 @@ static struct patch_mode patch_mode_add = { N_("Stage mode change [y,n,q,a,d%s,?]? "), N_("Stage deletion [y,n,q,a,d%s,?]? "), N_("Stage addition [y,n,q,a,d%s,?]? "), - N_("Stage this hunk [y,n,q,a,d%s,?]? ") + N_("Stage this hunk [y,n,q,a,d%s,?]? "), }, .edit_hunk_hint = N_("If the patch applies cleanly, the edited hunk " "will immediately be marked for staging."), @@ -260,7 +264,10 @@ struct hunk { }; struct add_p_state { - struct add_i_state s; + struct repository *r; + struct index_state *index; + const char *index_file; + struct interactive_config cfg; struct strbuf answer, buf; /* parsed diff */ @@ -278,6 +285,122 @@ struct add_p_state { const char *revision; }; +static void init_color(struct repository *r, + enum git_colorbool use_color, + const char *section_and_slot, char *dst, + const char *default_color) +{ + char *key = xstrfmt("color.%s", section_and_slot); + const char *value; + + if (!want_color(use_color)) + dst[0] = '\0'; + else if (repo_config_get_value(r, key, &value) || + color_parse(value, dst)) + strlcpy(dst, default_color, COLOR_MAXLEN); + + free(key); +} + +static enum git_colorbool check_color_config(struct repository *r, const char *var) +{ + const char *value; + enum git_colorbool ret; + + if (repo_config_get_value(r, var, &value)) + ret = GIT_COLOR_UNKNOWN; + else + ret = git_config_colorbool(var, value); + + /* + * Do not rely on want_color() to fall back to color.ui for us. It uses + * the value parsed by git_color_config(), which may not have been + * called by the main command. + */ + if (ret == GIT_COLOR_UNKNOWN && + !repo_config_get_value(r, "color.ui", &value)) + ret = git_config_colorbool("color.ui", value); + + return ret; +} + +void interactive_config_init(struct interactive_config *cfg, + struct repository *r, + struct interactive_options *opts) +{ + cfg->context = -1; + cfg->interhunkcontext = -1; + + cfg->use_color_interactive = check_color_config(r, "color.interactive"); + + init_color(r, cfg->use_color_interactive, "interactive.header", + cfg->header_color, GIT_COLOR_BOLD); + init_color(r, cfg->use_color_interactive, "interactive.help", + cfg->help_color, GIT_COLOR_BOLD_RED); + init_color(r, cfg->use_color_interactive, "interactive.prompt", + cfg->prompt_color, GIT_COLOR_BOLD_BLUE); + init_color(r, cfg->use_color_interactive, "interactive.error", + cfg->error_color, GIT_COLOR_BOLD_RED); + strlcpy(cfg->reset_color_interactive, + want_color(cfg->use_color_interactive) ? GIT_COLOR_RESET : "", COLOR_MAXLEN); + + cfg->use_color_diff = check_color_config(r, "color.diff"); + + init_color(r, cfg->use_color_diff, "diff.frag", cfg->fraginfo_color, + diff_get_color(cfg->use_color_diff, DIFF_FRAGINFO)); + init_color(r, cfg->use_color_diff, "diff.context", cfg->context_color, + "fall back"); + if (!strcmp(cfg->context_color, "fall back")) + init_color(r, cfg->use_color_diff, "diff.plain", + cfg->context_color, + diff_get_color(cfg->use_color_diff, DIFF_CONTEXT)); + init_color(r, cfg->use_color_diff, "diff.old", cfg->file_old_color, + diff_get_color(cfg->use_color_diff, DIFF_FILE_OLD)); + init_color(r, cfg->use_color_diff, "diff.new", cfg->file_new_color, + diff_get_color(cfg->use_color_diff, DIFF_FILE_NEW)); + strlcpy(cfg->reset_color_diff, + want_color(cfg->use_color_diff) ? GIT_COLOR_RESET : "", COLOR_MAXLEN); + + FREE_AND_NULL(cfg->interactive_diff_filter); + repo_config_get_string(r, "interactive.difffilter", + &cfg->interactive_diff_filter); + + FREE_AND_NULL(cfg->interactive_diff_algorithm); + repo_config_get_string(r, "diff.algorithm", + &cfg->interactive_diff_algorithm); + + if (!repo_config_get_int(r, "diff.context", &cfg->context)) + if (cfg->context < 0) + die(_("%s cannot be negative"), "diff.context"); + if (!repo_config_get_int(r, "diff.interHunkContext", &cfg->interhunkcontext)) + if (cfg->interhunkcontext < 0) + die(_("%s cannot be negative"), "diff.interHunkContext"); + + repo_config_get_bool(r, "interactive.singlekey", &cfg->use_single_key); + if (cfg->use_single_key) + setbuf(stdin, NULL); + + if (opts->context != -1) { + if (opts->context < 0) + die(_("%s cannot be negative"), "--unified"); + cfg->context = opts->context; + } + if (opts->interhunkcontext != -1) { + if (opts->interhunkcontext < 0) + die(_("%s cannot be negative"), "--inter-hunk-context"); + cfg->interhunkcontext = opts->interhunkcontext; + } +} + +void interactive_config_clear(struct interactive_config *cfg) +{ + FREE_AND_NULL(cfg->interactive_diff_filter); + FREE_AND_NULL(cfg->interactive_diff_algorithm); + memset(cfg, 0, sizeof(*cfg)); + cfg->use_color_interactive = GIT_COLOR_UNKNOWN; + cfg->use_color_diff = GIT_COLOR_UNKNOWN; +} + static void add_p_state_clear(struct add_p_state *s) { size_t i; @@ -289,7 +412,7 @@ static void add_p_state_clear(struct add_p_state *s) for (i = 0; i < s->file_diff_nr; i++) free(s->file_diff[i].hunk); free(s->file_diff); - clear_add_i_state(&s->s); + interactive_config_clear(&s->cfg); } __attribute__((format (printf, 2, 3))) @@ -298,9 +421,9 @@ static void err(struct add_p_state *s, const char *fmt, ...) va_list args; va_start(args, fmt); - fputs(s->s.error_color, stdout); + fputs(s->cfg.error_color, stdout); vprintf(fmt, args); - puts(s->s.reset_color_interactive); + puts(s->cfg.reset_color_interactive); va_end(args); } @@ -318,7 +441,7 @@ static void setup_child_process(struct add_p_state *s, cp->git_cmd = 1; strvec_pushf(&cp->env, - INDEX_ENVIRONMENT "=%s", s->s.r->index_file); + INDEX_ENVIRONMENT "=%s", s->index_file); } static int parse_range(const char **p, @@ -423,12 +546,12 @@ static int parse_diff(struct add_p_state *s, const struct pathspec *ps) int res; strvec_pushv(&args, s->mode->diff_cmd); - if (s->s.context != -1) - strvec_pushf(&args, "--unified=%i", s->s.context); - if (s->s.interhunkcontext != -1) - strvec_pushf(&args, "--inter-hunk-context=%i", s->s.interhunkcontext); - if (s->s.interactive_diff_algorithm) - strvec_pushf(&args, "--diff-algorithm=%s", s->s.interactive_diff_algorithm); + if (s->cfg.context != -1) + strvec_pushf(&args, "--unified=%i", s->cfg.context); + if (s->cfg.interhunkcontext != -1) + strvec_pushf(&args, "--inter-hunk-context=%i", s->cfg.interhunkcontext); + if (s->cfg.interactive_diff_algorithm) + strvec_pushf(&args, "--diff-algorithm=%s", s->cfg.interactive_diff_algorithm); if (s->revision) { struct object_id oid; strvec_push(&args, @@ -457,9 +580,9 @@ static int parse_diff(struct add_p_state *s, const struct pathspec *ps) } strbuf_complete_line(plain); - if (want_color_fd(1, s->s.use_color_diff)) { + if (want_color_fd(1, s->cfg.use_color_diff)) { struct child_process colored_cp = CHILD_PROCESS_INIT; - const char *diff_filter = s->s.interactive_diff_filter; + const char *diff_filter = s->cfg.interactive_diff_filter; setup_child_process(s, &colored_cp, NULL); xsnprintf((char *)args.v[color_arg_index], 8, "--color"); @@ -692,7 +815,7 @@ static void render_hunk(struct add_p_state *s, struct hunk *hunk, hunk->colored_end - hunk->colored_start); return; } else { - strbuf_addstr(out, s->s.fraginfo_color); + strbuf_addstr(out, s->cfg.fraginfo_color); p = s->colored.buf + header->colored_extra_start; len = header->colored_extra_end - header->colored_extra_start; @@ -714,7 +837,7 @@ static void render_hunk(struct add_p_state *s, struct hunk *hunk, if (len) strbuf_add(out, p, len); else if (colored) - strbuf_addf(out, "%s\n", s->s.reset_color_diff); + strbuf_addf(out, "%s\n", s->cfg.reset_color_diff); else strbuf_addch(out, '\n'); } @@ -1103,12 +1226,12 @@ static void recolor_hunk(struct add_p_state *s, struct hunk *hunk) strbuf_addstr(&s->colored, plain[current] == '-' ? - s->s.file_old_color : + s->cfg.file_old_color : plain[current] == '+' ? - s->s.file_new_color : - s->s.context_color); + s->cfg.file_new_color : + s->cfg.context_color); strbuf_add(&s->colored, plain + current, eol - current); - strbuf_addstr(&s->colored, s->s.reset_color_diff); + strbuf_addstr(&s->colored, s->cfg.reset_color_diff); if (next > eol) strbuf_add(&s->colored, plain + eol, next - eol); current = next; @@ -1237,7 +1360,7 @@ static int run_apply_check(struct add_p_state *s, static int read_single_character(struct add_p_state *s) { - if (s->s.use_single_key) { + if (s->cfg.use_single_key) { int res = read_key_without_echo(&s->answer); printf("%s\n", res == EOF ? "" : s->answer.buf); return res; @@ -1251,7 +1374,7 @@ static int read_single_character(struct add_p_state *s) static int prompt_yesno(struct add_p_state *s, const char *prompt) { for (;;) { - color_fprintf(stdout, s->s.prompt_color, "%s", _(prompt)); + color_fprintf(stdout, s->cfg.prompt_color, "%s", _(prompt)); fflush(stdout); if (read_single_character(s) == EOF) return -1; @@ -1416,7 +1539,8 @@ N_("j - go to the next undecided hunk, roll over at the bottom\n" "/ - search for a hunk matching the given regex\n" "s - split the current hunk into smaller hunks\n" "e - manually edit the current hunk\n" - "p - print the current hunk, 'P' to use the pager\n" + "p - print the current hunk\n" + "P - print the current hunk using the pager\n" "? - print help\n"); static size_t dec_mod(size_t a, size_t m) @@ -1547,7 +1671,7 @@ static int patch_update_file(struct add_p_state *s, permitted |= ALLOW_EDIT; strbuf_addstr(&s->buf, ",e"); } - strbuf_addstr(&s->buf, ",p"); + strbuf_addstr(&s->buf, ",p,P"); } if (file_diff->deleted) prompt_mode_type = PROMPT_DELETION; @@ -1558,18 +1682,20 @@ static int patch_update_file(struct add_p_state *s, else prompt_mode_type = PROMPT_HUNK; - printf("%s(%"PRIuMAX"/%"PRIuMAX") ", s->s.prompt_color, + printf("%s(%"PRIuMAX"/%"PRIuMAX") ", s->cfg.prompt_color, (uintmax_t)hunk_index + 1, (uintmax_t)(file_diff->hunk_nr ? file_diff->hunk_nr : 1)); printf(_(s->mode->prompt_mode[prompt_mode_type]), s->buf.buf); - if (*s->s.reset_color_interactive) - fputs(s->s.reset_color_interactive, stdout); + if (*s->cfg.reset_color_interactive) + fputs(s->cfg.reset_color_interactive, stdout); fflush(stdout); - if (read_single_character(s) == EOF) + if (read_single_character(s) == EOF) { + quit = 1; break; + } if (!s->answer.len) continue; @@ -1600,7 +1726,7 @@ soft_increment: } else if (hunk->use == UNDECIDED_HUNK) { hunk->use = USE_HUNK; } - } else if (ch == 'd' || ch == 'q') { + } else if (ch == 'd') { if (file_diff->hunk_nr) { for (; hunk_index < file_diff->hunk_nr; hunk_index++) { hunk = file_diff->hunk + hunk_index; @@ -1612,10 +1738,9 @@ soft_increment: } else if (hunk->use == UNDECIDED_HUNK) { hunk->use = SKIP_HUNK; } - if (ch == 'q') { - quit = 1; - break; - } + } else if (ch == 'q') { + quit = 1; + break; } else if (s->answer.buf[0] == 'K') { if (permitted & ALLOW_GOTO_PREVIOUS_HUNK) hunk_index = dec_mod(hunk_index, @@ -1728,7 +1853,7 @@ soft_increment: err(s, _("Sorry, cannot split this hunk")); } else if (!split_hunk(s, file_diff, hunk - file_diff->hunk)) { - color_fprintf_ln(stdout, s->s.header_color, + color_fprintf_ln(stdout, s->cfg.header_color, _("Split into %d hunks."), (int)splittable_into); rendered_hunk_index = -1; @@ -1746,7 +1871,7 @@ soft_increment: } else if (s->answer.buf[0] == '?') { const char *p = _(help_patch_remainder), *eol = p; - color_fprintf(stdout, s->s.help_color, "%s", + color_fprintf(stdout, s->cfg.help_color, "%s", _(s->mode->help_patch_text)); /* @@ -1764,7 +1889,7 @@ soft_increment: if (*p != '?' && !strchr(s->buf.buf, *p)) continue; - color_fprintf_ln(stdout, s->s.help_color, + color_fprintf_ln(stdout, s->cfg.help_color, "%.*s", (int)(eol - p), p); } } else { @@ -1784,7 +1909,7 @@ soft_increment: strbuf_reset(&s->buf); reassemble_patch(s, file_diff, 0, &s->buf); - discard_index(s->s.r->index); + discard_index(s->index); if (s->mode->apply_for_checkout) apply_for_checkout(s, &s->buf, s->mode->is_reverse); @@ -1795,9 +1920,11 @@ soft_increment: NULL, 0, NULL, 0)) error(_("'git apply' failed")); } - if (repo_read_index(s->s.r) >= 0) - repo_refresh_and_write_index(s->s.r, REFRESH_QUIET, 0, + if (read_index_from(s->index, s->index_file, s->r->gitdir) >= 0 && + s->index == s->r->index) { + repo_refresh_and_write_index(s->r, REFRESH_QUIET, 0, 1, NULL, NULL, NULL); + } } putchar('\n'); @@ -1805,15 +1932,21 @@ soft_increment: } int run_add_p(struct repository *r, enum add_p_mode mode, - struct add_p_opt *o, const char *revision, + struct interactive_options *opts, const char *revision, const struct pathspec *ps) { struct add_p_state s = { - { r }, STRBUF_INIT, STRBUF_INIT, STRBUF_INIT, STRBUF_INIT + .r = r, + .index = r->index, + .index_file = r->index_file, + .answer = STRBUF_INIT, + .buf = STRBUF_INIT, + .plain = STRBUF_INIT, + .colored = STRBUF_INIT, }; size_t i, binary_count = 0; - init_add_i_state(&s.s, r, o); + interactive_config_init(&s.cfg, r, opts); if (mode == ADD_P_STASH) s.mode = &patch_mode_stash; @@ -1864,3 +1997,99 @@ int run_add_p(struct repository *r, enum add_p_mode mode, add_p_state_clear(&s); return 0; } + +int run_add_p_index(struct repository *r, + struct index_state *index, + const char *index_file, + struct interactive_options *opts, + const char *revision, + const struct pathspec *ps) +{ + struct patch_mode mode = { + .apply_args = { "--cached", NULL }, + .apply_check_args = { "--cached", NULL }, + .prompt_mode = { + N_("Stage mode change [y,n,q,a,d%s,?]? "), + N_("Stage deletion [y,n,q,a,d%s,?]? "), + N_("Stage addition [y,n,q,a,d%s,?]? "), + N_("Stage this hunk [y,n,q,a,d%s,?]? ") + }, + .edit_hunk_hint = N_("If the patch applies cleanly, the edited hunk " + "will immediately be marked for staging."), + .help_patch_text = + N_("y - stage this hunk\n" + "n - do not stage this hunk\n" + "q - quit; do not stage this hunk or any of the remaining " + "ones\n" + "a - stage this hunk and all later hunks in the file\n" + "d - do not stage this hunk or any of the later hunks in " + "the file\n"), + .index_only = 1, + }; + struct add_p_state s = { + .r = r, + .index = index, + .index_file = index_file, + .answer = STRBUF_INIT, + .buf = STRBUF_INIT, + .plain = STRBUF_INIT, + .colored = STRBUF_INIT, + .mode = &mode, + .revision = revision, + }; + struct strbuf parent_revision = STRBUF_INIT; + char parent_tree_oid[GIT_MAX_HEXSZ + 1]; + size_t binary_count = 0; + struct commit *commit; + int ret; + + commit = lookup_commit_reference_by_name(revision); + if (!commit) { + err(&s, _("Revision does not refer to a commit")); + ret = -1; + goto out; + } + + if (commit->parents) + oid_to_hex_r(parent_tree_oid, get_commit_tree_oid(commit->parents->item)); + else + oid_to_hex_r(parent_tree_oid, r->hash_algo->empty_tree); + + strbuf_addf(&parent_revision, "%s~", revision); + mode.diff_cmd[0] = "diff-tree"; + mode.diff_cmd[1] = "-r"; + mode.diff_cmd[2] = parent_tree_oid; + + interactive_config_init(&s.cfg, r, opts); + + if (parse_diff(&s, ps) < 0) { + ret = -1; + goto out; + } + + for (size_t i = 0; i < s.file_diff_nr; i++) { + if (s.file_diff[i].binary && !s.file_diff[i].hunk_nr) + binary_count++; + else if (patch_update_file(&s, s.file_diff + i)) + break; + } + + if (s.file_diff_nr == 0) { + err(&s, _("No changes.")); + ret = -1; + goto out; + } + + if (binary_count == s.file_diff_nr) { + err(&s, _("Only binary files changed.")); + ret = -1; + goto out; + } + + ret = 0; + +out: + strbuf_release(&parent_revision); + add_p_state_clear(&s); + return ret; +} diff --git a/add-patch.h b/add-patch.h new file mode 100644 index 0000000000..901c42fd7b --- /dev/null +++ b/add-patch.h @@ -0,0 +1,64 @@ +#ifndef ADD_PATCH_H +#define ADD_PATCH_H + +#include "color.h" + +struct index_state; +struct pathspec; +struct repository; + +struct interactive_options { + int context; + int interhunkcontext; +}; + +#define INTERACTIVE_OPTIONS_INIT { \ + .context = -1, \ + .interhunkcontext = -1, \ +} + +struct interactive_config { + enum git_colorbool use_color_interactive; + enum git_colorbool use_color_diff; + char header_color[COLOR_MAXLEN]; + char help_color[COLOR_MAXLEN]; + char prompt_color[COLOR_MAXLEN]; + char error_color[COLOR_MAXLEN]; + char reset_color_interactive[COLOR_MAXLEN]; + + char fraginfo_color[COLOR_MAXLEN]; + char context_color[COLOR_MAXLEN]; + char file_old_color[COLOR_MAXLEN]; + char file_new_color[COLOR_MAXLEN]; + char reset_color_diff[COLOR_MAXLEN]; + + int use_single_key; + char *interactive_diff_filter, *interactive_diff_algorithm; + int context, interhunkcontext; +}; + +void interactive_config_init(struct interactive_config *cfg, + struct repository *r, + struct interactive_options *opts); +void interactive_config_clear(struct interactive_config *cfg); + +enum add_p_mode { + ADD_P_ADD, + ADD_P_STASH, + ADD_P_RESET, + ADD_P_CHECKOUT, + ADD_P_WORKTREE, +}; + +int run_add_p(struct repository *r, enum add_p_mode mode, + struct interactive_options *opts, const char *revision, + const struct pathspec *ps); + +int run_add_p_index(struct repository *r, + struct index_state *index, + const char *index_file, + struct interactive_options *opts, + const char *revision, + const struct pathspec *ps); + +#endif @@ -1640,6 +1640,14 @@ static void record_ws_error(struct apply_state *state, state->squelch_whitespace_errors < state->whitespace_error) return; + /* + * line[len] for an incomplete line points at the "\n" at the end + * of patch input line, so "%.*s" would drop the last letter on line; + * compensate for it. + */ + if (result & WS_INCOMPLETE_LINE) + len++; + err = whitespace_error_string(result); if (state->apply_verbosity > verbosity_silent) fprintf(stderr, "%s:%d: %s.\n%.*s\n", @@ -1671,6 +1679,35 @@ static void check_old_for_crlf(struct patch *patch, const char *line, int len) /* + * Just saw a single line in a fragment. If it is a part of this hunk + * that is a context " ", an added "+", or a removed "-" line, it may + * be followed by "\\ No newline..." to signal that the last "\n" on + * this line needs to be dropped. Depending on locale settings when + * the patch was produced we don't know what this line would exactly + * say. The only thing we do know is that it begins with "\ ". + * Checking for 12 is just for sanity check; "\ No newline..." would + * be at least that long in any l10n. + * + * Return 0 if the line we saw is not followed by "\ No newline...", + * or length of that line. The caller will use it to skip over the + * "\ No newline..." line. + */ +static int adjust_incomplete(const char *line, int len, + unsigned long size) +{ + int nextlen; + + if (*line != '\n' && *line != ' ' && *line != '+' && *line != '-') + return 0; + if (size - len < 12 || memcmp(line + len, "\\ ", 2)) + return 0; + nextlen = linelen(line + len, size - len); + if (nextlen < 12) + return 0; + return nextlen; +} + +/* * Parse a unified diff. Note that this really needs to parse each * fragment separately, since the only way to know the difference * between a "---" that is part of a patch, and a "---" that starts @@ -1684,6 +1721,7 @@ static int parse_fragment(struct apply_state *state, { int added, deleted; int len = linelen(line, size), offset; + int skip_len = 0; unsigned long oldlines, newlines; unsigned long leading, trailing; @@ -1710,6 +1748,22 @@ static int parse_fragment(struct apply_state *state, len = linelen(line, size); if (!len || line[len-1] != '\n') return -1; + + /* + * For an incomplete line, skip_len counts the bytes + * on "\\ No newline..." marker line that comes next + * to the current line. + * + * Reduce "len" to drop the newline at the end of + * line[], but add one to "skip_len", which will be + * added back to "len" for the next iteration, to + * compensate. + */ + skip_len = adjust_incomplete(line, len, size); + if (skip_len) { + len--; + skip_len++; + } switch (*line) { default: return -1; @@ -1745,19 +1799,12 @@ static int parse_fragment(struct apply_state *state, newlines--; trailing = 0; break; + } - /* - * We allow "\ No newline at end of file". Depending - * on locale settings when the patch was produced we - * don't know what this line looks like. The only - * thing we do know is that it begins with "\ ". - * Checking for 12 is just for sanity check -- any - * l10n of "\ No newline..." is at least that long. - */ - case '\\': - if (len < 12 || memcmp(line, "\\ ", 2)) - return -1; - break; + /* eat the "\\ No newline..." as well, if exists */ + if (skip_len) { + len += skip_len; + state->linenr++; } } if (oldlines || newlines) @@ -1768,14 +1815,6 @@ static int parse_fragment(struct apply_state *state, fragment->leading = leading; fragment->trailing = trailing; - /* - * If a fragment ends with an incomplete line, we failed to include - * it in the above loop because we hit oldlines == newlines == 0 - * before seeing it. - */ - if (12 < size && !memcmp(line, "\\ ", 2)) - offset += linelen(line, size); - patch->lines_added += added; patch->lines_deleted += deleted; @@ -450,21 +450,20 @@ void find_bisection(struct commit_list **commit_list, int *reaches, clear_commit_weight(&commit_weight); } -static int register_ref(const char *refname, const char *referent UNUSED, const struct object_id *oid, - int flags UNUSED, void *cb_data UNUSED) +static int register_ref(const struct reference *ref, void *cb_data UNUSED) { struct strbuf good_prefix = STRBUF_INIT; strbuf_addstr(&good_prefix, term_good); strbuf_addstr(&good_prefix, "-"); - if (!strcmp(refname, term_bad)) { + if (!strcmp(ref->name, term_bad)) { free(current_bad_oid); current_bad_oid = xmalloc(sizeof(*current_bad_oid)); - oidcpy(current_bad_oid, oid); - } else if (starts_with(refname, good_prefix.buf)) { - oid_array_append(&good_revs, oid); - } else if (starts_with(refname, "skip-")) { - oid_array_append(&skipped_revs, oid); + oidcpy(current_bad_oid, ref->oid); + } else if (starts_with(ref->name, good_prefix.buf)) { + oid_array_append(&good_revs, ref->oid); + } else if (starts_with(ref->name, "skip-")) { + oid_array_append(&skipped_revs, ref->oid); } strbuf_release(&good_prefix); @@ -1178,14 +1177,11 @@ int estimate_bisect_steps(int all) return (e < 3 * x) ? n : n - 1; } -static int mark_for_removal(const char *refname, - const char *referent UNUSED, - const struct object_id *oid UNUSED, - int flag UNUSED, void *cb_data) +static int mark_for_removal(const struct reference *ref, void *cb_data) { struct string_list *refs = cb_data; - char *ref = xstrfmt("refs/bisect%s", refname); - string_list_append(refs, ref); + char *bisect_ref = xstrfmt("refs/bisect%s", ref->name); + string_list_append(refs, bisect_ref); return 0; } diff --git a/build.rs b/build.rs new file mode 100644 index 0000000000..3228367b5d --- /dev/null +++ b/build.rs @@ -0,0 +1,21 @@ +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation: version 2 of the License, dated June 1991. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, see <https://www.gnu.org/licenses/>. + +fn main() { + println!("cargo:rustc-link-search=."); + println!("cargo:rustc-link-search=reftable"); + println!("cargo:rustc-link-search=xdiff"); + println!("cargo:rustc-link-lib=git"); + println!("cargo:rustc-link-lib=reftable"); + println!("cargo:rustc-link-lib=z"); + println!("cargo:rustc-link-lib=xdiff"); +} @@ -172,6 +172,7 @@ int cmd_get_tar_commit_id(int argc, const char **argv, const char *prefix, struc int cmd_grep(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_hash_object(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_help(int argc, const char **argv, const char *prefix, struct repository *repo); +int cmd_history(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_hook(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_index_pack(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_init_db(int argc, const char **argv, const char *prefix, struct repository *repo); diff --git a/builtin/add.c b/builtin/add.c index 32709794b3..6f1e213052 100644 --- a/builtin/add.c +++ b/builtin/add.c @@ -31,7 +31,7 @@ static const char * const builtin_add_usage[] = { NULL }; static int patch_interactive, add_interactive, edit_interactive; -static struct add_p_opt add_p_opt = ADD_P_OPT_INIT; +static struct interactive_options interactive_opts = INTERACTIVE_OPTIONS_INIT; static int take_worktree_changes; static int add_renormalize; static int pathspec_file_nul; @@ -160,7 +160,7 @@ static int refresh(struct repository *repo, int verbose, const struct pathspec * int interactive_add(struct repository *repo, const char **argv, const char *prefix, - int patch, struct add_p_opt *add_p_opt) + int patch, struct interactive_options *interactive_opts) { struct pathspec pathspec; int ret; @@ -172,9 +172,9 @@ int interactive_add(struct repository *repo, prefix, argv); if (patch) - ret = !!run_add_p(repo, ADD_P_ADD, add_p_opt, NULL, &pathspec); + ret = !!run_add_p(repo, ADD_P_ADD, interactive_opts, NULL, &pathspec); else - ret = !!run_add_i(repo, &pathspec, add_p_opt); + ret = !!run_add_i(repo, &pathspec, interactive_opts); clear_pathspec(&pathspec); return ret; @@ -256,8 +256,8 @@ static struct option builtin_add_options[] = { OPT_GROUP(""), OPT_BOOL('i', "interactive", &add_interactive, N_("interactive picking")), OPT_BOOL('p', "patch", &patch_interactive, N_("select hunks interactively")), - OPT_DIFF_UNIFIED(&add_p_opt.context), - OPT_DIFF_INTERHUNK_CONTEXT(&add_p_opt.interhunkcontext), + OPT_DIFF_UNIFIED(&interactive_opts.context), + OPT_DIFF_INTERHUNK_CONTEXT(&interactive_opts.interhunkcontext), OPT_BOOL('e', "edit", &edit_interactive, N_("edit current diff and apply")), OPT__FORCE(&ignored_too, N_("allow adding otherwise ignored files"), 0), OPT_BOOL('u', "update", &take_worktree_changes, N_("update tracked files")), @@ -400,9 +400,9 @@ int cmd_add(int argc, prepare_repo_settings(repo); repo->settings.command_requires_full_index = 0; - if (add_p_opt.context < -1) + if (interactive_opts.context < -1) die(_("'%s' cannot be negative"), "--unified"); - if (add_p_opt.interhunkcontext < -1) + if (interactive_opts.interhunkcontext < -1) die(_("'%s' cannot be negative"), "--inter-hunk-context"); if (patch_interactive) @@ -412,11 +412,11 @@ int cmd_add(int argc, die(_("options '%s' and '%s' cannot be used together"), "--dry-run", "--interactive/--patch"); if (pathspec_from_file) die(_("options '%s' and '%s' cannot be used together"), "--pathspec-from-file", "--interactive/--patch"); - exit(interactive_add(repo, argv + 1, prefix, patch_interactive, &add_p_opt)); + exit(interactive_add(repo, argv + 1, prefix, patch_interactive, &interactive_opts)); } else { - if (add_p_opt.context != -1) + if (interactive_opts.context != -1) die(_("the option '%s' requires '%s'"), "--unified", "--interactive/--patch"); - if (add_p_opt.interhunkcontext != -1) + if (interactive_opts.interhunkcontext != -1) die(_("the option '%s' requires '%s'"), "--inter-hunk-context", "--interactive/--patch"); } diff --git a/builtin/bisect.c b/builtin/bisect.c index 8b8d870cd1..4cc118fb57 100644 --- a/builtin/bisect.c +++ b/builtin/bisect.c @@ -27,13 +27,14 @@ static GIT_PATH_FUNC(git_path_bisect_first_parent, "BISECT_FIRST_PARENT") static GIT_PATH_FUNC(git_path_bisect_run, "BISECT_RUN") #define BUILTIN_GIT_BISECT_START_USAGE \ - N_("git bisect start [--term-(new|bad)=<term> --term-(old|good)=<term>]" \ - " [--no-checkout] [--first-parent] [<bad> [<good>...]] [--]" \ - " [<pathspec>...]") -#define BUILTIN_GIT_BISECT_STATE_USAGE \ - N_("git bisect (good|bad) [<rev>...]") + N_("git bisect start [--term-(bad|new)=<term-new> --term-(good|old)=<term-old>]\n" \ + " [--no-checkout] [--first-parent] [<bad> [<good>...]] [--] [<pathspec>...]") +#define BUILTIN_GIT_BISECT_BAD_USAGE \ + N_("git bisect (bad|new|<term-new>) [<rev>]") +#define BUILTIN_GIT_BISECT_GOOD_USAGE \ + N_("git bisect (good|old|<term-old>) [<rev>...]") #define BUILTIN_GIT_BISECT_TERMS_USAGE \ - "git bisect terms [--term-good | --term-bad]" + "git bisect terms [--term-(good|old) | --term-(bad|new)]" #define BUILTIN_GIT_BISECT_SKIP_USAGE \ N_("git bisect skip [(<rev>|<range>)...]") #define BUILTIN_GIT_BISECT_NEXT_USAGE \ @@ -41,17 +42,20 @@ static GIT_PATH_FUNC(git_path_bisect_run, "BISECT_RUN") #define BUILTIN_GIT_BISECT_RESET_USAGE \ N_("git bisect reset [<commit>]") #define BUILTIN_GIT_BISECT_VISUALIZE_USAGE \ - "git bisect visualize" + "git bisect (visualize|view)" #define BUILTIN_GIT_BISECT_REPLAY_USAGE \ N_("git bisect replay <logfile>") #define BUILTIN_GIT_BISECT_LOG_USAGE \ "git bisect log" #define BUILTIN_GIT_BISECT_RUN_USAGE \ N_("git bisect run <cmd> [<arg>...]") +#define BUILTIN_GIT_BISECT_HELP_USAGE \ + "git bisect help" static const char * const git_bisect_usage[] = { BUILTIN_GIT_BISECT_START_USAGE, - BUILTIN_GIT_BISECT_STATE_USAGE, + BUILTIN_GIT_BISECT_BAD_USAGE, + BUILTIN_GIT_BISECT_GOOD_USAGE, BUILTIN_GIT_BISECT_TERMS_USAGE, BUILTIN_GIT_BISECT_SKIP_USAGE, BUILTIN_GIT_BISECT_NEXT_USAGE, @@ -60,6 +64,7 @@ static const char * const git_bisect_usage[] = { BUILTIN_GIT_BISECT_REPLAY_USAGE, BUILTIN_GIT_BISECT_LOG_USAGE, BUILTIN_GIT_BISECT_RUN_USAGE, + BUILTIN_GIT_BISECT_HELP_USAGE, NULL }; @@ -358,10 +363,7 @@ static int check_and_set_terms(struct bisect_terms *terms, const char *cmd) return 0; } -static int inc_nr(const char *refname UNUSED, - const char *referent UNUSED, - const struct object_id *oid UNUSED, - int flag UNUSED, void *cb_data) +static int inc_nr(const struct reference *ref UNUSED, void *cb_data) { unsigned int *nr = (unsigned int *)cb_data; (*nr)++; @@ -549,12 +551,11 @@ finish: return res; } -static int add_bisect_ref(const char *refname, const char *referent UNUSED, const struct object_id *oid, - int flags UNUSED, void *cb) +static int add_bisect_ref(const struct reference *ref, void *cb) { struct add_bisect_ref_data *data = cb; - add_pending_oid(data->revs, refname, oid, data->object_flags); + add_pending_oid(data->revs, ref->name, ref->oid, data->object_flags); return 0; } @@ -1165,12 +1166,9 @@ static int bisect_visualize(struct bisect_terms *terms, int argc, return run_command(&cmd); } -static int get_first_good(const char *refname UNUSED, - const char *referent UNUSED, - const struct object_id *oid, - int flag UNUSED, void *cb_data) +static int get_first_good(const struct reference *ref, void *cb_data) { - oidcpy(cb_data, oid); + oidcpy(cb_data, ref->oid); return 1; } @@ -1453,9 +1451,13 @@ int cmd_bisect(int argc, if (!argc) usage_msg_opt(_("need a command"), git_bisect_usage, options); + if (!strcmp(argv[0], "help")) + usage_with_options(git_bisect_usage, options); + set_terms(&terms, "bad", "good"); get_terms(&terms); - if (check_and_set_terms(&terms, argv[0])) + if (check_and_set_terms(&terms, argv[0]) || + !one_of(argv[0], terms.term_good, terms.term_bad, NULL)) usage_msg_optf(_("unknown command: '%s'"), git_bisect_usage, options, argv[0]); res = bisect_state(&terms, argc, argv); diff --git a/builtin/blame.c b/builtin/blame.c index 2703820258..27b513d27f 100644 --- a/builtin/blame.c +++ b/builtin/blame.c @@ -779,6 +779,19 @@ static int git_blame_config(const char *var, const char *value, } } + if (!strcmp(var, "diff.algorithm")) { + long diff_algorithm; + if (!value) + return config_error_nonbool(var); + diff_algorithm = parse_algorithm_value(value); + if (diff_algorithm < 0) + return error(_("unknown value for config '%s': %s"), + var, value); + xdl_opts &= ~XDF_DIFF_ALGORITHM_MASK; + xdl_opts |= diff_algorithm; + return 0; + } + if (git_diff_heuristic_config(var, value, cb) < 0) return -1; if (userdiff_config(var, value) < 0) @@ -824,6 +837,38 @@ static int blame_move_callback(const struct option *option, const char *arg, int return 0; } +static int blame_diff_algorithm_minimal(const struct option *option, + const char *arg, int unset) +{ + int *opt = option->value; + + BUG_ON_OPT_ARG(arg); + + *opt &= ~XDF_DIFF_ALGORITHM_MASK; + if (!unset) + *opt |= XDF_NEED_MINIMAL; + + return 0; +} + +static int blame_diff_algorithm_callback(const struct option *option, + const char *arg, int unset) +{ + int *opt = option->value; + long value = parse_algorithm_value(arg); + + BUG_ON_OPT_NEG(unset); + + if (value < 0) + return error(_("option diff-algorithm accepts \"myers\", " + "\"minimal\", \"patience\" and \"histogram\"")); + + *opt &= ~XDF_DIFF_ALGORITHM_MASK; + *opt |= value; + + return 0; +} + static int is_a_rev(const char *name) { struct object_id oid; @@ -915,11 +960,16 @@ int cmd_blame(int argc, OPT_BIT('s', NULL, &output_option, N_("suppress author name and timestamp (Default: off)"), OUTPUT_NO_AUTHOR), OPT_BIT('e', "show-email", &output_option, N_("show author email instead of name (Default: off)"), OUTPUT_SHOW_EMAIL), OPT_BIT('w', NULL, &xdl_opts, N_("ignore whitespace differences"), XDF_IGNORE_WHITESPACE), + OPT_CALLBACK_F(0, "diff-algorithm", &xdl_opts, N_("<algorithm>"), + N_("choose a diff algorithm"), + PARSE_OPT_NONEG, blame_diff_algorithm_callback), OPT_STRING_LIST(0, "ignore-rev", &ignore_rev_list, N_("rev"), N_("ignore <rev> when blaming")), OPT_STRING_LIST(0, "ignore-revs-file", &ignore_revs_file_list, N_("file"), N_("ignore revisions from <file>")), OPT_BIT(0, "color-lines", &output_option, N_("color redundant metadata from previous line differently"), OUTPUT_COLOR_LINE), OPT_BIT(0, "color-by-age", &output_option, N_("color lines by age"), OUTPUT_SHOW_AGE_WITH_COLOR), - OPT_BIT(0, "minimal", &xdl_opts, N_("spend extra cycles to find better match"), XDF_NEED_MINIMAL), + OPT_CALLBACK_F(0, "minimal", &xdl_opts, NULL, + N_("spend extra cycles to find a better match"), + PARSE_OPT_NOARG | PARSE_OPT_HIDDEN, blame_diff_algorithm_minimal), OPT_STRING('S', NULL, &revs_file, N_("file"), N_("use revisions from <file> instead of calling git-rev-list")), OPT_STRING(0, "contents", &contents_from, N_("file"), N_("use <file>'s contents as the final image")), OPT_CALLBACK_F('C', NULL, &opt, N_("score"), N_("find line copies within and across files"), PARSE_OPT_OPTARG, blame_copy_callback), diff --git a/builtin/cat-file.c b/builtin/cat-file.c index ee6715fa52..983ecec837 100644 --- a/builtin/cat-file.c +++ b/builtin/cat-file.c @@ -495,7 +495,7 @@ static void batch_object_write(const char *obj_name, OBJECT_INFO_LOOKUP_REPLACE); if (ret < 0) { if (data->mode == S_IFGITLINK) - report_object_status(opt, oid_to_hex(&data->oid), &data->oid, "submodule"); + report_object_status(opt, NULL, &data->oid, "submodule"); else report_object_status(opt, obj_name, &data->oid, "missing"); return; @@ -852,10 +852,9 @@ static void batch_each_object(struct batch_options *opt, if (bitmap && !for_each_bitmapped_object(bitmap, &opt->objects_filter, batch_one_object_bitmapped, &payload)) { - struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *pack; - for (pack = packfile_store_get_all_packs(packs); pack; pack = pack->next) { + repo_for_each_pack(the_repository, pack) { if (bitmap_index_contains_pack(bitmap, pack) || open_pack_index(pack)) continue; diff --git a/builtin/checkout.c b/builtin/checkout.c index f9453473fe..c09065cc61 100644 --- a/builtin/checkout.c +++ b/builtin/checkout.c @@ -546,7 +546,7 @@ static int checkout_paths(const struct checkout_opts *opts, if (opts->patch_mode) { enum add_p_mode patch_mode; - struct add_p_opt add_p_opt = { + struct interactive_options interactive_opts = { .context = opts->patch_context, .interhunkcontext = opts->patch_interhunk_context, }; @@ -575,7 +575,7 @@ static int checkout_paths(const struct checkout_opts *opts, else BUG("either flag must have been set, worktree=%d, index=%d", opts->checkout_worktree, opts->checkout_index); - return !!run_add_p(the_repository, patch_mode, &add_p_opt, + return !!run_add_p(the_repository, patch_mode, &interactive_opts, rev, &opts->pathspec); } @@ -902,7 +902,8 @@ static int merge_working_tree(const struct checkout_opts *opts, 0); init_ui_merge_options(&o, the_repository); o.verbosity = 0; - work = write_in_core_index_as_tree(the_repository); + work = write_in_core_index_as_tree(the_repository, + the_repository->index); ret = reset_tree(new_tree, opts, 1, @@ -1063,11 +1064,9 @@ static void update_refs_for_switch(const struct checkout_opts *opts, report_tracking(new_branch_info); } -static int add_pending_uninteresting_ref(const char *refname, const char *referent UNUSED, - const struct object_id *oid, - int flags UNUSED, void *cb_data) +static int add_pending_uninteresting_ref(const struct reference *ref, void *cb_data) { - add_pending_oid(cb_data, refname, oid, UNINTERESTING); + add_pending_oid(cb_data, ref->name, ref->oid, UNINTERESTING); return 0; } diff --git a/builtin/commit-graph.c b/builtin/commit-graph.c index fe3ebaadad..d62005edc0 100644 --- a/builtin/commit-graph.c +++ b/builtin/commit-graph.c @@ -210,6 +210,8 @@ static int git_commit_graph_write_config(const char *var, const char *value, { if (!strcmp(var, "commitgraph.maxnewfilters")) write_opts.max_new_filters = git_config_int(var, value, ctx->kvi); + else if (!strcmp(var, "commitgraph.changedpaths")) + opts.enable_changed_paths = git_config_bool(var, value) ? 1 : -1; /* * No need to fall-back to 'git_default_config', since this was already * called in 'cmd_commit_graph()'. diff --git a/builtin/commit.c b/builtin/commit.c index 0243f17d53..78e8fb91f4 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -123,7 +123,7 @@ static const char *edit_message, *use_message; static char *fixup_message, *fixup_commit, *squash_message; static const char *fixup_prefix; static int all, also, interactive, patch_interactive, only, amend, signoff; -static struct add_p_opt add_p_opt = ADD_P_OPT_INIT; +static struct interactive_options interactive_opts = INTERACTIVE_OPTIONS_INIT; static int edit_flag = -1; /* unspecified */ static int quiet, verbose, no_verify, allow_empty, dry_run, renew_authorship; static int config_commit_verbose = -1; /* unspecified */ @@ -356,9 +356,9 @@ static const char *prepare_index(const char **argv, const char *prefix, const char *ret; char *path = NULL; - if (add_p_opt.context < -1) + if (interactive_opts.context < -1) die(_("'%s' cannot be negative"), "--unified"); - if (add_p_opt.interhunkcontext < -1) + if (interactive_opts.interhunkcontext < -1) die(_("'%s' cannot be negative"), "--inter-hunk-context"); if (is_status) @@ -407,7 +407,7 @@ static const char *prepare_index(const char **argv, const char *prefix, old_index_env = xstrdup_or_null(getenv(INDEX_ENVIRONMENT)); setenv(INDEX_ENVIRONMENT, the_repository->index_file, 1); - if (interactive_add(the_repository, argv, prefix, patch_interactive, &add_p_opt) != 0) + if (interactive_add(the_repository, argv, prefix, patch_interactive, &interactive_opts) != 0) die(_("interactive add failed")); the_repository->index_file = old_repo_index_file; @@ -432,9 +432,9 @@ static const char *prepare_index(const char **argv, const char *prefix, ret = get_lock_file_path(&index_lock); goto out; } else { - if (add_p_opt.context != -1) + if (interactive_opts.context != -1) die(_("the option '%s' requires '%s'"), "--unified", "--interactive/--patch"); - if (add_p_opt.interhunkcontext != -1) + if (interactive_opts.interhunkcontext != -1) die(_("the option '%s' requires '%s'"), "--inter-hunk-context", "--interactive/--patch"); } @@ -1719,7 +1719,7 @@ int cmd_commit(int argc, OPT_STRING(0, "fixup", &fixup_message, N_("[(amend|reword):]commit"), N_("use autosquash formatted message to fixup or amend/reword specified commit")), OPT_STRING(0, "squash", &squash_message, N_("commit"), N_("use autosquash formatted message to squash specified commit")), OPT_BOOL(0, "reset-author", &renew_authorship, N_("the commit is authored by me now (used with -C/-c/--amend)")), - OPT_PASSTHRU_ARGV(0, "trailer", &trailer_args, N_("trailer"), N_("add custom trailer(s)"), PARSE_OPT_NONEG), + OPT_CALLBACK_F(0, "trailer", &trailer_args, N_("trailer"), N_("add custom trailer(s)"), PARSE_OPT_NONEG, parse_opt_strvec), OPT_BOOL('s', "signoff", &signoff, N_("add a Signed-off-by trailer")), OPT_FILENAME('t', "template", &template_file, N_("use specified template file")), OPT_BOOL('e', "edit", &edit_flag, N_("force edit of commit")), @@ -1742,8 +1742,8 @@ int cmd_commit(int argc, OPT_BOOL('i', "include", &also, N_("add specified files to index for commit")), OPT_BOOL(0, "interactive", &interactive, N_("interactively add files")), OPT_BOOL('p', "patch", &patch_interactive, N_("interactively add changes")), - OPT_DIFF_UNIFIED(&add_p_opt.context), - OPT_DIFF_INTERHUNK_CONTEXT(&add_p_opt.interhunkcontext), + OPT_DIFF_UNIFIED(&interactive_opts.context), + OPT_DIFF_INTERHUNK_CONTEXT(&interactive_opts.interhunkcontext), OPT_BOOL('o', "only", &only, N_("commit only specified files")), OPT_BOOL('n', "no-verify", &no_verify, N_("bypass pre-commit and commit-msg hooks")), OPT_BOOL(0, "dry-run", &dry_run, N_("show what would be committed")), diff --git a/builtin/count-objects.c b/builtin/count-objects.c index f2f407c2a7..18f6e33b6f 100644 --- a/builtin/count-objects.c +++ b/builtin/count-objects.c @@ -122,7 +122,6 @@ int cmd_count_objects(int argc, count_loose, count_cruft, NULL, NULL); if (verbose) { - struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *p; unsigned long num_pack = 0; off_t size_pack = 0; @@ -130,7 +129,7 @@ int cmd_count_objects(int argc, struct strbuf pack_buf = STRBUF_INIT; struct strbuf garbage_buf = STRBUF_INIT; - for (p = packfile_store_get_all_packs(packs); p; p = p->next) { + repo_for_each_pack(the_repository, p) { if (!p->pack_local) continue; if (open_pack_index(p)) diff --git a/builtin/credential-store.c b/builtin/credential-store.c index b74e06cc93..bc1453c6b2 100644 --- a/builtin/credential-store.c +++ b/builtin/credential-store.c @@ -7,6 +7,7 @@ #include "path.h" #include "string-list.h" #include "parse-options.h" +#include "url.h" #include "write-or-die.h" static struct lock_file credential_lock; @@ -76,12 +77,6 @@ static void rewrite_credential_file(const char *fn, struct credential *c, die_errno("unable to write credential store"); } -static int is_rfc3986_unreserved(char ch) -{ - return isalnum(ch) || - ch == '-' || ch == '_' || ch == '.' || ch == '~'; -} - static int is_rfc3986_reserved_or_unreserved(char ch) { if (is_rfc3986_unreserved(ch)) diff --git a/builtin/describe.c b/builtin/describe.c index ffaf8d9f0a..443546aaac 100644 --- a/builtin/describe.c +++ b/builtin/describe.c @@ -154,20 +154,19 @@ static void add_to_known_names(const char *path, } } -static int get_name(const char *path, const char *referent UNUSED, const struct object_id *oid, - int flag UNUSED, void *cb_data UNUSED) +static int get_name(const struct reference *ref, void *cb_data UNUSED) { int is_tag = 0; struct object_id peeled; int is_annotated, prio; const char *path_to_match = NULL; - if (skip_prefix(path, "refs/tags/", &path_to_match)) { + if (skip_prefix(ref->name, "refs/tags/", &path_to_match)) { is_tag = 1; } else if (all) { if ((exclude_patterns.nr || patterns.nr) && - !skip_prefix(path, "refs/heads/", &path_to_match) && - !skip_prefix(path, "refs/remotes/", &path_to_match)) { + !skip_prefix(ref->name, "refs/heads/", &path_to_match) && + !skip_prefix(ref->name, "refs/remotes/", &path_to_match)) { /* Only accept reference of known type if there are match/exclude patterns */ return 0; } @@ -209,10 +208,10 @@ static int get_name(const char *path, const char *referent UNUSED, const struct } /* Is it annotated? */ - if (!peel_iterated_oid(the_repository, oid, &peeled)) { - is_annotated = !oideq(oid, &peeled); + if (!reference_get_peeled_oid(the_repository, ref, &peeled)) { + is_annotated = !oideq(ref->oid, &peeled); } else { - oidcpy(&peeled, oid); + oidcpy(&peeled, ref->oid); is_annotated = 0; } @@ -229,7 +228,8 @@ static int get_name(const char *path, const char *referent UNUSED, const struct else prio = 0; - add_to_known_names(all ? path + 5 : path + 10, &peeled, prio, oid); + add_to_known_names(all ? ref->name + 5 : ref->name + 10, + &peeled, prio, ref->oid); return 0; } diff --git a/builtin/fast-export.c b/builtin/fast-export.c index dc2486f9a8..0421360ab7 100644 --- a/builtin/fast-export.c +++ b/builtin/fast-export.c @@ -65,7 +65,7 @@ static int parse_opt_sign_mode(const struct option *opt, return 0; if (parse_sign_mode(arg, val)) - return error("Unknown %s mode: %s", opt->long_name, arg); + return error(_("unknown %s mode: %s"), opt->long_name, arg); return 0; } @@ -82,7 +82,7 @@ static int parse_opt_tag_of_filtered_mode(const struct option *opt, else if (!strcmp(arg, "rewrite")) *val = REWRITE; else - return error("Unknown tag-of-filtered mode: %s", arg); + return error(_("unknown tag-of-filtered mode: %s"), arg); return 0; } @@ -107,7 +107,7 @@ static int parse_opt_reencode_mode(const struct option *opt, if (!strcasecmp(arg, "abort")) *val = REENCODE_ABORT; else - return error("Unknown reencoding mode: %s", arg); + return error(_("unknown reencoding mode: %s"), arg); } return 0; @@ -318,16 +318,16 @@ static void export_blob(const struct object_id *oid) } else { buf = odb_read_object(the_repository->objects, oid, &type, &size); if (!buf) - die("could not read blob %s", oid_to_hex(oid)); + die(_("could not read blob %s"), oid_to_hex(oid)); if (check_object_signature(the_repository, oid, buf, size, type) < 0) - die("oid mismatch in blob %s", oid_to_hex(oid)); + die(_("oid mismatch in blob %s"), oid_to_hex(oid)); object = parse_object_buffer(the_repository, oid, type, size, buf, &eaten); } if (!object) - die("Could not read blob %s", oid_to_hex(oid)); + die(_("could not read blob %s"), oid_to_hex(oid)); mark_next_object(object); @@ -336,7 +336,7 @@ static void export_blob(const struct object_id *oid) printf("original-oid %s\n", oid_to_hex(oid)); printf("data %"PRIuMAX"\n", (uintmax_t)size); if (size && fwrite(buf, size, 1, stdout) != 1) - die_errno("could not write blob '%s'", oid_to_hex(oid)); + die_errno(_("could not write blob '%s'"), oid_to_hex(oid)); printf("\n"); show_progress(); @@ -499,10 +499,10 @@ static void show_filemodify(struct diff_queue_struct *q, break; default: - die("Unexpected comparison status '%c' for %s, %s", - q->queue[i]->status, - ospec->path ? ospec->path : "none", - spec->path ? spec->path : "none"); + die(_("unexpected comparison status '%c' for %s, %s"), + q->queue[i]->status, + ospec->path ? ospec->path : _("none"), + spec->path ? spec->path : _("none")); } } } @@ -699,14 +699,14 @@ static void handle_commit(struct commit *commit, struct rev_info *rev, author = strstr(commit_buffer_cursor, "\nauthor "); if (!author) - die("could not find author in commit %s", + die(_("could not find author in commit %s"), oid_to_hex(&commit->object.oid)); author++; commit_buffer_cursor = author_end = strchrnul(author, '\n'); committer = strstr(commit_buffer_cursor, "\ncommitter "); if (!committer) - die("could not find committer in commit %s", + die(_("could not find committer in commit %s"), oid_to_hex(&commit->object.oid)); committer++; commit_buffer_cursor = committer_end = strchrnul(committer, '\n'); @@ -781,8 +781,8 @@ static void handle_commit(struct commit *commit, struct rev_info *rev, case REENCODE_NO: break; case REENCODE_ABORT: - die("Encountered commit-specific encoding %.*s in commit " - "%s; use --reencode=[yes|no] to handle it", + die(_("encountered commit-specific encoding %.*s in commit " + "%s; use --reencode=[yes|no] to handle it"), (int)encoding_len, encoding, oid_to_hex(&commit->object.oid)); } @@ -798,11 +798,11 @@ static void handle_commit(struct commit *commit, struct rev_info *rev, if (signatures.nr) { switch (signed_commit_mode) { case SIGN_ABORT: - die("encountered signed commit %s; use " - "--signed-commits=<mode> to handle it", + die(_("encountered signed commit %s; use " + "--signed-commits=<mode> to handle it"), oid_to_hex(&commit->object.oid)); case SIGN_WARN_VERBATIM: - warning("exporting %"PRIuMAX" signature(s) for commit %s", + warning(_("exporting %"PRIuMAX" signature(s) for commit %s"), (uintmax_t)signatures.nr, oid_to_hex(&commit->object.oid)); /* fallthru */ case SIGN_VERBATIM: @@ -812,7 +812,7 @@ static void handle_commit(struct commit *commit, struct rev_info *rev, } break; case SIGN_WARN_STRIP: - warning("stripping signature(s) from commit %s", + warning(_("stripping signature(s) from commit %s"), oid_to_hex(&commit->object.oid)); /* fallthru */ case SIGN_STRIP: @@ -890,7 +890,8 @@ static void handle_tag(const char *name, struct tag *tag) tagged = ((struct tag *)tagged)->tagged; } if (tagged->type == OBJ_TREE) { - warning("Omitting tag %s,\nsince tags of trees (or tags of tags of trees, etc.) are not supported.", + warning(_("omitting tag %s,\nsince tags of trees (or tags " + "of tags of trees, etc.) are not supported."), oid_to_hex(&tag->object.oid)); return; } @@ -898,7 +899,7 @@ static void handle_tag(const char *name, struct tag *tag) buf = odb_read_object(the_repository->objects, &tag->object.oid, &type, &size); if (!buf) - die("could not read tag %s", oid_to_hex(&tag->object.oid)); + die(_("could not read tag %s"), oid_to_hex(&tag->object.oid)); message = memmem(buf, size, "\n\n", 2); if (message) { message += 2; @@ -931,26 +932,25 @@ static void handle_tag(const char *name, struct tag *tag) /* handle signed tags */ if (message) { - const char *signature = strstr(message, - "\n-----BEGIN PGP SIGNATURE-----\n"); - if (signature) + size_t sig_offset = parse_signed_buffer(message, message_size); + if (sig_offset < message_size) switch (signed_tag_mode) { case SIGN_ABORT: - die("encountered signed tag %s; use " - "--signed-tags=<mode> to handle it", + die(_("encountered signed tag %s; use " + "--signed-tags=<mode> to handle it"), oid_to_hex(&tag->object.oid)); case SIGN_WARN_VERBATIM: - warning("exporting signed tag %s", + warning(_("exporting signed tag %s"), oid_to_hex(&tag->object.oid)); /* fallthru */ case SIGN_VERBATIM: break; case SIGN_WARN_STRIP: - warning("stripping signature from tag %s", + warning(_("stripping signature from tag %s"), oid_to_hex(&tag->object.oid)); /* fallthru */ case SIGN_STRIP: - message_size = signature + 1 - message; + message_size = sig_offset; break; } } @@ -961,8 +961,8 @@ static void handle_tag(const char *name, struct tag *tag) if (!tagged_mark) { switch (tag_of_filtered_mode) { case TAG_FILTERING_ABORT: - die("tag %s tags unexported object; use " - "--tag-of-filtered-object=<mode> to handle it", + die(_("tag %s tags unexported object; use " + "--tag-of-filtered-object=<mode> to handle it"), oid_to_hex(&tag->object.oid)); case DROP: /* Ignore this tag altogether */ @@ -970,7 +970,7 @@ static void handle_tag(const char *name, struct tag *tag) return; case REWRITE: if (tagged->type == OBJ_TAG && !mark_tags) { - die(_("Error: Cannot export nested tags unless --mark-tags is specified.")); + die(_("cannot export nested tags unless --mark-tags is specified.")); } else if (tagged->type == OBJ_COMMIT) { p = rewrite_commit((struct commit *)tagged); if (!p) { @@ -1026,7 +1026,7 @@ static struct commit *get_commit(struct rev_cmdline_entry *e, const char *full_n tag = (struct tag *)tag->tagged; } if (!tag) - die("Tag %s points nowhere?", e->name); + die(_("tag %s points nowhere?"), e->name); return (struct commit *)tag; } default: @@ -1064,7 +1064,7 @@ static void get_tags_and_duplicates(struct rev_cmdline_info *info) commit = get_commit(e, full_name); if (!commit) { - warning("%s: Unexpected object of type %s, skipping.", + warning(_("%s: unexpected object of type %s, skipping."), e->name, type_name(e->item->type)); free(full_name); @@ -1079,7 +1079,7 @@ static void get_tags_and_duplicates(struct rev_cmdline_info *info) free(full_name); continue; default: /* OBJ_TAG (nested tags) is already handled */ - warning("Tag points to object of unexpected type %s, skipping.", + warning(_("tag points to object of unexpected type %s, skipping."), type_name(commit->object.type)); free(full_name); continue; @@ -1175,7 +1175,7 @@ static void export_marks(char *file) f = fopen_for_writing(file); if (!f) - die_errno("Unable to open marks file %s for writing.", file); + die_errno(_("unable to open marks file %s for writing."), file); for (i = 0; i < idnums.size; i++) { if (deco->base && deco->base->type == 1) { @@ -1192,7 +1192,7 @@ static void export_marks(char *file) e |= ferror(f); e |= fclose(f); if (e) - error("Unable to write marks file %s.", file); + error(_("unable to write marks file %s."), file); } static void import_marks(char *input_file, int check_exists) @@ -1215,20 +1215,20 @@ static void import_marks(char *input_file, int check_exists) line_end = strchr(line, '\n'); if (line[0] != ':' || !line_end) - die("corrupt mark line: %s", line); + die(_("corrupt mark line: %s"), line); *line_end = '\0'; mark = strtoumax(line + 1, &mark_end, 10); if (!mark || mark_end == line + 1 || *mark_end != ' ' || get_oid_hex(mark_end + 1, &oid)) - die("corrupt mark line: %s", line); + die(_("corrupt mark line: %s"), line); if (last_idnum < mark) last_idnum = mark; type = odb_read_object_info(the_repository->objects, &oid, NULL); if (type < 0) - die("object not found: %s", oid_to_hex(&oid)); + die(_("object not found: %s"), oid_to_hex(&oid)); if (type != OBJ_COMMIT) /* only commits */ @@ -1236,12 +1236,12 @@ static void import_marks(char *input_file, int check_exists) commit = lookup_commit(the_repository, &oid); if (!commit) - die("not a commit? can't happen: %s", oid_to_hex(&oid)); + die(_("not a commit? can't happen: %s"), oid_to_hex(&oid)); object = &commit->object; if (object->flags & SHOWN) - error("Object %s already has a mark", oid_to_hex(&oid)); + error(_("object %s already has a mark"), oid_to_hex(&oid)); mark_object(object, mark); @@ -1395,7 +1395,7 @@ int cmd_fast_export(int argc, get_tags_and_duplicates(&revs.cmdline); if (prepare_revision_walk(&revs)) - die("revision walk setup failed"); + die(_("revision walk setup failed")); revs.reverse = 1; revs.diffopt.format_callback = show_filemodify; diff --git a/builtin/fast-import.c b/builtin/fast-import.c index 606c6aea82..4cd0b079b6 100644 --- a/builtin/fast-import.c +++ b/builtin/fast-import.c @@ -188,6 +188,7 @@ static int global_argc; static const char **global_argv; static const char *global_prefix; +static enum sign_mode signed_tag_mode = SIGN_VERBATIM; static enum sign_mode signed_commit_mode = SIGN_VERBATIM; /* Memory pools */ @@ -338,12 +339,12 @@ static void write_crash_report(const char *err) struct recent_command *rc; if (!rpt) { - error_errno("can't write crash report %s", loc); + error_errno(_("can't write crash report %s"), loc); free(loc); return; } - fprintf(stderr, "fast-import: dumping crash report to %s\n", loc); + fprintf(stderr, _("fast-import: dumping crash report to %s\n"), loc); fprintf(rpt, "fast-import crash report:\n"); fprintf(rpt, " fast-import process: %"PRIuMAX"\n", (uintmax_t) getpid()); @@ -587,7 +588,7 @@ static void *find_mark(struct mark_set *s, uintmax_t idnum) oe = s->data.marked[idnum]; } if (!oe) - die("mark :%" PRIuMAX " not declared", orig_idnum); + die(_("mark :%" PRIuMAX " not declared"), orig_idnum); return oe; } @@ -627,9 +628,9 @@ static struct branch *new_branch(const char *name) struct branch *b = lookup_branch(name); if (b) - die("Invalid attempt to create duplicate branch: %s", name); + die(_("invalid attempt to create duplicate branch: %s"), name); if (check_refname_format(name, REFNAME_ALLOW_ONELEVEL)) - die("Branch name doesn't conform to GIT standards: %s", name); + die(_("branch name doesn't conform to Git standards: %s"), name); b = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct branch)); b->name = mem_pool_strdup(&fi_mem_pool, name); @@ -800,7 +801,7 @@ static const char *create_index(void) *c++ = &e->idx; last = idx + object_count; if (c != last) - die("internal consistency error creating the index"); + die(_("internal consistency error creating the index")); tmpfile = write_idx_file(the_repository, NULL, idx, object_count, &pack_idx_opts, pack_data->hash); @@ -818,18 +819,18 @@ static char *keep_pack(const char *curr_index_name) keep_fd = safe_create_file_with_leading_directories(pack_data->repo, name.buf); if (keep_fd < 0) - die_errno("cannot create keep file"); + die_errno(_("cannot create keep file")); write_or_die(keep_fd, keep_msg, strlen(keep_msg)); if (close(keep_fd)) - die_errno("failed to write keep file"); + die_errno(_("failed to write keep file")); odb_pack_name(pack_data->repo, &name, pack_data->hash, "pack"); if (finalize_object_file(pack_data->repo, pack_data->pack_name, name.buf)) - die("cannot store pack file"); + die(_("cannot store pack file")); odb_pack_name(pack_data->repo, &name, pack_data->hash, "idx"); if (finalize_object_file(pack_data->repo, curr_index_name, name.buf)) - die("cannot store index file"); + die(_("cannot store index file")); free((void *)curr_index_name); return strbuf_detach(&name, NULL); } @@ -852,7 +853,7 @@ static int loosen_small_pack(const struct packed_git *p) struct child_process unpack = CHILD_PROCESS_INIT; if (lseek(p->pack_fd, 0, SEEK_SET) < 0) - die_errno("Failed seeking to start of '%s'", p->pack_name); + die_errno(_("failed seeking to start of '%s'"), p->pack_name); unpack.in = p->pack_fd; unpack.git_cmd = 1; @@ -902,7 +903,7 @@ static void end_packfile(void) new_p = packfile_store_load_pack(pack_data->repo->objects->packfiles, idx_name, 1); if (!new_p) - die("core git rejected index %s", idx_name); + die(_("core Git rejected index %s"), idx_name); all_packs[pack_id] = new_p; free(idx_name); @@ -978,7 +979,7 @@ static int store_object( if (e->idx.offset) { duplicate_count_by_type[type]++; return 1; - } else if (find_oid_pack(&oid, packfile_store_get_all_packs(packs))) { + } else if (packfile_list_find_oid(packfile_store_get_packs(packs), &oid)) { e->type = type; e->pack_id = MAX_PACK_ID; e->idx.offset = 1; /* just not zero! */ @@ -1089,7 +1090,7 @@ static int store_object( static void truncate_pack(struct hashfile_checkpoint *checkpoint) { if (hashfile_truncate(pack_file, checkpoint)) - die_errno("cannot truncate pack to skip duplicate"); + die_errno(_("cannot truncate pack to skip duplicate")); pack_size = checkpoint->offset; } @@ -1137,7 +1138,7 @@ static void stream_blob(uintmax_t len, struct object_id *oidout, uintmax_t mark) size_t cnt = in_sz < len ? in_sz : (size_t)len; size_t n = fread(in_buf, 1, cnt, stdin); if (!n && feof(stdin)) - die("EOF in data (%" PRIuMAX " bytes remaining)", len); + die(_("EOF in data (%" PRIuMAX " bytes remaining)"), len); git_hash_update(&c, in_buf, n); s.next_in = in_buf; @@ -1161,7 +1162,7 @@ static void stream_blob(uintmax_t len, struct object_id *oidout, uintmax_t mark) case Z_STREAM_END: continue; default: - die("unexpected deflate failure: %d", status); + die(_("unexpected deflate failure: %d"), status); } } git_deflate_end(&s); @@ -1179,7 +1180,7 @@ static void stream_blob(uintmax_t len, struct object_id *oidout, uintmax_t mark) duplicate_count_by_type[OBJ_BLOB]++; truncate_pack(&checkpoint); - } else if (find_oid_pack(&oid, packfile_store_get_all_packs(packs))) { + } else if (packfile_list_find_oid(packfile_store_get_packs(packs), &oid)) { e->type = OBJ_BLOB; e->pack_id = MAX_PACK_ID; e->idx.offset = 1; /* just not zero! */ @@ -1263,16 +1264,16 @@ static void load_tree(struct tree_entry *root) myoe = find_object(oid); if (myoe && myoe->pack_id != MAX_PACK_ID) { if (myoe->type != OBJ_TREE) - die("Not a tree: %s", oid_to_hex(oid)); + die(_("not a tree: %s"), oid_to_hex(oid)); t->delta_depth = myoe->depth; buf = gfi_unpack_entry(myoe, &size); if (!buf) - die("Can't load tree %s", oid_to_hex(oid)); + die(_("can't load tree %s"), oid_to_hex(oid)); } else { enum object_type type; buf = odb_read_object(the_repository->objects, oid, &type, &size); if (!buf || type != OBJ_TREE) - die("Can't load tree %s", oid_to_hex(oid)); + die(_("can't load tree %s"), oid_to_hex(oid)); } c = buf; @@ -1286,7 +1287,7 @@ static void load_tree(struct tree_entry *root) e->tree = NULL; c = parse_mode(c, &e->versions[1].mode); if (!c) - die("Corrupt mode in %s", oid_to_hex(oid)); + die(_("corrupt mode in %s"), oid_to_hex(oid)); e->versions[0].mode = e->versions[1].mode; e->name = to_atom(c, strlen(c)); c += e->name->str_len + 1; @@ -1398,7 +1399,7 @@ static void tree_content_replace( struct tree_content *newtree) { if (!S_ISDIR(mode)) - die("Root cannot be a non-directory"); + die(_("root cannot be a non-directory")); oidclr(&root->versions[0].oid, the_repository->hash_algo); oidcpy(&root->versions[1].oid, oid); if (root->tree) @@ -1421,9 +1422,9 @@ static int tree_content_set( slash1 = strchrnul(p, '/'); n = slash1 - p; if (!n) - die("Empty path component found in input"); + die(_("empty path component found in input")); if (!*slash1 && !S_ISDIR(mode) && subtree) - die("Non-directories cannot have subtrees"); + die(_("non-directories cannot have subtrees")); if (!root->tree) load_tree(root); @@ -1575,7 +1576,7 @@ static int tree_content_get( slash1 = strchrnul(p, '/'); n = slash1 - p; if (!n && !allow_root) - die("Empty path component found in input"); + die(_("empty path component found in input")); if (!root->tree) load_tree(root); @@ -1621,8 +1622,8 @@ static int update_branch(struct branch *b) !strcmp(b->name + strlen(replace_prefix), oid_to_hex(&b->oid))) { if (!quiet) - warning("Dropping %s since it would point to " - "itself (i.e. to %s)", + warning(_("dropping %s since it would point to " + "itself (i.e. to %s)"), b->name, oid_to_hex(&b->oid)); refs_delete_ref(get_main_ref_store(the_repository), NULL, b->name, NULL, 0); @@ -1645,14 +1646,14 @@ static int update_branch(struct branch *b) new_cmit = lookup_commit_reference_gently(the_repository, &b->oid, 0); if (!old_cmit || !new_cmit) - return error("Branch %s is missing commits.", b->name); + return error(_("branch %s is missing commits."), b->name); ret = repo_in_merge_bases(the_repository, old_cmit, new_cmit); if (ret < 0) exit(128); if (!ret) { - warning("Not updating %s" - " (new tip %s does not contain %s)", + warning(_("not updating %s" + " (new tip %s does not contain %s)"), b->name, oid_to_hex(&b->oid), oid_to_hex(&old_oid)); return -1; @@ -1728,13 +1729,13 @@ static void dump_marks(void) return; if (safe_create_leading_directories_const(the_repository, export_marks_file)) { - failure |= error_errno("unable to create leading directories of %s", + failure |= error_errno(_("unable to create leading directories of %s"), export_marks_file); return; } if (hold_lock_file_for_update(&mark_lock, export_marks_file, 0) < 0) { - failure |= error_errno("Unable to write marks file %s", + failure |= error_errno(_("unable to write marks file %s"), export_marks_file); return; } @@ -1743,14 +1744,14 @@ static void dump_marks(void) if (!f) { int saved_errno = errno; rollback_lock_file(&mark_lock); - failure |= error("Unable to write marks file %s: %s", + failure |= error(_("unable to write marks file %s: %s"), export_marks_file, strerror(saved_errno)); return; } for_each_mark(marks, 0, dump_marks_fn, f); if (commit_lock_file(&mark_lock)) { - failure |= error_errno("Unable to write file %s", + failure |= error_errno(_("unable to write file %s"), export_marks_file); return; } @@ -1764,7 +1765,7 @@ static void insert_object_entry(struct mark_set **s, struct object_id *oid, uint enum object_type type = odb_read_object_info(the_repository->objects, oid, NULL); if (type < 0) - die("object not found: %s", oid_to_hex(oid)); + die(_("object not found: %s"), oid_to_hex(oid)); e = insert_object(oid); e->type = type; e->pack_id = MAX_PACK_ID; @@ -1791,13 +1792,13 @@ static void read_mark_file(struct mark_set **s, FILE *f, mark_set_inserter_t ins end = strchr(line, '\n'); if (line[0] != ':' || !end) - die("corrupt mark line: %s", line); + die(_("corrupt mark line: %s"), line); *end = 0; mark = strtoumax(line + 1, &end, 10); if (!mark || end == line + 1 || *end != ' ' || get_oid_hex_any(end + 1, &oid) == GIT_HASH_UNKNOWN) - die("corrupt mark line: %s", line); + die(_("corrupt mark line: %s"), line); inserter(s, &oid, mark); } } @@ -1810,7 +1811,7 @@ static void read_marks(void) else if (import_marks_file_ignore_missing && errno == ENOENT) goto done; /* Marks file does not exist */ else - die_errno("cannot read '%s'", import_marks_file); + die_errno(_("cannot read '%s'"), import_marks_file); read_mark_file(&marks, f, insert_object_entry); fclose(f); done: @@ -1896,7 +1897,7 @@ static int parse_data(struct strbuf *sb, uintmax_t limit, uintmax_t *len_res) strbuf_reset(sb); if (!skip_prefix(command_buf.buf, "data ", &data)) - die("Expected 'data n' command, found: %s", command_buf.buf); + die(_("expected 'data n' command, found: %s"), command_buf.buf); if (skip_prefix(data, "<<", &data)) { char *term = xstrdup(data); @@ -1904,7 +1905,7 @@ static int parse_data(struct strbuf *sb, uintmax_t limit, uintmax_t *len_res) for (;;) { if (strbuf_getline_lf(&command_buf, stdin) == EOF) - die("EOF in data (terminator '%s' not found)", term); + die(_("EOF in data (terminator '%s' not found)"), term); if (term_len == command_buf.len && !strcmp(term, command_buf.buf)) break; @@ -1922,12 +1923,12 @@ static int parse_data(struct strbuf *sb, uintmax_t limit, uintmax_t *len_res) return 0; } if (length < len) - die("data is too large to use in this context"); + die(_("data is too large to use in this context")); while (n < length) { size_t s = strbuf_fread(sb, length - n, stdin); if (!s && feof(stdin)) - die("EOF in data (%lu bytes remaining)", + die(_("EOF in data (%lu bytes remaining)"), (unsigned long)(length - n)); n += s; } @@ -1984,15 +1985,15 @@ static char *parse_ident(const char *buf) ltgt = buf + strcspn(buf, "<>"); if (*ltgt != '<') - die("Missing < in ident string: %s", buf); + die(_("missing < in ident string: %s"), buf); if (ltgt != buf && ltgt[-1] != ' ') - die("Missing space before < in ident string: %s", buf); + die(_("missing space before < in ident string: %s"), buf); ltgt = ltgt + 1 + strcspn(ltgt + 1, "<>"); if (*ltgt != '>') - die("Missing > in ident string: %s", buf); + die(_("missing > in ident string: %s"), buf); ltgt++; if (*ltgt != ' ') - die("Missing space after > in ident string: %s", buf); + die(_("missing space after > in ident string: %s"), buf); ltgt++; name_len = ltgt - buf; strbuf_add(&ident, buf, name_len); @@ -2000,19 +2001,19 @@ static char *parse_ident(const char *buf) switch (whenspec) { case WHENSPEC_RAW: if (validate_raw_date(ltgt, &ident, 1) < 0) - die("Invalid raw date \"%s\" in ident: %s", ltgt, buf); + die(_("invalid raw date \"%s\" in ident: %s"), ltgt, buf); break; case WHENSPEC_RAW_PERMISSIVE: if (validate_raw_date(ltgt, &ident, 0) < 0) - die("Invalid raw date \"%s\" in ident: %s", ltgt, buf); + die(_("invalid raw date \"%s\" in ident: %s"), ltgt, buf); break; case WHENSPEC_RFC2822: if (parse_date(ltgt, &ident) < 0) - die("Invalid rfc2822 date \"%s\" in ident: %s", ltgt, buf); + die(_("invalid rfc2822 date \"%s\" in ident: %s"), ltgt, buf); break; case WHENSPEC_NOW: if (strcmp("now", ltgt)) - die("Date in ident must be 'now': %s", buf); + die(_("date in ident must be 'now': %s"), buf); datestamp(&ident); break; } @@ -2106,7 +2107,7 @@ static void construct_path_with_fanout(const char *hex_sha1, { unsigned int i = 0, j = 0; if (fanout >= the_hash_algo->rawsz) - die("Too large fanout (%u)", fanout); + die(_("too large fanout (%u)"), fanout); while (fanout) { path[i++] = hex_sha1[j++]; path[i++] = hex_sha1[j++]; @@ -2180,7 +2181,7 @@ static uintmax_t do_change_note_fanout( /* Rename fullpath to realpath */ if (!tree_content_remove(orig_root, fullpath, &leaf, 0)) - die("Failed to remove path %s", fullpath); + die(_("failed to remove path %s"), fullpath); tree_content_set(orig_root, realpath, &leaf.versions[1].oid, leaf.versions[1].mode, @@ -2253,7 +2254,7 @@ static uintmax_t parse_mark_ref(const char *p, char **endptr) p++; mark = strtoumax(p, endptr, 10); if (*endptr == p) - die("No value after ':' in mark: %s", command_buf.buf); + die(_("no value after ':' in mark: %s"), command_buf.buf); return mark; } @@ -2268,7 +2269,7 @@ static uintmax_t parse_mark_ref_eol(const char *p) mark = parse_mark_ref(p, &end); if (*end != '\0') - die("Garbage after mark: %s", command_buf.buf); + die(_("garbage after mark: %s"), command_buf.buf); return mark; } @@ -2283,7 +2284,7 @@ static uintmax_t parse_mark_ref_space(const char **p) mark = parse_mark_ref(*p, &end); if (*end++ != ' ') - die("Missing space after mark: %s", command_buf.buf); + die(_("missing space after mark: %s"), command_buf.buf); *p = end; return mark; } @@ -2299,9 +2300,9 @@ static void parse_path(struct strbuf *sb, const char *p, const char **endp, { if (*p == '"') { if (unquote_c_style(sb, p, endp)) - die("Invalid %s: %s", field, command_buf.buf); + die(_("invalid %s: %s"), field, command_buf.buf); if (strlen(sb->buf) != sb->len) - die("NUL in %s: %s", field, command_buf.buf); + die(_("NUL in %s: %s"), field, command_buf.buf); } else { /* * Unless we are parsing the last field of a line, @@ -2324,7 +2325,7 @@ static void parse_path_eol(struct strbuf *sb, const char *p, const char *field) parse_path(sb, p, &end, 1, field); if (*end) - die("Garbage after %s: %s", field, command_buf.buf); + die(_("garbage after %s: %s"), field, command_buf.buf); } /* @@ -2337,7 +2338,7 @@ static void parse_path_space(struct strbuf *sb, const char *p, { parse_path(sb, p, endp, 0, field); if (**endp != ' ') - die("Missing space after %s: %s", field, command_buf.buf); + die(_("missing space after %s: %s"), field, command_buf.buf); (*endp)++; } @@ -2350,7 +2351,7 @@ static void file_change_m(const char *p, struct branch *b) p = parse_mode(p, &mode); if (!p) - die("Corrupt mode: %s", command_buf.buf); + die(_("corrupt mode: %s"), command_buf.buf); switch (mode) { case 0644: case 0755: @@ -2363,7 +2364,7 @@ static void file_change_m(const char *p, struct branch *b) /* ok */ break; default: - die("Corrupt mode: %s", command_buf.buf); + die(_("corrupt mode: %s"), command_buf.buf); } if (*p == ':') { @@ -2374,10 +2375,10 @@ static void file_change_m(const char *p, struct branch *b) oe = NULL; /* not used with inline_data, but makes gcc happy */ } else { if (parse_mapped_oid_hex(p, &oid, &p)) - die("Invalid dataref: %s", command_buf.buf); + die(_("invalid dataref: %s"), command_buf.buf); oe = find_object(&oid); if (*p++ != ' ') - die("Missing space after SHA1: %s", command_buf.buf); + die(_("missing space after SHA1: %s"), command_buf.buf); } strbuf_reset(&path); @@ -2393,11 +2394,11 @@ static void file_change_m(const char *p, struct branch *b) if (S_ISGITLINK(mode)) { if (inline_data) - die("Git links cannot be specified 'inline': %s", + die(_("Git links cannot be specified 'inline': %s"), command_buf.buf); else if (oe) { if (oe->type != OBJ_COMMIT) - die("Not a commit (actually a %s): %s", + die(_("not a commit (actually a %s): %s"), type_name(oe->type), command_buf.buf); } /* @@ -2406,7 +2407,7 @@ static void file_change_m(const char *p, struct branch *b) */ } else if (inline_data) { if (S_ISDIR(mode)) - die("Directories cannot be specified 'inline': %s", + die(_("directories cannot be specified 'inline': %s"), command_buf.buf); while (read_next_command() != EOF) { const char *v; @@ -2424,11 +2425,11 @@ static void file_change_m(const char *p, struct branch *b) odb_read_object_info(the_repository->objects, &oid, NULL); if (type < 0) - die("%s not found: %s", - S_ISDIR(mode) ? "Tree" : "Blob", - command_buf.buf); + die(_("%s not found: %s"), + S_ISDIR(mode) ? _("tree") : _("blob"), + command_buf.buf); if (type != expected) - die("Not a %s (actually a %s): %s", + die(_("not a %s (actually a %s): %s"), type_name(expected), type_name(type), command_buf.buf); } @@ -2439,7 +2440,7 @@ static void file_change_m(const char *p, struct branch *b) } if (!verify_path(path.buf, mode)) - die("invalid path '%s'", path.buf); + die(_("invalid path '%s'"), path.buf); tree_content_set(&b->branch_tree, path.buf, &oid, mode, NULL); } @@ -2469,7 +2470,7 @@ static void file_change_cr(const char *p, struct branch *b, int rename) else tree_content_get(&b->branch_tree, source.buf, &leaf, 1); if (!leaf.versions[1].mode) - die("Path %s not in branch", source.buf); + die(_("path %s not in branch"), source.buf); if (!*dest.buf) { /* C "path/to/subdir" "" */ tree_content_replace(&b->branch_tree, &leaf.versions[1].oid, @@ -2478,7 +2479,7 @@ static void file_change_cr(const char *p, struct branch *b, int rename) return; } if (!verify_path(dest.buf, leaf.versions[1].mode)) - die("invalid path '%s'", dest.buf); + die(_("invalid path '%s'"), dest.buf); tree_content_set(&b->branch_tree, dest.buf, &leaf.versions[1].oid, leaf.versions[1].mode, @@ -2520,23 +2521,23 @@ static void note_change_n(const char *p, struct branch *b, unsigned char *old_fa oe = NULL; /* not used with inline_data, but makes gcc happy */ } else { if (parse_mapped_oid_hex(p, &oid, &p)) - die("Invalid dataref: %s", command_buf.buf); + die(_("invalid dataref: %s"), command_buf.buf); oe = find_object(&oid); if (*p++ != ' ') - die("Missing space after SHA1: %s", command_buf.buf); + die(_("missing space after SHA1: %s"), command_buf.buf); } /* <commit-ish> */ s = lookup_branch(p); if (s) { if (is_null_oid(&s->oid)) - die("Can't add a note on empty branch."); + die(_("can't add a note on empty branch.")); oidcpy(&commit_oid, &s->oid); } else if (*p == ':') { uintmax_t commit_mark = parse_mark_ref_eol(p); struct object_entry *commit_oe = find_mark(marks, commit_mark); if (commit_oe->type != OBJ_COMMIT) - die("Mark :%" PRIuMAX " not a commit", commit_mark); + die(_("mark :%" PRIuMAX " not a commit"), commit_mark); oidcpy(&commit_oid, &commit_oe->idx.oid); } else if (!repo_get_oid(the_repository, p, &commit_oid)) { unsigned long size; @@ -2544,25 +2545,25 @@ static void note_change_n(const char *p, struct branch *b, unsigned char *old_fa &commit_oid, OBJ_COMMIT, &size, &commit_oid); if (!buf || size < the_hash_algo->hexsz + 6) - die("Not a valid commit: %s", p); + die(_("not a valid commit: %s"), p); free(buf); } else - die("Invalid ref name or SHA1 expression: %s", p); + die(_("invalid ref name or SHA1 expression: %s"), p); if (inline_data) { read_next_command(); parse_and_store_blob(&last_blob, &oid, 0); } else if (oe) { if (oe->type != OBJ_BLOB) - die("Not a blob (actually a %s): %s", + die(_("not a blob (actually a %s): %s"), type_name(oe->type), command_buf.buf); } else if (!is_null_oid(&oid)) { enum object_type type = odb_read_object_info(the_repository->objects, &oid, NULL); if (type < 0) - die("Blob not found: %s", command_buf.buf); + die(_("blob not found: %s"), command_buf.buf); if (type != OBJ_BLOB) - die("Not a blob (actually a %s): %s", + die(_("not a blob (actually a %s): %s"), type_name(type), command_buf.buf); } @@ -2591,10 +2592,10 @@ static void file_change_deleteall(struct branch *b) static void parse_from_commit(struct branch *b, char *buf, unsigned long size) { if (!buf || size < the_hash_algo->hexsz + 6) - die("Not a valid commit: %s", oid_to_hex(&b->oid)); + die(_("not a valid commit: %s"), oid_to_hex(&b->oid)); if (memcmp("tree ", buf, 5) || get_oid_hex(buf + 5, &b->branch_tree.versions[1].oid)) - die("The commit %s is corrupt", oid_to_hex(&b->oid)); + die(_("the commit %s is corrupt"), oid_to_hex(&b->oid)); oidcpy(&b->branch_tree.versions[0].oid, &b->branch_tree.versions[1].oid); } @@ -2624,7 +2625,7 @@ static int parse_objectish(struct branch *b, const char *objectish) s = lookup_branch(objectish); if (b == s) - die("Can't create a branch from itself: %s", b->name); + die(_("can't create a branch from itself: %s"), b->name); else if (s) { struct object_id *t = &s->branch_tree.versions[1].oid; oidcpy(&b->oid, &s->oid); @@ -2634,7 +2635,7 @@ static int parse_objectish(struct branch *b, const char *objectish) uintmax_t idnum = parse_mark_ref_eol(objectish); struct object_entry *oe = find_mark(marks, idnum); if (oe->type != OBJ_COMMIT) - die("Mark :%" PRIuMAX " not a commit", idnum); + die(_("mark :%" PRIuMAX " not a commit"), idnum); if (!oideq(&b->oid, &oe->idx.oid)) { oidcpy(&b->oid, &oe->idx.oid); if (oe->pack_id != MAX_PACK_ID) { @@ -2651,7 +2652,7 @@ static int parse_objectish(struct branch *b, const char *objectish) b->delete = 1; } else - die("Invalid ref name or SHA1 expression: %s", objectish); + die(_("invalid ref name or SHA1 expression: %s"), objectish); if (b->branch_tree.tree && !oideq(&oid, &b->branch_tree.versions[1].oid)) { release_tree_content_recursive(b->branch_tree.tree); @@ -2698,7 +2699,7 @@ static struct hash_list *parse_merge(unsigned int *count) uintmax_t idnum = parse_mark_ref_eol(from); struct object_entry *oe = find_mark(marks, idnum); if (oe->type != OBJ_COMMIT) - die("Mark :%" PRIuMAX " not a commit", idnum); + die(_("mark :%" PRIuMAX " not a commit"), idnum); oidcpy(&n->oid, &oe->idx.oid); } else if (!repo_get_oid(the_repository, from, &n->oid)) { unsigned long size; @@ -2706,10 +2707,10 @@ static struct hash_list *parse_merge(unsigned int *count) &n->oid, OBJ_COMMIT, &size, &n->oid); if (!buf || size < the_hash_algo->hexsz + 6) - die("Not a valid commit: %s", from); + die(_("not a valid commit: %s"), from); free(buf); } else - die("Invalid ref name or SHA1 expression: %s", from); + die(_("invalid ref name or SHA1 expression: %s"), from); n->next = NULL; *tail = n; @@ -2733,8 +2734,8 @@ static void parse_one_signature(struct signature_data *sig, const char *v) char *space = strchr(args, ' '); if (!space) - die("Expected gpgsig format: 'gpgsig <hash-algo> <signature-format>', " - "got 'gpgsig %s'", args); + die(_("expected gpgsig format: 'gpgsig <hash-algo> <signature-format>', " + "got 'gpgsig %s'"), args); *space = '\0'; sig->hash_algo = args; @@ -2743,13 +2744,13 @@ static void parse_one_signature(struct signature_data *sig, const char *v) /* Validate hash algorithm */ if (strcmp(sig->hash_algo, "sha1") && strcmp(sig->hash_algo, "sha256")) - die("Unknown git hash algorithm in gpgsig: '%s'", sig->hash_algo); + die(_("unknown git hash algorithm in gpgsig: '%s'"), sig->hash_algo); /* Validate signature format */ if (!valid_signature_format(sig->sig_format)) - die("Invalid signature format in gpgsig: '%s'", sig->sig_format); + die(_("invalid signature format in gpgsig: '%s'"), sig->sig_format); if (!strcmp(sig->sig_format, "unknown")) - warning("'unknown' signature format in gpgsig"); + warning(_("'unknown' signature format in gpgsig")); /* Read signature data */ read_next_command(); @@ -2788,8 +2789,8 @@ static void store_signature(struct signature_data *stored_sig, const char *hash_type) { if (stored_sig->hash_algo) { - warning("multiple %s signatures found, " - "ignoring additional signature", + warning(_("multiple %s signatures found, " + "ignoring additional signature"), hash_type); strbuf_release(&new_sig->data); free(new_sig->hash_algo); @@ -2844,15 +2845,15 @@ static void parse_new_commit(const char *arg) read_next_command(); } if (!committer) - die("Expected committer but didn't get one"); + die(_("expected committer but didn't get one")); while (skip_prefix(command_buf.buf, "gpgsig ", &v)) { switch (signed_commit_mode) { /* First, modes that don't need the signature to be parsed */ case SIGN_ABORT: - die("encountered signed commit; use " - "--signed-commits=<mode> to handle it"); + die(_("encountered signed commit; use " + "--signed-commits=<mode> to handle it")); case SIGN_WARN_STRIP: warning(_("stripping a commit signature")); /* fallthru */ @@ -2963,6 +2964,43 @@ static void parse_new_commit(const char *arg) b->last_commit = object_count_by_type[OBJ_COMMIT]; } +static void handle_tag_signature(struct strbuf *msg, const char *name) +{ + size_t sig_offset = parse_signed_buffer(msg->buf, msg->len); + + /* If there is no signature, there is nothing to do. */ + if (sig_offset >= msg->len) + return; + + switch (signed_tag_mode) { + + /* First, modes that don't change anything */ + case SIGN_ABORT: + die(_("encountered signed tag; use " + "--signed-tags=<mode> to handle it")); + case SIGN_WARN_VERBATIM: + warning(_("importing a tag signature verbatim for tag '%s'"), name); + /* fallthru */ + case SIGN_VERBATIM: + /* Nothing to do, the signature will be put into the imported tag. */ + break; + + /* Second, modes that remove the signature */ + case SIGN_WARN_STRIP: + warning(_("stripping a tag signature for tag '%s'"), name); + /* fallthru */ + case SIGN_STRIP: + /* Truncate the buffer to remove the signature */ + strbuf_setlen(msg, sig_offset); + break; + + /* Third, BUG */ + default: + BUG("invalid signed_tag_mode value %d from tag '%s'", + signed_tag_mode, name); + } +} + static void parse_new_tag(const char *arg) { static struct strbuf msg = STRBUF_INIT; @@ -2987,11 +3025,11 @@ static void parse_new_tag(const char *arg) /* from ... */ if (!skip_prefix(command_buf.buf, "from ", &from)) - die("Expected from command, got %s", command_buf.buf); + die(_("expected 'from' command, got '%s'"), command_buf.buf); s = lookup_branch(from); if (s) { if (is_null_oid(&s->oid)) - die("Can't tag an empty branch."); + die(_("can't tag an empty branch.")); oidcpy(&oid, &s->oid); type = OBJ_COMMIT; } else if (*from == ':') { @@ -3006,11 +3044,11 @@ static void parse_new_tag(const char *arg) type = odb_read_object_info(the_repository->objects, &oid, NULL); if (type < 0) - die("Not a valid object: %s", from); + die(_("not a valid object: %s"), from); } else type = oe->type; } else - die("Invalid ref name or SHA1 expression: %s", from); + die(_("invalid ref name or SHA1 expression: %s"), from); read_next_command(); /* original-oid ... */ @@ -3026,6 +3064,8 @@ static void parse_new_tag(const char *arg) /* tag payload/message */ parse_data(&msg, 0, NULL); + handle_tag_signature(&msg, t->name); + /* build the tag object */ strbuf_reset(&new_data); @@ -3099,7 +3139,7 @@ static void parse_reset_branch(const char *arg) static void cat_blob_write(const char *buf, unsigned long size) { if (write_in_full(cat_blob_fd, buf, size) < 0) - die_errno("Write to frontend failed"); + die_errno(_("write to frontend failed")); } static void cat_blob(struct object_entry *oe, struct object_id *oid) @@ -3128,9 +3168,9 @@ static void cat_blob(struct object_entry *oe, struct object_id *oid) return; } if (!buf) - die("Can't read object %s", oid_to_hex(oid)); + die(_("can't read object %s"), oid_to_hex(oid)); if (type != OBJ_BLOB) - die("Object %s is a %s but a blob was expected.", + die(_("object %s is a %s but a blob was expected."), oid_to_hex(oid), type_name(type)); strbuf_reset(&line); strbuf_addf(&line, "%s %s %"PRIuMAX"\n", oid_to_hex(oid), @@ -3154,11 +3194,11 @@ static void parse_get_mark(const char *p) /* get-mark SP <object> LF */ if (*p != ':') - die("Not a mark: %s", p); + die(_("not a mark: %s"), p); oe = find_mark(marks, parse_mark_ref_eol(p)); if (!oe) - die("Unknown mark: %s", command_buf.buf); + die(_("unknown mark: %s"), command_buf.buf); xsnprintf(output, sizeof(output), "%s\n", oid_to_hex(&oe->idx.oid)); cat_blob_write(output, the_hash_algo->hexsz + 1); @@ -3173,13 +3213,13 @@ static void parse_cat_blob(const char *p) if (*p == ':') { oe = find_mark(marks, parse_mark_ref_eol(p)); if (!oe) - die("Unknown mark: %s", command_buf.buf); + die(_("unknown mark: %s"), command_buf.buf); oidcpy(&oid, &oe->idx.oid); } else { if (parse_mapped_oid_hex(p, &oid, &p)) - die("Invalid dataref: %s", command_buf.buf); + die(_("invalid dataref: %s"), command_buf.buf); if (*p) - die("Garbage after SHA1: %s", command_buf.buf); + die(_("garbage after SHA1: %s"), command_buf.buf); oe = find_object(&oid); } @@ -3197,7 +3237,7 @@ static struct object_entry *dereference(struct object_entry *oe, enum object_type type = odb_read_object_info(the_repository->objects, oid, NULL); if (type < 0) - die("object not found: %s", oid_to_hex(oid)); + die(_("object not found: %s"), oid_to_hex(oid)); /* cache it! */ oe = insert_object(oid); oe->type = type; @@ -3211,7 +3251,7 @@ static struct object_entry *dereference(struct object_entry *oe, case OBJ_TAG: break; default: - die("Not a tree-ish: %s", command_buf.buf); + die(_("not a tree-ish: %s"), command_buf.buf); } if (oe->pack_id != MAX_PACK_ID) { /* in a pack being written */ @@ -3222,19 +3262,19 @@ static struct object_entry *dereference(struct object_entry *oe, &unused, &size); } if (!buf) - die("Can't load object %s", oid_to_hex(oid)); + die(_("can't load object %s"), oid_to_hex(oid)); /* Peel one layer. */ switch (oe->type) { case OBJ_TAG: if (size < hexsz + strlen("object ") || get_oid_hex(buf + strlen("object "), oid)) - die("Invalid SHA1 in tag: %s", command_buf.buf); + die(_("invalid SHA1 in tag: %s"), command_buf.buf); break; case OBJ_COMMIT: if (size < hexsz + strlen("tree ") || get_oid_hex(buf + strlen("tree "), oid)) - die("Invalid SHA1 in commit: %s", command_buf.buf); + die(_("invalid SHA1 in commit: %s"), command_buf.buf); } free(buf); @@ -3269,9 +3309,9 @@ static void build_mark_map(struct string_list *from, struct string_list *to) for_each_string_list_item(fromp, from) { top = string_list_lookup(to, fromp->string); if (!fromp->util) { - die(_("Missing from marks for submodule '%s'"), fromp->string); + die(_("missing from marks for submodule '%s'"), fromp->string); } else if (!top || !top->util) { - die(_("Missing to marks for submodule '%s'"), fromp->string); + die(_("missing to marks for submodule '%s'"), fromp->string); } build_mark_map_one(fromp->util, top->util); } @@ -3285,14 +3325,14 @@ static struct object_entry *parse_treeish_dataref(const char **p) if (**p == ':') { /* <mark> */ e = find_mark(marks, parse_mark_ref_space(p)); if (!e) - die("Unknown mark: %s", command_buf.buf); + die(_("unknown mark: %s"), command_buf.buf); oidcpy(&oid, &e->idx.oid); } else { /* <sha1> */ if (parse_mapped_oid_hex(*p, &oid, p)) - die("Invalid dataref: %s", command_buf.buf); + die(_("invalid dataref: %s"), command_buf.buf); e = find_object(&oid); if (*(*p)++ != ' ') - die("Missing space after tree-ish: %s", command_buf.buf); + die(_("missing space after tree-ish: %s"), command_buf.buf); } while (!e || e->type != OBJ_TREE) @@ -3336,7 +3376,7 @@ static void parse_ls(const char *p, struct branch *b) /* ls SP (<tree-ish> SP)? <path> */ if (*p == '"') { if (!b) - die("Not in a commit: %s", command_buf.buf); + die(_("not in a commit: %s"), command_buf.buf); root = &b->branch_tree; } else { struct object_entry *e = parse_treeish_dataref(&p); @@ -3399,12 +3439,12 @@ static void parse_alias(void) /* mark ... */ parse_mark(); if (!next_mark) - die(_("Expected 'mark' command, got %s"), command_buf.buf); + die(_("expected 'mark' command, got %s"), command_buf.buf); /* to ... */ memset(&b, 0, sizeof(b)); if (!parse_objectish_with_prefix(&b, "to ")) - die(_("Expected 'to' command, got %s"), command_buf.buf); + die(_("expected 'to' command, got %s"), command_buf.buf); e = find_object(&b.oid); assert(e); insert_mark(&marks, next_mark, e); @@ -3422,7 +3462,7 @@ static void option_import_marks(const char *marks, { if (import_marks_file) { if (from_stream) - die("Only one import-marks command allowed per stream"); + die(_("only one import-marks command allowed per stream")); /* read previous mark file */ if(!import_marks_file_from_stream) @@ -3446,7 +3486,7 @@ static void option_date_format(const char *fmt) else if (!strcmp(fmt, "now")) whenspec = WHENSPEC_NOW; else - die("unknown --date-format argument %s", fmt); + die(_("unknown --date-format argument %s"), fmt); } static unsigned long ulong_arg(const char *option, const char *arg) @@ -3454,7 +3494,7 @@ static unsigned long ulong_arg(const char *option, const char *arg) char *endptr; unsigned long rv = strtoul(arg, &endptr, 0); if (strchr(arg, '-') || endptr == arg || *endptr) - die("%s: argument must be a non-negative integer", option); + die(_("%s: argument must be a non-negative integer"), option); return rv; } @@ -3462,7 +3502,7 @@ static void option_depth(const char *depth) { max_depth = ulong_arg("--depth", depth); if (max_depth > MAX_DEPTH) - die("--depth cannot exceed %u", MAX_DEPTH); + die(_("--depth cannot exceed %u"), MAX_DEPTH); } static void option_active_branches(const char *branches) @@ -3480,7 +3520,7 @@ static void option_cat_blob_fd(const char *fd) { unsigned long n = ulong_arg("--cat-blob-fd", fd); if (n > (unsigned long) INT_MAX) - die("--cat-blob-fd cannot exceed %d", INT_MAX); + die(_("--cat-blob-fd cannot exceed %d"), INT_MAX); cat_blob_fd = (int) n; } @@ -3500,7 +3540,7 @@ static void option_rewrite_submodules(const char *arg, struct string_list *list) char *s = xstrdup(arg); char *f = strchr(s, ':'); if (!f) - die(_("Expected format name:filename for submodule rewrite option")); + die(_("expected format name:filename for submodule rewrite option")); *f = '\0'; f++; CALLOC_ARRAY(ms, 1); @@ -3508,7 +3548,7 @@ static void option_rewrite_submodules(const char *arg, struct string_list *list) f = prefix_filename(global_prefix, f); fp = fopen(f, "r"); if (!fp) - die_errno("cannot read '%s'", f); + die_errno(_("cannot read '%s'"), f); read_mark_file(&ms, fp, insert_oid_entry); fclose(fp); free(f); @@ -3525,10 +3565,10 @@ static int parse_one_option(const char *option) if (!git_parse_ulong(option, &v)) return 0; if (v < 8192) { - warning("max-pack-size is now in bytes, assuming --max-pack-size=%lum", v); + warning(_("max-pack-size is now in bytes, assuming --max-pack-size=%lum"), v); v *= 1024 * 1024; } else if (v < 1024 * 1024) { - warning("minimum max-pack-size is 1 MiB"); + warning(_("minimum max-pack-size is 1 MiB")); v = 1024 * 1024; } max_packsize = v; @@ -3546,6 +3586,9 @@ static int parse_one_option(const char *option) } else if (skip_prefix(option, "signed-commits=", &option)) { if (parse_sign_mode(option, &signed_commit_mode)) usagef(_("unknown --signed-commits mode '%s'"), option); + } else if (skip_prefix(option, "signed-tags=", &option)) { + if (parse_sign_mode(option, &signed_tag_mode)) + usagef(_("unknown --signed-tags mode '%s'"), option); } else if (!strcmp(option, "quiet")) { show_stats = 0; quiet = 1; @@ -3612,23 +3655,23 @@ static int parse_one_feature(const char *feature, int from_stream) static void parse_feature(const char *feature) { if (seen_data_command) - die("Got feature command '%s' after data command", feature); + die(_("got feature command '%s' after data command"), feature); if (parse_one_feature(feature, 1)) return; - die("This version of fast-import does not support feature %s.", feature); + die(_("this version of fast-import does not support feature %s."), feature); } static void parse_option(const char *option) { if (seen_data_command) - die("Got option command '%s' after data command", option); + die(_("got option command '%s' after data command"), option); if (parse_one_option(option)) return; - die("This version of fast-import does not support option: %s", option); + die(_("this version of fast-import does not support option: %s"), option); } static void git_pack_config(void) @@ -3672,7 +3715,7 @@ static void parse_argv(void) break; if (!skip_prefix(a, "--", &a)) - die("unknown option %s", a); + die(_("unknown option %s"), a); if (parse_one_option(a)) continue; @@ -3685,7 +3728,7 @@ static void parse_argv(void) continue; } - die("unknown option --%s", a); + die(_("unknown option --%s"), a); } if (i != global_argc) usage(fast_import_usage); @@ -3774,7 +3817,7 @@ int cmd_fast_import(int argc, else if (starts_with(command_buf.buf, "option ")) /* ignore non-git options*/; else - die("Unsupported command: %s", command_buf.buf); + die(_("unsupported command: %s"), command_buf.buf); if (checkpoint_requested) checkpoint(); @@ -3785,7 +3828,7 @@ int cmd_fast_import(int argc, parse_argv(); if (require_explicit_termination && feof(stdin)) - die("stream ends early"); + die(_("stream ends early")); end_packfile(); diff --git a/builtin/fetch.c b/builtin/fetch.c index c7ff3480fb..7052e6ff21 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -289,13 +289,11 @@ static struct refname_hash_entry *refname_hash_add(struct hashmap *map, return ent; } -static int add_one_refname(const char *refname, const char *referent UNUSED, - const struct object_id *oid, - int flag UNUSED, void *cbdata) +static int add_one_refname(const struct reference *ref, void *cbdata) { struct hashmap *refname_map = cbdata; - (void) refname_hash_add(refname_map, refname, oid); + (void) refname_hash_add(refname_map, ref->name, ref->oid); return 0; } @@ -1416,14 +1414,11 @@ static void set_option(struct transport *transport, const char *name, const char } -static int add_oid(const char *refname UNUSED, - const char *referent UNUSED, - const struct object_id *oid, - int flags UNUSED, void *cb_data) +static int add_oid(const struct reference *ref, void *cb_data) { struct oid_array *oids = cb_data; - oid_array_append(oids, oid); + oid_array_append(oids, ref->oid); return 0; } diff --git a/builtin/fsck.c b/builtin/fsck.c index 8ee95e0d67..c489582faa 100644 --- a/builtin/fsck.c +++ b/builtin/fsck.c @@ -530,14 +530,13 @@ static int fsck_handle_reflog(const char *logname, void *cb_data) return 0; } -static int fsck_handle_ref(const char *refname, const char *referent UNUSED, const struct object_id *oid, - int flag UNUSED, void *cb_data UNUSED) +static int fsck_handle_ref(const struct reference *ref, void *cb_data UNUSED) { struct object *obj; - obj = parse_object(the_repository, oid); + obj = parse_object(the_repository, ref->oid); if (!obj) { - if (is_promisor_object(the_repository, oid)) { + if (is_promisor_object(the_repository, ref->oid)) { /* * Increment default_refs anyway, because this is a * valid ref. @@ -546,19 +545,19 @@ static int fsck_handle_ref(const char *refname, const char *referent UNUSED, con return 0; } error(_("%s: invalid sha1 pointer %s"), - refname, oid_to_hex(oid)); + ref->name, oid_to_hex(ref->oid)); errors_found |= ERROR_REACHABLE; /* We'll continue with the rest despite the error.. */ return 0; } - if (obj->type != OBJ_COMMIT && is_branch(refname)) { - error(_("%s: not a commit"), refname); + if (obj->type != OBJ_COMMIT && is_branch(ref->name)) { + error(_("%s: not a commit"), ref->name); errors_found |= ERROR_REFS; } default_refs++; obj->flags |= USED; fsck_put_object_name(&fsck_walk_options, - oid, "%s", refname); + ref->oid, "%s", ref->name); mark_object_reachable(obj); return 0; @@ -580,13 +579,19 @@ static void get_default_heads(void) worktrees = get_worktrees(); for (p = worktrees; *p; p++) { struct worktree *wt = *p; - struct strbuf ref = STRBUF_INIT; + struct strbuf refname = STRBUF_INIT; - strbuf_worktree_ref(wt, &ref, "HEAD"); - fsck_head_link(ref.buf, &head_points_at, &head_oid); - if (head_points_at && !is_null_oid(&head_oid)) - fsck_handle_ref(ref.buf, NULL, &head_oid, 0, NULL); - strbuf_release(&ref); + strbuf_worktree_ref(wt, &refname, "HEAD"); + fsck_head_link(refname.buf, &head_points_at, &head_oid); + if (head_points_at && !is_null_oid(&head_oid)) { + struct reference ref = { + .name = refname.buf, + .oid = &head_oid, + }; + + fsck_handle_ref(&ref, NULL); + } + strbuf_release(&refname); if (include_reflogs) refs_for_each_reflog(get_worktree_ref_store(wt), @@ -867,20 +872,20 @@ static int mark_packed_for_connectivity(const struct object_id *oid, static int check_pack_rev_indexes(struct repository *r, int show_progress) { - struct packfile_store *packs = r->objects->packfiles; struct progress *progress = NULL; + struct packed_git *p; uint32_t pack_count = 0; int res = 0; if (show_progress) { - for (struct packed_git *p = packfile_store_get_all_packs(packs); p; p = p->next) + repo_for_each_pack(r, p) pack_count++; progress = start_delayed_progress(the_repository, "Verifying reverse pack-indexes", pack_count); pack_count = 0; } - for (struct packed_git *p = packfile_store_get_all_packs(packs); p; p = p->next) { + repo_for_each_pack(r, p) { int load_error = load_pack_revindex_from_disk(p); if (load_error < 0) { @@ -1000,8 +1005,6 @@ int cmd_fsck(int argc, for_each_packed_object(the_repository, mark_packed_for_connectivity, NULL, 0); } else { - struct packfile_store *packs = the_repository->objects->packfiles; - odb_prepare_alternates(the_repository->objects); for (source = the_repository->objects->sources; source; source = source->next) fsck_source(source); @@ -1012,8 +1015,7 @@ int cmd_fsck(int argc, struct progress *progress = NULL; if (show_progress) { - for (p = packfile_store_get_all_packs(packs); p; - p = p->next) { + repo_for_each_pack(the_repository, p) { if (open_pack_index(p)) continue; total += p->num_objects; @@ -1022,8 +1024,8 @@ int cmd_fsck(int argc, progress = start_progress(the_repository, _("Checking objects"), total); } - for (p = packfile_store_get_all_packs(packs); p; - p = p->next) { + + repo_for_each_pack(the_repository, p) { /* verify gives error messages itself */ if (verify_pack(the_repository, p, fsck_obj_buffer, diff --git a/builtin/gc.c b/builtin/gc.c index e19e13d978..5fa0653ed0 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -34,7 +34,9 @@ #include "pack-objects.h" #include "path.h" #include "reflog.h" +#include "repack.h" #include "rerere.h" +#include "revision.h" #include "blob.h" #include "tree.h" #include "promisor-remote.h" @@ -55,7 +57,6 @@ static const char * const builtin_gc_usage[] = { }; static timestamp_t gc_log_expire_time; -static struct strvec repack = STRVEC_INIT; static struct tempfile *pidfile; static struct lock_file log_lock; static struct string_list pack_garbage = STRING_LIST_INIT_DUP; @@ -255,6 +256,7 @@ enum maintenance_task_label { TASK_PREFETCH, TASK_LOOSE_OBJECTS, TASK_INCREMENTAL_REPACK, + TASK_GEOMETRIC_REPACK, TASK_GC, TASK_COMMIT_GRAPH, TASK_PACK_REFS, @@ -285,12 +287,26 @@ static void maintenance_run_opts_release(struct maintenance_run_opts *opts) static int pack_refs_condition(UNUSED struct gc_config *cfg) { - /* - * The auto-repacking logic for refs is handled by the ref backends and - * exposed via `git pack-refs --auto`. We thus always return truish - * here and let the backend decide for us. - */ - return 1; + struct string_list included_refs = STRING_LIST_INIT_NODUP; + struct ref_exclusions excludes = REF_EXCLUSIONS_INIT; + struct refs_optimize_opts optimize_opts = { + .exclusions = &excludes, + .includes = &included_refs, + .flags = REFS_OPTIMIZE_PRUNE | REFS_OPTIMIZE_AUTO, + }; + bool required; + + /* Check for all refs, similar to 'git refs optimize --all'. */ + string_list_append(optimize_opts.includes, "*"); + + if (refs_optimize_required(get_main_ref_store(the_repository), + &optimize_opts, &required)) + return 0; + + clear_ref_exclusions(&excludes); + string_list_clear(&included_refs, 0); + + return required == true; } static int maintenance_task_pack_refs(struct maintenance_run_opts *opts, @@ -448,7 +464,7 @@ out: return should_gc; } -static int too_many_loose_objects(struct gc_config *cfg) +static int too_many_loose_objects(int limit) { /* * Quickly check if a "gc" is needed, by estimating how @@ -470,7 +486,7 @@ static int too_many_loose_objects(struct gc_config *cfg) if (!dir) return 0; - auto_threshold = DIV_ROUND_UP(cfg->gc_auto_threshold, 256); + auto_threshold = DIV_ROUND_UP(limit, 256); while ((ent = readdir(dir)) != NULL) { if (strspn(ent->d_name, "0123456789abcdef") != hexsz_loose || ent->d_name[hexsz_loose] != '\0') @@ -487,10 +503,9 @@ static int too_many_loose_objects(struct gc_config *cfg) static struct packed_git *find_base_packs(struct string_list *packs, unsigned long limit) { - struct packfile_store *packfiles = the_repository->objects->packfiles; struct packed_git *p, *base = NULL; - for (p = packfile_store_get_all_packs(packfiles); p; p = p->next) { + repo_for_each_pack(the_repository, p) { if (!p->pack_local || p->is_cruft) continue; if (limit) { @@ -509,14 +524,13 @@ static struct packed_git *find_base_packs(struct string_list *packs, static int too_many_packs(struct gc_config *cfg) { - struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *p; - int cnt; + int cnt = 0; if (cfg->gc_auto_pack_limit <= 0) return 0; - for (cnt = 0, p = packfile_store_get_all_packs(packs); p; p = p->next) { + repo_for_each_pack(the_repository, p) { if (!p->pack_local) continue; if (p->pack_keep) @@ -618,48 +632,50 @@ static uint64_t estimate_repack_memory(struct gc_config *cfg, return os_cache + heap; } -static int keep_one_pack(struct string_list_item *item, void *data UNUSED) +static int keep_one_pack(struct string_list_item *item, void *data) { - strvec_pushf(&repack, "--keep-pack=%s", basename(item->string)); + struct strvec *args = data; + strvec_pushf(args, "--keep-pack=%s", basename(item->string)); return 0; } static void add_repack_all_option(struct gc_config *cfg, - struct string_list *keep_pack) + struct string_list *keep_pack, + struct strvec *args) { if (cfg->prune_expire && !strcmp(cfg->prune_expire, "now") && !(cfg->cruft_packs && cfg->repack_expire_to)) - strvec_push(&repack, "-a"); + strvec_push(args, "-a"); else if (cfg->cruft_packs) { - strvec_push(&repack, "--cruft"); + strvec_push(args, "--cruft"); if (cfg->prune_expire) - strvec_pushf(&repack, "--cruft-expiration=%s", cfg->prune_expire); + strvec_pushf(args, "--cruft-expiration=%s", cfg->prune_expire); if (cfg->max_cruft_size) - strvec_pushf(&repack, "--max-cruft-size=%lu", + strvec_pushf(args, "--max-cruft-size=%lu", cfg->max_cruft_size); if (cfg->repack_expire_to) - strvec_pushf(&repack, "--expire-to=%s", cfg->repack_expire_to); + strvec_pushf(args, "--expire-to=%s", cfg->repack_expire_to); } else { - strvec_push(&repack, "-A"); + strvec_push(args, "-A"); if (cfg->prune_expire) - strvec_pushf(&repack, "--unpack-unreachable=%s", cfg->prune_expire); + strvec_pushf(args, "--unpack-unreachable=%s", cfg->prune_expire); } if (keep_pack) - for_each_string_list(keep_pack, keep_one_pack, NULL); + for_each_string_list(keep_pack, keep_one_pack, args); if (cfg->repack_filter && *cfg->repack_filter) - strvec_pushf(&repack, "--filter=%s", cfg->repack_filter); + strvec_pushf(args, "--filter=%s", cfg->repack_filter); if (cfg->repack_filter_to && *cfg->repack_filter_to) - strvec_pushf(&repack, "--filter-to=%s", cfg->repack_filter_to); + strvec_pushf(args, "--filter-to=%s", cfg->repack_filter_to); } -static void add_repack_incremental_option(void) +static void add_repack_incremental_option(struct strvec *args) { - strvec_push(&repack, "--no-write-bitmap-index"); + strvec_push(args, "--no-write-bitmap-index"); } -static int need_to_gc(struct gc_config *cfg) +static int need_to_gc(struct gc_config *cfg, struct strvec *repack_args) { /* * Setting gc.auto to 0 or negative can disable the @@ -700,10 +716,10 @@ static int need_to_gc(struct gc_config *cfg) string_list_clear(&keep_pack, 0); } - add_repack_all_option(cfg, &keep_pack); + add_repack_all_option(cfg, &keep_pack, repack_args); string_list_clear(&keep_pack, 0); - } else if (too_many_loose_objects(cfg)) - add_repack_incremental_option(); + } else if (too_many_loose_objects(cfg->gc_auto_threshold)) + add_repack_incremental_option(repack_args); else return 0; @@ -852,6 +868,7 @@ int cmd_gc(int argc, int keep_largest_pack = -1; int skip_foreground_tasks = 0; timestamp_t dummy; + struct strvec repack_args = STRVEC_INIT; struct maintenance_run_opts opts = MAINTENANCE_RUN_OPTS_INIT; struct gc_config cfg = GC_CONFIG_INIT; const char *prune_expire_sentinel = "sentinel"; @@ -891,7 +908,7 @@ int cmd_gc(int argc, show_usage_with_options_if_asked(argc, argv, builtin_gc_usage, builtin_gc_options); - strvec_pushl(&repack, "repack", "-d", "-l", NULL); + strvec_pushl(&repack_args, "repack", "-d", "-l", NULL); gc_config(&cfg); @@ -914,14 +931,14 @@ int cmd_gc(int argc, die(_("failed to parse prune expiry value %s"), cfg.prune_expire); if (aggressive) { - strvec_push(&repack, "-f"); + strvec_push(&repack_args, "-f"); if (cfg.aggressive_depth > 0) - strvec_pushf(&repack, "--depth=%d", cfg.aggressive_depth); + strvec_pushf(&repack_args, "--depth=%d", cfg.aggressive_depth); if (cfg.aggressive_window > 0) - strvec_pushf(&repack, "--window=%d", cfg.aggressive_window); + strvec_pushf(&repack_args, "--window=%d", cfg.aggressive_window); } if (opts.quiet) - strvec_push(&repack, "-q"); + strvec_push(&repack_args, "-q"); if (opts.auto_flag) { if (cfg.detach_auto && opts.detach < 0) @@ -930,7 +947,7 @@ int cmd_gc(int argc, /* * Auto-gc should be least intrusive as possible. */ - if (!need_to_gc(&cfg)) { + if (!need_to_gc(&cfg, &repack_args)) { ret = 0; goto out; } @@ -952,7 +969,7 @@ int cmd_gc(int argc, find_base_packs(&keep_pack, cfg.big_pack_threshold); } - add_repack_all_option(&cfg, &keep_pack); + add_repack_all_option(&cfg, &keep_pack, &repack_args); string_list_clear(&keep_pack, 0); } @@ -1014,9 +1031,9 @@ int cmd_gc(int argc, repack_cmd.git_cmd = 1; repack_cmd.close_object_store = 1; - strvec_pushv(&repack_cmd.args, repack.v); + strvec_pushv(&repack_cmd.args, repack_args.v); if (run_command(&repack_cmd)) - die(FAILED_RUN, repack.v[0]); + die(FAILED_RUN, repack_args.v[0]); if (cfg.prune_expire) { struct child_process prune_cmd = CHILD_PROCESS_INIT; @@ -1055,7 +1072,7 @@ int cmd_gc(int argc, !opts.quiet && !daemonized ? COMMIT_GRAPH_WRITE_PROGRESS : 0, NULL); - if (opts.auto_flag && too_many_loose_objects(&cfg)) + if (opts.auto_flag && too_many_loose_objects(cfg.gc_auto_threshold)) warning(_("There are too many unreachable loose objects; " "run 'git prune' to remove them.")); @@ -1067,6 +1084,7 @@ int cmd_gc(int argc, out: maintenance_run_opts_release(&opts); + strvec_clear(&repack_args); gc_config_release(&cfg); return 0; } @@ -1092,32 +1110,26 @@ static int maintenance_opt_schedule(const struct option *opt, const char *arg, return 0; } -/* Remember to update object flag allocation in object.h */ -#define SEEN (1u<<0) - struct cg_auto_data { int num_not_in_graph; int limit; }; -static int dfs_on_ref(const char *refname UNUSED, - const char *referent UNUSED, - const struct object_id *oid, - int flags UNUSED, - void *cb_data) +static int dfs_on_ref(const struct reference *ref, void *cb_data) { struct cg_auto_data *data = (struct cg_auto_data *)cb_data; int result = 0; + const struct object_id *maybe_peeled = ref->oid; struct object_id peeled; struct commit_list *stack = NULL; struct commit *commit; - if (!peel_iterated_oid(the_repository, oid, &peeled)) - oid = &peeled; - if (odb_read_object_info(the_repository->objects, oid, NULL) != OBJ_COMMIT) + if (!reference_get_peeled_oid(the_repository, ref, &peeled)) + maybe_peeled = &peeled; + if (odb_read_object_info(the_repository->objects, maybe_peeled, NULL) != OBJ_COMMIT) return 0; - commit = lookup_commit(the_repository, oid); + commit = lookup_commit(the_repository, maybe_peeled); if (!commit) return 0; if (repo_parse_commit(the_repository, commit) || @@ -1269,6 +1281,19 @@ static int maintenance_task_gc_background(struct maintenance_run_opts *opts, return run_command(&child); } +static int gc_condition(struct gc_config *cfg) +{ + /* + * Note that it's fine to drop the repack arguments here, as we execute + * git-gc(1) as a separate child process anyway. So it knows to compute + * these arguments again. + */ + struct strvec repack_args = STRVEC_INIT; + int ret = need_to_gc(cfg, &repack_args); + strvec_clear(&repack_args); + return ret; +} + static int prune_packed(struct maintenance_run_opts *opts) { struct child_process child = CHILD_PROCESS_INIT; @@ -1425,9 +1450,9 @@ static int incremental_repack_auto_condition(struct gc_config *cfg UNUSED) if (incremental_repack_auto_limit < 0) return 1; - for (p = packfile_store_get_packs(the_repository->objects->packfiles); - count < incremental_repack_auto_limit && p; - p = p->next) { + repo_for_each_pack(the_repository, p) { + if (count >= incremental_repack_auto_limit) + break; if (!p->multi_pack_index) count++; } @@ -1494,7 +1519,7 @@ static off_t get_auto_pack_size(void) struct repository *r = the_repository; odb_reprepare(r->objects); - for (p = packfile_store_get_all_packs(r->objects->packfiles); p; p = p->next) { + repo_for_each_pack(r, p) { if (p->pack_size > max_size) { second_largest_size = max_size; max_size = p->pack_size; @@ -1550,6 +1575,108 @@ static int maintenance_task_incremental_repack(struct maintenance_run_opts *opts return 0; } +static int maintenance_task_geometric_repack(struct maintenance_run_opts *opts, + struct gc_config *cfg) +{ + struct pack_geometry geometry = { + .split_factor = 2, + }; + struct pack_objects_args po_args = { + .local = 1, + }; + struct existing_packs existing_packs = EXISTING_PACKS_INIT; + struct string_list kept_packs = STRING_LIST_INIT_DUP; + struct child_process child = CHILD_PROCESS_INIT; + int ret; + + repo_config_get_int(the_repository, "maintenance.geometric-repack.splitFactor", + &geometry.split_factor); + + existing_packs.repo = the_repository; + existing_packs_collect(&existing_packs, &kept_packs); + pack_geometry_init(&geometry, &existing_packs, &po_args); + pack_geometry_split(&geometry); + + child.git_cmd = 1; + + strvec_pushl(&child.args, "repack", "-d", "-l", NULL); + if (geometry.split < geometry.pack_nr) + strvec_pushf(&child.args, "--geometric=%d", + geometry.split_factor); + else + add_repack_all_option(cfg, NULL, &child.args); + if (opts->quiet) + strvec_push(&child.args, "--quiet"); + if (the_repository->settings.core_multi_pack_index) + strvec_push(&child.args, "--write-midx"); + + if (run_command(&child)) { + ret = error(_("failed to perform geometric repack")); + goto out; + } + + ret = 0; + +out: + existing_packs_release(&existing_packs); + pack_geometry_release(&geometry); + return ret; +} + +static int geometric_repack_auto_condition(struct gc_config *cfg UNUSED) +{ + struct pack_geometry geometry = { + .split_factor = 2, + }; + struct pack_objects_args po_args = { + .local = 1, + }; + struct existing_packs existing_packs = EXISTING_PACKS_INIT; + struct string_list kept_packs = STRING_LIST_INIT_DUP; + int auto_value = 100; + int ret; + + repo_config_get_int(the_repository, "maintenance.geometric-repack.auto", + &auto_value); + if (!auto_value) + return 0; + if (auto_value < 0) + return 1; + + repo_config_get_int(the_repository, "maintenance.geometric-repack.splitFactor", + &geometry.split_factor); + + existing_packs.repo = the_repository; + existing_packs_collect(&existing_packs, &kept_packs); + pack_geometry_init(&geometry, &existing_packs, &po_args); + pack_geometry_split(&geometry); + + /* + * When we'd merge at least two packs with one another we always + * perform the repack. + */ + if (geometry.split) { + ret = 1; + goto out; + } + + /* + * Otherwise, we estimate the number of loose objects to determine + * whether we want to create a new packfile or not. + */ + if (too_many_loose_objects(auto_value)) { + ret = 1; + goto out; + } + + ret = 0; + +out: + existing_packs_release(&existing_packs); + pack_geometry_release(&geometry); + return ret; +} + typedef int (*maintenance_task_fn)(struct maintenance_run_opts *opts, struct gc_config *cfg); typedef int (*maintenance_auto_fn)(struct gc_config *cfg); @@ -1592,11 +1719,16 @@ static const struct maintenance_task tasks[] = { .background = maintenance_task_incremental_repack, .auto_condition = incremental_repack_auto_condition, }, + [TASK_GEOMETRIC_REPACK] = { + .name = "geometric-repack", + .background = maintenance_task_geometric_repack, + .auto_condition = geometric_repack_auto_condition, + }, [TASK_GC] = { .name = "gc", .foreground = maintenance_task_gc_foreground, .background = maintenance_task_gc_background, - .auto_condition = need_to_gc, + .auto_condition = gc_condition, }, [TASK_COMMIT_GRAPH] = { .name = "commit-graph", @@ -1702,39 +1834,116 @@ static int maintenance_run_tasks(struct maintenance_run_opts *opts, return result; } +enum maintenance_type { + /* As invoked via `git maintenance run --schedule=`. */ + MAINTENANCE_TYPE_SCHEDULED = (1 << 0), + /* As invoked via `git maintenance run` and with `--auto`. */ + MAINTENANCE_TYPE_MANUAL = (1 << 1), +}; + struct maintenance_strategy { struct { - int enabled; + unsigned type; enum schedule_priority schedule; } tasks[TASK__COUNT]; }; static const struct maintenance_strategy none_strategy = { 0 }; -static const struct maintenance_strategy default_strategy = { + +static const struct maintenance_strategy gc_strategy = { .tasks = { - [TASK_GC].enabled = 1, + [TASK_GC] = { + .type = MAINTENANCE_TYPE_MANUAL | MAINTENANCE_TYPE_SCHEDULED, + .schedule = SCHEDULE_DAILY, + }, }, }; + static const struct maintenance_strategy incremental_strategy = { .tasks = { - [TASK_COMMIT_GRAPH].enabled = 1, - [TASK_COMMIT_GRAPH].schedule = SCHEDULE_HOURLY, - [TASK_PREFETCH].enabled = 1, - [TASK_PREFETCH].schedule = SCHEDULE_HOURLY, - [TASK_INCREMENTAL_REPACK].enabled = 1, - [TASK_INCREMENTAL_REPACK].schedule = SCHEDULE_DAILY, - [TASK_LOOSE_OBJECTS].enabled = 1, - [TASK_LOOSE_OBJECTS].schedule = SCHEDULE_DAILY, - [TASK_PACK_REFS].enabled = 1, - [TASK_PACK_REFS].schedule = SCHEDULE_WEEKLY, + [TASK_COMMIT_GRAPH] = { + .type = MAINTENANCE_TYPE_SCHEDULED, + .schedule = SCHEDULE_HOURLY, + }, + [TASK_PREFETCH] = { + .type = MAINTENANCE_TYPE_SCHEDULED, + .schedule = SCHEDULE_HOURLY, + }, + [TASK_INCREMENTAL_REPACK] = { + .type = MAINTENANCE_TYPE_SCHEDULED, + .schedule = SCHEDULE_DAILY, + }, + [TASK_LOOSE_OBJECTS] = { + .type = MAINTENANCE_TYPE_SCHEDULED, + .schedule = SCHEDULE_DAILY, + }, + [TASK_PACK_REFS] = { + .type = MAINTENANCE_TYPE_SCHEDULED, + .schedule = SCHEDULE_WEEKLY, + }, + /* + * Historically, the "incremental" strategy was only available + * in the context of scheduled maintenance when set up via + * "maintenance.strategy". We have later expanded that config + * to also cover manual maintenance. + * + * To retain backwards compatibility with the previous status + * quo we thus run git-gc(1) in case manual maintenance was + * requested. This is the same as the default strategy, which + * would have been in use beforehand. + */ + [TASK_GC] = { + .type = MAINTENANCE_TYPE_MANUAL, + }, + }, +}; + +static const struct maintenance_strategy geometric_strategy = { + .tasks = { + [TASK_COMMIT_GRAPH] = { + .type = MAINTENANCE_TYPE_SCHEDULED | MAINTENANCE_TYPE_MANUAL, + .schedule = SCHEDULE_HOURLY, + }, + [TASK_GEOMETRIC_REPACK] = { + .type = MAINTENANCE_TYPE_SCHEDULED | MAINTENANCE_TYPE_MANUAL, + .schedule = SCHEDULE_DAILY, + }, + [TASK_PACK_REFS] = { + .type = MAINTENANCE_TYPE_SCHEDULED | MAINTENANCE_TYPE_MANUAL, + .schedule = SCHEDULE_DAILY, + }, + [TASK_RERERE_GC] = { + .type = MAINTENANCE_TYPE_SCHEDULED | MAINTENANCE_TYPE_MANUAL, + .schedule = SCHEDULE_WEEKLY, + }, + [TASK_REFLOG_EXPIRE] = { + .type = MAINTENANCE_TYPE_SCHEDULED | MAINTENANCE_TYPE_MANUAL, + .schedule = SCHEDULE_WEEKLY, + }, + [TASK_WORKTREE_PRUNE] = { + .type = MAINTENANCE_TYPE_SCHEDULED | MAINTENANCE_TYPE_MANUAL, + .schedule = SCHEDULE_WEEKLY, + }, }, }; +static struct maintenance_strategy parse_maintenance_strategy(const char *name) +{ + if (!strcasecmp(name, "incremental")) + return incremental_strategy; + if (!strcasecmp(name, "gc")) + return gc_strategy; + if (!strcasecmp(name, "geometric")) + return geometric_strategy; + die(_("unknown maintenance strategy: '%s'"), name); +} + static void initialize_task_config(struct maintenance_run_opts *opts, const struct string_list *selected_tasks) { struct strbuf config_name = STRBUF_INIT; struct maintenance_strategy strategy; + enum maintenance_type type; const char *config_str; /* @@ -1762,19 +1971,20 @@ static void initialize_task_config(struct maintenance_run_opts *opts, * - Unscheduled maintenance uses our default strategy. * * Both of these are affected by the gitconfig though, which may - * override specific aspects of our strategy. + * override specific aspects of our strategy. Furthermore, both + * strategies can be overridden by setting "maintenance.strategy". */ if (opts->schedule) { strategy = none_strategy; - - if (!repo_config_get_string_tmp(the_repository, "maintenance.strategy", &config_str)) { - if (!strcasecmp(config_str, "incremental")) - strategy = incremental_strategy; - } + type = MAINTENANCE_TYPE_SCHEDULED; } else { - strategy = default_strategy; + strategy = gc_strategy; + type = MAINTENANCE_TYPE_MANUAL; } + if (!repo_config_get_string_tmp(the_repository, "maintenance.strategy", &config_str)) + strategy = parse_maintenance_strategy(config_str); + for (size_t i = 0; i < TASK__COUNT; i++) { int config_value; @@ -1782,8 +1992,8 @@ static void initialize_task_config(struct maintenance_run_opts *opts, strbuf_addf(&config_name, "maintenance.%s.enabled", tasks[i].name); if (!repo_config_get_bool(the_repository, config_name.buf, &config_value)) - strategy.tasks[i].enabled = config_value; - if (!strategy.tasks[i].enabled) + strategy.tasks[i].type = config_value ? type : 0; + if (!(strategy.tasks[i].type & type)) continue; if (opts->schedule) { @@ -3246,7 +3456,59 @@ static int maintenance_stop(int argc, const char **argv, const char *prefix, return update_background_schedule(NULL, 0); } -static const char * const builtin_maintenance_usage[] = { +static const char *const builtin_maintenance_is_needed_usage[] = { + "git maintenance is-needed [--task=<task>] [--schedule]", + NULL +}; + +static int maintenance_is_needed(int argc, const char **argv, const char *prefix, + struct repository *repo UNUSED) +{ + struct maintenance_run_opts opts = MAINTENANCE_RUN_OPTS_INIT; + struct string_list selected_tasks = STRING_LIST_INIT_DUP; + struct gc_config cfg = GC_CONFIG_INIT; + struct option options[] = { + OPT_BOOL(0, "auto", &opts.auto_flag, + N_("run tasks based on the state of the repository")), + OPT_CALLBACK_F(0, "task", &selected_tasks, N_("task"), + N_("check a specific task"), + PARSE_OPT_NONEG, task_option_parse), + OPT_END() + }; + bool is_needed = false; + + argc = parse_options(argc, argv, prefix, options, + builtin_maintenance_is_needed_usage, + PARSE_OPT_STOP_AT_NON_OPTION); + if (argc) + usage_with_options(builtin_maintenance_is_needed_usage, options); + + gc_config(&cfg); + initialize_task_config(&opts, &selected_tasks); + + if (opts.auto_flag) { + for (size_t i = 0; i < opts.tasks_nr; i++) { + if (tasks[opts.tasks[i]].auto_condition && + tasks[opts.tasks[i]].auto_condition(&cfg)) { + is_needed = true; + break; + } + } + } else { + /* When not using --auto, we should always require maintenance. */ + is_needed = true; + } + + string_list_clear(&selected_tasks, 0); + maintenance_run_opts_release(&opts); + gc_config_release(&cfg); + + if (is_needed) + return 0; + return 1; +} + +static const char *const builtin_maintenance_usage[] = { N_("git maintenance <subcommand> [<options>]"), NULL, }; @@ -3263,6 +3525,7 @@ int cmd_maintenance(int argc, OPT_SUBCOMMAND("stop", &fn, maintenance_stop), OPT_SUBCOMMAND("register", &fn, maintenance_register), OPT_SUBCOMMAND("unregister", &fn, maintenance_unregister), + OPT_SUBCOMMAND("is-needed", &fn, maintenance_is_needed), OPT_END(), }; diff --git a/builtin/grep.c b/builtin/grep.c index 13841fbf00..53cccf2d25 100644 --- a/builtin/grep.c +++ b/builtin/grep.c @@ -1214,7 +1214,7 @@ int cmd_grep(int argc, if (recurse_submodules) repo_read_gitmodules(the_repository, 1); if (startup_info->have_repository) - (void)packfile_store_get_packs(the_repository->objects->packfiles); + packfile_store_prepare(the_repository->objects->packfiles); start_threads(&opt); } else { diff --git a/builtin/history.c b/builtin/history.c new file mode 100644 index 0000000000..cae841707d --- /dev/null +++ b/builtin/history.c @@ -0,0 +1,561 @@ +#define USE_THE_REPOSITORY_VARIABLE + +#include "builtin.h" +#include "cache-tree.h" +#include "commit-reach.h" +#include "commit.h" +#include "config.h" +#include "editor.h" +#include "environment.h" +#include "gettext.h" +#include "hex.h" +#include "oidmap.h" +#include "parse-options.h" +#include "path.h" +#include "read-cache.h" +#include "refs.h" +#include "replay.h" +#include "reset.h" +#include "revision.h" +#include "run-command.h" +#include "sequencer.h" +#include "strvec.h" +#include "tree.h" +#include "wt-status.h" + +#define GIT_HISTORY_REWORD_USAGE N_("git history reword <commit>") +#define GIT_HISTORY_SPLIT_USAGE N_("git history split <commit> [--] [<pathspec>...]") + +static int collect_commits(struct repository *repo, + struct commit *old_commit, + struct commit *new_commit, + struct strvec *out) +{ + struct setup_revision_opt revision_opts = { + .assume_dashdash = 1, + }; + struct strvec revisions = STRVEC_INIT; + struct commit *child; + struct rev_info rev = { 0 }; + int ret; + + repo_init_revisions(repo, &rev, NULL); + strvec_push(&revisions, ""); + strvec_push(&revisions, oid_to_hex(&new_commit->object.oid)); + if (old_commit) + strvec_pushf(&revisions, "^%s", oid_to_hex(&old_commit->object.oid)); + + setup_revisions_from_strvec(&revisions, &rev, &revision_opts); + if (revisions.nr != 1 || prepare_revision_walk(&rev)) { + ret = error(_("revision walk setup failed")); + goto out; + } + + while ((child = get_revision(&rev))) { + if (old_commit && !child->parents) + BUG("revision walk did not find child commit"); + if (child->parents && child->parents->next) { + ret = error(_("cannot rearrange commit history with merges")); + goto out; + } + + strvec_push(out, oid_to_hex(&child->object.oid)); + + if (child->parents && old_commit && + commit_list_contains(old_commit, child->parents)) + break; + } + + /* + * Revisions are in newest-order-first. We have to reverse the + * array though so that we pick the oldest commits first. + */ + for (size_t i = 0, j = out->nr - 1; i < j; i++, j--) + SWAP(out->v[i], out->v[j]); + + ret = 0; + +out: + strvec_clear(&revisions); + release_revisions(&rev); + reset_revision_walk(); + return ret; +} + +static void replace_commits(struct strvec *commits, + const struct object_id *commit_to_replace, + const struct object_id *replacements, + size_t replacements_nr) +{ + char commit_to_replace_oid[GIT_MAX_HEXSZ + 1]; + struct strvec replacement_oids = STRVEC_INIT; + bool found = false; + + oid_to_hex_r(commit_to_replace_oid, commit_to_replace); + for (size_t i = 0; i < replacements_nr; i++) + strvec_push(&replacement_oids, oid_to_hex(&replacements[i])); + + for (size_t i = 0; i < commits->nr; i++) { + if (strcmp(commits->v[i], commit_to_replace_oid)) + continue; + strvec_splice(commits, i, 1, replacement_oids.v, replacement_oids.nr); + found = true; + break; + } + if (!found) + BUG("could not find commit to replace"); + + strvec_clear(&replacement_oids); +} + +static int apply_commits(struct repository *repo, + const struct strvec *commits, + struct commit *onto, + struct commit *orig_head, + const char *action) +{ + struct reset_head_opts reset_opts = { 0 }; + struct strbuf buf = STRBUF_INIT; + int ret; + + for (size_t i = 0; i < commits->nr; i++) { + struct object_id commit_id; + struct commit *commit; + const char *end; + + if (parse_oid_hex_algop(commits->v[i], &commit_id, &end, + repo->hash_algo)) { + ret = error(_("invalid object ID: %s"), commits->v[i]); + goto out; + } + + commit = lookup_commit(repo, &commit_id); + if (!commit || repo_parse_commit(repo, commit)) { + ret = error(_("failed to look up commit: %s"), oid_to_hex(&commit_id)); + goto out; + } + + if (!onto) { + onto = commit; + } else { + struct tree *tree = repo_get_commit_tree(repo, commit); + onto = replay_create_commit(repo, tree, commit, onto); + if (!onto) + break; + } + } + + reset_opts.oid = &onto->object.oid; + strbuf_addf(&buf, "%s: switch to rewritten %s", action, oid_to_hex(reset_opts.oid)); + reset_opts.flags = RESET_HEAD_REFS_ONLY | RESET_ORIG_HEAD; + reset_opts.orig_head = &orig_head->object.oid; + reset_opts.default_reflog_action = action; + if (reset_head(repo, &reset_opts) < 0) { + ret = error(_("could not switch to %s"), oid_to_hex(reset_opts.oid)); + goto out; + } + + ret = 0; + +out: + strbuf_release(&buf); + return ret; +} + +static void change_data_free(void *util, const char *str UNUSED) +{ + struct wt_status_change_data *d = util; + free(d->rename_source); + free(d); +} + +static int fill_commit_message(struct repository *repo, + const struct object_id *old_tree, + const struct object_id *new_tree, + const char *default_message, + const char *action, + struct strbuf *out) +{ + const char *path = git_path_commit_editmsg(); + const char *hint = + _("Please enter the commit message for the %s changes." + " Lines starting\nwith '%s' will be ignored.\n"); + struct wt_status s; + + strbuf_addstr(out, default_message); + strbuf_addch(out, '\n'); + strbuf_commented_addf(out, comment_line_str, hint, action, comment_line_str); + write_file_buf(path, out->buf, out->len); + + wt_status_prepare(repo, &s); + FREE_AND_NULL(s.branch); + s.ahead_behind_flags = AHEAD_BEHIND_QUICK; + s.commit_template = 1; + s.colopts = 0; + s.display_comment_prefix = 1; + s.hints = 0; + s.use_color = 0; + s.whence = FROM_COMMIT; + s.committable = 1; + + s.fp = fopen(git_path_commit_editmsg(), "a"); + if (!s.fp) + return error_errno(_("could not open '%s'"), git_path_commit_editmsg()); + + wt_status_collect_changes_trees(&s, old_tree, new_tree); + wt_status_print(&s); + wt_status_collect_free_buffers(&s); + string_list_clear_func(&s.change, change_data_free); + + strbuf_reset(out); + if (launch_editor(path, out, NULL)) { + fprintf(stderr, _("Please supply the message using the -m option.\n")); + return -1; + } + strbuf_stripspace(out, comment_line_str); + + cleanup_message(out, COMMIT_MSG_CLEANUP_ALL, 0); + + if (!out->len) { + fprintf(stderr, _("Aborting commit due to empty commit message.\n")); + return -1; + } + + return 0; +} + +static int cmd_history_reword(int argc, + const char **argv, + const char *prefix, + struct repository *repo) +{ + const char * const usage[] = { + GIT_HISTORY_REWORD_USAGE, + NULL, + }; + struct option options[] = { + OPT_END(), + }; + struct strbuf final_message = STRBUF_INIT; + struct commit *original_commit, *parent, *head; + struct strvec commits = STRVEC_INIT; + struct object_id parent_tree_oid, original_commit_tree_oid; + struct object_id rewritten_commit; + struct commit_list *from_list = NULL; + const char *original_message, *original_body, *ptr; + char *original_author = NULL; + size_t len; + int ret; + + argc = parse_options(argc, argv, prefix, options, usage, 0); + if (argc != 1) { + ret = error(_("command expects a single revision")); + goto out; + } + repo_config(repo, git_default_config, NULL); + + original_commit = lookup_commit_reference_by_name(argv[0]); + if (!original_commit) { + ret = error(_("commit to be reworded cannot be found: %s"), argv[0]); + goto out; + } + original_commit_tree_oid = repo_get_commit_tree(repo, original_commit)->object.oid; + + parent = original_commit->parents ? original_commit->parents->item : NULL; + if (parent) { + if (repo_parse_commit(repo, parent)) { + ret = error(_("unable to parse commit %s"), + oid_to_hex(&parent->object.oid)); + goto out; + } + parent_tree_oid = repo_get_commit_tree(repo, parent)->object.oid; + } else { + oidcpy(&parent_tree_oid, repo->hash_algo->empty_tree); + } + + head = lookup_commit_reference_by_name("HEAD"); + if (!head) { + ret = error(_("could not resolve HEAD to a commit")); + goto out; + } + + commit_list_append(original_commit, &from_list); + if (!repo_is_descendant_of(repo, head, from_list)) { + ret = error (_("split commit must be reachable from current HEAD commit")); + goto out; + } + + /* + * Collect the list of commits that we'll have to reapply now already. + * This ensures that we'll abort early on in case the range of commits + * contains merges, which we do not yet handle. + */ + ret = collect_commits(repo, parent, head, &commits); + if (ret < 0) + goto out; + + /* We retain authorship of the original commit. */ + original_message = repo_logmsg_reencode(repo, original_commit, NULL, NULL); + ptr = find_commit_header(original_message, "author", &len); + if (ptr) + original_author = xmemdupz(ptr, len); + find_commit_subject(original_message, &original_body); + + ret = fill_commit_message(repo, &parent_tree_oid, &original_commit_tree_oid, + original_body, "reworded", &final_message); + if (ret < 0) + goto out; + + ret = commit_tree(final_message.buf, final_message.len, &original_commit_tree_oid, + original_commit->parents, &rewritten_commit, original_author, NULL); + if (ret < 0) { + ret = error(_("failed writing reworded commit")); + goto out; + } + + replace_commits(&commits, &original_commit->object.oid, &rewritten_commit, 1); + + ret = apply_commits(repo, &commits, parent, head, "reword"); + if (ret < 0) + goto out; + + ret = 0; + +out: + strbuf_release(&final_message); + free_commit_list(from_list); + strvec_clear(&commits); + free(original_author); + return ret; +} + +static int split_commit(struct repository *repo, + struct commit *original_commit, + struct pathspec *pathspec, + struct object_id *out) +{ + struct interactive_options interactive_opts = INTERACTIVE_OPTIONS_INIT; + struct strbuf index_file = STRBUF_INIT, split_message = STRBUF_INIT; + struct child_process read_tree_cmd = CHILD_PROCESS_INIT; + struct index_state index = INDEX_STATE_INIT(repo); + struct object_id original_commit_tree_oid, parent_tree_oid; + const char *original_message, *original_body, *ptr; + char original_commit_oid[GIT_MAX_HEXSZ + 1]; + char *original_author = NULL; + struct commit_list *parents = NULL; + struct commit *first_commit; + struct tree *split_tree; + size_t len; + int ret; + + if (original_commit->parents) + parent_tree_oid = *get_commit_tree_oid(original_commit->parents->item); + else + oidcpy(&parent_tree_oid, repo->hash_algo->empty_tree); + original_commit_tree_oid = *get_commit_tree_oid(original_commit); + + /* + * Construct the first commit. This is done by taking the original + * commit parent's tree and selectively patching changes from the diff + * between that parent and its child. + */ + repo_git_path_replace(repo, &index_file, "%s", "history-split.index"); + + read_tree_cmd.git_cmd = 1; + strvec_pushf(&read_tree_cmd.env, "GIT_INDEX_FILE=%s", index_file.buf); + strvec_push(&read_tree_cmd.args, "read-tree"); + strvec_push(&read_tree_cmd.args, oid_to_hex(&parent_tree_oid)); + ret = run_command(&read_tree_cmd); + if (ret < 0) + goto out; + + ret = read_index_from(&index, index_file.buf, repo->gitdir); + if (ret < 0) { + ret = error(_("failed reading temporary index")); + goto out; + } + + oid_to_hex_r(original_commit_oid, &original_commit->object.oid); + ret = run_add_p_index(repo, &index, index_file.buf, &interactive_opts, + original_commit_oid, pathspec); + if (ret < 0) + goto out; + + split_tree = write_in_core_index_as_tree(repo, &index); + if (!split_tree) { + ret = error(_("failed split tree")); + goto out; + } + + unlink(index_file.buf); + + /* + * We disallow the cases where either the split-out commit or the + * original commit would become empty. Consequently, if we see that the + * new tree ID matches either of those trees we abort. + */ + if (oideq(&split_tree->object.oid, &parent_tree_oid)) { + ret = error(_("split commit is empty")); + goto out; + } else if (oideq(&split_tree->object.oid, &original_commit_tree_oid)) { + ret = error(_("split commit tree matches original commit")); + goto out; + } + + /* We retain authorship of the original commit. */ + original_message = repo_logmsg_reencode(repo, original_commit, NULL, NULL); + ptr = find_commit_header(original_message, "author", &len); + if (ptr) + original_author = xmemdupz(ptr, len); + + ret = fill_commit_message(repo, &parent_tree_oid, &split_tree->object.oid, + "", "split-out", &split_message); + if (ret < 0) + goto out; + + ret = commit_tree(split_message.buf, split_message.len, &split_tree->object.oid, + original_commit->parents, &out[0], original_author, NULL); + if (ret < 0) { + ret = error(_("failed writing split-out commit")); + goto out; + } + + /* + * The second commit is much simpler to construct, as we can simply use + * the original commit details, except that we adjust its parent to be + * the newly split-out commit. + */ + find_commit_subject(original_message, &original_body); + first_commit = lookup_commit_reference(repo, &out[0]); + commit_list_append(first_commit, &parents); + + ret = commit_tree(original_body, strlen(original_body), &original_commit_tree_oid, + parents, &out[1], original_author, NULL); + if (ret < 0) { + ret = error(_("failed writing second commit")); + goto out; + } + + ret = 0; + +out: + if (index_file.len) + unlink(index_file.buf); + strbuf_release(&split_message); + strbuf_release(&index_file); + free_commit_list(parents); + free(original_author); + release_index(&index); + return ret; +} + +static int cmd_history_split(int argc, + const char **argv, + const char *prefix, + struct repository *repo) +{ + const char * const usage[] = { + GIT_HISTORY_SPLIT_USAGE, + NULL, + }; + struct option options[] = { + OPT_END(), + }; + struct oidmap rewritten_commits = OIDMAP_INIT; + struct commit *original_commit, *parent, *head; + struct strvec commits = STRVEC_INIT; + struct commit_list *from_list = NULL; + struct object_id split_commits[2]; + struct pathspec pathspec = { 0 }; + int ret; + + argc = parse_options(argc, argv, prefix, options, usage, 0); + if (argc < 1) { + ret = error(_("command expects a revision")); + goto out; + } + repo_config(repo, git_default_config, NULL); + + original_commit = lookup_commit_reference_by_name(argv[0]); + if (!original_commit) { + ret = error(_("commit to be split cannot be found: %s"), argv[0]); + goto out; + } + + parent = original_commit->parents ? original_commit->parents->item : NULL; + if (parent && repo_parse_commit(repo, parent)) { + ret = error(_("unable to parse commit %s"), + oid_to_hex(&parent->object.oid)); + goto out; + } + + head = lookup_commit_reference_by_name("HEAD"); + if (!head) { + ret = error(_("could not resolve HEAD to a commit")); + goto out; + } + + commit_list_append(original_commit, &from_list); + if (!repo_is_descendant_of(repo, head, from_list)) { + ret = error(_("split commit must be reachable from current HEAD commit")); + goto out; + } + + parse_pathspec(&pathspec, 0, + PATHSPEC_PREFER_FULL | PATHSPEC_SYMLINK_LEADING_PATH | PATHSPEC_PREFIX_ORIGIN, + prefix, argv + 1); + + /* + * Collect the list of commits that we'll have to reapply now already. + * This ensures that we'll abort early on in case the range of commits + * contains merges, which we do not yet handle. + */ + ret = collect_commits(repo, parent, head, &commits); + if (ret < 0) + goto out; + + /* + * Then we split up the commit and replace the original commit with the + * new ones. + */ + ret = split_commit(repo, original_commit, &pathspec, split_commits); + if (ret < 0) + goto out; + + replace_commits(&commits, &original_commit->object.oid, + split_commits, ARRAY_SIZE(split_commits)); + + ret = apply_commits(repo, &commits, parent, head, "split"); + if (ret < 0) + goto out; + + ret = 0; + +out: + oidmap_clear(&rewritten_commits, 0); + free_commit_list(from_list); + clear_pathspec(&pathspec); + strvec_clear(&commits); + return ret; +} + +int cmd_history(int argc, + const char **argv, + const char *prefix, + struct repository *repo) +{ + const char * const usage[] = { + GIT_HISTORY_REWORD_USAGE, + GIT_HISTORY_SPLIT_USAGE, + NULL, + }; + parse_opt_subcommand_fn *fn = NULL; + struct option options[] = { + OPT_SUBCOMMAND("reword", &fn, cmd_history_reword), + OPT_SUBCOMMAND("split", &fn, cmd_history_split), + OPT_END(), + }; + + argc = parse_options(argc, argv, prefix, options, usage, 0); + return fn(argc, argv, prefix, repo); +} diff --git a/builtin/hook.c b/builtin/hook.c index 7afec380d2..73e7b8c2e8 100644 --- a/builtin/hook.c +++ b/builtin/hook.c @@ -44,6 +44,12 @@ static int run(int argc, const char **argv, const char *prefix, goto usage; /* + * All current "hook run" use-cases require ungrouped child output. + * If this changes, a hook run argument can be added to toggle it. + */ + opt.ungroup = 1; + + /* * Having a -- for "run" when providing <hook-args> is * mandatory. */ diff --git a/builtin/interpret-trailers.c b/builtin/interpret-trailers.c index 41b0750e5a..268a43372b 100644 --- a/builtin/interpret-trailers.c +++ b/builtin/interpret-trailers.c @@ -10,7 +10,6 @@ #include "gettext.h" #include "parse-options.h" #include "string-list.h" -#include "tempfile.h" #include "trailer.h" #include "config.h" @@ -93,37 +92,6 @@ static int parse_opt_parse(const struct option *opt, const char *arg, return 0; } -static struct tempfile *trailers_tempfile; - -static FILE *create_in_place_tempfile(const char *file) -{ - struct stat st; - struct strbuf filename_template = STRBUF_INIT; - const char *tail; - FILE *outfile; - - if (stat(file, &st)) - die_errno(_("could not stat %s"), file); - if (!S_ISREG(st.st_mode)) - die(_("file %s is not a regular file"), file); - if (!(st.st_mode & S_IWUSR)) - die(_("file %s is not writable by user"), file); - - /* Create temporary file in the same directory as the original */ - tail = strrchr(file, '/'); - if (tail) - strbuf_add(&filename_template, file, tail - file + 1); - strbuf_addstr(&filename_template, "git-interpret-trailers-XXXXXX"); - - trailers_tempfile = xmks_tempfile_m(filename_template.buf, st.st_mode); - strbuf_release(&filename_template); - outfile = fdopen_tempfile(trailers_tempfile, "w"); - if (!outfile) - die_errno(_("could not open temporary file")); - - return outfile; -} - static void read_input_file(struct strbuf *sb, const char *file) { if (file) { @@ -140,55 +108,20 @@ static void interpret_trailers(const struct process_trailer_options *opts, struct list_head *new_trailer_head, const char *file) { - LIST_HEAD(head); struct strbuf sb = STRBUF_INIT; - struct strbuf trailer_block_sb = STRBUF_INIT; - struct trailer_block *trailer_block; - FILE *outfile = stdout; - - trailer_config_init(); + struct strbuf out = STRBUF_INIT; read_input_file(&sb, file); - if (opts->in_place) - outfile = create_in_place_tempfile(file); - - trailer_block = parse_trailers(opts, sb.buf, &head); - - /* Print the lines before the trailer block */ - if (!opts->only_trailers) - fwrite(sb.buf, 1, trailer_block_start(trailer_block), outfile); - - if (!opts->only_trailers && !blank_line_before_trailer_block(trailer_block)) - fprintf(outfile, "\n"); - - - if (!opts->only_input) { - LIST_HEAD(config_head); - LIST_HEAD(arg_head); - parse_trailers_from_config(&config_head); - parse_trailers_from_command_line_args(&arg_head, new_trailer_head); - list_splice(&config_head, &arg_head); - process_trailers_lists(&head, &arg_head); - } - - /* Print trailer block. */ - format_trailers(opts, &head, &trailer_block_sb); - free_trailers(&head); - fwrite(trailer_block_sb.buf, 1, trailer_block_sb.len, outfile); - strbuf_release(&trailer_block_sb); - - /* Print the lines after the trailer block as is. */ - if (!opts->only_trailers) - fwrite(sb.buf + trailer_block_end(trailer_block), 1, - sb.len - trailer_block_end(trailer_block), outfile); - trailer_block_release(trailer_block); + process_trailers(opts, new_trailer_head, &sb, &out); if (opts->in_place) - if (rename_tempfile(&trailers_tempfile, file)) - die_errno(_("could not rename temporary file to %s"), file); + write_file_buf(file, out.buf, out.len); + else + strbuf_write(&out, stdout); strbuf_release(&sb); + strbuf_release(&out); } int cmd_interpret_trailers(int argc, @@ -232,6 +165,8 @@ int cmd_interpret_trailers(int argc, git_interpret_trailers_usage, options); + trailer_config_init(); + if (argc) { int i; for (i = 0; i < argc; i++) diff --git a/builtin/last-modified.c b/builtin/last-modified.c index ae8b36a2c3..b0ecbdc540 100644 --- a/builtin/last-modified.c +++ b/builtin/last-modified.c @@ -2,26 +2,32 @@ #include "bloom.h" #include "builtin.h" #include "commit-graph.h" +#include "commit-slab.h" #include "commit.h" #include "config.h" -#include "environment.h" #include "diff.h" #include "diffcore.h" #include "environment.h" +#include "ewah/ewok.h" #include "hashmap.h" #include "hex.h" -#include "log-tree.h" #include "object-name.h" #include "object.h" #include "parse-options.h" +#include "prio-queue.h" #include "quote.h" #include "repository.h" #include "revision.h" +/* Remember to update object flag allocation in object.h */ +#define PARENT1 (1u<<16) /* used instead of SEEN */ +#define PARENT2 (1u<<17) /* used instead of BOTTOM, BOUNDARY */ + struct last_modified_entry { struct hashmap_entry hashent; struct object_id oid; struct bloom_key key; + size_t diff_idx; const char path[FLEX_ARRAY]; }; @@ -37,13 +43,45 @@ static int last_modified_entry_hashcmp(const void *unused UNUSED, return strcmp(ent1->path, path ? path : ent2->path); } +/* + * Hold a bitmap for each commit we're working with. In the bitmap, each bit + * represents a path in `lm->all_paths`. An active bit indicates the path still + * needs to be associated to a commit. + */ +define_commit_slab(active_paths_for_commit, struct bitmap *); + struct last_modified { struct hashmap paths; struct rev_info rev; bool recursive; bool show_trees; + + const char **all_paths; + size_t all_paths_nr; + struct active_paths_for_commit active_paths; + + /* 'scratch' to avoid allocating a bitmap every process_parent() */ + struct bitmap *scratch; }; +static struct bitmap *active_paths_for(struct last_modified *lm, struct commit *c) +{ + struct bitmap **bitmap = active_paths_for_commit_at(&lm->active_paths, c); + if (!*bitmap) + *bitmap = bitmap_word_alloc(lm->all_paths_nr / BITS_IN_EWORD + 1); + + return *bitmap; +} + +static void active_paths_free(struct last_modified *lm, struct commit *c) +{ + struct bitmap **bitmap = active_paths_for_commit_at(&lm->active_paths, c); + if (*bitmap) { + bitmap_free(*bitmap); + *bitmap = NULL; + } +} + static void last_modified_release(struct last_modified *lm) { struct hashmap_iter iter; @@ -54,6 +92,8 @@ static void last_modified_release(struct last_modified *lm) hashmap_clear_and_free(&lm->paths, struct last_modified_entry, hashent); release_revisions(&lm->rev); + + free(lm->all_paths); } struct last_modified_callback_data { @@ -146,7 +186,7 @@ static void mark_path(const char *path, const struct object_id *oid, * Is it arriving at a version of interest, or is it from a side branch * which did not contribute to the final state? */ - if (!oideq(oid, &ent->oid)) + if (oid && !oideq(oid, &ent->oid)) return; last_modified_emit(data->lm, path, data->commit); @@ -196,7 +236,17 @@ static void last_modified_diff(struct diff_queue_struct *q, } } -static bool maybe_changed_path(struct last_modified *lm, struct commit *origin) +static void pass_to_parent(struct bitmap *c, + struct bitmap *p, + size_t pos) +{ + bitmap_unset(c, pos); + bitmap_set(p, pos); +} + +static bool maybe_changed_path(struct last_modified *lm, + struct commit *origin, + struct bitmap *active) { struct bloom_filter *filter; struct last_modified_entry *ent; @@ -213,6 +263,9 @@ static bool maybe_changed_path(struct last_modified *lm, struct commit *origin) return true; hashmap_for_each_entry(&lm->paths, &iter, ent, hashent) { + if (active && !bitmap_get(active, ent->diff_idx)) + continue; + if (bloom_filter_contains(filter, &ent->key, lm->rev.bloom_filter_settings)) return true; @@ -220,42 +273,202 @@ static bool maybe_changed_path(struct last_modified *lm, struct commit *origin) return false; } +static void process_parent(struct last_modified *lm, + struct prio_queue *queue, + struct commit *c, struct bitmap *active_c, + struct commit *parent, int parent_i) +{ + struct bitmap *active_p; + + repo_parse_commit(lm->rev.repo, parent); + active_p = active_paths_for(lm, parent); + + /* + * The first time entering this function for this commit (i.e. first parent) + * see if Bloom filters will tell us it's worth to do the diff. + */ + if (parent_i || maybe_changed_path(lm, c, active_c)) { + diff_tree_oid(&parent->object.oid, + &c->object.oid, "", &lm->rev.diffopt); + diffcore_std(&lm->rev.diffopt); + } + + /* + * Test each path for TREESAME-ness against the parent. If a path is + * TREESAME, pass it on to this parent. + * + * First, collect all paths that are *not* TREESAME in 'scratch'. + * Then, pass paths that *are* TREESAME and active to the parent. + */ + for (int i = 0; i < diff_queued_diff.nr; i++) { + struct diff_filepair *fp = diff_queued_diff.queue[i]; + const char *path = fp->two->path; + struct last_modified_entry *ent = + hashmap_get_entry_from_hash(&lm->paths, strhash(path), path, + struct last_modified_entry, hashent); + if (ent) { + size_t k = ent->diff_idx; + if (bitmap_get(active_c, k)) + bitmap_set(lm->scratch, k); + } + } + for (size_t i = 0; i < lm->all_paths_nr; i++) { + if (bitmap_get(active_c, i) && !bitmap_get(lm->scratch, i)) + pass_to_parent(active_c, active_p, i); + } + + /* + * If parent has any active paths, put it on the queue (if not already). + */ + if (!bitmap_is_empty(active_p) && !(parent->object.flags & PARENT1)) { + parent->object.flags |= PARENT1; + prio_queue_put(queue, parent); + } + if (!(parent->object.flags & PARENT1)) + active_paths_free(lm, parent); + + memset(lm->scratch->words, 0x0, lm->scratch->word_alloc); + diff_queue_clear(&diff_queued_diff); +} + static int last_modified_run(struct last_modified *lm) { + int max_count, queue_popped = 0; + struct prio_queue queue = { compare_commits_by_gen_then_commit_date }; + struct prio_queue not_queue = { compare_commits_by_gen_then_commit_date }; + struct commit_list *list; struct last_modified_callback_data data = { .lm = lm }; lm->rev.diffopt.output_format = DIFF_FORMAT_CALLBACK; lm->rev.diffopt.format_callback = last_modified_diff; lm->rev.diffopt.format_callback_data = &data; + lm->rev.no_walk = 1; prepare_revision_walk(&lm->rev); - while (hashmap_get_size(&lm->paths)) { - data.commit = get_revision(&lm->rev); - if (!data.commit) - BUG("paths remaining beyond boundary in last-modified"); + max_count = lm->rev.max_count; + + init_active_paths_for_commit(&lm->active_paths); + lm->scratch = bitmap_word_alloc(lm->all_paths_nr); + + /* + * lm->rev.commits holds the set of boundary commits for our walk. + * + * Loop through each such commit, and place it in the appropriate queue. + */ + for (list = lm->rev.commits; list; list = list->next) { + struct commit *c = list->item; + + if (c->object.flags & BOTTOM) { + prio_queue_put(¬_queue, c); + c->object.flags |= PARENT2; + } else if (!(c->object.flags & PARENT1)) { + /* + * If the commit is a starting point (and hasn't been + * seen yet), then initialize the set of interesting + * paths, too. + */ + struct bitmap *active; + + prio_queue_put(&queue, c); + c->object.flags |= PARENT1; + + active = active_paths_for(lm, c); + for (size_t i = 0; i < lm->all_paths_nr; i++) + bitmap_set(active, i); + } + } - if (data.commit->object.flags & BOUNDARY) { + while (queue.nr) { + int parent_i; + struct commit_list *p; + struct commit *c = prio_queue_get(&queue); + struct bitmap *active_c = active_paths_for(lm, c); + + if ((0 <= max_count && max_count < ++queue_popped) || + (c->object.flags & PARENT2)) { + /* + * Either a boundary commit, or we have already seen too + * many others. Either way, stop here. + */ + c->object.flags |= PARENT2 | BOUNDARY; + data.commit = c; diff_tree_oid(lm->rev.repo->hash_algo->empty_tree, - &data.commit->object.oid, "", - &lm->rev.diffopt); + &c->object.oid, + "", &lm->rev.diffopt); diff_flush(&lm->rev.diffopt); + goto cleanup; + } - break; + /* + * Otherwise, make sure that 'c' isn't reachable from anything + * in the '--not' queue. + */ + repo_parse_commit(lm->rev.repo, c); + + while (not_queue.nr) { + struct commit_list *np; + struct commit *n = prio_queue_get(¬_queue); + + repo_parse_commit(lm->rev.repo, n); + + for (np = n->parents; np; np = np->next) { + if (!(np->item->object.flags & PARENT2)) { + prio_queue_put(¬_queue, np->item); + np->item->object.flags |= PARENT2; + } + } + + if (commit_graph_generation(n) < commit_graph_generation(c)) + break; } - if (!maybe_changed_path(lm, data.commit)) - continue; + /* + * Look at each parent and pass on each path that's TREESAME + * with that parent. Stop early when no active paths remain. + */ + for (p = c->parents, parent_i = 0; p; p = p->next, parent_i++) { + process_parent(lm, &queue, + c, active_c, + p->item, parent_i); + + if (bitmap_is_empty(active_c)) + break; + } + + /* + * Paths that remain active, or not TREESAME with any parent, + * were changed by 'c'. + */ + if (!bitmap_is_empty(active_c)) { + data.commit = c; + for (size_t i = 0; i < lm->all_paths_nr; i++) { + if (bitmap_get(active_c, i)) + mark_path(lm->all_paths[i], NULL, &data); + } + } - log_tree_commit(&lm->rev, data.commit); +cleanup: + active_paths_free(lm, c); } + if (hashmap_get_size(&lm->paths)) + BUG("paths remaining beyond boundary in last-modified"); + + clear_prio_queue(¬_queue); + clear_prio_queue(&queue); + clear_active_paths_for_commit(&lm->active_paths); + bitmap_free(lm->scratch); + return 0; } static int last_modified_init(struct last_modified *lm, struct repository *r, const char *prefix, int argc, const char **argv) { + struct hashmap_iter iter; + struct last_modified_entry *ent; + hashmap_init(&lm->paths, last_modified_entry_hashcmp, NULL, 0); repo_init_revisions(r, &lm->rev, prefix); @@ -280,6 +493,13 @@ static int last_modified_init(struct last_modified *lm, struct repository *r, if (populate_paths_from_revs(lm) < 0) return error(_("unable to setup last-modified")); + CALLOC_ARRAY(lm->all_paths, hashmap_get_size(&lm->paths)); + lm->all_paths_nr = 0; + hashmap_for_each_entry(&lm->paths, &iter, ent, hashent) { + ent->diff_idx = lm->all_paths_nr++; + lm->all_paths[ent->diff_idx] = ent->path; + } + return 0; } diff --git a/builtin/ls-remote.c b/builtin/ls-remote.c index df09000b30..fe77829557 100644 --- a/builtin/ls-remote.c +++ b/builtin/ls-remote.c @@ -156,7 +156,7 @@ int cmd_ls_remote(int argc, continue; if (!tail_match(&pattern, ref->name)) continue; - item = ref_array_push(&ref_array, ref->name, &ref->old_oid); + item = ref_array_push(&ref_array, ref->name, &ref->old_oid, NULL); item->symref = xstrdup_or_null(ref->symref); } diff --git a/builtin/name-rev.c b/builtin/name-rev.c index 74512e54a3..615f7d1aae 100644 --- a/builtin/name-rev.c +++ b/builtin/name-rev.c @@ -339,10 +339,9 @@ static int cmp_by_tag_and_age(const void *a_, const void *b_) return a->taggerdate != b->taggerdate; } -static int name_ref(const char *path, const char *referent UNUSED, const struct object_id *oid, - int flags UNUSED, void *cb_data) +static int name_ref(const struct reference *ref, void *cb_data) { - struct object *o = parse_object(the_repository, oid); + struct object *o = parse_object(the_repository, ref->oid); struct name_ref_data *data = cb_data; int can_abbreviate_output = data->tags_only && data->name_only; int deref = 0; @@ -350,14 +349,14 @@ static int name_ref(const char *path, const char *referent UNUSED, const struct struct commit *commit = NULL; timestamp_t taggerdate = TIME_MAX; - if (data->tags_only && !starts_with(path, "refs/tags/")) + if (data->tags_only && !starts_with(ref->name, "refs/tags/")) return 0; if (data->exclude_filters.nr) { struct string_list_item *item; for_each_string_list_item(item, &data->exclude_filters) { - if (subpath_matches(path, item->string) >= 0) + if (subpath_matches(ref->name, item->string) >= 0) return 0; } } @@ -378,7 +377,7 @@ static int name_ref(const char *path, const char *referent UNUSED, const struct * shouldn't stop when seeing 'refs/tags/v1.4' matches * 'refs/tags/v*'. We should show it as 'v1.4'. */ - switch (subpath_matches(path, item->string)) { + switch (subpath_matches(ref->name, item->string)) { case -1: /* did not match */ break; case 0: /* matched fully */ @@ -406,13 +405,13 @@ static int name_ref(const char *path, const char *referent UNUSED, const struct } if (o && o->type == OBJ_COMMIT) { commit = (struct commit *)o; - from_tag = starts_with(path, "refs/tags/"); + from_tag = starts_with(ref->name, "refs/tags/"); if (taggerdate == TIME_MAX) taggerdate = commit->date; } - add_to_tip_table(oid, path, can_abbreviate_output, commit, taggerdate, - from_tag, deref); + add_to_tip_table(ref->oid, ref->name, can_abbreviate_output, + commit, taggerdate, from_tag, deref); return 0; } diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 5bdc44fb2d..7937106ec5 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -831,15 +831,14 @@ static enum write_one_status write_one(struct hashfile *f, return WRITE_ONE_WRITTEN; } -static int mark_tagged(const char *path UNUSED, const char *referent UNUSED, const struct object_id *oid, - int flag UNUSED, void *cb_data UNUSED) +static int mark_tagged(const struct reference *ref, void *cb_data UNUSED) { struct object_id peeled; - struct object_entry *entry = packlist_find(&to_pack, oid); + struct object_entry *entry = packlist_find(&to_pack, ref->oid); if (entry) entry->tagged = 1; - if (!peel_iterated_oid(the_repository, oid, &peeled)) { + if (!reference_get_peeled_oid(the_repository, ref, &peeled)) { entry = packlist_find(&to_pack, &peeled); if (entry) entry->tagged = 1; @@ -1706,8 +1705,8 @@ static int want_object_in_pack_mtime(const struct object_id *oid, uint32_t found_mtime) { int want; + struct packfile_list_entry *e; struct odb_source *source; - struct list_head *pos; if (!exclude && local) { /* @@ -1716,7 +1715,7 @@ static int want_object_in_pack_mtime(const struct object_id *oid, */ struct odb_source *source = the_repository->objects->sources->next; for (; source; source = source->next) - if (has_loose_object(source, oid)) + if (odb_source_loose_has_object(source, oid)) return 0; } @@ -1748,12 +1747,11 @@ static int want_object_in_pack_mtime(const struct object_id *oid, } } - list_for_each(pos, packfile_store_get_packs_mru(the_repository->objects->packfiles)) { - struct packed_git *p = list_entry(pos, struct packed_git, mru); + for (e = the_repository->objects->packfiles->packs.head; e; e = e->next) { + struct packed_git *p = e->pack; want = want_object_in_pack_one(p, oid, exclude, found_pack, found_offset, found_mtime); if (!exclude && want > 0) - list_move(&p->mru, - packfile_store_get_packs_mru(the_repository->objects->packfiles)); + packfile_list_prepend(&the_repository->objects->packfiles->packs, p); if (want != -1) return want; } @@ -3306,13 +3304,13 @@ static void add_tag_chain(const struct object_id *oid) } } -static int add_ref_tag(const char *tag UNUSED, const char *referent UNUSED, const struct object_id *oid, - int flag UNUSED, void *cb_data UNUSED) +static int add_ref_tag(const struct reference *ref, void *cb_data UNUSED) { struct object_id peeled; - if (!peel_iterated_oid(the_repository, oid, &peeled) && obj_is_packed(&peeled)) - add_tag_chain(oid); + if (!reference_get_peeled_oid(the_repository, ref, &peeled) && + obj_is_packed(&peeled)) + add_tag_chain(ref->oid); return 0; } @@ -3831,12 +3829,10 @@ static int pack_mtime_cmp(const void *_a, const void *_b) static void read_packs_list_from_stdin(struct rev_info *revs) { - struct packfile_store *packs = the_repository->objects->packfiles; struct strbuf buf = STRBUF_INIT; struct string_list include_packs = STRING_LIST_INIT_DUP; struct string_list exclude_packs = STRING_LIST_INIT_DUP; struct string_list_item *item = NULL; - struct packed_git *p; while (strbuf_getline(&buf, stdin) != EOF) { @@ -3856,7 +3852,7 @@ static void read_packs_list_from_stdin(struct rev_info *revs) string_list_sort(&exclude_packs); string_list_remove_duplicates(&exclude_packs, 0); - for (p = packfile_store_get_all_packs(packs); p; p = p->next) { + repo_for_each_pack(the_repository, p) { const char *pack_name = pack_basename(p); if ((item = string_list_lookup(&include_packs, pack_name))) @@ -3980,7 +3976,7 @@ static void add_cruft_object_entry(const struct object_id *oid, enum object_type int found = 0; for (; !found && source; source = source->next) - if (has_loose_object(source, oid)) + if (odb_source_loose_has_object(source, oid)) found = 1; /* @@ -4077,7 +4073,6 @@ static void enumerate_cruft_objects(void) static void enumerate_and_traverse_cruft_objects(struct string_list *fresh_packs) { - struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *p; struct rev_info revs; int ret; @@ -4107,7 +4102,7 @@ static void enumerate_and_traverse_cruft_objects(struct string_list *fresh_packs * Re-mark only the fresh packs as kept so that objects in * unknown packs do not halt the reachability traversal early. */ - for (p = packfile_store_get_all_packs(packs); p; p = p->next) + repo_for_each_pack(the_repository, p) p->pack_keep_in_core = 0; mark_pack_kept_in_core(fresh_packs, 1); @@ -4124,7 +4119,6 @@ static void enumerate_and_traverse_cruft_objects(struct string_list *fresh_packs static void read_cruft_objects(void) { - struct packfile_store *packs = the_repository->objects->packfiles; struct strbuf buf = STRBUF_INIT; struct string_list discard_packs = STRING_LIST_INIT_DUP; struct string_list fresh_packs = STRING_LIST_INIT_DUP; @@ -4145,7 +4139,7 @@ static void read_cruft_objects(void) string_list_sort(&discard_packs); string_list_sort(&fresh_packs); - for (p = packfile_store_get_all_packs(packs); p; p = p->next) { + repo_for_each_pack(the_repository, p) { const char *pack_name = pack_basename(p); struct string_list_item *item; @@ -4393,27 +4387,27 @@ static void add_unreachable_loose_objects(struct rev_info *revs) static int has_sha1_pack_kept_or_nonlocal(const struct object_id *oid) { - struct packfile_store *packs = the_repository->objects->packfiles; - static struct packed_git *last_found = (void *)1; + static struct packed_git *last_found = NULL; struct packed_git *p; - p = (last_found != (void *)1) ? last_found : - packfile_store_get_all_packs(packs); + if (last_found && find_pack_entry_one(oid, last_found)) + return 1; - while (p) { - if ((!p->pack_local || p->pack_keep || - p->pack_keep_in_core) && - find_pack_entry_one(oid, p)) { + repo_for_each_pack(the_repository, p) { + /* + * We have already checked `last_found`, so there is no need to + * re-check here. + */ + if (p == last_found) + continue; + + if ((!p->pack_local || p->pack_keep || p->pack_keep_in_core) && + find_pack_entry_one(oid, p)) { last_found = p; return 1; } - if (p == last_found) - p = packfile_store_get_all_packs(packs); - else - p = p->next; - if (p == last_found) - p = p->next; } + return 0; } @@ -4440,13 +4434,12 @@ static int loosened_object_can_be_discarded(const struct object_id *oid, static void loosen_unused_packed_objects(void) { - struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *p; uint32_t i; uint32_t loosened_objects_nr = 0; struct object_id oid; - for (p = packfile_store_get_all_packs(packs); p; p = p->next) { + repo_for_each_pack(the_repository, p) { if (!p->pack_local || p->pack_keep || p->pack_keep_in_core) continue; @@ -4533,19 +4526,16 @@ static void record_recent_commit(struct commit *commit, void *data UNUSED) oid_array_append(&recent_objects, &commit->object.oid); } -static int mark_bitmap_preferred_tip(const char *refname, - const char *referent UNUSED, - const struct object_id *oid, - int flags UNUSED, - void *data UNUSED) +static int mark_bitmap_preferred_tip(const struct reference *ref, void *data UNUSED) { + const struct object_id *maybe_peeled = ref->oid; struct object_id peeled; struct object *object; - if (!peel_iterated_oid(the_repository, oid, &peeled)) - oid = &peeled; + if (!reference_get_peeled_oid(the_repository, ref, &peeled)) + maybe_peeled = &peeled; - object = parse_object_or_die(the_repository, oid, refname); + object = parse_object_or_die(the_repository, maybe_peeled, ref->name); if (object->type == OBJ_COMMIT) object->flags |= NEEDS_BITMAP; @@ -4747,13 +4737,12 @@ static void get_object_list(struct rev_info *revs, struct strvec *argv) static void add_extra_kept_packs(const struct string_list *names) { - struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *p; if (!names->nr) return; - for (p = packfile_store_get_all_packs(packs); p; p = p->next) { + repo_for_each_pack(the_repository, p) { const char *name = basename(p->pack_name); int i; @@ -5191,10 +5180,9 @@ int cmd_pack_objects(int argc, add_extra_kept_packs(&keep_pack_list); if (ignore_packed_keep_on_disk) { - struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *p; - for (p = packfile_store_get_all_packs(packs); p; p = p->next) + repo_for_each_pack(the_repository, p) if (p->pack_local && p->pack_keep) break; if (!p) /* no keep-able packs found */ @@ -5206,10 +5194,9 @@ int cmd_pack_objects(int argc, * want to unset "local" based on looking at packs, as * it also covers non-local objects */ - struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *p; - for (p = packfile_store_get_all_packs(packs); p; p = p->next) { + repo_for_each_pack(the_repository, p) { if (!p->pack_local) { have_non_local_packs = 1; break; diff --git a/builtin/pack-redundant.c b/builtin/pack-redundant.c index 80743d8806..e4ecf774ca 100644 --- a/builtin/pack-redundant.c +++ b/builtin/pack-redundant.c @@ -566,29 +566,23 @@ static struct pack_list * add_pack(struct packed_git *p) static struct pack_list * add_pack_file(const char *filename) { - struct packfile_store *packs = the_repository->objects->packfiles; - struct packed_git *p = packfile_store_get_all_packs(packs); + struct packed_git *p; if (strlen(filename) < 40) die("Bad pack filename: %s", filename); - while (p) { + repo_for_each_pack(the_repository, p) if (strstr(p->pack_name, filename)) return add_pack(p); - p = p->next; - } die("Filename %s not found in packed_git", filename); } static void load_all(void) { - struct packfile_store *packs = the_repository->objects->packfiles; - struct packed_git *p = packfile_store_get_all_packs(packs); + struct packed_git *p; - while (p) { + repo_for_each_pack(the_repository, p) add_pack(p); - p = p->next; - } } int cmd_pack_redundant(int argc, const char **argv, const char *prefix UNUSED, struct repository *repo UNUSED) { diff --git a/builtin/rebase.c b/builtin/rebase.c index c468828189..a88abe08b4 100644 --- a/builtin/rebase.c +++ b/builtin/rebase.c @@ -36,6 +36,7 @@ #include "reset.h" #include "trace2.h" #include "hook.h" +#include "trailer.h" static char const * const builtin_rebase_usage[] = { N_("git rebase [-i] [options] [--exec <cmd>] " @@ -113,6 +114,7 @@ struct rebase_options { enum action action; char *reflog_action; int signoff; + struct strvec trailer_args; int allow_rerere_autoupdate; int keep_empty; int autosquash; @@ -143,6 +145,7 @@ struct rebase_options { .flags = REBASE_NO_QUIET, \ .git_am_opts = STRVEC_INIT, \ .exec = STRING_LIST_INIT_NODUP, \ + .trailer_args = STRVEC_INIT, \ .git_format_patch_opt = STRBUF_INIT, \ .fork_point = -1, \ .reapply_cherry_picks = -1, \ @@ -166,6 +169,7 @@ static void rebase_options_release(struct rebase_options *opts) free(opts->strategy); string_list_clear(&opts->strategy_opts, 0); strbuf_release(&opts->git_format_patch_opt); + strvec_clear(&opts->trailer_args); } static struct replay_opts get_replay_opts(const struct rebase_options *opts) @@ -177,6 +181,10 @@ static struct replay_opts get_replay_opts(const struct rebase_options *opts) sequencer_init_config(&replay); replay.signoff = opts->signoff; + + for (size_t i = 0; i < opts->trailer_args.nr; i++) + strvec_push(&replay.trailer_args, opts->trailer_args.v[i]); + replay.allow_ff = !(opts->flags & REBASE_FORCE); if (opts->allow_rerere_autoupdate) replay.allow_rerere_auto = opts->allow_rerere_autoupdate; @@ -500,6 +508,23 @@ static int read_basic_state(struct rebase_options *opts) opts->gpg_sign_opt = xstrdup(buf.buf); } + strbuf_reset(&buf); + + if (strbuf_read_file(&buf, state_dir_path("trailer", opts), 0) >= 0) { + const char *p = buf.buf, *end = buf.buf + buf.len; + + while (p < end) { + char *nl = memchr(p, '\n', end - p); + if (!nl) + die("nl shouldn't be NULL"); + *nl = '\0'; + + if (*p) + strvec_push(&opts->trailer_args, p); + + p = nl + 1; + } + } strbuf_release(&buf); return 0; @@ -528,6 +553,21 @@ static int rebase_write_basic_state(struct rebase_options *opts) if (opts->signoff) write_file(state_dir_path("signoff", opts), "--signoff"); + /* + * save opts->trailer_args into state_dir/trailer + */ + if (opts->trailer_args.nr) { + struct strbuf buf = STRBUF_INIT; + + for (size_t i = 0; i < opts->trailer_args.nr; i++) { + strbuf_addstr(&buf, opts->trailer_args.v[i]); + strbuf_addch(&buf, '\n'); + } + write_file(state_dir_path("trailer", opts), + "%s", buf.buf); + strbuf_release(&buf); + } + return 0; } @@ -1132,6 +1172,8 @@ int cmd_rebase(int argc, .flags = PARSE_OPT_NOARG, .defval = REBASE_DIFFSTAT, }, + OPT_STRVEC(0, "trailer", &options.trailer_args, N_("trailer"), + N_("add custom trailer(s)")), OPT_BOOL(0, "signoff", &options.signoff, N_("add a Signed-off-by trailer to each commit")), OPT_BOOL(0, "committer-date-is-author-date", @@ -1285,6 +1327,11 @@ int cmd_rebase(int argc, builtin_rebase_options, builtin_rebase_usage, 0); + if (options.trailer_args.nr) { + validate_trailer_args_after_config(&options.trailer_args); + options.flags |= REBASE_FORCE; + } + if (preserve_merges_selected) die(_("--preserve-merges was replaced by --rebase-merges\n" "Note: Your `pull.rebase` configuration may also be set to 'preserve',\n" @@ -1542,6 +1589,9 @@ int cmd_rebase(int argc, if (options.root && !options.onto_name) imply_merge(&options, "--root without --onto"); + if (options.trailer_args.nr) + imply_merge(&options, "--trailer"); + if (isatty(2) && options.flags & REBASE_NO_QUIET) strbuf_addstr(&options.git_format_patch_opt, " --progress"); diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c index c9288a9c7e..cfc1eeedd5 100644 --- a/builtin/receive-pack.c +++ b/builtin/receive-pack.c @@ -305,13 +305,12 @@ static void show_ref(const char *path, const struct object_id *oid) } } -static int show_ref_cb(const char *path_full, const char *referent UNUSED, const struct object_id *oid, - int flag UNUSED, void *data) +static int show_ref_cb(const struct reference *ref, void *data) { struct oidset *seen = data; - const char *path = strip_namespace(path_full); + const char *path = strip_namespace(ref->name); - if (ref_is_hidden(path, path_full, &hidden_refs)) + if (ref_is_hidden(path, ref->name, &hidden_refs)) return 0; /* @@ -320,13 +319,13 @@ static int show_ref_cb(const char *path_full, const char *referent UNUSED, const * transfer but will otherwise ignore them. */ if (!path) { - if (oidset_insert(seen, oid)) + if (oidset_insert(seen, ref->oid)) return 0; path = ".have"; } else { - oidset_insert(seen, oid); + oidset_insert(seen, ref->oid); } - show_ref(path, oid); + show_ref(path, ref->oid); return 0; } @@ -749,7 +748,7 @@ static int check_cert_push_options(const struct string_list *push_options) return retval; } -static void prepare_push_cert_sha1(struct child_process *proc) +static void prepare_push_cert_sha1(struct run_hooks_opt *opt) { static int already_done; @@ -775,147 +774,132 @@ static void prepare_push_cert_sha1(struct child_process *proc) nonce_status = check_nonce(sigcheck.payload); } if (!is_null_oid(&push_cert_oid)) { - strvec_pushf(&proc->env, "GIT_PUSH_CERT=%s", + strvec_pushf(&opt->env, "GIT_PUSH_CERT=%s", oid_to_hex(&push_cert_oid)); - strvec_pushf(&proc->env, "GIT_PUSH_CERT_SIGNER=%s", + strvec_pushf(&opt->env, "GIT_PUSH_CERT_SIGNER=%s", sigcheck.signer ? sigcheck.signer : ""); - strvec_pushf(&proc->env, "GIT_PUSH_CERT_KEY=%s", + strvec_pushf(&opt->env, "GIT_PUSH_CERT_KEY=%s", sigcheck.key ? sigcheck.key : ""); - strvec_pushf(&proc->env, "GIT_PUSH_CERT_STATUS=%c", + strvec_pushf(&opt->env, "GIT_PUSH_CERT_STATUS=%c", sigcheck.result); if (push_cert_nonce) { - strvec_pushf(&proc->env, + strvec_pushf(&opt->env, "GIT_PUSH_CERT_NONCE=%s", push_cert_nonce); - strvec_pushf(&proc->env, + strvec_pushf(&opt->env, "GIT_PUSH_CERT_NONCE_STATUS=%s", nonce_status); if (nonce_status == NONCE_SLOP) - strvec_pushf(&proc->env, + strvec_pushf(&opt->env, "GIT_PUSH_CERT_NONCE_SLOP=%ld", nonce_stamp_slop); } } } +struct receive_hook_feed_context { + struct command *cmd; + int skip_broken; +}; + struct receive_hook_feed_state { struct command *cmd; struct ref_push_report *report; int skip_broken; struct strbuf buf; - const struct string_list *push_options; }; -typedef int (*feed_fn)(void *, const char **, size_t *); -static int run_and_feed_hook(const char *hook_name, feed_fn feed, - struct receive_hook_feed_state *feed_state) +static int feed_receive_hook(int hook_stdin_fd, struct receive_hook_feed_state *state, int lines_batch_size) { - struct child_process proc = CHILD_PROCESS_INIT; - struct async muxer; - int code; - const char *hook_path = find_hook(the_repository, hook_name); + struct command *cmd = state->cmd; - if (!hook_path) - return 0; + strbuf_reset(&state->buf); - strvec_push(&proc.args, hook_path); - proc.in = -1; - proc.stdout_to_stderr = 1; - proc.trace2_hook_name = hook_name; - - if (feed_state->push_options) { - size_t i; - for (i = 0; i < feed_state->push_options->nr; i++) - strvec_pushf(&proc.env, - "GIT_PUSH_OPTION_%"PRIuMAX"=%s", - (uintmax_t)i, - feed_state->push_options->items[i].string); - strvec_pushf(&proc.env, "GIT_PUSH_OPTION_COUNT=%"PRIuMAX"", - (uintmax_t)feed_state->push_options->nr); - } else - strvec_pushf(&proc.env, "GIT_PUSH_OPTION_COUNT"); + /* batch lines to avoid going through run-command's ppoll for each line */ + for (int i = 0; i < lines_batch_size; i++) { + while (cmd && + state->skip_broken && (cmd->error_string || cmd->did_not_exist)) + cmd = cmd->next; - if (tmp_objdir) - strvec_pushv(&proc.env, tmp_objdir_env(tmp_objdir)); + if (!cmd) + break; /* no more commands left */ - if (use_sideband) { - memset(&muxer, 0, sizeof(muxer)); - muxer.proc = copy_to_sideband; - muxer.in = -1; - code = start_async(&muxer); - if (code) - return code; - proc.err = muxer.in; - } + if (!state->report) + state->report = cmd->report; - prepare_push_cert_sha1(&proc); + if (state->report) { + struct object_id *old_oid; + struct object_id *new_oid; + const char *ref_name; - code = start_command(&proc); - if (code) { - if (use_sideband) - finish_async(&muxer); - return code; - } + old_oid = state->report->old_oid ? state->report->old_oid : &cmd->old_oid; + new_oid = state->report->new_oid ? state->report->new_oid : &cmd->new_oid; + ref_name = state->report->ref_name ? state->report->ref_name : cmd->ref_name; - sigchain_push(SIGPIPE, SIG_IGN); + strbuf_addf(&state->buf, "%s %s %s\n", + oid_to_hex(old_oid), oid_to_hex(new_oid), + ref_name); - while (1) { - const char *buf; - size_t n; - if (feed(feed_state, &buf, &n)) - break; - if (write_in_full(proc.in, buf, n) < 0) - break; + state->report = state->report->next; + if (!state->report) + cmd = cmd->next; + } else { + strbuf_addf(&state->buf, "%s %s %s\n", + oid_to_hex(&cmd->old_oid), oid_to_hex(&cmd->new_oid), + cmd->ref_name); + cmd = cmd->next; + } } - close(proc.in); - if (use_sideband) - finish_async(&muxer); - sigchain_pop(SIGPIPE); + state->cmd = cmd; - return finish_command(&proc); + if (state->buf.len > 0) { + int ret = write_in_full(hook_stdin_fd, state->buf.buf, state->buf.len); + if (ret < 0) { + if (errno == EPIPE) + return 1; /* child closed pipe */ + return ret; + } + } + + return state->cmd ? 0 : 1; /* 0 = more to come, 1 = EOF */ } -static int feed_receive_hook(void *state_, const char **bufp, size_t *sizep) +static int feed_receive_hook_cb(int hook_stdin_fd, void *pp_cb, void *pp_task_cb UNUSED) { - struct receive_hook_feed_state *state = state_; - struct command *cmd = state->cmd; + struct hook_cb_data *hook_cb = pp_cb; + struct receive_hook_feed_state *feed_state = hook_cb->options->feed_pipe_cb_data; - while (cmd && - state->skip_broken && (cmd->error_string || cmd->did_not_exist)) - cmd = cmd->next; - if (!cmd) - return -1; /* EOF */ - if (!bufp) - return 0; /* OK, can feed something. */ - strbuf_reset(&state->buf); - if (!state->report) - state->report = cmd->report; - if (state->report) { - struct object_id *old_oid; - struct object_id *new_oid; - const char *ref_name; - - old_oid = state->report->old_oid ? state->report->old_oid : &cmd->old_oid; - new_oid = state->report->new_oid ? state->report->new_oid : &cmd->new_oid; - ref_name = state->report->ref_name ? state->report->ref_name : cmd->ref_name; - strbuf_addf(&state->buf, "%s %s %s\n", - oid_to_hex(old_oid), oid_to_hex(new_oid), - ref_name); - state->report = state->report->next; - if (!state->report) - state->cmd = cmd->next; - } else { - strbuf_addf(&state->buf, "%s %s %s\n", - oid_to_hex(&cmd->old_oid), oid_to_hex(&cmd->new_oid), - cmd->ref_name); - state->cmd = cmd->next; - } - if (bufp) { - *bufp = state->buf.buf; - *sizep = state->buf.len; + /* first-time setup */ + if (!hook_cb->options->feed_pipe_cb_data) { + struct receive_hook_feed_context *ctx = hook_cb->options->feed_pipe_ctx; + if (!ctx) + BUG("run_hooks_opt.feed_pipe_ctx required for receive hook"); + + hook_cb->options->feed_pipe_cb_data = xmalloc(sizeof(struct receive_hook_feed_state)); + feed_state = hook_cb->options->feed_pipe_cb_data; + strbuf_init(&feed_state->buf, 0); + feed_state->cmd = ctx->cmd; + feed_state->skip_broken = ctx->skip_broken; + feed_state->report = NULL; } - return 0; + + /* batch 500 lines at once to avoid going through the run-command ppoll loop too often */ + if (feed_receive_hook(hook_stdin_fd, feed_state, 500) == 0) + return 0; /* still have more data to feed */ + + strbuf_release(&feed_state->buf); + + if (hook_cb->options->feed_pipe_cb_data) + FREE_AND_NULL(hook_cb->options->feed_pipe_cb_data); + + return 1; /* done feeding, run-command can close pipe */ +} + +static void hook_output_to_sideband(struct strbuf *output, void *cb_data UNUSED) +{ + if (output && output->len) + send_sideband(1, 2, output->buf, output->len, use_sideband); } static int run_receive_hook(struct command *commands, @@ -923,47 +907,58 @@ static int run_receive_hook(struct command *commands, int skip_broken, const struct string_list *push_options) { - struct receive_hook_feed_state state; - int status; + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; + struct receive_hook_feed_context ctx; + struct command *iter = commands; - strbuf_init(&state.buf, 0); - state.cmd = commands; - state.skip_broken = skip_broken; - state.report = NULL; - if (feed_receive_hook(&state, NULL, NULL)) + /* if there are no valid commands, don't invoke the hook at all. */ + while (iter && skip_broken && (iter->error_string || iter->did_not_exist)) + iter = iter->next; + if (!iter) return 0; - state.cmd = commands; - state.push_options = push_options; - status = run_and_feed_hook(hook_name, feed_receive_hook, &state); - strbuf_release(&state.buf); - return status; + + if (push_options) { + int i; + for (i = 0; i < push_options->nr; i++) + strvec_pushf(&opt.env, "GIT_PUSH_OPTION_%d=%s", i, + push_options->items[i].string); + strvec_pushf(&opt.env, "GIT_PUSH_OPTION_COUNT=%"PRIuMAX"", + (uintmax_t)push_options->nr); + } else + strvec_push(&opt.env, "GIT_PUSH_OPTION_COUNT"); + + if (tmp_objdir) + strvec_pushv(&opt.env, tmp_objdir_env(tmp_objdir)); + + prepare_push_cert_sha1(&opt); + + /* set up sideband printer */ + if (use_sideband) + opt.consume_sideband = hook_output_to_sideband; + + /* set up stdin callback */ + ctx.cmd = commands; + ctx.skip_broken = skip_broken; + opt.feed_pipe = feed_receive_hook_cb; + opt.feed_pipe_ctx = &ctx; + + return run_hooks_opt(the_repository, hook_name, &opt); } static int run_update_hook(struct command *cmd) { - struct child_process proc = CHILD_PROCESS_INIT; - int code; - const char *hook_path = find_hook(the_repository, "update"); - - if (!hook_path) - return 0; - - strvec_push(&proc.args, hook_path); - strvec_push(&proc.args, cmd->ref_name); - strvec_push(&proc.args, oid_to_hex(&cmd->old_oid)); - strvec_push(&proc.args, oid_to_hex(&cmd->new_oid)); + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; - proc.no_stdin = 1; - proc.stdout_to_stderr = 1; - proc.err = use_sideband ? -1 : 0; - proc.trace2_hook_name = "update"; + strvec_pushl(&opt.args, + cmd->ref_name, + oid_to_hex(&cmd->old_oid), + oid_to_hex(&cmd->new_oid), + NULL); - code = start_command(&proc); - if (code) - return code; if (use_sideband) - copy_to_sideband(proc.err, -1, NULL); - return finish_command(&proc); + opt.consume_sideband = hook_output_to_sideband; + + return run_hooks_opt(the_repository, "update", &opt); } static struct command *find_command_by_refname(struct command *list, @@ -1640,33 +1635,20 @@ out: static void run_update_post_hook(struct command *commands) { struct command *cmd; - struct child_process proc = CHILD_PROCESS_INIT; - const char *hook; - - hook = find_hook(the_repository, "post-update"); - if (!hook) - return; + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; for (cmd = commands; cmd; cmd = cmd->next) { if (cmd->error_string || cmd->did_not_exist) continue; - if (!proc.args.nr) - strvec_push(&proc.args, hook); - strvec_push(&proc.args, cmd->ref_name); + strvec_push(&opt.args, cmd->ref_name); } - if (!proc.args.nr) + if (!opt.args.nr) return; - proc.no_stdin = 1; - proc.stdout_to_stderr = 1; - proc.err = use_sideband ? -1 : 0; - proc.trace2_hook_name = "post-update"; + if (use_sideband) + opt.consume_sideband = hook_output_to_sideband; - if (!start_command(&proc)) { - if (use_sideband) - copy_to_sideband(proc.err, -1, NULL); - finish_command(&proc); - } + run_hooks_opt(the_repository, "post-update", &opt); } static void check_aliased_update_internal(struct command *cmd, diff --git a/builtin/remote.c b/builtin/remote.c index 8a7ed4299a..7ffc14ba15 100644 --- a/builtin/remote.c +++ b/builtin/remote.c @@ -570,17 +570,14 @@ struct branches_for_remote { struct known_remotes *keep; }; -static int add_branch_for_removal(const char *refname, - const char *referent UNUSED, - const struct object_id *oid UNUSED, - int flags UNUSED, void *cb_data) +static int add_branch_for_removal(const struct reference *ref, void *cb_data) { struct branches_for_remote *branches = cb_data; struct refspec_item refspec; struct known_remote *kr; memset(&refspec, 0, sizeof(refspec)); - refspec.dst = (char *)refname; + refspec.dst = (char *)ref->name; if (remote_find_tracking(branches->remote, &refspec)) return 0; free(refspec.src); @@ -588,7 +585,7 @@ static int add_branch_for_removal(const char *refname, /* don't delete a branch if another remote also uses it */ for (kr = branches->keep->list; kr; kr = kr->next) { memset(&refspec, 0, sizeof(refspec)); - refspec.dst = (char *)refname; + refspec.dst = (char *)ref->name; if (!remote_find_tracking(kr->remote, &refspec)) { free(refspec.src); return 0; @@ -596,16 +593,16 @@ static int add_branch_for_removal(const char *refname, } /* don't delete non-remote-tracking refs */ - if (!starts_with(refname, "refs/remotes/")) { + if (!starts_with(ref->name, "refs/remotes/")) { /* advise user how to delete local branches */ - if (starts_with(refname, "refs/heads/")) + if (starts_with(ref->name, "refs/heads/")) string_list_append(branches->skipped, - abbrev_branch(refname)); + abbrev_branch(ref->name)); /* silently skip over other non-remote refs */ return 0; } - string_list_append(branches->branches, refname); + string_list_append(branches->branches, ref->name); return 0; } @@ -713,18 +710,18 @@ out: return error; } -static int rename_one_ref(const char *old_refname, const char *referent, - const struct object_id *oid, - int flags, void *cb_data) +static int rename_one_ref(const struct reference *ref, void *cb_data) { struct strbuf new_referent = STRBUF_INIT; struct strbuf new_refname = STRBUF_INIT; struct rename_info *rename = cb_data; + const struct object_id *oid = ref->oid; + const char *referent = ref->target; int error; - compute_renamed_ref(rename, old_refname, &new_refname); + compute_renamed_ref(rename, ref->name, &new_refname); - if (flags & REF_ISSYMREF) { + if (ref->flags & REF_ISSYMREF) { /* * Stupidly enough `referent` is not pointing to the immediate * target of a symref, but it's the recursively resolved value. @@ -732,25 +729,25 @@ static int rename_one_ref(const char *old_refname, const char *referent, * unborn symrefs don't have any value for the `referent` at all. */ referent = refs_resolve_ref_unsafe(get_main_ref_store(the_repository), - old_refname, RESOLVE_REF_NO_RECURSE, + ref->name, RESOLVE_REF_NO_RECURSE, NULL, NULL); compute_renamed_ref(rename, referent, &new_referent); oid = NULL; } - error = ref_transaction_delete(rename->transaction, old_refname, + error = ref_transaction_delete(rename->transaction, ref->name, oid, referent, REF_NO_DEREF, NULL, rename->err); if (error < 0) goto out; error = ref_transaction_update(rename->transaction, new_refname.buf, oid, null_oid(the_hash_algo), - (flags & REF_ISSYMREF) ? new_referent.buf : NULL, NULL, + (ref->flags & REF_ISSYMREF) ? new_referent.buf : NULL, NULL, REF_SKIP_CREATE_REFLOG | REF_NO_DEREF | REF_SKIP_OID_VERIFICATION, NULL, rename->err); if (error < 0) goto out; - error = rename_one_reflog(old_refname, oid, rename); + error = rename_one_reflog(ref->name, oid, rename); if (error < 0) goto out; @@ -1125,19 +1122,16 @@ static void free_remote_ref_states(struct ref_states *states) string_list_clear_func(&states->push, clear_push_info); } -static int append_ref_to_tracked_list(const char *refname, - const char *referent UNUSED, - const struct object_id *oid UNUSED, - int flags, void *cb_data) +static int append_ref_to_tracked_list(const struct reference *ref, void *cb_data) { struct ref_states *states = cb_data; struct refspec_item refspec; - if (flags & REF_ISSYMREF) + if (ref->flags & REF_ISSYMREF) return 0; memset(&refspec, 0, sizeof(refspec)); - refspec.dst = (char *)refname; + refspec.dst = (char *)ref->name; if (!remote_find_tracking(states->remote, &refspec)) { string_list_append(&states->tracked, abbrev_branch(refspec.src)); free(refspec.src); diff --git a/builtin/repack.c b/builtin/repack.c index e8730808c5..cfdb4c0920 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -3,27 +3,18 @@ #include "builtin.h" #include "config.h" -#include "dir.h" #include "environment.h" -#include "gettext.h" -#include "hex.h" #include "parse-options.h" #include "path.h" #include "run-command.h" #include "server-info.h" -#include "strbuf.h" #include "string-list.h" -#include "strvec.h" #include "midx.h" #include "packfile.h" #include "prune-packed.h" -#include "odb.h" #include "promisor-remote.h" +#include "repack.h" #include "shallow.h" -#include "pack.h" -#include "pack-bitmap.h" -#include "refs.h" -#include "list-objects-filter-options.h" #define ALL_INTO_ONE 1 #define LOOSEN_UNREACHABLE 2 @@ -33,8 +24,6 @@ #define RETAIN_PACK 2 static int pack_everything; -static int delta_base_offset = 1; -static int pack_kept_objects = -1; static int write_bitmaps = -1; static int use_delta_islands; static int run_update_server_info = 1; @@ -53,31 +42,23 @@ static const char incremental_bitmap_conflict_error[] = N_( "--no-write-bitmap-index or disable the pack.writeBitmaps configuration." ); -struct pack_objects_args { - char *window; - char *window_memory; - char *depth; - char *threads; - unsigned long max_pack_size; - int no_reuse_delta; - int no_reuse_object; - int quiet; - int local; - int name_hash_version; - int path_walk; - struct list_objects_filter_options filter_options; +struct repack_config_ctx { + struct pack_objects_args *po_args; + struct pack_objects_args *cruft_po_args; }; static int repack_config(const char *var, const char *value, const struct config_context *ctx, void *cb) { - struct pack_objects_args *cruft_po_args = cb; + struct repack_config_ctx *repack_ctx = cb; + struct pack_objects_args *po_args = repack_ctx->po_args; + struct pack_objects_args *cruft_po_args = repack_ctx->cruft_po_args; if (!strcmp(var, "repack.usedeltabaseoffset")) { - delta_base_offset = git_config_bool(var, value); + po_args->delta_base_offset = git_config_bool(var, value); return 0; } if (!strcmp(var, "repack.packkeptobjects")) { - pack_kept_objects = git_config_bool(var, value); + po_args->pack_kept_objects = git_config_bool(var, value); return 0; } if (!strcmp(var, "repack.writebitmaps") || @@ -116,1138 +97,10 @@ static int repack_config(const char *var, const char *value, return git_default_config(var, value, ctx, cb); } -static void pack_objects_args_release(struct pack_objects_args *args) -{ - free(args->window); - free(args->window_memory); - free(args->depth); - free(args->threads); - list_objects_filter_release(&args->filter_options); -} - -struct existing_packs { - struct string_list kept_packs; - struct string_list non_kept_packs; - struct string_list cruft_packs; -}; - -#define EXISTING_PACKS_INIT { \ - .kept_packs = STRING_LIST_INIT_DUP, \ - .non_kept_packs = STRING_LIST_INIT_DUP, \ - .cruft_packs = STRING_LIST_INIT_DUP, \ -} - -static int has_existing_non_kept_packs(const struct existing_packs *existing) -{ - return existing->non_kept_packs.nr || existing->cruft_packs.nr; -} - -static void pack_mark_for_deletion(struct string_list_item *item) -{ - item->util = (void*)((uintptr_t)item->util | DELETE_PACK); -} - -static void pack_unmark_for_deletion(struct string_list_item *item) -{ - item->util = (void*)((uintptr_t)item->util & ~DELETE_PACK); -} - -static int pack_is_marked_for_deletion(struct string_list_item *item) -{ - return (uintptr_t)item->util & DELETE_PACK; -} - -static void pack_mark_retained(struct string_list_item *item) -{ - item->util = (void*)((uintptr_t)item->util | RETAIN_PACK); -} - -static int pack_is_retained(struct string_list_item *item) -{ - return (uintptr_t)item->util & RETAIN_PACK; -} - -static void mark_packs_for_deletion_1(struct string_list *names, - struct string_list *list) -{ - struct string_list_item *item; - const int hexsz = the_hash_algo->hexsz; - - for_each_string_list_item(item, list) { - char *sha1; - size_t len = strlen(item->string); - if (len < hexsz) - continue; - sha1 = item->string + len - hexsz; - - if (pack_is_retained(item)) { - pack_unmark_for_deletion(item); - } else if (!string_list_has_string(names, sha1)) { - /* - * Mark this pack for deletion, which ensures - * that this pack won't be included in a MIDX - * (if `--write-midx` was given) and that we - * will actually delete this pack (if `-d` was - * given). - */ - pack_mark_for_deletion(item); - } - } -} - -static void retain_cruft_pack(struct existing_packs *existing, - struct packed_git *cruft) -{ - struct strbuf buf = STRBUF_INIT; - struct string_list_item *item; - - strbuf_addstr(&buf, pack_basename(cruft)); - strbuf_strip_suffix(&buf, ".pack"); - - item = string_list_lookup(&existing->cruft_packs, buf.buf); - if (!item) - BUG("could not find cruft pack '%s'", pack_basename(cruft)); - - pack_mark_retained(item); - strbuf_release(&buf); -} - -static void mark_packs_for_deletion(struct existing_packs *existing, - struct string_list *names) - -{ - mark_packs_for_deletion_1(names, &existing->non_kept_packs); - mark_packs_for_deletion_1(names, &existing->cruft_packs); -} - -static void remove_redundant_pack(const char *dir_name, const char *base_name) -{ - struct strbuf buf = STRBUF_INIT; - struct odb_source *source = the_repository->objects->sources; - struct multi_pack_index *m = get_multi_pack_index(source); - strbuf_addf(&buf, "%s.pack", base_name); - if (m && source->local && midx_contains_pack(m, buf.buf)) - clear_midx_file(the_repository); - strbuf_insertf(&buf, 0, "%s/", dir_name); - unlink_pack_path(buf.buf, 1); - strbuf_release(&buf); -} - -static void remove_redundant_packs_1(struct string_list *packs) -{ - struct string_list_item *item; - for_each_string_list_item(item, packs) { - if (!pack_is_marked_for_deletion(item)) - continue; - remove_redundant_pack(packdir, item->string); - } -} - -static void remove_redundant_existing_packs(struct existing_packs *existing) -{ - remove_redundant_packs_1(&existing->non_kept_packs); - remove_redundant_packs_1(&existing->cruft_packs); -} - -static void existing_packs_release(struct existing_packs *existing) -{ - string_list_clear(&existing->kept_packs, 0); - string_list_clear(&existing->non_kept_packs, 0); - string_list_clear(&existing->cruft_packs, 0); -} - -/* - * Adds all packs hex strings (pack-$HASH) to either packs->non_kept - * or packs->kept based on whether each pack has a corresponding - * .keep file or not. Packs without a .keep file are not to be kept - * if we are going to pack everything into one file. - */ -static void collect_pack_filenames(struct existing_packs *existing, - const struct string_list *extra_keep) -{ - struct packfile_store *packs = the_repository->objects->packfiles; - struct packed_git *p; - struct strbuf buf = STRBUF_INIT; - - for (p = packfile_store_get_all_packs(packs); p; p = p->next) { - int i; - const char *base; - - if (!p->pack_local) - continue; - - base = pack_basename(p); - - for (i = 0; i < extra_keep->nr; i++) - if (!fspathcmp(base, extra_keep->items[i].string)) - break; - - strbuf_reset(&buf); - strbuf_addstr(&buf, base); - strbuf_strip_suffix(&buf, ".pack"); - - if ((extra_keep->nr > 0 && i < extra_keep->nr) || p->pack_keep) - string_list_append(&existing->kept_packs, buf.buf); - else if (p->is_cruft) - string_list_append(&existing->cruft_packs, buf.buf); - else - string_list_append(&existing->non_kept_packs, buf.buf); - } - - string_list_sort(&existing->kept_packs); - string_list_sort(&existing->non_kept_packs); - string_list_sort(&existing->cruft_packs); - strbuf_release(&buf); -} - -static void prepare_pack_objects(struct child_process *cmd, - const struct pack_objects_args *args, - const char *out) -{ - strvec_push(&cmd->args, "pack-objects"); - if (args->window) - strvec_pushf(&cmd->args, "--window=%s", args->window); - if (args->window_memory) - strvec_pushf(&cmd->args, "--window-memory=%s", args->window_memory); - if (args->depth) - strvec_pushf(&cmd->args, "--depth=%s", args->depth); - if (args->threads) - strvec_pushf(&cmd->args, "--threads=%s", args->threads); - if (args->max_pack_size) - strvec_pushf(&cmd->args, "--max-pack-size=%lu", args->max_pack_size); - if (args->no_reuse_delta) - strvec_pushf(&cmd->args, "--no-reuse-delta"); - if (args->no_reuse_object) - strvec_pushf(&cmd->args, "--no-reuse-object"); - if (args->name_hash_version) - strvec_pushf(&cmd->args, "--name-hash-version=%d", args->name_hash_version); - if (args->path_walk) - strvec_pushf(&cmd->args, "--path-walk"); - if (args->local) - strvec_push(&cmd->args, "--local"); - if (args->quiet) - strvec_push(&cmd->args, "--quiet"); - if (delta_base_offset) - strvec_push(&cmd->args, "--delta-base-offset"); - strvec_push(&cmd->args, out); - cmd->git_cmd = 1; - cmd->out = -1; -} - -/* - * Write oid to the given struct child_process's stdin, starting it first if - * necessary. - */ -static int write_oid(const struct object_id *oid, - struct packed_git *pack UNUSED, - uint32_t pos UNUSED, void *data) -{ - struct child_process *cmd = data; - - if (cmd->in == -1) { - if (start_command(cmd)) - die(_("could not start pack-objects to repack promisor objects")); - } - - if (write_in_full(cmd->in, oid_to_hex(oid), the_hash_algo->hexsz) < 0 || - write_in_full(cmd->in, "\n", 1) < 0) - die(_("failed to feed promisor objects to pack-objects")); - return 0; -} - -static struct { - const char *name; - unsigned optional:1; -} exts[] = { - {".pack"}, - {".rev", 1}, - {".mtimes", 1}, - {".bitmap", 1}, - {".promisor", 1}, - {".idx"}, -}; - -struct generated_pack_data { - struct tempfile *tempfiles[ARRAY_SIZE(exts)]; -}; - -static struct generated_pack_data *populate_pack_exts(const char *name) -{ - struct stat statbuf; - struct strbuf path = STRBUF_INIT; - struct generated_pack_data *data = xcalloc(1, sizeof(*data)); - int i; - - for (i = 0; i < ARRAY_SIZE(exts); i++) { - strbuf_reset(&path); - strbuf_addf(&path, "%s-%s%s", packtmp, name, exts[i].name); - - if (stat(path.buf, &statbuf)) - continue; - - data->tempfiles[i] = register_tempfile(path.buf); - } - - strbuf_release(&path); - return data; -} - -static int has_pack_ext(const struct generated_pack_data *data, - const char *ext) -{ - int i; - for (i = 0; i < ARRAY_SIZE(exts); i++) { - if (strcmp(exts[i].name, ext)) - continue; - return !!data->tempfiles[i]; - } - BUG("unknown pack extension: '%s'", ext); -} - -static void repack_promisor_objects(const struct pack_objects_args *args, - struct string_list *names) -{ - struct child_process cmd = CHILD_PROCESS_INIT; - FILE *out; - struct strbuf line = STRBUF_INIT; - - prepare_pack_objects(&cmd, args, packtmp); - cmd.in = -1; - - /* - * NEEDSWORK: Giving pack-objects only the OIDs without any ordering - * hints may result in suboptimal deltas in the resulting pack. See if - * the OIDs can be sent with fake paths such that pack-objects can use a - * {type -> existing pack order} ordering when computing deltas instead - * of a {type -> size} ordering, which may produce better deltas. - */ - for_each_packed_object(the_repository, write_oid, &cmd, - FOR_EACH_OBJECT_PROMISOR_ONLY); - - if (cmd.in == -1) { - /* No packed objects; cmd was never started */ - child_process_clear(&cmd); - return; - } - - close(cmd.in); - - out = xfdopen(cmd.out, "r"); - while (strbuf_getline_lf(&line, out) != EOF) { - struct string_list_item *item; - char *promisor_name; - - if (line.len != the_hash_algo->hexsz) - die(_("repack: Expecting full hex object ID lines only from pack-objects.")); - item = string_list_append(names, line.buf); - - /* - * pack-objects creates the .pack and .idx files, but not the - * .promisor file. Create the .promisor file, which is empty. - * - * NEEDSWORK: fetch-pack sometimes generates non-empty - * .promisor files containing the ref names and associated - * hashes at the point of generation of the corresponding - * packfile, but this would not preserve their contents. Maybe - * concatenate the contents of all .promisor files instead of - * just creating a new empty file. - */ - promisor_name = mkpathdup("%s-%s.promisor", packtmp, - line.buf); - write_promisor_file(promisor_name, NULL, 0); - - item->util = populate_pack_exts(item->string); - - free(promisor_name); - } - - fclose(out); - if (finish_command(&cmd)) - die(_("could not finish pack-objects to repack promisor objects")); - strbuf_release(&line); -} - -struct pack_geometry { - struct packed_git **pack; - uint32_t pack_nr, pack_alloc; - uint32_t split; - - int split_factor; -}; - -static uint32_t geometry_pack_weight(struct packed_git *p) -{ - if (open_pack_index(p)) - die(_("cannot open index for %s"), p->pack_name); - return p->num_objects; -} - -static int geometry_cmp(const void *va, const void *vb) -{ - uint32_t aw = geometry_pack_weight(*(struct packed_git **)va), - bw = geometry_pack_weight(*(struct packed_git **)vb); - - if (aw < bw) - return -1; - if (aw > bw) - return 1; - return 0; -} - -static void init_pack_geometry(struct pack_geometry *geometry, - struct existing_packs *existing, - const struct pack_objects_args *args) -{ - struct packfile_store *packs = the_repository->objects->packfiles; - struct packed_git *p; - struct strbuf buf = STRBUF_INIT; - - for (p = packfile_store_get_all_packs(packs); p; p = p->next) { - if (args->local && !p->pack_local) - /* - * When asked to only repack local packfiles we skip - * over any packfiles that are borrowed from alternate - * object directories. - */ - continue; - - if (!pack_kept_objects) { - /* - * Any pack that has its pack_keep bit set will - * appear in existing->kept_packs below, but - * this saves us from doing a more expensive - * check. - */ - if (p->pack_keep) - continue; - - /* - * The pack may be kept via the --keep-pack - * option; check 'existing->kept_packs' to - * determine whether to ignore it. - */ - strbuf_reset(&buf); - strbuf_addstr(&buf, pack_basename(p)); - strbuf_strip_suffix(&buf, ".pack"); - - if (string_list_has_string(&existing->kept_packs, buf.buf)) - continue; - } - if (p->is_cruft) - continue; - - ALLOC_GROW(geometry->pack, - geometry->pack_nr + 1, - geometry->pack_alloc); - - geometry->pack[geometry->pack_nr] = p; - geometry->pack_nr++; - } - - QSORT(geometry->pack, geometry->pack_nr, geometry_cmp); - strbuf_release(&buf); -} - -static void split_pack_geometry(struct pack_geometry *geometry) -{ - uint32_t i; - uint32_t split; - off_t total_size = 0; - - if (!geometry->pack_nr) { - geometry->split = geometry->pack_nr; - return; - } - - /* - * First, count the number of packs (in descending order of size) which - * already form a geometric progression. - */ - for (i = geometry->pack_nr - 1; i > 0; i--) { - struct packed_git *ours = geometry->pack[i]; - struct packed_git *prev = geometry->pack[i - 1]; - - if (unsigned_mult_overflows(geometry->split_factor, - geometry_pack_weight(prev))) - die(_("pack %s too large to consider in geometric " - "progression"), - prev->pack_name); - - if (geometry_pack_weight(ours) < - geometry->split_factor * geometry_pack_weight(prev)) - break; - } - - split = i; - - if (split) { - /* - * Move the split one to the right, since the top element in the - * last-compared pair can't be in the progression. Only do this - * when we split in the middle of the array (otherwise if we got - * to the end, then the split is in the right place). - */ - split++; - } - - /* - * Then, anything to the left of 'split' must be in a new pack. But, - * creating that new pack may cause packs in the heavy half to no longer - * form a geometric progression. - * - * Compute an expected size of the new pack, and then determine how many - * packs in the heavy half need to be joined into it (if any) to restore - * the geometric progression. - */ - for (i = 0; i < split; i++) { - struct packed_git *p = geometry->pack[i]; - - if (unsigned_add_overflows(total_size, geometry_pack_weight(p))) - die(_("pack %s too large to roll up"), p->pack_name); - total_size += geometry_pack_weight(p); - } - for (i = split; i < geometry->pack_nr; i++) { - struct packed_git *ours = geometry->pack[i]; - - if (unsigned_mult_overflows(geometry->split_factor, - total_size)) - die(_("pack %s too large to roll up"), ours->pack_name); - - if (geometry_pack_weight(ours) < - geometry->split_factor * total_size) { - if (unsigned_add_overflows(total_size, - geometry_pack_weight(ours))) - die(_("pack %s too large to roll up"), - ours->pack_name); - - split++; - total_size += geometry_pack_weight(ours); - } else - break; - } - - geometry->split = split; -} - -static struct packed_git *get_preferred_pack(struct pack_geometry *geometry) -{ - uint32_t i; - - if (!geometry) { - /* - * No geometry means either an all-into-one repack (in which - * case there is only one pack left and it is the largest) or an - * incremental one. - * - * If repacking incrementally, then we could check the size of - * all packs to determine which should be preferred, but leave - * this for later. - */ - return NULL; - } - if (geometry->split == geometry->pack_nr) - return NULL; - - /* - * The preferred pack is the largest pack above the split line. In - * other words, it is the largest pack that does not get rolled up in - * the geometric repack. - */ - for (i = geometry->pack_nr; i > geometry->split; i--) - /* - * A pack that is not local would never be included in a - * multi-pack index. We thus skip over any non-local packs. - */ - if (geometry->pack[i - 1]->pack_local) - return geometry->pack[i - 1]; - - return NULL; -} - -static void geometry_remove_redundant_packs(struct pack_geometry *geometry, - struct string_list *names, - struct existing_packs *existing) -{ - struct strbuf buf = STRBUF_INIT; - uint32_t i; - - for (i = 0; i < geometry->split; i++) { - struct packed_git *p = geometry->pack[i]; - if (string_list_has_string(names, hash_to_hex(p->hash))) - continue; - - strbuf_reset(&buf); - strbuf_addstr(&buf, pack_basename(p)); - strbuf_strip_suffix(&buf, ".pack"); - - if ((p->pack_keep) || - (string_list_has_string(&existing->kept_packs, buf.buf))) - continue; - - remove_redundant_pack(packdir, buf.buf); - } - - strbuf_release(&buf); -} - -static void free_pack_geometry(struct pack_geometry *geometry) -{ - if (!geometry) - return; - - free(geometry->pack); -} - -static int midx_has_unknown_packs(char **midx_pack_names, - size_t midx_pack_names_nr, - struct string_list *include, - struct pack_geometry *geometry, - struct existing_packs *existing) -{ - size_t i; - - string_list_sort(include); - - for (i = 0; i < midx_pack_names_nr; i++) { - const char *pack_name = midx_pack_names[i]; - - /* - * Determine whether or not each MIDX'd pack from the existing - * MIDX (if any) is represented in the new MIDX. For each pack - * in the MIDX, it must either be: - * - * - In the "include" list of packs to be included in the new - * MIDX. Note this function is called before the include - * list is populated with any cruft pack(s). - * - * - Below the geometric split line (if using pack geometry), - * indicating that the pack won't be included in the new - * MIDX, but its contents were rolled up as part of the - * geometric repack. - * - * - In the existing non-kept packs list (if not using pack - * geometry), and marked as non-deleted. - */ - if (string_list_has_string(include, pack_name)) { - continue; - } else if (geometry) { - struct strbuf buf = STRBUF_INIT; - uint32_t j; - - for (j = 0; j < geometry->split; j++) { - strbuf_reset(&buf); - strbuf_addstr(&buf, pack_basename(geometry->pack[j])); - strbuf_strip_suffix(&buf, ".pack"); - strbuf_addstr(&buf, ".idx"); - - if (!strcmp(pack_name, buf.buf)) { - strbuf_release(&buf); - break; - } - } - - strbuf_release(&buf); - - if (j < geometry->split) - continue; - } else { - struct string_list_item *item; - - item = string_list_lookup(&existing->non_kept_packs, - pack_name); - if (item && !pack_is_marked_for_deletion(item)) - continue; - } - - /* - * If we got to this point, the MIDX includes some pack that we - * don't know about. - */ - return 1; - } - - return 0; -} - -struct midx_snapshot_ref_data { - struct tempfile *f; - struct oidset seen; - int preferred; -}; - -static int midx_snapshot_ref_one(const char *refname UNUSED, - const char *referent UNUSED, - const struct object_id *oid, - int flag UNUSED, void *_data) -{ - struct midx_snapshot_ref_data *data = _data; - struct object_id peeled; - - if (!peel_iterated_oid(the_repository, oid, &peeled)) - oid = &peeled; - - if (oidset_insert(&data->seen, oid)) - return 0; /* already seen */ - - if (odb_read_object_info(the_repository->objects, oid, NULL) != OBJ_COMMIT) - return 0; - - fprintf(data->f->fp, "%s%s\n", data->preferred ? "+" : "", - oid_to_hex(oid)); - - return 0; -} - -static void midx_snapshot_refs(struct tempfile *f) -{ - struct midx_snapshot_ref_data data; - const struct string_list *preferred = bitmap_preferred_tips(the_repository); - - data.f = f; - data.preferred = 0; - oidset_init(&data.seen, 0); - - if (!fdopen_tempfile(f, "w")) - die(_("could not open tempfile %s for writing"), - get_tempfile_path(f)); - - if (preferred) { - struct string_list_item *item; - - data.preferred = 1; - for_each_string_list_item(item, preferred) - refs_for_each_ref_in(get_main_ref_store(the_repository), - item->string, - midx_snapshot_ref_one, &data); - data.preferred = 0; - } - - refs_for_each_ref(get_main_ref_store(the_repository), - midx_snapshot_ref_one, &data); - - if (close_tempfile_gently(f)) { - int save_errno = errno; - delete_tempfile(&f); - errno = save_errno; - die_errno(_("could not close refs snapshot tempfile")); - } - - oidset_clear(&data.seen); -} - -static void midx_included_packs(struct string_list *include, - struct existing_packs *existing, - char **midx_pack_names, - size_t midx_pack_names_nr, - struct string_list *names, - struct pack_geometry *geometry) -{ - struct string_list_item *item; - struct strbuf buf = STRBUF_INIT; - - for_each_string_list_item(item, &existing->kept_packs) { - strbuf_reset(&buf); - strbuf_addf(&buf, "%s.idx", item->string); - string_list_insert(include, buf.buf); - } - - for_each_string_list_item(item, names) { - strbuf_reset(&buf); - strbuf_addf(&buf, "pack-%s.idx", item->string); - string_list_insert(include, buf.buf); - } - - if (geometry->split_factor) { - uint32_t i; - - for (i = geometry->split; i < geometry->pack_nr; i++) { - struct packed_git *p = geometry->pack[i]; - - /* - * The multi-pack index never refers to packfiles part - * of an alternate object database, so we skip these. - * While git-multi-pack-index(1) would silently ignore - * them anyway, this allows us to skip executing the - * command completely when we have only non-local - * packfiles. - */ - if (!p->pack_local) - continue; - - strbuf_reset(&buf); - strbuf_addstr(&buf, pack_basename(p)); - strbuf_strip_suffix(&buf, ".pack"); - strbuf_addstr(&buf, ".idx"); - - string_list_insert(include, buf.buf); - } - } else { - for_each_string_list_item(item, &existing->non_kept_packs) { - if (pack_is_marked_for_deletion(item)) - continue; - - strbuf_reset(&buf); - strbuf_addf(&buf, "%s.idx", item->string); - string_list_insert(include, buf.buf); - } - } - - if (midx_must_contain_cruft || - midx_has_unknown_packs(midx_pack_names, midx_pack_names_nr, - include, geometry, existing)) { - /* - * If there are one or more unknown pack(s) present (see - * midx_has_unknown_packs() for what makes a pack - * "unknown") in the MIDX before the repack, keep them - * as they may be required to form a reachability - * closure if the MIDX is bitmapped. - * - * For example, a cruft pack can be required to form a - * reachability closure if the MIDX is bitmapped and one - * or more of the bitmap's selected commits reaches a - * once-cruft object that was later made reachable. - */ - for_each_string_list_item(item, &existing->cruft_packs) { - /* - * When doing a --geometric repack, there is no - * need to check for deleted packs, since we're - * by definition not doing an ALL_INTO_ONE - * repack (hence no packs will be deleted). - * Otherwise we must check for and exclude any - * packs which are enqueued for deletion. - * - * So we could omit the conditional below in the - * --geometric case, but doing so is unnecessary - * since no packs are marked as pending - * deletion (since we only call - * `mark_packs_for_deletion()` when doing an - * all-into-one repack). - */ - if (pack_is_marked_for_deletion(item)) - continue; - - strbuf_reset(&buf); - strbuf_addf(&buf, "%s.idx", item->string); - string_list_insert(include, buf.buf); - } - } else { - /* - * Modern versions of Git (with the appropriate - * configuration setting) will write new copies of - * once-cruft objects when doing a --geometric repack. - * - * If the MIDX has no cruft pack, new packs written - * during a --geometric repack will not rely on the - * cruft pack to form a reachability closure, so we can - * avoid including them in the MIDX in that case. - */ - ; - } - - strbuf_release(&buf); -} - -static int write_midx_included_packs(struct string_list *include, - struct pack_geometry *geometry, - struct string_list *names, - const char *refs_snapshot, - int show_progress, int write_bitmaps) -{ - struct child_process cmd = CHILD_PROCESS_INIT; - struct string_list_item *item; - struct packed_git *preferred = get_preferred_pack(geometry); - FILE *in; - int ret; - - if (!include->nr) - return 0; - - cmd.in = -1; - cmd.git_cmd = 1; - - strvec_push(&cmd.args, "multi-pack-index"); - strvec_pushl(&cmd.args, "write", "--stdin-packs", NULL); - - if (show_progress) - strvec_push(&cmd.args, "--progress"); - else - strvec_push(&cmd.args, "--no-progress"); - - if (write_bitmaps) - strvec_push(&cmd.args, "--bitmap"); - - if (preferred) - strvec_pushf(&cmd.args, "--preferred-pack=%s", - pack_basename(preferred)); - else if (names->nr) { - /* The largest pack was repacked, meaning that either - * one or two packs exist depending on whether the - * repository has a cruft pack or not. - * - * Select the non-cruft one as preferred to encourage - * pack-reuse among packs containing reachable objects - * over unreachable ones. - * - * (Note we could write multiple packs here if - * `--max-pack-size` was given, but any one of them - * will suffice, so pick the first one.) - */ - for_each_string_list_item(item, names) { - struct generated_pack_data *data = item->util; - if (has_pack_ext(data, ".mtimes")) - continue; - - strvec_pushf(&cmd.args, "--preferred-pack=pack-%s.pack", - item->string); - break; - } - } else { - /* - * No packs were kept, and no packs were written. The - * only thing remaining are .keep packs (unless - * --pack-kept-objects was given). - * - * Set the `--preferred-pack` arbitrarily here. - */ - ; - } - - if (refs_snapshot) - strvec_pushf(&cmd.args, "--refs-snapshot=%s", refs_snapshot); - - ret = start_command(&cmd); - if (ret) - return ret; - - in = xfdopen(cmd.in, "w"); - for_each_string_list_item(item, include) - fprintf(in, "%s\n", item->string); - fclose(in); - - return finish_command(&cmd); -} - -static void remove_redundant_bitmaps(struct string_list *include, - const char *packdir) -{ - struct strbuf path = STRBUF_INIT; - struct string_list_item *item; - size_t packdir_len; - - strbuf_addstr(&path, packdir); - strbuf_addch(&path, '/'); - packdir_len = path.len; - - /* - * Remove any pack bitmaps corresponding to packs which are now - * included in the MIDX. - */ - for_each_string_list_item(item, include) { - strbuf_addstr(&path, item->string); - strbuf_strip_suffix(&path, ".idx"); - strbuf_addstr(&path, ".bitmap"); - - if (unlink(path.buf) && errno != ENOENT) - warning_errno(_("could not remove stale bitmap: %s"), - path.buf); - - strbuf_setlen(&path, packdir_len); - } - strbuf_release(&path); -} - -static int finish_pack_objects_cmd(struct child_process *cmd, - struct string_list *names, - int local) -{ - FILE *out; - struct strbuf line = STRBUF_INIT; - - out = xfdopen(cmd->out, "r"); - while (strbuf_getline_lf(&line, out) != EOF) { - struct string_list_item *item; - - if (line.len != the_hash_algo->hexsz) - die(_("repack: Expecting full hex object ID lines only " - "from pack-objects.")); - /* - * Avoid putting packs written outside of the repository in the - * list of names. - */ - if (local) { - item = string_list_append(names, line.buf); - item->util = populate_pack_exts(line.buf); - } - } - fclose(out); - - strbuf_release(&line); - - return finish_command(cmd); -} - -static int write_filtered_pack(const struct pack_objects_args *args, - const char *destination, - const char *pack_prefix, - struct existing_packs *existing, - struct string_list *names) -{ - struct child_process cmd = CHILD_PROCESS_INIT; - struct string_list_item *item; - FILE *in; - int ret; - const char *caret; - const char *scratch; - int local = skip_prefix(destination, packdir, &scratch); - - prepare_pack_objects(&cmd, args, destination); - - strvec_push(&cmd.args, "--stdin-packs"); - - if (!pack_kept_objects) - strvec_push(&cmd.args, "--honor-pack-keep"); - for_each_string_list_item(item, &existing->kept_packs) - strvec_pushf(&cmd.args, "--keep-pack=%s", item->string); - - cmd.in = -1; - - ret = start_command(&cmd); - if (ret) - return ret; - - /* - * Here 'names' contains only the pack(s) that were just - * written, which is exactly the packs we want to keep. Also - * 'existing_kept_packs' already contains the packs in - * 'keep_pack_list'. - */ - in = xfdopen(cmd.in, "w"); - for_each_string_list_item(item, names) - fprintf(in, "^%s-%s.pack\n", pack_prefix, item->string); - for_each_string_list_item(item, &existing->non_kept_packs) - fprintf(in, "%s.pack\n", item->string); - for_each_string_list_item(item, &existing->cruft_packs) - fprintf(in, "%s.pack\n", item->string); - caret = pack_kept_objects ? "" : "^"; - for_each_string_list_item(item, &existing->kept_packs) - fprintf(in, "%s%s.pack\n", caret, item->string); - fclose(in); - - return finish_pack_objects_cmd(&cmd, names, local); -} - -static void combine_small_cruft_packs(FILE *in, size_t combine_cruft_below_size, - struct existing_packs *existing) -{ - struct packfile_store *packs = the_repository->objects->packfiles; - struct packed_git *p; - struct strbuf buf = STRBUF_INIT; - size_t i; - - for (p = packfile_store_get_all_packs(packs); p; p = p->next) { - if (!(p->is_cruft && p->pack_local)) - continue; - - strbuf_reset(&buf); - strbuf_addstr(&buf, pack_basename(p)); - strbuf_strip_suffix(&buf, ".pack"); - - if (!string_list_has_string(&existing->cruft_packs, buf.buf)) - continue; - - if (p->pack_size < combine_cruft_below_size) { - fprintf(in, "-%s\n", pack_basename(p)); - } else { - retain_cruft_pack(existing, p); - fprintf(in, "%s\n", pack_basename(p)); - } - } - - for (i = 0; i < existing->non_kept_packs.nr; i++) - fprintf(in, "-%s.pack\n", - existing->non_kept_packs.items[i].string); - - strbuf_release(&buf); -} - -static int write_cruft_pack(const struct pack_objects_args *args, - const char *destination, - const char *pack_prefix, - const char *cruft_expiration, - unsigned long combine_cruft_below_size, - struct string_list *names, - struct existing_packs *existing) -{ - struct child_process cmd = CHILD_PROCESS_INIT; - struct string_list_item *item; - FILE *in; - int ret; - const char *scratch; - int local = skip_prefix(destination, packdir, &scratch); - - prepare_pack_objects(&cmd, args, destination); - - strvec_push(&cmd.args, "--cruft"); - if (cruft_expiration) - strvec_pushf(&cmd.args, "--cruft-expiration=%s", - cruft_expiration); - - strvec_push(&cmd.args, "--honor-pack-keep"); - strvec_push(&cmd.args, "--non-empty"); - - cmd.in = -1; - - ret = start_command(&cmd); - if (ret) - return ret; - - /* - * names has a confusing double use: it both provides the list - * of just-written new packs, and accepts the name of the cruft - * pack we are writing. - * - * By the time it is read here, it contains only the pack(s) - * that were just written, which is exactly the set of packs we - * want to consider kept. - * - * If `--expire-to` is given, the double-use served by `names` - * ensures that the pack written to `--expire-to` excludes any - * objects contained in the cruft pack. - */ - in = xfdopen(cmd.in, "w"); - for_each_string_list_item(item, names) - fprintf(in, "%s-%s.pack\n", pack_prefix, item->string); - if (combine_cruft_below_size && !cruft_expiration) { - combine_small_cruft_packs(in, combine_cruft_below_size, - existing); - } else { - for_each_string_list_item(item, &existing->non_kept_packs) - fprintf(in, "-%s.pack\n", item->string); - for_each_string_list_item(item, &existing->cruft_packs) - fprintf(in, "-%s.pack\n", item->string); - } - for_each_string_list_item(item, &existing->kept_packs) - fprintf(in, "%s.pack\n", item->string); - fclose(in); - - return finish_pack_objects_cmd(&cmd, names, local); -} - -static const char *find_pack_prefix(const char *packdir, const char *packtmp) -{ - const char *pack_prefix; - if (!skip_prefix(packtmp, packdir, &pack_prefix)) - die(_("pack prefix %s does not begin with objdir %s"), - packtmp, packdir); - if (*pack_prefix == '/') - pack_prefix++; - return pack_prefix; -} - int cmd_repack(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { struct child_process cmd = CHILD_PROCESS_INIT; struct string_list_item *item; @@ -1255,18 +108,17 @@ int cmd_repack(int argc, struct existing_packs existing = EXISTING_PACKS_INIT; struct pack_geometry geometry = { 0 }; struct tempfile *refs_snapshot = NULL; - int i, ext, ret; + int i, ret; int show_progress; - char **midx_pack_names = NULL; - size_t midx_pack_names_nr = 0; /* variables to be filled by option parsing */ + struct repack_config_ctx config_ctx; int delete_redundant = 0; const char *unpack_unreachable = NULL; int keep_unreachable = 0; struct string_list keep_pack_list = STRING_LIST_INIT_NODUP; - struct pack_objects_args po_args = { 0 }; - struct pack_objects_args cruft_po_args = { 0 }; + struct pack_objects_args po_args = PACK_OBJECTS_ARGS_INIT; + struct pack_objects_args cruft_po_args = PACK_OBJECTS_ARGS_INIT; int write_midx = 0; const char *cruft_expiration = NULL; const char *expire_to = NULL; @@ -1327,7 +179,7 @@ int cmd_repack(int argc, OPT_UNSIGNED(0, "max-pack-size", &po_args.max_pack_size, N_("maximum size of each packfile")), OPT_PARSE_LIST_OBJECTS_FILTER(&po_args.filter_options), - OPT_BOOL(0, "pack-kept-objects", &pack_kept_objects, + OPT_BOOL(0, "pack-kept-objects", &po_args.pack_kept_objects, N_("repack objects in packs marked with .keep")), OPT_STRING_LIST(0, "keep-pack", &keep_pack_list, N_("name"), N_("do not repack this pack")), @@ -1344,7 +196,11 @@ int cmd_repack(int argc, list_objects_filter_init(&po_args.filter_options); - repo_config(the_repository, repack_config, &cruft_po_args); + memset(&config_ctx, 0, sizeof(config_ctx)); + config_ctx.po_args = &po_args; + config_ctx.cruft_po_args = &cruft_po_args; + + repo_config(repo, repack_config, &config_ctx); argc = parse_options(argc, argv, prefix, builtin_repack_options, git_repack_usage, 0); @@ -1354,7 +210,7 @@ int cmd_repack(int argc, po_args.depth = xstrdup_or_null(opt_depth); po_args.threads = xstrdup_or_null(opt_threads); - if (delete_redundant && the_repository->repository_format_precious_objects) + if (delete_redundant && repo->repository_format_precious_objects) die(_("cannot delete packs in a precious-objects repo")); die_for_incompatible_opt3(unpack_unreachable || (pack_everything & LOOSEN_UNREACHABLE), "-A", @@ -1369,14 +225,14 @@ int cmd_repack(int argc, (!(pack_everything & ALL_INTO_ONE) || !is_bare_repository())) write_bitmaps = 0; } - if (pack_kept_objects < 0) - pack_kept_objects = write_bitmaps > 0 && !write_midx; + if (po_args.pack_kept_objects < 0) + po_args.pack_kept_objects = write_bitmaps > 0 && !write_midx; if (write_bitmaps && !(pack_everything & ALL_INTO_ONE) && !write_midx) die(_(incremental_bitmap_conflict_error)); if (write_bitmaps && po_args.local && - odb_has_alternates(the_repository->objects)) { + odb_has_alternates(repo->objects)) { /* * When asked to do a local repack, but we have * packfiles that are inherited from an alternate, then @@ -1391,26 +247,28 @@ int cmd_repack(int argc, if (write_midx && write_bitmaps) { struct strbuf path = STRBUF_INIT; - strbuf_addf(&path, "%s/%s_XXXXXX", repo_get_object_directory(the_repository), + strbuf_addf(&path, "%s/%s_XXXXXX", + repo_get_object_directory(repo), "bitmap-ref-tips"); refs_snapshot = xmks_tempfile(path.buf); - midx_snapshot_refs(refs_snapshot); + midx_snapshot_refs(repo, refs_snapshot); strbuf_release(&path); } - packdir = mkpathdup("%s/pack", repo_get_object_directory(the_repository)); + packdir = mkpathdup("%s/pack", repo_get_object_directory(repo)); packtmp_name = xstrfmt(".tmp-%d-pack", (int)getpid()); packtmp = mkpathdup("%s/%s", packdir, packtmp_name); - collect_pack_filenames(&existing, &keep_pack_list); + existing.repo = repo; + existing_packs_collect(&existing, &keep_pack_list); if (geometry.split_factor) { if (pack_everything) die(_("options '%s' and '%s' cannot be used together"), "--geometric", "-A/-a"); - init_pack_geometry(&geometry, &existing, &po_args); - split_pack_geometry(&geometry); + pack_geometry_init(&geometry, &existing, &po_args); + pack_geometry_split(&geometry); } prepare_pack_objects(&cmd, &po_args, packtmp); @@ -1418,8 +276,6 @@ int cmd_repack(int argc, show_progress = !po_args.quiet && isatty(2); strvec_push(&cmd.args, "--keep-true-parents"); - if (!pack_kept_objects) - strvec_push(&cmd.args, "--honor-pack-keep"); for (i = 0; i < keep_pack_list.nr; i++) strvec_pushf(&cmd.args, "--keep-pack=%s", keep_pack_list.items[i].string); @@ -1439,7 +295,7 @@ int cmd_repack(int argc, strvec_push(&cmd.args, "--reflog"); strvec_push(&cmd.args, "--indexed-objects"); } - if (repo_has_promisor_remote(the_repository)) + if (repo_has_promisor_remote(repo)) strvec_push(&cmd.args, "--exclude-promisor-objects"); if (!write_midx) { if (write_bitmaps > 0) @@ -1451,9 +307,9 @@ int cmd_repack(int argc, strvec_push(&cmd.args, "--delta-islands"); if (pack_everything & ALL_INTO_ONE) { - repack_promisor_objects(&po_args, &names); + repack_promisor_objects(repo, &po_args, &names, packtmp); - if (has_existing_non_kept_packs(&existing) && + if (existing_packs_has_non_kept(&existing) && delete_redundant && !(pack_everything & PACK_CRUFT)) { for_each_string_list_item(item, &names) { @@ -1515,9 +371,17 @@ int cmd_repack(int argc, fclose(in); } - ret = finish_pack_objects_cmd(&cmd, &names, 1); - if (ret) - goto cleanup; + { + struct write_pack_opts opts = { + .packdir = packdir, + .destination = packdir, + .packtmp = packtmp, + }; + ret = finish_pack_objects_cmd(repo->hash_algo, &opts, &cmd, + &names); + if (ret) + goto cleanup; + } if (!names.nr) { if (!po_args.quiet) @@ -1535,12 +399,17 @@ int cmd_repack(int argc, * midx_has_unknown_packs() will make the decision for * us. */ - if (!get_multi_pack_index(the_repository->objects->sources)) + if (!get_multi_pack_index(repo->objects->sources)) midx_must_contain_cruft = 1; } if (pack_everything & PACK_CRUFT) { - const char *pack_prefix = find_pack_prefix(packdir, packtmp); + struct write_pack_opts opts = { + .po_args = &cruft_po_args, + .destination = packtmp, + .packtmp = packtmp, + .packdir = packdir, + }; if (!cruft_po_args.window) cruft_po_args.window = xstrdup_or_null(po_args.window); @@ -1555,9 +424,10 @@ int cmd_repack(int argc, cruft_po_args.local = po_args.local; cruft_po_args.quiet = po_args.quiet; + cruft_po_args.delta_base_offset = po_args.delta_base_offset; + cruft_po_args.pack_kept_objects = 0; - ret = write_cruft_pack(&cruft_po_args, packtmp, pack_prefix, - cruft_expiration, + ret = write_cruft_pack(&opts, cruft_expiration, combine_cruft_below_size, &names, &existing); if (ret) @@ -1592,11 +462,8 @@ int cmd_repack(int argc, * pack, but rather removing all cruft packs from the * main repository regardless of size. */ - ret = write_cruft_pack(&cruft_po_args, expire_to, - pack_prefix, - NULL, - 0ul, - &names, + opts.destination = expire_to; + ret = write_cruft_pack(&opts, NULL, 0ul, &names, &existing); if (ret) goto cleanup; @@ -1604,99 +471,63 @@ int cmd_repack(int argc, } if (po_args.filter_options.choice) { - if (!filter_to) - filter_to = packtmp; - - ret = write_filtered_pack(&po_args, - filter_to, - find_pack_prefix(packdir, packtmp), - &existing, - &names); + struct write_pack_opts opts = { + .po_args = &po_args, + .destination = filter_to, + .packdir = packdir, + .packtmp = packtmp, + }; + + if (!opts.destination) + opts.destination = packtmp; + + ret = write_filtered_pack(&opts, &existing, &names); if (ret) goto cleanup; } string_list_sort(&names); - if (get_multi_pack_index(the_repository->objects->sources)) { - struct multi_pack_index *m = - get_multi_pack_index(the_repository->objects->sources); - - ALLOC_ARRAY(midx_pack_names, - m->num_packs + m->num_packs_in_base); - - for (; m; m = m->base_midx) - for (uint32_t i = 0; i < m->num_packs; i++) - midx_pack_names[midx_pack_names_nr++] = - xstrdup(m->pack_names[i]); - } - - close_object_store(the_repository->objects); + close_object_store(repo->objects); /* * Ok we have prepared all new packfiles. */ - for_each_string_list_item(item, &names) { - struct generated_pack_data *data = item->util; - - for (ext = 0; ext < ARRAY_SIZE(exts); ext++) { - char *fname; - - fname = mkpathdup("%s/pack-%s%s", - packdir, item->string, exts[ext].name); - - if (data->tempfiles[ext]) { - const char *fname_old = get_tempfile_path(data->tempfiles[ext]); - struct stat statbuffer; - - if (!stat(fname_old, &statbuffer)) { - statbuffer.st_mode &= ~(S_IWUSR | S_IWGRP | S_IWOTH); - chmod(fname_old, statbuffer.st_mode); - } - - if (rename_tempfile(&data->tempfiles[ext], fname)) - die_errno(_("renaming pack to '%s' failed"), fname); - } else if (!exts[ext].optional) - die(_("pack-objects did not write a '%s' file for pack %s-%s"), - exts[ext].name, packtmp, item->string); - else if (unlink(fname) < 0 && errno != ENOENT) - die_errno(_("could not unlink: %s"), fname); - - free(fname); - } - } + for_each_string_list_item(item, &names) + generated_pack_install(item->util, item->string, packdir, + packtmp); /* End of pack replacement. */ if (delete_redundant && pack_everything & ALL_INTO_ONE) - mark_packs_for_deletion(&existing, &names); + existing_packs_mark_for_deletion(&existing, &names); if (write_midx) { - struct string_list include = STRING_LIST_INIT_DUP; - midx_included_packs(&include, &existing, midx_pack_names, - midx_pack_names_nr, &names, &geometry); - - ret = write_midx_included_packs(&include, &geometry, &names, - refs_snapshot ? get_tempfile_path(refs_snapshot) : NULL, - show_progress, write_bitmaps > 0); - - if (!ret && write_bitmaps) - remove_redundant_bitmaps(&include, packdir); - - string_list_clear(&include, 0); + struct repack_write_midx_opts opts = { + .existing = &existing, + .geometry = &geometry, + .names = &names, + .refs_snapshot = refs_snapshot ? get_tempfile_path(refs_snapshot) : NULL, + .packdir = packdir, + .show_progress = show_progress, + .write_bitmaps = write_bitmaps > 0, + .midx_must_contain_cruft = midx_must_contain_cruft + }; + + ret = write_midx_included_packs(&opts); if (ret) goto cleanup; } - odb_reprepare(the_repository->objects); + odb_reprepare(repo->objects); if (delete_redundant) { int opts = 0; - remove_redundant_existing_packs(&existing); + existing_packs_remove_redundant(&existing, packdir); if (geometry.split_factor) - geometry_remove_redundant_packs(&geometry, &names, - &existing); + pack_geometry_remove_redundant(&geometry, &names, + &existing, packdir); if (show_progress) opts |= PRUNE_PACKED_VERBOSE; prune_packed_objects(opts); @@ -1704,18 +535,18 @@ int cmd_repack(int argc, if (!keep_unreachable && (!(pack_everything & LOOSEN_UNREACHABLE) || unpack_unreachable) && - is_repository_shallow(the_repository)) + is_repository_shallow(repo)) prune_shallow(PRUNE_QUICK); } if (run_update_server_info) - update_server_info(the_repository, 0); + update_server_info(repo, 0); if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX, 0)) { unsigned flags = 0; if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL, 0)) flags |= MIDX_WRITE_INCREMENTAL; - write_midx_file(the_repository->objects->sources, + write_midx_file(repo->objects->sources, NULL, NULL, flags); } @@ -1723,10 +554,7 @@ cleanup: string_list_clear(&keep_pack_list, 0); string_list_clear(&names, 1); existing_packs_release(&existing); - free_pack_geometry(&geometry); - for (size_t i = 0; i < midx_pack_names_nr; i++) - free(midx_pack_names[i]); - free(midx_pack_names); + pack_geometry_release(&geometry); pack_objects_args_release(&po_args); pack_objects_args_release(&cruft_po_args); diff --git a/builtin/replace.c b/builtin/replace.c index 900b560a77..4c62c5ab58 100644 --- a/builtin/replace.c +++ b/builtin/replace.c @@ -47,30 +47,27 @@ struct show_data { enum replace_format format; }; -static int show_reference(const char *refname, - const char *referent UNUSED, - const struct object_id *oid, - int flag UNUSED, void *cb_data) +static int show_reference(const struct reference *ref, void *cb_data) { struct show_data *data = cb_data; - if (!wildmatch(data->pattern, refname, 0)) { + if (!wildmatch(data->pattern, ref->name, 0)) { if (data->format == REPLACE_FORMAT_SHORT) - printf("%s\n", refname); + printf("%s\n", ref->name); else if (data->format == REPLACE_FORMAT_MEDIUM) - printf("%s -> %s\n", refname, oid_to_hex(oid)); + printf("%s -> %s\n", ref->name, oid_to_hex(ref->oid)); else { /* data->format == REPLACE_FORMAT_LONG */ struct object_id object; enum object_type obj_type, repl_type; - if (repo_get_oid(data->repo, refname, &object)) - return error(_("failed to resolve '%s' as a valid ref"), refname); + if (repo_get_oid(data->repo, ref->name, &object)) + return error(_("failed to resolve '%s' as a valid ref"), ref->name); obj_type = odb_read_object_info(data->repo->objects, &object, NULL); - repl_type = odb_read_object_info(data->repo->objects, oid, NULL); + repl_type = odb_read_object_info(data->repo->objects, ref->oid, NULL); - printf("%s (%s) -> %s (%s)\n", refname, type_name(obj_type), - oid_to_hex(oid), type_name(repl_type)); + printf("%s (%s) -> %s (%s)\n", ref->name, type_name(obj_type), + oid_to_hex(ref->oid), type_name(repl_type)); } } diff --git a/builtin/replay.c b/builtin/replay.c index 6172c8aacc..f974a8c963 100644 --- a/builtin/replay.c +++ b/builtin/replay.c @@ -2,12 +2,12 @@ * "git replay" builtin command */ -#define USE_THE_REPOSITORY_VARIABLE #define DISABLE_SIGN_COMPARE_WARNINGS #include "git-compat-util.h" #include "builtin.h" +#include "config.h" #include "environment.h" #include "hex.h" #include "lockfile.h" @@ -15,17 +15,16 @@ #include "object-name.h" #include "parse-options.h" #include "refs.h" +#include "replay.h" #include "revision.h" #include "strmap.h" #include <oidset.h> #include <tree.h> -static const char *short_commit_name(struct repository *repo, - struct commit *commit) -{ - return repo_find_unique_abbrev(repo, &commit->object.oid, - DEFAULT_ABBREV); -} +enum ref_action_mode { + REF_ACTION_UPDATE, + REF_ACTION_PRINT, +}; static struct commit *peel_committish(struct repository *repo, const char *name) { @@ -39,59 +38,6 @@ static struct commit *peel_committish(struct repository *repo, const char *name) OBJ_COMMIT); } -static char *get_author(const char *message) -{ - size_t len; - const char *a; - - a = find_commit_header(message, "author", &len); - if (a) - return xmemdupz(a, len); - - return NULL; -} - -static struct commit *create_commit(struct repository *repo, - struct tree *tree, - struct commit *based_on, - struct commit *parent) -{ - struct object_id ret; - struct object *obj = NULL; - struct commit_list *parents = NULL; - char *author; - char *sign_commit = NULL; /* FIXME: cli users might want to sign again */ - struct commit_extra_header *extra = NULL; - struct strbuf msg = STRBUF_INIT; - const char *out_enc = get_commit_output_encoding(); - const char *message = repo_logmsg_reencode(repo, based_on, - NULL, out_enc); - const char *orig_message = NULL; - const char *exclude_gpgsig[] = { "gpgsig", NULL }; - - commit_list_insert(parent, &parents); - extra = read_commit_extra_headers(based_on, exclude_gpgsig); - find_commit_subject(message, &orig_message); - strbuf_addstr(&msg, orig_message); - author = get_author(message); - reset_ident_date(); - if (commit_tree_extended(msg.buf, msg.len, &tree->object.oid, parents, - &ret, author, NULL, sign_commit, extra)) { - error(_("failed to write commit object")); - goto out; - } - - obj = parse_object(repo, &ret); - -out: - repo_unuse_commit_buffer(the_repository, based_on, message); - free_commit_extra_headers(extra); - free_commit_list(parents); - strbuf_release(&msg); - free(author); - return (struct commit *)obj; -} - struct ref_info { struct commit *onto; struct strset positive_refs; @@ -240,48 +186,52 @@ static void determine_replay_mode(struct repository *repo, strset_clear(&rinfo.positive_refs); } -static struct commit *mapped_commit(kh_oid_map_t *replayed_commits, - struct commit *commit, - struct commit *fallback) +static enum ref_action_mode parse_ref_action_mode(const char *ref_action, const char *source) { - khint_t pos = kh_get_oid_map(replayed_commits, commit->object.oid); - if (pos == kh_end(replayed_commits)) - return fallback; - return kh_value(replayed_commits, pos); + if (!ref_action || !strcmp(ref_action, "update")) + return REF_ACTION_UPDATE; + if (!strcmp(ref_action, "print")) + return REF_ACTION_PRINT; + die(_("invalid %s value: '%s'"), source, ref_action); } -static struct commit *pick_regular_commit(struct repository *repo, - struct commit *pickme, - kh_oid_map_t *replayed_commits, - struct commit *onto, - struct merge_options *merge_opt, - struct merge_result *result) +static enum ref_action_mode get_ref_action_mode(struct repository *repo, const char *ref_action) { - struct commit *base, *replayed_base; - struct tree *pickme_tree, *base_tree; - - base = pickme->parents->item; - replayed_base = mapped_commit(replayed_commits, base, onto); + const char *config_value = NULL; - result->tree = repo_get_commit_tree(repo, replayed_base); - pickme_tree = repo_get_commit_tree(repo, pickme); - base_tree = repo_get_commit_tree(repo, base); + /* Command line option takes precedence */ + if (ref_action) + return parse_ref_action_mode(ref_action, "--ref-action"); - merge_opt->branch1 = short_commit_name(repo, replayed_base); - merge_opt->branch2 = short_commit_name(repo, pickme); - merge_opt->ancestor = xstrfmt("parent of %s", merge_opt->branch2); + /* Check config value */ + if (!repo_config_get_string_tmp(repo, "replay.refAction", &config_value)) + return parse_ref_action_mode(config_value, "replay.refAction"); - merge_incore_nonrecursive(merge_opt, - base_tree, - result->tree, - pickme_tree, - result); + /* Default to update mode */ + return REF_ACTION_UPDATE; +} - free((char*)merge_opt->ancestor); - merge_opt->ancestor = NULL; - if (!result->clean) - return NULL; - return create_commit(repo, result->tree, pickme, replayed_base); +static int handle_ref_update(enum ref_action_mode mode, + struct ref_transaction *transaction, + const char *refname, + const struct object_id *new_oid, + const struct object_id *old_oid, + const char *reflog_msg, + struct strbuf *err) +{ + switch (mode) { + case REF_ACTION_PRINT: + printf("update %s %s %s\n", + refname, + oid_to_hex(new_oid), + oid_to_hex(old_oid)); + return 0; + case REF_ACTION_UPDATE: + return ref_transaction_update(transaction, refname, new_oid, old_oid, + NULL, NULL, 0, reflog_msg, err); + default: + BUG("unknown ref_action_mode %d", mode); + } } int cmd_replay(int argc, @@ -294,6 +244,8 @@ int cmd_replay(int argc, struct commit *onto = NULL; const char *onto_name = NULL; int contained = 0; + const char *ref_action = NULL; + enum ref_action_mode ref_mode; struct rev_info revs; struct commit *last_commit = NULL; @@ -302,12 +254,15 @@ int cmd_replay(int argc, struct merge_result result; struct strset *update_refs = NULL; kh_oid_map_t *replayed_commits; + struct ref_transaction *transaction = NULL; + struct strbuf transaction_err = STRBUF_INIT; + struct strbuf reflog_msg = STRBUF_INIT; int ret = 0; - const char * const replay_usage[] = { + const char *const replay_usage[] = { N_("(EXPERIMENTAL!) git replay " "([--contained] --onto <newbase> | --advance <branch>) " - "<revision-range>..."), + "[--ref-action[=<mode>]] <revision-range>..."), NULL }; struct option replay_options[] = { @@ -319,6 +274,9 @@ int cmd_replay(int argc, N_("replay onto given commit")), OPT_BOOL(0, "contained", &contained, N_("advance all branches contained in revision-range")), + OPT_STRING(0, "ref-action", &ref_action, + N_("mode"), + N_("control ref update behavior (update|print)")), OPT_END() }; @@ -330,9 +288,12 @@ int cmd_replay(int argc, usage_with_options(replay_usage, replay_options); } - if (advance_name_opt && contained) - die(_("options '%s' and '%s' cannot be used together"), - "--advance", "--contained"); + die_for_incompatible_opt2(!!advance_name_opt, "--advance", + contained, "--contained"); + + /* Parse ref action mode from command line or config */ + ref_mode = get_ref_action_mode(repo, ref_action); + advance_name = xstrdup_or_null(advance_name_opt); repo_init_revisions(repo, &revs, prefix); @@ -389,6 +350,24 @@ int cmd_replay(int argc, determine_replay_mode(repo, &revs.cmdline, onto_name, &advance_name, &onto, &update_refs); + /* Build reflog message */ + if (advance_name_opt) + strbuf_addf(&reflog_msg, "replay --advance %s", advance_name_opt); + else + strbuf_addf(&reflog_msg, "replay --onto %s", + oid_to_hex(&onto->object.oid)); + + /* Initialize ref transaction if using update mode */ + if (ref_mode == REF_ACTION_UPDATE) { + transaction = ref_store_transaction_begin(get_main_ref_store(repo), + 0, &transaction_err); + if (!transaction) { + ret = error(_("failed to begin ref transaction: %s"), + transaction_err.buf); + goto cleanup; + } + } + if (!onto) /* FIXME: Should handle replaying down to root commit */ die("Replaying down to root commit is not supported yet!"); @@ -412,8 +391,8 @@ int cmd_replay(int argc, if (commit->parents->next) die(_("replaying merge commits is not supported yet!")); - last_commit = pick_regular_commit(repo, commit, replayed_commits, - onto, &merge_opt, &result); + last_commit = replay_pick_regular_commit(repo, commit, replayed_commits, + onto, &merge_opt, &result); if (!last_commit) break; @@ -434,10 +413,16 @@ int cmd_replay(int argc, if (decoration->type == DECORATION_REF_LOCAL && (contained || strset_contains(update_refs, decoration->name))) { - printf("update %s %s %s\n", - decoration->name, - oid_to_hex(&last_commit->object.oid), - oid_to_hex(&commit->object.oid)); + if (handle_ref_update(ref_mode, transaction, + decoration->name, + &last_commit->object.oid, + &commit->object.oid, + reflog_msg.buf, + &transaction_err) < 0) { + ret = error(_("failed to update ref '%s': %s"), + decoration->name, transaction_err.buf); + goto cleanup; + } } decoration = decoration->next; } @@ -445,10 +430,24 @@ int cmd_replay(int argc, /* In --advance mode, advance the target ref */ if (result.clean == 1 && advance_name) { - printf("update %s %s %s\n", - advance_name, - oid_to_hex(&last_commit->object.oid), - oid_to_hex(&onto->object.oid)); + if (handle_ref_update(ref_mode, transaction, advance_name, + &last_commit->object.oid, + &onto->object.oid, + reflog_msg.buf, + &transaction_err) < 0) { + ret = error(_("failed to update ref '%s': %s"), + advance_name, transaction_err.buf); + goto cleanup; + } + } + + /* Commit the ref transaction if we have one */ + if (transaction && result.clean == 1) { + if (ref_transaction_commit(transaction, &transaction_err)) { + ret = error(_("failed to commit ref transaction: %s"), + transaction_err.buf); + goto cleanup; + } } merge_finalize(&merge_opt, &result); @@ -460,6 +459,10 @@ int cmd_replay(int argc, ret = result.clean; cleanup: + if (transaction) + ref_transaction_free(transaction); + strbuf_release(&transaction_err); + strbuf_release(&reflog_msg); release_revisions(&revs); free(advance_name); diff --git a/builtin/repo.c b/builtin/repo.c index bbb0966f2d..9c3da1a975 100644 --- a/builtin/repo.c +++ b/builtin/repo.c @@ -3,19 +3,27 @@ #include "builtin.h" #include "environment.h" #include "parse-options.h" +#include "path-walk.h" +#include "progress.h" #include "quote.h" +#include "ref-filter.h" #include "refs.h" +#include "revision.h" #include "strbuf.h" +#include "string-list.h" #include "shallow.h" +#include "utf8.h" static const char *const repo_usage[] = { - "git repo info [--format=(keyvalue|nul)] [-z] [<key>...]", + "git repo info [--format=(keyvalue|nul)] [-z] [--all | <key>...]", + "git repo structure [--format=(table|keyvalue|nul)]", NULL }; typedef int get_value_fn(struct repository *repo, struct strbuf *buf); enum output_format { + FORMAT_TABLE, FORMAT_KEYVALUE, FORMAT_NUL_TERMINATED, }; @@ -77,6 +85,24 @@ static get_value_fn *get_value_fn_for_key(const char *key) return found ? found->get_value : NULL; } +static void print_field(enum output_format format, const char *key, + struct strbuf *valbuf, struct strbuf *quotbuf) +{ + strbuf_reset(quotbuf); + + switch (format) { + case FORMAT_KEYVALUE: + quote_c_style(valbuf->buf, quotbuf, NULL, 0); + printf("%s=%s\n", key, quotbuf->buf); + break; + case FORMAT_NUL_TERMINATED: + printf("%s\n%s%c", key, valbuf->buf, '\0'); + break; + default: + BUG("not a valid output format: %d", format); + } +} + static int print_fields(int argc, const char **argv, struct repository *repo, enum output_format format) @@ -97,21 +123,8 @@ static int print_fields(int argc, const char **argv, } strbuf_reset(&valbuf); - strbuf_reset("buf); - get_value(repo, &valbuf); - - switch (format) { - case FORMAT_KEYVALUE: - quote_c_style(valbuf.buf, "buf, NULL, 0); - printf("%s=%s\n", key, quotbuf.buf); - break; - case FORMAT_NUL_TERMINATED: - printf("%s\n%s%c", key, valbuf.buf, '\0'); - break; - default: - BUG("not a valid output format: %d", format); - } + print_field(format, key, &valbuf, "buf); } strbuf_release(&valbuf); @@ -119,6 +132,24 @@ static int print_fields(int argc, const char **argv, return ret; } +static void print_all_fields(struct repository *repo, + enum output_format format) +{ + struct strbuf valbuf = STRBUF_INIT; + struct strbuf quotbuf = STRBUF_INIT; + + for (unsigned long i = 0; i < ARRAY_SIZE(repo_info_fields); i++) { + struct field field = repo_info_fields[i]; + + strbuf_reset(&valbuf); + field.get_value(repo, &valbuf); + print_field(format, field.key, &valbuf, "buf); + } + + strbuf_release(&valbuf); + strbuf_release("buf); +} + static int parse_format_cb(const struct option *opt, const char *arg, int unset UNUSED) { @@ -130,16 +161,19 @@ static int parse_format_cb(const struct option *opt, *format = FORMAT_NUL_TERMINATED; else if (!strcmp(arg, "keyvalue")) *format = FORMAT_KEYVALUE; + else if (!strcmp(arg, "table")) + *format = FORMAT_TABLE; else die(_("invalid format '%s'"), arg); return 0; } -static int repo_info(int argc, const char **argv, const char *prefix, - struct repository *repo) +static int cmd_repo_info(int argc, const char **argv, const char *prefix, + struct repository *repo) { enum output_format format = FORMAT_KEYVALUE; + int all_keys = 0; struct option options[] = { OPT_CALLBACK_F(0, "format", &format, N_("format"), N_("output format"), @@ -148,20 +182,390 @@ static int repo_info(int argc, const char **argv, const char *prefix, N_("synonym for --format=nul"), PARSE_OPT_NONEG | PARSE_OPT_NOARG, parse_format_cb), + OPT_BOOL(0, "all", &all_keys, N_("return all keys")), OPT_END() }; argc = parse_options(argc, argv, prefix, options, repo_usage, 0); + if (format != FORMAT_KEYVALUE && format != FORMAT_NUL_TERMINATED) + die(_("unsupported output format")); + + if (all_keys) { + if (argc) + die(_("--all and <key> cannot be used together")); + + print_all_fields(repo, format); + return 0; + } return print_fields(argc, argv, repo, format); } +struct ref_stats { + size_t branches; + size_t remotes; + size_t tags; + size_t others; +}; + +struct object_stats { + size_t tags; + size_t commits; + size_t trees; + size_t blobs; +}; + +struct repo_structure { + struct ref_stats refs; + struct object_stats objects; +}; + +struct stats_table { + struct string_list rows; + + int name_col_width; + int value_col_width; +}; + +/* + * Holds column data that gets stored for each row. + */ +struct stats_table_entry { + char *value; +}; + +static void stats_table_vaddf(struct stats_table *table, + struct stats_table_entry *entry, + const char *format, va_list ap) +{ + struct strbuf buf = STRBUF_INIT; + struct string_list_item *item; + char *formatted_name; + int name_width; + + strbuf_vaddf(&buf, format, ap); + formatted_name = strbuf_detach(&buf, NULL); + name_width = utf8_strwidth(formatted_name); + + item = string_list_append_nodup(&table->rows, formatted_name); + item->util = entry; + + if (name_width > table->name_col_width) + table->name_col_width = name_width; + if (entry) { + int value_width = utf8_strwidth(entry->value); + if (value_width > table->value_col_width) + table->value_col_width = value_width; + } +} + +static void stats_table_addf(struct stats_table *table, const char *format, ...) +{ + va_list ap; + + va_start(ap, format); + stats_table_vaddf(table, NULL, format, ap); + va_end(ap); +} + +static void stats_table_count_addf(struct stats_table *table, size_t value, + const char *format, ...) +{ + struct stats_table_entry *entry; + va_list ap; + + CALLOC_ARRAY(entry, 1); + entry->value = xstrfmt("%" PRIuMAX, (uintmax_t)value); + + va_start(ap, format); + stats_table_vaddf(table, entry, format, ap); + va_end(ap); +} + +static inline size_t get_total_reference_count(struct ref_stats *stats) +{ + return stats->branches + stats->remotes + stats->tags + stats->others; +} + +static inline size_t get_total_object_count(struct object_stats *stats) +{ + return stats->tags + stats->commits + stats->trees + stats->blobs; +} + +static void stats_table_setup_structure(struct stats_table *table, + struct repo_structure *stats) +{ + struct object_stats *objects = &stats->objects; + struct ref_stats *refs = &stats->refs; + size_t object_total; + size_t ref_total; + + ref_total = get_total_reference_count(refs); + stats_table_addf(table, "* %s", _("References")); + stats_table_count_addf(table, ref_total, " * %s", _("Count")); + stats_table_count_addf(table, refs->branches, " * %s", _("Branches")); + stats_table_count_addf(table, refs->tags, " * %s", _("Tags")); + stats_table_count_addf(table, refs->remotes, " * %s", _("Remotes")); + stats_table_count_addf(table, refs->others, " * %s", _("Others")); + + object_total = get_total_object_count(objects); + stats_table_addf(table, ""); + stats_table_addf(table, "* %s", _("Reachable objects")); + stats_table_count_addf(table, object_total, " * %s", _("Count")); + stats_table_count_addf(table, objects->commits, " * %s", _("Commits")); + stats_table_count_addf(table, objects->trees, " * %s", _("Trees")); + stats_table_count_addf(table, objects->blobs, " * %s", _("Blobs")); + stats_table_count_addf(table, objects->tags, " * %s", _("Tags")); +} + +static void stats_table_print_structure(const struct stats_table *table) +{ + const char *name_col_title = _("Repository structure"); + const char *value_col_title = _("Value"); + int name_col_width = utf8_strwidth(name_col_title); + int value_col_width = utf8_strwidth(value_col_title); + struct string_list_item *item; + + if (table->name_col_width > name_col_width) + name_col_width = table->name_col_width; + if (table->value_col_width > value_col_width) + value_col_width = table->value_col_width; + + printf("| %-*s | %-*s |\n", name_col_width, name_col_title, + value_col_width, value_col_title); + printf("| "); + for (int i = 0; i < name_col_width; i++) + putchar('-'); + printf(" | "); + for (int i = 0; i < value_col_width; i++) + putchar('-'); + printf(" |\n"); + + for_each_string_list_item(item, &table->rows) { + struct stats_table_entry *entry = item->util; + const char *value = ""; + + if (entry) { + struct stats_table_entry *entry = item->util; + value = entry->value; + } + + printf("| %-*s | %*s |\n", name_col_width, item->string, + value_col_width, value); + } +} + +static void stats_table_clear(struct stats_table *table) +{ + struct stats_table_entry *entry; + struct string_list_item *item; + + for_each_string_list_item(item, &table->rows) { + entry = item->util; + if (entry) + free(entry->value); + } + + string_list_clear(&table->rows, 1); +} + +static void structure_keyvalue_print(struct repo_structure *stats, + char key_delim, char value_delim) +{ + printf("references.branches.count%c%" PRIuMAX "%c", key_delim, + (uintmax_t)stats->refs.branches, value_delim); + printf("references.tags.count%c%" PRIuMAX "%c", key_delim, + (uintmax_t)stats->refs.tags, value_delim); + printf("references.remotes.count%c%" PRIuMAX "%c", key_delim, + (uintmax_t)stats->refs.remotes, value_delim); + printf("references.others.count%c%" PRIuMAX "%c", key_delim, + (uintmax_t)stats->refs.others, value_delim); + + printf("objects.commits.count%c%" PRIuMAX "%c", key_delim, + (uintmax_t)stats->objects.commits, value_delim); + printf("objects.trees.count%c%" PRIuMAX "%c", key_delim, + (uintmax_t)stats->objects.trees, value_delim); + printf("objects.blobs.count%c%" PRIuMAX "%c", key_delim, + (uintmax_t)stats->objects.blobs, value_delim); + printf("objects.tags.count%c%" PRIuMAX "%c", key_delim, + (uintmax_t)stats->objects.tags, value_delim); + + fflush(stdout); +} + +struct count_references_data { + struct ref_stats *stats; + struct rev_info *revs; + struct progress *progress; +}; + +static int count_references(const struct reference *ref, void *cb_data) +{ + struct count_references_data *data = cb_data; + struct ref_stats *stats = data->stats; + size_t ref_count; + + switch (ref_kind_from_refname(ref->name)) { + case FILTER_REFS_BRANCHES: + stats->branches++; + break; + case FILTER_REFS_REMOTES: + stats->remotes++; + break; + case FILTER_REFS_TAGS: + stats->tags++; + break; + case FILTER_REFS_OTHERS: + stats->others++; + break; + default: + BUG("unexpected reference type"); + } + + /* + * While iterating through references for counting, also add OIDs in + * preparation for the path walk. + */ + add_pending_oid(data->revs, NULL, ref->oid, 0); + + ref_count = get_total_reference_count(stats); + display_progress(data->progress, ref_count); + + return 0; +} + +static void structure_count_references(struct ref_stats *stats, + struct rev_info *revs, + struct repository *repo, + int show_progress) +{ + struct count_references_data data = { + .stats = stats, + .revs = revs, + }; + + if (show_progress) + data.progress = start_delayed_progress(repo, + _("Counting references"), 0); + + refs_for_each_ref(get_main_ref_store(repo), count_references, &data); + stop_progress(&data.progress); +} + +struct count_objects_data { + struct object_stats *stats; + struct progress *progress; +}; + +static int count_objects(const char *path UNUSED, struct oid_array *oids, + enum object_type type, void *cb_data) +{ + struct count_objects_data *data = cb_data; + struct object_stats *stats = data->stats; + size_t object_count; + + switch (type) { + case OBJ_TAG: + stats->tags += oids->nr; + break; + case OBJ_COMMIT: + stats->commits += oids->nr; + break; + case OBJ_TREE: + stats->trees += oids->nr; + break; + case OBJ_BLOB: + stats->blobs += oids->nr; + break; + default: + BUG("invalid object type"); + } + + object_count = get_total_object_count(stats); + display_progress(data->progress, object_count); + + return 0; +} + +static void structure_count_objects(struct object_stats *stats, + struct rev_info *revs, + struct repository *repo, int show_progress) +{ + struct path_walk_info info = PATH_WALK_INFO_INIT; + struct count_objects_data data = { + .stats = stats, + }; + + info.revs = revs; + info.path_fn = count_objects; + info.path_fn_data = &data; + + if (show_progress) + data.progress = start_delayed_progress(repo, _("Counting objects"), 0); + + walk_objects_by_path(&info); + path_walk_info_clear(&info); + stop_progress(&data.progress); +} + +static int cmd_repo_structure(int argc, const char **argv, const char *prefix, + struct repository *repo) +{ + struct stats_table table = { + .rows = STRING_LIST_INIT_DUP, + }; + enum output_format format = FORMAT_TABLE; + struct repo_structure stats = { 0 }; + struct rev_info revs; + int show_progress = -1; + struct option options[] = { + OPT_CALLBACK_F(0, "format", &format, N_("format"), + N_("output format"), + PARSE_OPT_NONEG, parse_format_cb), + OPT_BOOL(0, "progress", &show_progress, N_("show progress")), + OPT_END() + }; + + argc = parse_options(argc, argv, prefix, options, repo_usage, 0); + if (argc) + usage(_("too many arguments")); + + repo_init_revisions(repo, &revs, prefix); + + if (show_progress < 0) + show_progress = isatty(2); + + structure_count_references(&stats.refs, &revs, repo, show_progress); + structure_count_objects(&stats.objects, &revs, repo, show_progress); + + switch (format) { + case FORMAT_TABLE: + stats_table_setup_structure(&table, &stats); + stats_table_print_structure(&table); + break; + case FORMAT_KEYVALUE: + structure_keyvalue_print(&stats, '=', '\n'); + break; + case FORMAT_NUL_TERMINATED: + structure_keyvalue_print(&stats, '\n', '\0'); + break; + default: + BUG("invalid output format"); + } + + stats_table_clear(&table); + release_revisions(&revs); + + return 0; +} + int cmd_repo(int argc, const char **argv, const char *prefix, struct repository *repo) { parse_opt_subcommand_fn *fn = NULL; struct option options[] = { - OPT_SUBCOMMAND("info", &fn, repo_info), + OPT_SUBCOMMAND("info", &fn, cmd_repo_info), + OPT_SUBCOMMAND("structure", &fn, cmd_repo_structure), OPT_END() }; diff --git a/builtin/reset.c b/builtin/reset.c index ed35802af1..088449e120 100644 --- a/builtin/reset.c +++ b/builtin/reset.c @@ -346,7 +346,7 @@ int cmd_reset(int argc, struct object_id oid; struct pathspec pathspec; int intent_to_add = 0; - struct add_p_opt add_p_opt = ADD_P_OPT_INIT; + struct interactive_options interactive_opts = INTERACTIVE_OPTIONS_INIT; const struct option options[] = { OPT__QUIET(&quiet, N_("be quiet, only report errors")), OPT_BOOL(0, "no-refresh", &no_refresh, @@ -371,8 +371,8 @@ int cmd_reset(int argc, PARSE_OPT_OPTARG, option_parse_recurse_submodules_worktree_updater), OPT_BOOL('p', "patch", &patch_mode, N_("select hunks interactively")), - OPT_DIFF_UNIFIED(&add_p_opt.context), - OPT_DIFF_INTERHUNK_CONTEXT(&add_p_opt.interhunkcontext), + OPT_DIFF_UNIFIED(&interactive_opts.context), + OPT_DIFF_INTERHUNK_CONTEXT(&interactive_opts.interhunkcontext), OPT_BOOL('N', "intent-to-add", &intent_to_add, N_("record only the fact that removed paths will be added later")), OPT_PATHSPEC_FROM_FILE(&pathspec_from_file), @@ -423,9 +423,9 @@ int cmd_reset(int argc, oidcpy(&oid, &tree->object.oid); } - if (add_p_opt.context < -1) + if (interactive_opts.context < -1) die(_("'%s' cannot be negative"), "--unified"); - if (add_p_opt.interhunkcontext < -1) + if (interactive_opts.interhunkcontext < -1) die(_("'%s' cannot be negative"), "--inter-hunk-context"); prepare_repo_settings(the_repository); @@ -436,12 +436,12 @@ int cmd_reset(int argc, die(_("options '%s' and '%s' cannot be used together"), "--patch", "--{hard,mixed,soft}"); trace2_cmd_mode("patch-interactive"); update_ref_status = !!run_add_p(the_repository, ADD_P_RESET, - &add_p_opt, rev, &pathspec); + &interactive_opts, rev, &pathspec); goto cleanup; } else { - if (add_p_opt.context != -1) + if (interactive_opts.context != -1) die(_("the option '%s' requires '%s'"), "--unified", "--patch"); - if (add_p_opt.interhunkcontext != -1) + if (interactive_opts.interhunkcontext != -1) die(_("the option '%s' requires '%s'"), "--inter-hunk-context", "--patch"); } diff --git a/builtin/rev-parse.c b/builtin/rev-parse.c index 9da92b990d..9032cc6327 100644 --- a/builtin/rev-parse.c +++ b/builtin/rev-parse.c @@ -217,19 +217,17 @@ static int show_default(void) return 0; } -static int show_reference(const char *refname, const char *referent UNUSED, const struct object_id *oid, - int flag UNUSED, void *cb_data UNUSED) +static int show_reference(const struct reference *ref, void *cb_data UNUSED) { - if (ref_excluded(&ref_excludes, refname)) + if (ref_excluded(&ref_excludes, ref->name)) return 0; - show_rev(NORMAL, oid, refname); + show_rev(NORMAL, ref->oid, ref->name); return 0; } -static int anti_reference(const char *refname, const char *referent UNUSED, const struct object_id *oid, - int flag UNUSED, void *cb_data UNUSED) +static int anti_reference(const struct reference *ref, void *cb_data UNUSED) { - show_rev(REVERSED, oid, refname); + show_rev(REVERSED, ref->oid, ref->name); return 0; } @@ -1107,11 +1105,20 @@ int cmd_rev_parse(int argc, const char *val = arg ? arg : "storage"; if (strcmp(val, "storage") && + strcmp(val, "compat") && strcmp(val, "input") && strcmp(val, "output")) die(_("unknown mode for --show-object-format: %s"), arg); - puts(the_hash_algo->name); + + if (!strcmp(val, "compat")) { + if (the_repository->compat_hash_algo) + puts(the_repository->compat_hash_algo->name); + else + putchar('\n'); + } else { + puts(the_hash_algo->name); + } continue; } if (!strcmp(arg, "--show-ref-format")) { diff --git a/builtin/show-branch.c b/builtin/show-branch.c index 441babf2e3..10475a6b5e 100644 --- a/builtin/show-branch.c +++ b/builtin/show-branch.c @@ -413,34 +413,32 @@ static int append_ref(const char *refname, const struct object_id *oid, return 0; } -static int append_head_ref(const char *refname, const char *referent UNUSED, const struct object_id *oid, - int flag UNUSED, void *cb_data UNUSED) +static int append_head_ref(const struct reference *ref, void *cb_data UNUSED) { struct object_id tmp; int ofs = 11; - if (!starts_with(refname, "refs/heads/")) + if (!starts_with(ref->name, "refs/heads/")) return 0; /* If both heads/foo and tags/foo exists, get_sha1 would * get confused. */ - if (repo_get_oid(the_repository, refname + ofs, &tmp) || !oideq(&tmp, oid)) + if (repo_get_oid(the_repository, ref->name + ofs, &tmp) || !oideq(&tmp, ref->oid)) ofs = 5; - return append_ref(refname + ofs, oid, 0); + return append_ref(ref->name + ofs, ref->oid, 0); } -static int append_remote_ref(const char *refname, const char *referent UNUSED, const struct object_id *oid, - int flag UNUSED, void *cb_data UNUSED) +static int append_remote_ref(const struct reference *ref, void *cb_data UNUSED) { struct object_id tmp; int ofs = 13; - if (!starts_with(refname, "refs/remotes/")) + if (!starts_with(ref->name, "refs/remotes/")) return 0; /* If both heads/foo and tags/foo exists, get_sha1 would * get confused. */ - if (repo_get_oid(the_repository, refname + ofs, &tmp) || !oideq(&tmp, oid)) + if (repo_get_oid(the_repository, ref->name + ofs, &tmp) || !oideq(&tmp, ref->oid)) ofs = 5; - return append_ref(refname + ofs, oid, 0); + return append_ref(ref->name + ofs, ref->oid, 0); } static int append_tag_ref(const char *refname, const struct object_id *oid, @@ -454,27 +452,26 @@ static int append_tag_ref(const char *refname, const struct object_id *oid, static const char *match_ref_pattern = NULL; static int match_ref_slash = 0; -static int append_matching_ref(const char *refname, const char *referent UNUSED, const struct object_id *oid, - int flag, void *cb_data) +static int append_matching_ref(const struct reference *ref, void *cb_data) { /* we want to allow pattern hold/<asterisk> to show all * branches under refs/heads/hold/, and v0.99.9? to show * refs/tags/v0.99.9a and friends. */ const char *tail; - int slash = count_slashes(refname); - for (tail = refname; *tail && match_ref_slash < slash; ) + int slash = count_slashes(ref->name); + for (tail = ref->name; *tail && match_ref_slash < slash; ) if (*tail++ == '/') slash--; if (!*tail) return 0; if (wildmatch(match_ref_pattern, tail, 0)) return 0; - if (starts_with(refname, "refs/heads/")) - return append_head_ref(refname, NULL, oid, flag, cb_data); - if (starts_with(refname, "refs/tags/")) - return append_tag_ref(refname, oid, flag, cb_data); - return append_ref(refname, oid, 0); + if (starts_with(ref->name, "refs/heads/")) + return append_head_ref(ref, cb_data); + if (starts_with(ref->name, "refs/tags/")) + return append_tag_ref(ref->name, ref->oid, ref->flags, cb_data); + return append_ref(ref->name, ref->oid, 0); } static void snarf_refs(int head, int remotes) diff --git a/builtin/show-ref.c b/builtin/show-ref.c index 0b6f9edf86..4d4984e4e0 100644 --- a/builtin/show-ref.c +++ b/builtin/show-ref.c @@ -31,31 +31,31 @@ struct show_one_options { }; static void show_one(const struct show_one_options *opts, - const char *refname, const struct object_id *oid) + const struct reference *ref) { const char *hex; struct object_id peeled; - if (!odb_has_object(the_repository->objects, oid, + if (!odb_has_object(the_repository->objects, ref->oid, HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR)) - die("git show-ref: bad ref %s (%s)", refname, - oid_to_hex(oid)); + die("git show-ref: bad ref %s (%s)", ref->name, + oid_to_hex(ref->oid)); if (opts->quiet) return; - hex = repo_find_unique_abbrev(the_repository, oid, opts->abbrev); + hex = repo_find_unique_abbrev(the_repository, ref->oid, opts->abbrev); if (opts->hash_only) printf("%s\n", hex); else - printf("%s %s\n", hex, refname); + printf("%s %s\n", hex, ref->name); if (!opts->deref_tags) return; - if (!peel_iterated_oid(the_repository, oid, &peeled)) { + if (!reference_get_peeled_oid(the_repository, ref, &peeled)) { hex = repo_find_unique_abbrev(the_repository, &peeled, opts->abbrev); - printf("%s %s^{}\n", hex, refname); + printf("%s %s^{}\n", hex, ref->name); } } @@ -66,26 +66,25 @@ struct show_ref_data { int show_head; }; -static int show_ref(const char *refname, const char *referent UNUSED, const struct object_id *oid, - int flag UNUSED, void *cbdata) +static int show_ref(const struct reference *ref, void *cbdata) { struct show_ref_data *data = cbdata; - if (data->show_head && !strcmp(refname, "HEAD")) + if (data->show_head && !strcmp(ref->name, "HEAD")) goto match; if (data->patterns) { - int reflen = strlen(refname); + int reflen = strlen(ref->name); const char **p = data->patterns, *m; while ((m = *p++) != NULL) { int len = strlen(m); if (len > reflen) continue; - if (memcmp(m, refname + reflen - len, len)) + if (memcmp(m, ref->name + reflen - len, len)) continue; if (len == reflen) goto match; - if (refname[reflen - len - 1] == '/') + if (ref->name[reflen - len - 1] == '/') goto match; } return 0; @@ -94,18 +93,15 @@ static int show_ref(const char *refname, const char *referent UNUSED, const stru match: data->found_match++; - show_one(data->show_one_opts, refname, oid); + show_one(data->show_one_opts, ref); return 0; } -static int add_existing(const char *refname, - const char *referent UNUSED, - const struct object_id *oid UNUSED, - int flag UNUSED, void *cbdata) +static int add_existing(const struct reference *ref, void *cbdata) { struct string_list *list = (struct string_list *)cbdata; - string_list_insert(list, refname); + string_list_insert(list, ref->name); return 0; } @@ -179,12 +175,18 @@ static int cmd_show_ref__verify(const struct show_one_options *show_one_opts, if ((starts_with(*refs, "refs/") || refname_is_safe(*refs)) && !refs_read_ref(get_main_ref_store(the_repository), *refs, &oid)) { - show_one(show_one_opts, *refs, &oid); - } - else if (!show_one_opts->quiet) + struct reference ref = { + .name = *refs, + .oid = &oid, + }; + + show_one(show_one_opts, &ref); + } else if (!show_one_opts->quiet) { die("'%s' - not a valid ref", *refs); - else + } else { return 1; + } + refs++; } diff --git a/builtin/sparse-checkout.c b/builtin/sparse-checkout.c index 8c333b3e2e..15d51e60a8 100644 --- a/builtin/sparse-checkout.c +++ b/builtin/sparse-checkout.c @@ -2,6 +2,7 @@ #define DISABLE_SIGN_COMPARE_WARNINGS #include "builtin.h" +#include "abspath.h" #include "config.h" #include "dir.h" #include "environment.h" @@ -23,7 +24,7 @@ static const char *empty_base = ""; static char const * const builtin_sparse_checkout_usage[] = { - N_("git sparse-checkout (init | list | set | add | reapply | disable | check-rules) [<options>]"), + N_("git sparse-checkout (init | list | set | add | reapply | disable | check-rules | clean) [<options>]"), NULL }; @@ -204,12 +205,12 @@ static void clean_tracked_sparse_directories(struct repository *r) ensure_full_index(r->index); } -static int update_working_directory(struct pattern_list *pl) +static int update_working_directory(struct repository *r, + struct pattern_list *pl) { enum update_sparsity_result result; struct unpack_trees_options o; struct lock_file lock_file = LOCK_INIT; - struct repository *r = the_repository; struct pattern_list *old_pl; /* If no branch has been checked out, there are no updates to make. */ @@ -327,7 +328,8 @@ static void write_cone_to_file(FILE *fp, struct pattern_list *pl) string_list_clear(&sl, 0); } -static int write_patterns_and_update(struct pattern_list *pl) +static int write_patterns_and_update(struct repository *repo, + struct pattern_list *pl) { char *sparse_filename; FILE *fp; @@ -336,15 +338,15 @@ static int write_patterns_and_update(struct pattern_list *pl) sparse_filename = get_sparse_checkout_filename(); - if (safe_create_leading_directories(the_repository, sparse_filename)) + if (safe_create_leading_directories(repo, sparse_filename)) die(_("failed to create directory for sparse-checkout file")); hold_lock_file_for_update(&lk, sparse_filename, LOCK_DIE_ON_ERROR); - result = update_working_directory(pl); + result = update_working_directory(repo, pl); if (result) { rollback_lock_file(&lk); - update_working_directory(NULL); + update_working_directory(repo, NULL); goto out; } @@ -372,25 +374,26 @@ enum sparse_checkout_mode { MODE_CONE_PATTERNS = 2, }; -static int set_config(enum sparse_checkout_mode mode) +static int set_config(struct repository *repo, + enum sparse_checkout_mode mode) { /* Update to use worktree config, if not already. */ - if (init_worktree_config(the_repository)) { + if (init_worktree_config(repo)) { error(_("failed to initialize worktree config")); return 1; } - if (repo_config_set_worktree_gently(the_repository, + if (repo_config_set_worktree_gently(repo, "core.sparseCheckout", mode ? "true" : "false") || - repo_config_set_worktree_gently(the_repository, + repo_config_set_worktree_gently(repo, "core.sparseCheckoutCone", mode == MODE_CONE_PATTERNS ? "true" : "false")) return 1; if (mode == MODE_NO_PATTERNS) - return set_sparse_index_config(the_repository, 0); + return set_sparse_index_config(repo, 0); return 0; } @@ -410,7 +413,7 @@ static enum sparse_checkout_mode update_cone_mode(int *cone_mode) { return MODE_ALL_PATTERNS; } -static int update_modes(int *cone_mode, int *sparse_index) +static int update_modes(struct repository *repo, int *cone_mode, int *sparse_index) { int mode, record_mode; @@ -418,20 +421,20 @@ static int update_modes(int *cone_mode, int *sparse_index) record_mode = (*cone_mode != -1) || !core_apply_sparse_checkout; mode = update_cone_mode(cone_mode); - if (record_mode && set_config(mode)) + if (record_mode && set_config(repo, mode)) return 1; /* Set sparse-index/non-sparse-index mode if specified */ if (*sparse_index >= 0) { - if (set_sparse_index_config(the_repository, *sparse_index) < 0) + if (set_sparse_index_config(repo, *sparse_index) < 0) die(_("failed to modify sparse-index config")); /* force an index rewrite */ - repo_read_index(the_repository); - the_repository->index->updated_workdir = 1; + repo_read_index(repo); + repo->index->updated_workdir = 1; if (!*sparse_index) - ensure_full_index(the_repository->index); + ensure_full_index(repo->index); } return 0; @@ -448,7 +451,7 @@ static struct sparse_checkout_init_opts { } init_opts; static int sparse_checkout_init(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { struct pattern_list pl; char *sparse_filename; @@ -464,7 +467,7 @@ static int sparse_checkout_init(int argc, const char **argv, const char *prefix, }; setup_work_tree(); - repo_read_index(the_repository); + repo_read_index(repo); init_opts.cone_mode = -1; init_opts.sparse_index = -1; @@ -473,7 +476,7 @@ static int sparse_checkout_init(int argc, const char **argv, const char *prefix, builtin_sparse_checkout_init_options, builtin_sparse_checkout_init_usage, 0); - if (update_modes(&init_opts.cone_mode, &init_opts.sparse_index)) + if (update_modes(repo, &init_opts.cone_mode, &init_opts.sparse_index)) return 1; memset(&pl, 0, sizeof(pl)); @@ -485,14 +488,14 @@ static int sparse_checkout_init(int argc, const char **argv, const char *prefix, if (res >= 0) { free(sparse_filename); clear_pattern_list(&pl); - return update_working_directory(NULL); + return update_working_directory(repo, NULL); } - if (repo_get_oid(the_repository, "HEAD", &oid)) { + if (repo_get_oid(repo, "HEAD", &oid)) { FILE *fp; /* assume we are in a fresh repo, but update the sparse-checkout file */ - if (safe_create_leading_directories(the_repository, sparse_filename)) + if (safe_create_leading_directories(repo, sparse_filename)) die(_("unable to create leading directories of %s"), sparse_filename); fp = xfopen(sparse_filename, "w"); @@ -511,7 +514,7 @@ static int sparse_checkout_init(int argc, const char **argv, const char *prefix, add_pattern("!/*/", empty_base, 0, &pl, 0); pl.use_cone_patterns = init_opts.cone_mode; - return write_patterns_and_update(&pl); + return write_patterns_and_update(repo, &pl); } static void insert_recursive_pattern(struct pattern_list *pl, struct strbuf *path) @@ -674,7 +677,8 @@ static void add_patterns_literal(int argc, const char **argv, add_patterns_from_input(pl, argc, argv, use_stdin ? stdin : NULL); } -static int modify_pattern_list(struct strvec *args, int use_stdin, +static int modify_pattern_list(struct repository *repo, + struct strvec *args, int use_stdin, enum modify_type m) { int result; @@ -696,22 +700,23 @@ static int modify_pattern_list(struct strvec *args, int use_stdin, } if (!core_apply_sparse_checkout) { - set_config(MODE_ALL_PATTERNS); + set_config(repo, MODE_ALL_PATTERNS); core_apply_sparse_checkout = 1; changed_config = 1; } - result = write_patterns_and_update(pl); + result = write_patterns_and_update(repo, pl); if (result && changed_config) - set_config(MODE_NO_PATTERNS); + set_config(repo, MODE_NO_PATTERNS); clear_pattern_list(pl); free(pl); return result; } -static void sanitize_paths(struct strvec *args, +static void sanitize_paths(struct repository *repo, + struct strvec *args, const char *prefix, int skip_checks) { int i; @@ -752,7 +757,7 @@ static void sanitize_paths(struct strvec *args, for (i = 0; i < args->nr; i++) { struct cache_entry *ce; - struct index_state *index = the_repository->index; + struct index_state *index = repo->index; int pos = index_name_pos(index, args->v[i], strlen(args->v[i])); if (pos < 0) @@ -779,7 +784,7 @@ static struct sparse_checkout_add_opts { } add_opts; static int sparse_checkout_add(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { static struct option builtin_sparse_checkout_add_options[] = { OPT_BOOL_F(0, "skip-checks", &add_opts.skip_checks, @@ -796,7 +801,7 @@ static int sparse_checkout_add(int argc, const char **argv, const char *prefix, if (!core_apply_sparse_checkout) die(_("no sparse-checkout to add to")); - repo_read_index(the_repository); + repo_read_index(repo); argc = parse_options(argc, argv, prefix, builtin_sparse_checkout_add_options, @@ -804,9 +809,9 @@ static int sparse_checkout_add(int argc, const char **argv, const char *prefix, for (int i = 0; i < argc; i++) strvec_push(&patterns, argv[i]); - sanitize_paths(&patterns, prefix, add_opts.skip_checks); + sanitize_paths(repo, &patterns, prefix, add_opts.skip_checks); - ret = modify_pattern_list(&patterns, add_opts.use_stdin, ADD); + ret = modify_pattern_list(repo, &patterns, add_opts.use_stdin, ADD); strvec_clear(&patterns); return ret; @@ -825,7 +830,7 @@ static struct sparse_checkout_set_opts { } set_opts; static int sparse_checkout_set(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { int default_patterns_nr = 2; const char *default_patterns[] = {"/*", "!/*/", NULL}; @@ -847,7 +852,7 @@ static int sparse_checkout_set(int argc, const char **argv, const char *prefix, int ret; setup_work_tree(); - repo_read_index(the_repository); + repo_read_index(repo); set_opts.cone_mode = -1; set_opts.sparse_index = -1; @@ -856,7 +861,7 @@ static int sparse_checkout_set(int argc, const char **argv, const char *prefix, builtin_sparse_checkout_set_options, builtin_sparse_checkout_set_usage, 0); - if (update_modes(&set_opts.cone_mode, &set_opts.sparse_index)) + if (update_modes(repo, &set_opts.cone_mode, &set_opts.sparse_index)) return 1; /* @@ -870,10 +875,10 @@ static int sparse_checkout_set(int argc, const char **argv, const char *prefix, } else { for (int i = 0; i < argc; i++) strvec_push(&patterns, argv[i]); - sanitize_paths(&patterns, prefix, set_opts.skip_checks); + sanitize_paths(repo, &patterns, prefix, set_opts.skip_checks); } - ret = modify_pattern_list(&patterns, set_opts.use_stdin, REPLACE); + ret = modify_pattern_list(repo, &patterns, set_opts.use_stdin, REPLACE); strvec_clear(&patterns); return ret; @@ -891,7 +896,7 @@ static struct sparse_checkout_reapply_opts { static int sparse_checkout_reapply(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { static struct option builtin_sparse_checkout_reapply_options[] = { OPT_BOOL(0, "cone", &reapply_opts.cone_mode, @@ -912,12 +917,107 @@ static int sparse_checkout_reapply(int argc, const char **argv, builtin_sparse_checkout_reapply_options, builtin_sparse_checkout_reapply_usage, 0); - repo_read_index(the_repository); + repo_read_index(repo); - if (update_modes(&reapply_opts.cone_mode, &reapply_opts.sparse_index)) + if (update_modes(repo, &reapply_opts.cone_mode, &reapply_opts.sparse_index)) return 1; - return update_working_directory(NULL); + return update_working_directory(repo, NULL); +} + +static char const * const builtin_sparse_checkout_clean_usage[] = { + "git sparse-checkout clean [-n|--dry-run]", + NULL +}; + +static int list_file_iterator(const char *path, const void *data) +{ + const char *msg = data; + + printf(msg, path); + return 0; +} + +static void list_every_file_in_dir(const char *msg, + const char *directory) +{ + struct strbuf path = STRBUF_INIT; + + strbuf_addstr(&path, directory); + for_each_file_in_dir(&path, list_file_iterator, msg); + strbuf_release(&path); +} + +static const char *msg_remove = N_("Removing %s\n"); +static const char *msg_would_remove = N_("Would remove %s\n"); + +static int sparse_checkout_clean(int argc, const char **argv, + const char *prefix, + struct repository *repo) +{ + struct strbuf full_path = STRBUF_INIT; + const char *msg = msg_remove; + size_t worktree_len; + int force = 0, dry_run = 0, verbose = 0; + int require_force = 1; + + struct option builtin_sparse_checkout_clean_options[] = { + OPT__DRY_RUN(&dry_run, N_("dry run")), + OPT__FORCE(&force, N_("force"), PARSE_OPT_NOCOMPLETE), + OPT__VERBOSE(&verbose, N_("report each affected file, not just directories")), + OPT_END(), + }; + + setup_work_tree(); + if (!core_apply_sparse_checkout) + die(_("must be in a sparse-checkout to clean directories")); + if (!core_sparse_checkout_cone) + die(_("must be in a cone-mode sparse-checkout to clean directories")); + + argc = parse_options(argc, argv, prefix, + builtin_sparse_checkout_clean_options, + builtin_sparse_checkout_clean_usage, 0); + + repo_config_get_bool(repo, "clean.requireforce", &require_force); + if (require_force && !force && !dry_run) + die(_("for safety, refusing to clean without one of --force or --dry-run")); + + if (dry_run) + msg = msg_would_remove; + + if (repo_read_index(repo) < 0) + die(_("failed to read index")); + + if (convert_to_sparse(repo->index, SPARSE_INDEX_MEMORY_ONLY) || + repo->index->sparse_index == INDEX_EXPANDED) + die(_("failed to convert index to a sparse index; resolve merge conflicts and try again")); + + strbuf_addstr(&full_path, repo->worktree); + strbuf_addch(&full_path, '/'); + worktree_len = full_path.len; + + for (size_t i = 0; i < repo->index->cache_nr; i++) { + struct cache_entry *ce = repo->index->cache[i]; + if (!S_ISSPARSEDIR(ce->ce_mode)) + continue; + strbuf_setlen(&full_path, worktree_len); + strbuf_add(&full_path, ce->name, ce->ce_namelen); + + if (!is_directory(full_path.buf)) + continue; + + if (verbose) + list_every_file_in_dir(msg, ce->name); + else + printf(msg, ce->name); + + if (dry_run <= 0 && + remove_dir_recursively(&full_path, 0)) + warning_errno(_("failed to remove '%s'"), ce->name); + } + + strbuf_release(&full_path); + return 0; } static char const * const builtin_sparse_checkout_disable_usage[] = { @@ -927,7 +1027,7 @@ static char const * const builtin_sparse_checkout_disable_usage[] = { static int sparse_checkout_disable(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { static struct option builtin_sparse_checkout_disable_options[] = { OPT_END(), @@ -955,7 +1055,7 @@ static int sparse_checkout_disable(int argc, const char **argv, * are expecting to do that when disabling sparse-checkout. */ give_advice_on_expansion = 0; - repo_read_index(the_repository); + repo_read_index(repo); memset(&pl, 0, sizeof(pl)); hashmap_init(&pl.recursive_hashmap, pl_hashmap_cmp, NULL, 0); @@ -966,13 +1066,13 @@ static int sparse_checkout_disable(int argc, const char **argv, add_pattern("/*", empty_base, 0, &pl, 0); prepare_repo_settings(the_repository); - the_repository->settings.sparse_index = 0; + repo->settings.sparse_index = 0; - if (update_working_directory(&pl)) + if (update_working_directory(repo, &pl)) die(_("error while refreshing working directory")); clear_pattern_list(&pl); - return set_config(MODE_NO_PATTERNS); + return set_config(repo, MODE_NO_PATTERNS); } static char const * const builtin_sparse_checkout_check_rules_usage[] = { @@ -987,14 +1087,17 @@ static struct sparse_checkout_check_rules_opts { char *rules_file; } check_rules_opts; -static int check_rules(struct pattern_list *pl, int null_terminated) { +static int check_rules(struct repository *repo, + struct pattern_list *pl, + int null_terminated) +{ struct strbuf line = STRBUF_INIT; struct strbuf unquoted = STRBUF_INIT; char *path; int line_terminator = null_terminated ? 0 : '\n'; strbuf_getline_fn getline_fn = null_terminated ? strbuf_getline_nul : strbuf_getline; - the_repository->index->sparse_checkout_patterns = pl; + repo->index->sparse_checkout_patterns = pl; while (!getline_fn(&line, stdin)) { path = line.buf; if (!null_terminated && line.buf[0] == '"') { @@ -1006,7 +1109,7 @@ static int check_rules(struct pattern_list *pl, int null_terminated) { path = unquoted.buf; } - if (path_in_sparse_checkout(path, the_repository->index)) + if (path_in_sparse_checkout(path, repo->index)) write_name_quoted(path, stdout, line_terminator); } strbuf_release(&line); @@ -1016,7 +1119,7 @@ static int check_rules(struct pattern_list *pl, int null_terminated) { } static int sparse_checkout_check_rules(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { static struct option builtin_sparse_checkout_check_rules_options[] = { OPT_BOOL('z', NULL, &check_rules_opts.null_termination, @@ -1055,7 +1158,7 @@ static int sparse_checkout_check_rules(int argc, const char **argv, const char * free(sparse_filename); } - ret = check_rules(&pl, check_rules_opts.null_termination); + ret = check_rules(repo, &pl, check_rules_opts.null_termination); clear_pattern_list(&pl); free(check_rules_opts.rules_file); return ret; @@ -1073,6 +1176,7 @@ int cmd_sparse_checkout(int argc, OPT_SUBCOMMAND("set", &fn, sparse_checkout_set), OPT_SUBCOMMAND("add", &fn, sparse_checkout_add), OPT_SUBCOMMAND("reapply", &fn, sparse_checkout_reapply), + OPT_SUBCOMMAND("clean", &fn, sparse_checkout_clean), OPT_SUBCOMMAND("disable", &fn, sparse_checkout_disable), OPT_SUBCOMMAND("check-rules", &fn, sparse_checkout_check_rules), OPT_END(), @@ -1084,8 +1188,8 @@ int cmd_sparse_checkout(int argc, repo_config(the_repository, git_default_config, NULL); - prepare_repo_settings(the_repository); - the_repository->settings.command_requires_full_index = 0; + prepare_repo_settings(repo); + repo->settings.command_requires_full_index = 0; return fn(argc, argv, prefix, repo); } diff --git a/builtin/stash.c b/builtin/stash.c index 948eba06fb..3b50905233 100644 --- a/builtin/stash.c +++ b/builtin/stash.c @@ -1306,7 +1306,7 @@ done: static int stash_patch(struct stash_info *info, const struct pathspec *ps, struct strbuf *out_patch, int quiet, - struct add_p_opt *add_p_opt) + struct interactive_options *interactive_opts) { int ret = 0; struct child_process cp_read_tree = CHILD_PROCESS_INIT; @@ -1331,7 +1331,7 @@ static int stash_patch(struct stash_info *info, const struct pathspec *ps, old_index_env = xstrdup_or_null(getenv(INDEX_ENVIRONMENT)); setenv(INDEX_ENVIRONMENT, the_repository->index_file, 1); - ret = !!run_add_p(the_repository, ADD_P_STASH, add_p_opt, NULL, ps); + ret = !!run_add_p(the_repository, ADD_P_STASH, interactive_opts, NULL, ps); the_repository->index_file = old_repo_index_file; if (old_index_env && *old_index_env) @@ -1427,7 +1427,8 @@ done: } static int do_create_stash(const struct pathspec *ps, struct strbuf *stash_msg_buf, - int include_untracked, int patch_mode, struct add_p_opt *add_p_opt, + int include_untracked, int patch_mode, + struct interactive_options *interactive_opts, int only_staged, struct stash_info *info, struct strbuf *patch, int quiet) { @@ -1509,7 +1510,7 @@ static int do_create_stash(const struct pathspec *ps, struct strbuf *stash_msg_b untracked_commit_option = 1; } if (patch_mode) { - ret = stash_patch(info, ps, patch, quiet, add_p_opt); + ret = stash_patch(info, ps, patch, quiet, interactive_opts); if (ret < 0) { if (!quiet) fprintf_ln(stderr, _("Cannot save the current " @@ -1595,7 +1596,8 @@ static int create_stash(int argc, const char **argv, const char *prefix UNUSED, } static int do_push_stash(const struct pathspec *ps, const char *stash_msg, int quiet, - int keep_index, int patch_mode, struct add_p_opt *add_p_opt, + int keep_index, int patch_mode, + struct interactive_options *interactive_opts, int include_untracked, int only_staged) { int ret = 0; @@ -1667,7 +1669,7 @@ static int do_push_stash(const struct pathspec *ps, const char *stash_msg, int q if (stash_msg) strbuf_addstr(&stash_msg_buf, stash_msg); if (do_create_stash(ps, &stash_msg_buf, include_untracked, patch_mode, - add_p_opt, only_staged, &info, &patch, quiet)) { + interactive_opts, only_staged, &info, &patch, quiet)) { ret = -1; goto done; } @@ -1841,7 +1843,7 @@ static int push_stash(int argc, const char **argv, const char *prefix, const char *stash_msg = NULL; char *pathspec_from_file = NULL; struct pathspec ps; - struct add_p_opt add_p_opt = ADD_P_OPT_INIT; + struct interactive_options interactive_opts = INTERACTIVE_OPTIONS_INIT; struct option options[] = { OPT_BOOL('k', "keep-index", &keep_index, N_("keep index")), @@ -1849,8 +1851,8 @@ static int push_stash(int argc, const char **argv, const char *prefix, N_("stash staged changes only")), OPT_BOOL('p', "patch", &patch_mode, N_("stash in patch mode")), - OPT_DIFF_UNIFIED(&add_p_opt.context), - OPT_DIFF_INTERHUNK_CONTEXT(&add_p_opt.interhunkcontext), + OPT_DIFF_UNIFIED(&interactive_opts.context), + OPT_DIFF_INTERHUNK_CONTEXT(&interactive_opts.interhunkcontext), OPT__QUIET(&quiet, N_("quiet mode")), OPT_BOOL('u', "include-untracked", &include_untracked, N_("include untracked files in stash")), @@ -1907,19 +1909,19 @@ static int push_stash(int argc, const char **argv, const char *prefix, } if (!patch_mode) { - if (add_p_opt.context != -1) + if (interactive_opts.context != -1) die(_("the option '%s' requires '%s'"), "--unified", "--patch"); - if (add_p_opt.interhunkcontext != -1) + if (interactive_opts.interhunkcontext != -1) die(_("the option '%s' requires '%s'"), "--inter-hunk-context", "--patch"); } - if (add_p_opt.context < -1) + if (interactive_opts.context < -1) die(_("'%s' cannot be negative"), "--unified"); - if (add_p_opt.interhunkcontext < -1) + if (interactive_opts.interhunkcontext < -1) die(_("'%s' cannot be negative"), "--inter-hunk-context"); ret = do_push_stash(&ps, stash_msg, quiet, keep_index, patch_mode, - &add_p_opt, include_untracked, only_staged); + &interactive_opts, include_untracked, only_staged); clear_pathspec(&ps); free(pathspec_from_file); @@ -1944,7 +1946,7 @@ static int save_stash(int argc, const char **argv, const char *prefix, const char *stash_msg = NULL; struct pathspec ps; struct strbuf stash_msg_buf = STRBUF_INIT; - struct add_p_opt add_p_opt = ADD_P_OPT_INIT; + struct interactive_options interactive_opts = INTERACTIVE_OPTIONS_INIT; struct option options[] = { OPT_BOOL('k', "keep-index", &keep_index, N_("keep index")), @@ -1952,8 +1954,8 @@ static int save_stash(int argc, const char **argv, const char *prefix, N_("stash staged changes only")), OPT_BOOL('p', "patch", &patch_mode, N_("stash in patch mode")), - OPT_DIFF_UNIFIED(&add_p_opt.context), - OPT_DIFF_INTERHUNK_CONTEXT(&add_p_opt.interhunkcontext), + OPT_DIFF_UNIFIED(&interactive_opts.context), + OPT_DIFF_INTERHUNK_CONTEXT(&interactive_opts.interhunkcontext), OPT__QUIET(&quiet, N_("quiet mode")), OPT_BOOL('u', "include-untracked", &include_untracked, N_("include untracked files in stash")), @@ -1973,20 +1975,20 @@ static int save_stash(int argc, const char **argv, const char *prefix, memset(&ps, 0, sizeof(ps)); - if (add_p_opt.context < -1) + if (interactive_opts.context < -1) die(_("'%s' cannot be negative"), "--unified"); - if (add_p_opt.interhunkcontext < -1) + if (interactive_opts.interhunkcontext < -1) die(_("'%s' cannot be negative"), "--inter-hunk-context"); if (!patch_mode) { - if (add_p_opt.context != -1) + if (interactive_opts.context != -1) die(_("the option '%s' requires '%s'"), "--unified", "--patch"); - if (add_p_opt.interhunkcontext != -1) + if (interactive_opts.interhunkcontext != -1) die(_("the option '%s' requires '%s'"), "--inter-hunk-context", "--patch"); } ret = do_push_stash(&ps, stash_msg, quiet, keep_index, - patch_mode, &add_p_opt, include_untracked, + patch_mode, &interactive_opts, include_untracked, only_staged); strbuf_release(&stash_msg_buf); diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c index fcd73abe53..96d5301510 100644 --- a/builtin/submodule--helper.c +++ b/builtin/submodule--helper.c @@ -593,16 +593,12 @@ static void print_status(unsigned int flags, char state, const char *path, printf("\n"); } -static int handle_submodule_head_ref(const char *refname UNUSED, - const char *referent UNUSED, - const struct object_id *oid, - int flags UNUSED, - void *cb_data) +static int handle_submodule_head_ref(const struct reference *ref, void *cb_data) { struct object_id *output = cb_data; - if (oid) - oidcpy(output, oid); + if (ref->oid) + oidcpy(output, ref->oid); return 0; } @@ -1208,6 +1204,22 @@ static int module_summary(int argc, const char **argv, const char *prefix, return ret; } +static int module_gitdir(int argc, const char **argv, const char *prefix UNUSED, + struct repository *repo) +{ + struct strbuf gitdir = STRBUF_INIT; + + if (argc != 2) + usage(_("git submodule--helper gitdir <name>")); + + submodule_name_to_gitdir(&gitdir, repo, argv[1]); + + printf("%s\n", gitdir.buf); + + strbuf_release(&gitdir); + return 0; +} + struct sync_cb { const char *prefix; const char *super_prefix; @@ -3187,13 +3199,13 @@ static void append_fetch_remotes(struct strbuf *msg, const char *git_dir_path) static int add_submodule(const struct add_data *add_data) { - char *submod_gitdir_path; struct module_clone_data clone_data = MODULE_CLONE_DATA_INIT; struct string_list reference = STRING_LIST_INIT_NODUP; int ret = -1; /* perhaps the path already exists and is already a git repo, else clone it */ if (is_directory(add_data->sm_path)) { + char *submod_gitdir_path; struct strbuf sm_path = STRBUF_INIT; strbuf_addstr(&sm_path, add_data->sm_path); submod_gitdir_path = xstrfmt("%s/.git", add_data->sm_path); @@ -3207,10 +3219,11 @@ static int add_submodule(const struct add_data *add_data) free(submod_gitdir_path); } else { struct child_process cp = CHILD_PROCESS_INIT; + struct strbuf submod_gitdir = STRBUF_INIT; - submod_gitdir_path = xstrfmt(".git/modules/%s", add_data->sm_name); + submodule_name_to_gitdir(&submod_gitdir, the_repository, add_data->sm_name); - if (is_directory(submod_gitdir_path)) { + if (is_directory(submod_gitdir.buf)) { if (!add_data->force) { struct strbuf msg = STRBUF_INIT; char *die_msg; @@ -3219,8 +3232,8 @@ static int add_submodule(const struct add_data *add_data) "locally with remote(s):\n"), add_data->sm_name); - append_fetch_remotes(&msg, submod_gitdir_path); - free(submod_gitdir_path); + append_fetch_remotes(&msg, submod_gitdir.buf); + strbuf_release(&submod_gitdir); strbuf_addf(&msg, _("If you want to reuse this local git " "directory instead of cloning again from\n" @@ -3238,7 +3251,7 @@ static int add_submodule(const struct add_data *add_data) "submodule '%s'\n"), add_data->sm_name); } } - free(submod_gitdir_path); + strbuf_release(&submod_gitdir); clone_data.prefix = add_data->prefix; clone_data.path = add_data->sm_path; @@ -3590,6 +3603,7 @@ int cmd_submodule__helper(int argc, NULL }; struct option options[] = { + OPT_SUBCOMMAND("gitdir", &fn, module_gitdir), OPT_SUBCOMMAND("clone", &fn, module_clone), OPT_SUBCOMMAND("add", &fn, module_add), OPT_SUBCOMMAND("update", &fn, module_update), diff --git a/builtin/tag.c b/builtin/tag.c index f0665af3ac..81dedb9a4c 100644 --- a/builtin/tag.c +++ b/builtin/tag.c @@ -153,7 +153,7 @@ static int verify_tag(const char *name, const char *ref UNUSED, return -1; if (format->format) - pretty_print_ref(name, oid, format); + pretty_print_ref(name, oid, NULL, format); return 0; } @@ -499,8 +499,7 @@ int cmd_tag(int argc, OPT_CALLBACK_F('m', "message", &msg, N_("message"), N_("tag message"), PARSE_OPT_NONEG, parse_msg_arg), OPT_FILENAME('F', "file", &msgfile, N_("read message from file")), - OPT_PASSTHRU_ARGV(0, "trailer", &trailer_args, N_("trailer"), - N_("add custom trailer(s)"), PARSE_OPT_NONEG), + OPT_CALLBACK_F(0, "trailer", &trailer_args, N_("trailer"), N_("add custom trailer(s)"), PARSE_OPT_NONEG, parse_opt_strvec), OPT_BOOL('e', "edit", &edit_flag, N_("force edit of tag message")), OPT_BOOL('s', "sign", &opt.sign, N_("annotated and GPG-signed tag")), OPT_CLEANUP(&cleanup_arg), diff --git a/builtin/unpack-objects.c b/builtin/unpack-objects.c index ef79e43715..6fc64e9e4b 100644 --- a/builtin/unpack-objects.c +++ b/builtin/unpack-objects.c @@ -363,7 +363,7 @@ struct input_zstream_data { int status; }; -static const void *feed_input_zstream(struct input_stream *in_stream, +static const void *feed_input_zstream(struct odb_write_stream *in_stream, unsigned long *readlen) { struct input_zstream_data *data = in_stream->data; @@ -393,7 +393,7 @@ static void stream_blob(unsigned long size, unsigned nr) { git_zstream zstream = { 0 }; struct input_zstream_data data = { 0 }; - struct input_stream in_stream = { + struct odb_write_stream in_stream = { .read = feed_input_zstream, .data = &data, }; @@ -402,8 +402,7 @@ static void stream_blob(unsigned long size, unsigned nr) data.zstream = &zstream; git_inflate_init(&zstream); - if (stream_loose_object(the_repository->objects->sources, - &in_stream, size, &info->oid)) + if (odb_write_object_stream(the_repository->objects, &in_stream, size, &info->oid)) die(_("failed to write object in stream")); if (data.status != Z_STREAM_END) diff --git a/builtin/verify-tag.c b/builtin/verify-tag.c index cd6bc11095..558121eaa1 100644 --- a/builtin/verify-tag.c +++ b/builtin/verify-tag.c @@ -67,7 +67,7 @@ int cmd_verify_tag(int argc, } if (format.format) - pretty_print_ref(name, &oid, &format); + pretty_print_ref(name, &oid, NULL, &format); } return had_error; } diff --git a/builtin/worktree.c b/builtin/worktree.c index 812774a5ca..b7f323b5e4 100644 --- a/builtin/worktree.c +++ b/builtin/worktree.c @@ -635,11 +635,7 @@ static void print_preparing_worktree_line(int detach, * * Returns 0 on failure and non-zero on success. */ -static int first_valid_ref(const char *refname UNUSED, - const char *referent UNUSED, - const struct object_id *oid UNUSED, - int flags UNUSED, - void *cb_data UNUSED) +static int first_valid_ref(const struct reference *ref UNUSED, void *cb_data UNUSED) { return 1; } diff --git a/cache-tree.c b/cache-tree.c index 2aba47060e..b67d0d703d 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -699,11 +699,11 @@ static int write_index_as_tree_internal(struct object_id *oid, return 0; } -struct tree* write_in_core_index_as_tree(struct repository *repo) { +struct tree *write_in_core_index_as_tree(struct repository *repo, + struct index_state *index_state) { struct object_id o; int was_valid, ret; - struct index_state *index_state = repo->index; was_valid = index_state->cache_tree && cache_tree_fully_valid(index_state->cache_tree); @@ -723,7 +723,6 @@ struct tree* write_in_core_index_as_tree(struct repository *repo) { return lookup_tree(repo, &index_state->cache_tree->oid); } - int write_index_as_tree(struct object_id *oid, struct index_state *index_state, const char *index_path, int flags, const char *prefix) { int entries, was_valid; diff --git a/cache-tree.h b/cache-tree.h index b82c4963e7..f8bddae523 100644 --- a/cache-tree.h +++ b/cache-tree.h @@ -47,7 +47,8 @@ int cache_tree_verify(struct repository *, struct index_state *); #define WRITE_TREE_UNMERGED_INDEX (-2) #define WRITE_TREE_PREFIX_ERROR (-3) -struct tree* write_in_core_index_as_tree(struct repository *repo); +struct tree *write_in_core_index_as_tree(struct repository *repo, + struct index_state *index_state); int write_index_as_tree(struct object_id *oid, struct index_state *index_state, const char *index_path, int flags, const char *prefix); void prime_cache_tree(struct repository *, struct index_state *, struct tree *); diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index a8dcd9b9bc..50628ee2dd 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -10,6 +10,8 @@ begin_group "Install dependencies" P4WHENCE=https://cdist2.perforce.com/perforce/r23.2 LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION JGITWHENCE=https://repo1.maven.org/maven2/org/eclipse/jgit/org.eclipse.jgit.pgm/6.8.0.202311291450-r/org.eclipse.jgit.pgm-6.8.0.202311291450-r.sh +CARGO_MSRV_VERSION=0.18.4 +CARGO_MSRV_WHENCE=https://github.com/foresterre/cargo-msrv/releases/download/v$CARGO_MSRV_VERSION/cargo-msrv-x86_64-unknown-linux-musl-v$CARGO_MSRV_VERSION.tgz # Make sudo a no-op and execute the command directly when running as root. # While using sudo would be fine on most platforms when we are root already, @@ -129,21 +131,28 @@ esac case "$jobname" in ClangFormat) - sudo apt-get -q update sudo apt-get -q -y install clang-format ;; StaticAnalysis) - sudo apt-get -q update sudo apt-get -q -y install coccinelle libcurl4-openssl-dev libssl-dev \ libexpat-dev gettext make ;; +RustAnalysis) + sudo apt-get -q -y install rustup + rustup default stable + rustup component add clippy rustfmt + + wget -q "$CARGO_MSRV_WHENCE" -O "cargo-msvc.tgz" + sudo mkdir -p "$CUSTOM_PATH" + sudo tar -xf "cargo-msvc.tgz" --strip-components=1 \ + --directory "$CUSTOM_PATH" --wildcards "*/cargo-msrv" + sudo chmod a+x "$CUSTOM_PATH/cargo-msrv" + ;; sparse) - sudo apt-get -q update -q sudo apt-get -q -y install libssl-dev libcurl4-openssl-dev \ libexpat-dev gettext zlib1g-dev sparse ;; Documentation) - sudo apt-get -q update sudo apt-get -q -y install asciidoc xmlto docbook-xsl-ns make test -n "$ALREADY_HAVE_ASCIIDOCTOR" || diff --git a/ci/run-rust-checks.sh b/ci/run-rust-checks.sh new file mode 100755 index 0000000000..b5ad9e8dc6 --- /dev/null +++ b/ci/run-rust-checks.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +. ${0%/*}/lib.sh + +set +x + +if ! group "Check Rust formatting" cargo fmt --all --check +then + RET=1 +fi + +if ! group "Check for common Rust mistakes" cargo clippy --all-targets --all-features -- -Dwarnings +then + RET=1 +fi + +if ! group "Check for minimum required Rust version" cargo msrv verify +then + RET=1 +fi + +exit $RET diff --git a/command-list.txt b/command-list.txt index accd3d0c4b..f9005cf459 100644 --- a/command-list.txt +++ b/command-list.txt @@ -115,6 +115,7 @@ git-grep mainporcelain info git-gui mainporcelain git-hash-object plumbingmanipulators git-help ancillaryinterrogators complete +git-history mainporcelain history git-hook purehelpers git-http-backend synchingrepositories git-http-fetch synchelpers diff --git a/commit-graph.c b/commit-graph.c index 474454db73..80be2ff2c3 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -1851,18 +1851,16 @@ struct refs_cb_data { struct progress *progress; }; -static int add_ref_to_set(const char *refname UNUSED, - const char *referent UNUSED, - const struct object_id *oid, - int flags UNUSED, void *cb_data) +static int add_ref_to_set(const struct reference *ref, void *cb_data) { + const struct object_id *maybe_peeled = ref->oid; struct object_id peeled; struct refs_cb_data *data = (struct refs_cb_data *)cb_data; - if (!peel_iterated_oid(data->repo, oid, &peeled)) - oid = &peeled; - if (odb_read_object_info(data->repo->objects, oid, NULL) == OBJ_COMMIT) - oidset_insert(data->commits, oid); + if (!reference_get_peeled_oid(data->repo, ref, &peeled)) + maybe_peeled = &peeled; + if (odb_read_object_info(data->repo->objects, maybe_peeled, NULL) == OBJ_COMMIT) + oidset_insert(data->commits, maybe_peeled); display_progress(data->progress, oidset_size(data->commits)); diff --git a/commit-reach.c b/commit-reach.c index a339e41aa4..cc18c86d3b 100644 --- a/commit-reach.c +++ b/commit-reach.c @@ -60,6 +60,7 @@ static int paint_down_to_common(struct repository *r, struct prio_queue queue = { compare_commits_by_gen_then_commit_date }; int i; timestamp_t last_gen = GENERATION_NUMBER_INFINITY; + struct commit_list **tail = result; if (!min_generation && !corrected_commit_dates_enabled(r)) queue.compare = compare_commits_by_commit_date; @@ -95,7 +96,7 @@ static int paint_down_to_common(struct repository *r, if (flags == (PARENT1 | PARENT2)) { if (!(commit->object.flags & RESULT)) { commit->object.flags |= RESULT; - commit_list_insert_by_date(commit, result); + tail = commit_list_append(commit, tail); } /* Mark parents of a found merge stale */ flags |= STALE; @@ -128,6 +129,7 @@ static int paint_down_to_common(struct repository *r, } clear_prio_queue(&queue); + commit_list_sort_by_date(result); return 0; } @@ -136,7 +138,7 @@ static int merge_bases_many(struct repository *r, struct commit **twos, struct commit_list **result) { - struct commit_list *list = NULL; + struct commit_list *list = NULL, **tail = result; int i; for (i = 0; i < n; i++) { @@ -171,8 +173,9 @@ static int merge_bases_many(struct repository *r, while (list) { struct commit *commit = pop_commit(&list); if (!(commit->object.flags & STALE)) - commit_list_insert_by_date(commit, result); + tail = commit_list_append(commit, tail); } + commit_list_sort_by_date(result); return 0; } @@ -425,7 +428,7 @@ static int get_merge_bases_many_0(struct repository *r, int cleanup, struct commit_list **result) { - struct commit_list *list; + struct commit_list *list, **tail = result; struct commit **rslt; size_t cnt, i; int ret; @@ -461,7 +464,8 @@ static int get_merge_bases_many_0(struct repository *r, return -1; } for (i = 0; i < cnt; i++) - commit_list_insert_by_date(rslt[i], result); + tail = commit_list_append(rslt[i], tail); + commit_list_sort_by_date(result); free(rslt); return 0; } @@ -1965,6 +1965,9 @@ int run_commit_hook(int editor_is_used, const char *index_file, strvec_push(&opt.args, arg); va_end(args); + /* All commit hook use-cases require ungrouping child output. */ + opt.ungroup = 1; + opt.invoked_hook = invoked_hook; return run_hooks_opt(the_repository, name, &opt); } @@ -258,7 +258,7 @@ int for_each_commit_graft(each_commit_graft_fn, void *); int interactive_add(struct repository *repo, const char **argv, const char *prefix, - int patch, struct add_p_opt *add_p_opt); + int patch, struct interactive_options *opts); struct commit_extra_header { struct commit_extra_header *next; diff --git a/compat/mingw.c b/compat/mingw.c index 8538e3d172..736a07a028 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1,26 +1,26 @@ #define USE_THE_REPOSITORY_VARIABLE #define DISABLE_SIGN_COMPARE_WARNINGS -#include "../git-compat-util.h" +#include "git-compat-util.h" +#include "abspath.h" +#include "alloc.h" +#include "config.h" +#include "dir.h" +#include "environment.h" +#include "gettext.h" +#include "run-command.h" +#include "strbuf.h" +#include "symlinks.h" +#include "trace2.h" #include "win32.h" +#include "win32/lazyload.h" +#include "wrapper.h" #include <aclapi.h> -#include <sddl.h> #include <conio.h> -#include <wchar.h> -#include "../strbuf.h" -#include "../run-command.h" -#include "../abspath.h" -#include "../alloc.h" -#include "win32/lazyload.h" -#include "../config.h" -#include "../environment.h" -#include "../trace2.h" -#include "../symlinks.h" -#include "../wrapper.h" -#include "dir.h" -#include "gettext.h" +#include <sddl.h> #define SECURITY_WIN32 #include <sspi.h> +#include <wchar.h> #include <winternl.h> #define STATUS_DELETE_PENDING ((NTSTATUS) 0xC0000056) @@ -1278,7 +1278,7 @@ int git_config_string(char **dest, const char *var, const char *value) int git_config_pathname(char **dest, const char *var, const char *value) { - int is_optional; + bool is_optional; char *path; if (!value) diff --git a/connected.c b/connected.c index b288a18b17..79403108dd 100644 --- a/connected.c +++ b/connected.c @@ -74,10 +74,9 @@ int check_connected(oid_iterate_fn fn, void *cb_data, */ odb_reprepare(the_repository->objects); do { - struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *p; - for (p = packfile_store_get_all_packs(packs); p; p = p->next) { + repo_for_each_pack(the_repository, p) { if (!p->pack_promisor) continue; if (find_pack_entry_one(oid, p)) diff --git a/contrib/completion/git-completion.bash b/contrib/completion/git-completion.bash index e3d88b0672..73abea31b4 100644 --- a/contrib/completion/git-completion.bash +++ b/contrib/completion/git-completion.bash @@ -2218,7 +2218,7 @@ __git_log_gitk_options=" " # Options that go well for log and shortlog (not gitk) __git_log_shortlog_options=" - --author= --committer= --grep= + --author= --grep= --exclude= --all-match --invert-grep " # Options accepted by log and show @@ -2296,6 +2296,7 @@ __git_complete_log_opts () $__git_log_shortlog_options $__git_log_gitk_options $__git_log_show_options + --committer= --root --topo-order --date-order --reverse --follow --full-diff --abbrev-commit --no-abbrev-commit --abbrev= @@ -3229,7 +3230,7 @@ _git_shortlog () __gitcomp " $__git_log_common_options $__git_log_shortlog_options - --numbered --summary --email + --committer --numbered --summary --email " return ;; diff --git a/contrib/contacts/meson.build b/contrib/contacts/meson.build index c8fdb35ed9..4ae6b32a03 100644 --- a/contrib/contacts/meson.build +++ b/contrib/contacts/meson.build @@ -50,6 +50,6 @@ if get_option('docs').contains('html') input: 'git-contacts.adoc', output: 'git-contacts.html', install: true, - install_dir: get_option('datadir') / 'doc/git-doc', + install_dir: htmldir, ) endif diff --git a/contrib/credential/libsecret/Makefile b/contrib/credential/libsecret/Makefile index 97ce9c92fb..9309cfb78c 100644 --- a/contrib/credential/libsecret/Makefile +++ b/contrib/credential/libsecret/Makefile @@ -1,28 +1,32 @@ # The default target of this Makefile is... -all:: - -MAIN:=git-credential-libsecret -all:: $(MAIN) - -CC = gcc -RM = rm -f -CFLAGS = -g -O2 -Wall -PKG_CONFIG = pkg-config +all:: git-credential-libsecret -include ../../../config.mak.autogen -include ../../../config.mak +prefix ?= /usr/local +gitexecdir ?= $(prefix)/libexec/git-core + +CC ?= gcc +CFLAGS ?= -g -O2 -Wall +PKG_CONFIG ?= pkg-config +INSTALL ?= install +RM ?= rm -f + INCS:=$(shell $(PKG_CONFIG) --cflags libsecret-1 glib-2.0) LIBS:=$(shell $(PKG_CONFIG) --libs libsecret-1 glib-2.0) -SRCS:=$(MAIN).c -OBJS:=$(SRCS:.c=.o) - %.o: %.c $(CC) $(CFLAGS) $(CPPFLAGS) $(INCS) -o $@ -c $< -$(MAIN): $(OBJS) - $(CC) -o $@ $(LDFLAGS) $^ $(LIBS) +git-credential-libsecret: git-credential-libsecret.o + $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) $(LIBS) + +install: git-credential-libsecret + $(INSTALL) -d -m 755 $(DESTDIR)$(gitexecdir) + $(INSTALL) -m 755 $< $(DESTDIR)$(gitexecdir) clean: - @$(RM) $(MAIN) $(OBJS) + $(RM) git-credential-libsecret git-credential-libsecret.o + +.PHONY: all install clean diff --git a/contrib/credential/osxkeychain/Makefile b/contrib/credential/osxkeychain/Makefile index 0948297e20..9680717abe 100644 --- a/contrib/credential/osxkeychain/Makefile +++ b/contrib/credential/osxkeychain/Makefile @@ -1,19 +1,29 @@ # The default target of this Makefile is... all:: git-credential-osxkeychain -CC = gcc -RM = rm -f -CFLAGS = -g -O2 -Wall - -include ../../../config.mak.autogen -include ../../../config.mak +prefix ?= /usr/local +gitexecdir ?= $(prefix)/libexec/git-core + +CC ?= gcc +CFLAGS ?= -g -O2 -Wall +INSTALL ?= install +RM ?= rm -f + +%.o: %.c + $(CC) $(CFLAGS) $(CPPFLAGS) -o $@ -c $< + git-credential-osxkeychain: git-credential-osxkeychain.o - $(CC) $(CFLAGS) -o $@ $< $(LDFLAGS) \ + $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) \ -framework Security -framework CoreFoundation -git-credential-osxkeychain.o: git-credential-osxkeychain.c - $(CC) -c $(CFLAGS) $< +install: git-credential-osxkeychain + $(INSTALL) -d -m 755 $(DESTDIR)$(gitexecdir) + $(INSTALL) -m 755 $< $(DESTDIR)$(gitexecdir) clean: $(RM) git-credential-osxkeychain git-credential-osxkeychain.o + +.PHONY: all install clean diff --git a/contrib/credential/wincred/Makefile b/contrib/credential/wincred/Makefile index 5b795fc9fe..d92e721e24 100644 --- a/contrib/credential/wincred/Makefile +++ b/contrib/credential/wincred/Makefile @@ -4,20 +4,22 @@ all:: git-credential-wincred.exe -include ../../../config.mak.autogen -include ../../../config.mak -CC ?= gcc -RM ?= rm -f -CFLAGS ?= -O2 -Wall - prefix ?= /usr/local -libexecdir ?= $(prefix)/libexec/git-core +gitexecdir ?= $(prefix)/libexec/git-core +CC ?= gcc +CFLAGS ?= -O2 -Wall INSTALL ?= install +RM ?= rm -f -git-credential-wincred.exe : git-credential-wincred.c - $(LINK.c) $^ $(LOADLIBES) $(LDLIBS) -o $@ +git-credential-wincred.exe: git-credential-wincred.c + $(LINK.c) -o $@ $^ $(LDFLAGS) $(LDLIBS) install: git-credential-wincred.exe - $(INSTALL) -m 755 $^ $(libexecdir) + $(INSTALL) -d -m 755 $(DESTDIR)$(gitexecdir) + $(INSTALL) -m 755 $< $(DESTDIR)$(gitexecdir) clean: $(RM) git-credential-wincred.exe + +.PHONY: all install clean diff --git a/contrib/subtree/meson.build b/contrib/subtree/meson.build index 46cdbcc30c..161435abeb 100644 --- a/contrib/subtree/meson.build +++ b/contrib/subtree/meson.build @@ -68,6 +68,6 @@ if get_option('docs').contains('html') input: 'git-subtree.adoc', output: 'git-subtree.html', install: true, - install_dir: get_option('datadir') / 'doc/git-doc', + install_dir: htmldir, ) endif diff --git a/csum-file.c b/csum-file.c index 6e21e3cac8..3d3047c776 100644 --- a/csum-file.c +++ b/csum-file.c @@ -110,7 +110,7 @@ void discard_hashfile(struct hashfile *f) free_hashfile(f); } -void hashwrite(struct hashfile *f, const void *buf, unsigned int count) +void hashwrite(struct hashfile *f, const void *buf, uint32_t count) { while (count) { unsigned left = f->buffer_len - f->offset; diff --git a/csum-file.h b/csum-file.h index 07ae11024a..ecce9d27b0 100644 --- a/csum-file.h +++ b/csum-file.h @@ -63,7 +63,7 @@ void free_hashfile(struct hashfile *f); */ int finalize_hashfile(struct hashfile *, unsigned char *, enum fsync_component, unsigned int); void discard_hashfile(struct hashfile *); -void hashwrite(struct hashfile *, const void *, unsigned int); +void hashwrite(struct hashfile *, const void *, uint32_t); void hashflush(struct hashfile *f); void crc32_begin(struct hashfile *); uint32_t crc32_end(struct hashfile *); diff --git a/delta-islands.c b/delta-islands.c index 36c94799d6..7cfebc4162 100644 --- a/delta-islands.c +++ b/delta-islands.c @@ -390,8 +390,7 @@ static void add_ref_to_island(kh_str_t *remote_islands, const char *island_name, rl->hash += sha_core; } -static int find_island_for_ref(const char *refname, const char *referent UNUSED, const struct object_id *oid, - int flags UNUSED, void *cb) +static int find_island_for_ref(const struct reference *ref, void *cb) { struct island_load_data *ild = cb; @@ -406,7 +405,7 @@ static int find_island_for_ref(const char *refname, const char *referent UNUSED, /* walk backwards to get last-one-wins ordering */ for (i = ild->nr - 1; i >= 0; i--) { - if (!regexec(&ild->rx[i], refname, + if (!regexec(&ild->rx[i], ref->name, ARRAY_SIZE(matches), matches, 0)) break; } @@ -428,10 +427,10 @@ static int find_island_for_ref(const char *refname, const char *referent UNUSED, if (island_name.len) strbuf_addch(&island_name, '-'); - strbuf_add(&island_name, refname + match->rm_so, match->rm_eo - match->rm_so); + strbuf_add(&island_name, ref->name + match->rm_so, match->rm_eo - match->rm_so); } - add_ref_to_island(ild->remote_islands, island_name.buf, oid); + add_ref_to_island(ild->remote_islands, island_name.buf, ref->oid); strbuf_release(&island_name); return 0; } @@ -601,6 +601,7 @@ struct emit_callback { int blank_at_eof_in_postimage; int lno_in_preimage; int lno_in_postimage; + int last_line_kind; const char **label_path; struct diff_words_data *diff_words; struct diff_options *opt; @@ -796,21 +797,23 @@ enum diff_symbol { DIFF_SYMBOL_CONTEXT_INCOMPLETE, DIFF_SYMBOL_PLUS, DIFF_SYMBOL_MINUS, - DIFF_SYMBOL_NO_LF_EOF, DIFF_SYMBOL_CONTEXT_FRAGINFO, DIFF_SYMBOL_CONTEXT_MARKER, DIFF_SYMBOL_SEPARATOR }; + /* * Flags for content lines: - * 0..12 are whitespace rules - * 13-15 are WSEH_NEW | WSEH_OLD | WSEH_CONTEXT - * 16 is marking if the line is blank at EOF + * 0..15 are whitespace rules (see ws.h) + * 16..18 are WSEH_NEW | WSEH_CONTEXT | WSEH_OLD + * 19 is marking if the line is blank at EOF + * 20..22 are used for color-moved. */ -#define DIFF_SYMBOL_CONTENT_BLANK_LINE_EOF (1<<16) -#define DIFF_SYMBOL_MOVED_LINE (1<<17) -#define DIFF_SYMBOL_MOVED_LINE_ALT (1<<18) -#define DIFF_SYMBOL_MOVED_LINE_UNINTERESTING (1<<19) +#define DIFF_SYMBOL_CONTENT_BLANK_LINE_EOF (1<<19) +#define DIFF_SYMBOL_MOVED_LINE (1<<20) +#define DIFF_SYMBOL_MOVED_LINE_ALT (1<<21) +#define DIFF_SYMBOL_MOVED_LINE_UNINTERESTING (1<<22) + #define DIFF_SYMBOL_CONTENT_WS_MASK (WSEH_NEW | WSEH_OLD | WSEH_CONTEXT | WS_RULE_MASK) /* @@ -1318,20 +1321,25 @@ static void emit_line_ws_markup(struct diff_options *o, const char *ws = NULL; int sign = o->output_indicators[sign_index]; + if (diff_suppress_blank_empty && + sign_index == OUTPUT_INDICATOR_CONTEXT && + len == 1 && line[0] == '\n') + sign = 0; + if (o->ws_error_highlight & ws_rule) { ws = diff_get_color_opt(o, DIFF_WHITESPACE); if (!*ws) ws = NULL; } - if (!ws && !set_sign) + if (!ws && !set_sign) { emit_line_0(o, set, NULL, 0, reset, sign, line, len); - else if (!ws) { + } else if (!ws) { emit_line_0(o, set_sign, set, !!set_sign, reset, sign, line, len); - } else if (blank_at_eof) + } else if (blank_at_eof) { /* Blank line at EOF - paint '+' as well */ emit_line_0(o, ws, NULL, 0, reset, sign, line, len); - else { + } else { /* Emit just the prefix, then the rest. */ emit_line_0(o, set_sign ? set_sign : set, NULL, !!set_sign, reset, sign, "", 0); @@ -1343,7 +1351,6 @@ static void emit_line_ws_markup(struct diff_options *o, static void emit_diff_symbol_from_struct(struct diff_options *o, struct emitted_diff_symbol *eds) { - static const char *nneof = " No newline at end of file\n"; const char *context, *reset, *set, *set_sign, *meta, *fraginfo; enum diff_symbol s = eds->s; @@ -1351,14 +1358,10 @@ static void emit_diff_symbol_from_struct(struct diff_options *o, int len = eds->len; unsigned flags = eds->flags; + if (!o->file) + return; + switch (s) { - case DIFF_SYMBOL_NO_LF_EOF: - context = diff_get_color_opt(o, DIFF_CONTEXT); - reset = diff_get_color_opt(o, DIFF_RESET); - putc('\n', o->file); - emit_line_0(o, context, NULL, 0, reset, '\\', - nneof, strlen(nneof)); - break; case DIFF_SYMBOL_SUBMODULE_HEADER: case DIFF_SYMBOL_SUBMODULE_ERROR: case DIFF_SYMBOL_SUBMODULE_PIPETHROUGH: @@ -1370,6 +1373,14 @@ static void emit_diff_symbol_from_struct(struct diff_options *o, emit_line(o, "", "", line, len); break; case DIFF_SYMBOL_CONTEXT_INCOMPLETE: + if ((flags & WS_INCOMPLETE_LINE) && + (flags & o->ws_error_highlight)) + set = diff_get_color_opt(o, DIFF_WHITESPACE); + else + set = diff_get_color_opt(o, DIFF_CONTEXT); + reset = diff_get_color_opt(o, DIFF_RESET); + emit_line(o, set, reset, line, len); + break; case DIFF_SYMBOL_CONTEXT_MARKER: context = diff_get_color_opt(o, DIFF_CONTEXT); reset = diff_get_color_opt(o, DIFF_RESET); @@ -1495,15 +1506,9 @@ static void emit_diff_symbol_from_struct(struct diff_options *o, case DIFF_SYMBOL_WORDS: context = diff_get_color_opt(o, DIFF_CONTEXT); reset = diff_get_color_opt(o, DIFF_RESET); - /* - * Skip the prefix character, if any. With - * diff_suppress_blank_empty, there may be - * none. - */ - if (line[0] != '\n') { - line++; - len--; - } + + /* Skip the prefix character */ + line++; len--; emit_line(o, context, reset, line, len); break; case DIFF_SYMBOL_FILEPAIR_PLUS: @@ -1665,6 +1670,19 @@ static void emit_context_line(struct emit_callback *ecbdata, emit_diff_symbol(ecbdata->opt, DIFF_SYMBOL_CONTEXT, line, len, flags); } +static void emit_incomplete_line(struct emit_callback *ecbdata, + const char *line, int len) +{ + int last_line_kind = ecbdata->last_line_kind; + unsigned flags = (last_line_kind == '+' + ? WSEH_NEW + : last_line_kind == '-' + ? WSEH_OLD + : WSEH_CONTEXT) | ecbdata->ws_rule; + emit_diff_symbol(ecbdata->opt, DIFF_SYMBOL_CONTEXT_INCOMPLETE, + line, len, flags); +} + static void emit_hunk_header(struct emit_callback *ecbdata, const char *line, int len) { @@ -1766,28 +1784,42 @@ static void add_line_count(struct strbuf *out, int count) } } -static void emit_rewrite_lines(struct emit_callback *ecb, +static void emit_rewrite_lines(struct emit_callback *ecbdata, int prefix, const char *data, int size) { const char *endp = NULL; while (0 < size) { - int len; + int len, plen; + char *pdata = NULL; endp = memchr(data, '\n', size); len = endp ? (endp - data + 1) : size; + plen = len; + + if (!endp) { + plen = len + 1; + pdata = xmalloc(plen + 2); + memcpy(pdata, data, len); + pdata[len] = '\n'; + pdata[len + 1] = '\0'; + } if (prefix != '+') { - ecb->lno_in_preimage++; - emit_del_line(ecb, data, len); + ecbdata->lno_in_preimage++; + emit_del_line(ecbdata, pdata ? pdata : data, plen); } else { - ecb->lno_in_postimage++; - emit_add_line(ecb, data, len); + ecbdata->lno_in_postimage++; + emit_add_line(ecbdata, pdata ? pdata : data, plen); } + free(pdata); size -= len; data += len; } - if (!endp) - emit_diff_symbol(ecb->opt, DIFF_SYMBOL_NO_LF_EOF, NULL, 0, 0); + if (!endp) { + static const char nneof[] = "\\ No newline at end of file\n"; + ecbdata->last_line_kind = prefix; + emit_incomplete_line(ecbdata, nneof, sizeof(nneof) - 1); + } } static void emit_rewrite_diff(const char *name_a, @@ -2372,12 +2404,6 @@ static int fn_out_consume(void *priv, char *line, unsigned long len) ecbdata->label_path[0] = ecbdata->label_path[1] = NULL; } - if (diff_suppress_blank_empty - && len == 2 && line[0] == ' ' && line[1] == '\n') { - line[0] = '\n'; - len = 1; - } - if (line[0] == '@') { if (ecbdata->diff_words) diff_words_flush(ecbdata); @@ -2428,13 +2454,29 @@ static int fn_out_consume(void *priv, char *line, unsigned long len) ecbdata->lno_in_preimage++; emit_context_line(ecbdata, line + 1, len - 1); break; - default: + case '\\': /* incomplete line at the end */ - ecbdata->lno_in_preimage++; - emit_diff_symbol(o, DIFF_SYMBOL_CONTEXT_INCOMPLETE, - line, len, 0); + switch (ecbdata->last_line_kind) { + case '+': + ecbdata->lno_in_postimage++; + break; + case '-': + ecbdata->lno_in_preimage++; + break; + case ' ': + ecbdata->lno_in_preimage++; + ecbdata->lno_in_postimage++; + break; + default: + BUG("fn_out_consume: '\\No newline' after unknown line (%c)", + ecbdata->last_line_kind); + } + emit_incomplete_line(ecbdata, line, len); break; + default: + BUG("fn_out_consume: unknown line '%s'", line); } + ecbdata->last_line_kind = line[0]; return 0; } @@ -3228,6 +3270,7 @@ struct checkdiff_t { struct diff_options *o; unsigned ws_rule; unsigned status; + int last_line_kind; }; static int is_conflict_marker(const char *line, int marker_size, unsigned long len) @@ -3266,6 +3309,7 @@ static void checkdiff_consume_hunk(void *priv, static int checkdiff_consume(void *priv, char *line, unsigned long len) { struct checkdiff_t *data = priv; + int last_line_kind; int marker_size = data->conflict_marker_size; const char *ws = diff_get_color(data->o->use_color, DIFF_WHITESPACE); const char *reset = diff_get_color(data->o->use_color, DIFF_RESET); @@ -3276,6 +3320,8 @@ static int checkdiff_consume(void *priv, char *line, unsigned long len) assert(data->o); line_prefix = diff_line_prefix(data->o); + last_line_kind = data->last_line_kind; + data->last_line_kind = line[0]; if (line[0] == '+') { unsigned bad; data->lineno++; @@ -3298,6 +3344,17 @@ static int checkdiff_consume(void *priv, char *line, unsigned long len) data->o->file, set, reset, ws); } else if (line[0] == ' ') { data->lineno++; + } else if (line[0] == '\\') { + /* no newline at the end of the line */ + if ((data->ws_rule & WS_INCOMPLETE_LINE) && + (last_line_kind == '+')) { + unsigned bad = WS_INCOMPLETE_LINE; + data->status |= bad; + err = whitespace_error_string(bad); + fprintf(data->o->file, "%s%s:%d: %s.\n", + line_prefix, data->filename, data->lineno, err); + free(err); + } } return 0; } @@ -3526,8 +3583,6 @@ static int set_diff_algorithm(struct diff_options *opts, if (value < 0) return -1; - /* clear out previous settings */ - DIFF_XDL_CLR(opts, NEED_MINIMAL); opts->xdl_opts &= ~XDF_DIFF_ALGORITHM_MASK; opts->xdl_opts |= value; @@ -3762,9 +3817,9 @@ static void builtin_diff(const char *name_a, if (o->word_diff) init_diff_words_data(&ecbdata, o, one, two); - if (o->dry_run) { + if (!o->file) { /* - * Unlike the !dry_run case, we need to ignore the + * Unlike the normal output case, we need to ignore the * return value from xdi_diff_outf() here, because * xdi_diff_outf() takes non-zero return from its * callback function as a sign of error and returns @@ -4420,7 +4475,6 @@ static void run_external_diff(const struct external_diff *pgm, { struct child_process cmd = CHILD_PROCESS_INIT; struct diff_queue_struct *q = &diff_queued_diff; - int quiet = !(o->output_format & DIFF_FORMAT_PATCH); int rc; /* @@ -4429,7 +4483,7 @@ static void run_external_diff(const struct external_diff *pgm, * external diff program lacks the ability to tell us whether * it's empty then we consider it non-empty without even asking. */ - if (!pgm->trust_exit_code && quiet) { + if (!pgm->trust_exit_code && !o->file) { o->found_changes = 1; return; } @@ -4454,7 +4508,10 @@ static void run_external_diff(const struct external_diff *pgm, diff_free_filespec_data(one); diff_free_filespec_data(two); cmd.use_shell = 1; - cmd.no_stdout = quiet; + if (!o->file) + cmd.no_stdout = 1; + else if (o->file != stdout) + cmd.out = xdup(fileno(o->file)); rc = run_command(&cmd); if (!pgm->trust_exit_code && rc == 0) o->found_changes = 1; @@ -4615,7 +4672,8 @@ static void run_diff_cmd(const struct external_diff *pgm, p->status == DIFF_STATUS_RENAMED) o->found_changes = 1; } else { - fprintf(o->file, "* Unmerged path %s\n", name); + if (o->file) + fprintf(o->file, "* Unmerged path %s\n", name); o->found_changes = 1; } } @@ -6192,15 +6250,15 @@ static void diff_flush_patch(struct diff_filepair *p, struct diff_options *o) /* return 1 if any change is found; otherwise, return 0 */ static int diff_flush_patch_quietly(struct diff_filepair *p, struct diff_options *o) { - int saved_dry_run = o->dry_run; + FILE *saved_file = o->file; int saved_found_changes = o->found_changes; int ret; - o->dry_run = 1; + o->file = NULL; o->found_changes = 0; diff_flush_patch(p, o); ret = o->found_changes; - o->dry_run = saved_dry_run; + o->file = saved_file; o->found_changes |= saved_found_changes; return ret; } @@ -331,9 +331,9 @@ struct diff_options { int ita_invisible_in_index; /* white-space error highlighting */ -#define WSEH_NEW (1<<12) -#define WSEH_CONTEXT (1<<13) -#define WSEH_OLD (1<<14) +#define WSEH_NEW (1<<16) +#define WSEH_CONTEXT (1<<17) +#define WSEH_OLD (1<<18) unsigned ws_error_highlight; const char *prefix; int prefix_length; @@ -408,8 +408,6 @@ struct diff_options { #define COLOR_MOVED_WS_ERROR (1<<0) unsigned color_moved_ws_handling; - bool dry_run; - struct repository *repo; struct strmap *additional_path_headers; @@ -30,6 +30,7 @@ #include "read-cache-ll.h" #include "setup.h" #include "sparse-index.h" +#include "strbuf.h" #include "submodule-config.h" #include "symlinks.h" #include "trace2.h" @@ -87,6 +88,33 @@ struct dirent *readdir_skip_dot_and_dotdot(DIR *dirp) return e; } +int for_each_file_in_dir(struct strbuf *path, file_iterator fn, const void *data) +{ + struct dirent *e; + int res = 0; + size_t baselen = path->len; + DIR *dir = opendir(path->buf); + + if (!dir) + return 0; + + while (!res && (e = readdir_skip_dot_and_dotdot(dir)) != NULL) { + unsigned char dtype = get_dtype(e, path, 0); + strbuf_setlen(path, baselen); + strbuf_addstr(path, e->d_name); + + if (dtype == DT_REG) { + res = fn(path->buf, data); + } else if (dtype == DT_DIR) { + strbuf_addch(path, '/'); + res = for_each_file_in_dir(path, fn, data); + } + } + + closedir(dir); + return res; +} + int count_slashes(const char *s) { int cnt = 0; @@ -1360,18 +1388,25 @@ int match_pathname(const char *pathname, int pathlen, if (fspathncmp(pattern, name, prefix)) return 0; - pattern += prefix; - patternlen -= prefix; - name += prefix; - namelen -= prefix; /* * If the whole pattern did not have a wildcard, * then our prefix match is all we need; we * do not need to call fnmatch at all. */ - if (!patternlen && !namelen) + if (patternlen == prefix && namelen == prefix) return 1; + + /* + * Retain one character of the prefix to + * pass to fnmatch, which lets it distinguish + * the start of a directory component correctly. + */ + prefix--; + pattern += prefix; + patternlen -= prefix; + name += prefix; + namelen -= prefix; } return fnmatch_icase_mem(pattern, patternlen, @@ -2228,6 +2263,8 @@ static int exclude_matches_pathspec(const char *path, int pathlen, const struct pathspec *pathspec) { int i; + int matches_exclude_magic = 0; + int matches_pathspec_elem = 0; if (!pathspec || !pathspec->nr) return 0; @@ -2244,15 +2281,23 @@ static int exclude_matches_pathspec(const char *path, int pathlen, for (i = 0; i < pathspec->nr; i++) { const struct pathspec_item *item = &pathspec->items[i]; int len = item->nowildcard_len; + int *matches; + + if (item->magic & PATHSPEC_EXCLUDE) + matches = &matches_exclude_magic; + else + matches = &matches_pathspec_elem; if (len == pathlen && !ps_strncmp(item, item->match, path, pathlen)) - return 1; + *matches = 1; if (len > pathlen && item->match[pathlen] == '/' && !ps_strncmp(item, item->match, path, pathlen)) - return 1; + *matches = 1; } + if (matches_pathspec_elem && !matches_exclude_magic) + return 1; return 0; } @@ -537,6 +537,20 @@ int get_sparse_checkout_patterns(struct pattern_list *pl); int remove_dir_recursively(struct strbuf *path, int flag); /* + * This function pointer type is called on each file discovered in + * for_each_file_in_dir. The iteration stops if this method returns + * non-zero. + */ +typedef int (*file_iterator)(const char *path, const void *data); + +struct strbuf; +/* + * Given a directory path, recursively visit each file within, including + * within subdirectories. + */ +int for_each_file_in_dir(struct strbuf *path, file_iterator fn, const void *data); + +/* * Tries to remove the path, along with leading empty directories so long as * those empty directories are not startup_info->original_cwd. Ignores * ENOENT. diff --git a/fetch-pack.c b/fetch-pack.c index fe7a84bf2f..78c45d4a15 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -188,13 +188,9 @@ static int rev_list_insert_ref(struct fetch_negotiator *negotiator, return 0; } -static int rev_list_insert_ref_oid(const char *refname UNUSED, - const char *referent UNUSED, - const struct object_id *oid, - int flag UNUSED, - void *cb_data) +static int rev_list_insert_ref_oid(const struct reference *ref, void *cb_data) { - return rev_list_insert_ref(cb_data, oid); + return rev_list_insert_ref(cb_data, ref->oid); } enum ack_type { @@ -616,13 +612,9 @@ static int mark_complete(const struct object_id *oid) return 0; } -static int mark_complete_oid(const char *refname UNUSED, - const char *referent UNUSED, - const struct object_id *oid, - int flag UNUSED, - void *cb_data UNUSED) +static int mark_complete_oid(const struct reference *ref, void *cb_data UNUSED) { - return mark_complete(oid); + return mark_complete(ref->oid); } static void mark_recent_complete_commits(struct fetch_pack_args *args, @@ -1067,6 +1067,24 @@ int fsck_tag_standalone(const struct object_id *oid, const char *buffer, else ret = fsck_ident(&buffer, oid, OBJ_TAG, options); + if (buffer < buffer_end && (skip_prefix(buffer, "gpgsig ", &buffer) || skip_prefix(buffer, "gpgsig-sha256 ", &buffer))) { + eol = memchr(buffer, '\n', buffer_end - buffer); + if (!eol) { + ret = report(options, oid, OBJ_TAG, FSCK_MSG_BAD_GPGSIG, "invalid format - unexpected end after 'gpgsig' or 'gpgsig-sha256' line"); + goto done; + } + buffer = eol + 1; + + while (buffer < buffer_end && starts_with(buffer, " ")) { + eol = memchr(buffer, '\n', buffer_end - buffer); + if (!eol) { + ret = report(options, oid, OBJ_TAG, FSCK_MSG_BAD_HEADER_CONTINUATION, "invalid format - unexpected end in 'gpgsig' or 'gpgsig-sha256' continuation line"); + goto done; + } + buffer = eol + 1; + } + } + if (buffer < buffer_end && !starts_with(buffer, "\n")) { /* * The verify_headers() check will allow @@ -25,9 +25,11 @@ enum fsck_msg_type { FUNC(NUL_IN_HEADER, FATAL) \ FUNC(UNTERMINATED_HEADER, FATAL) \ /* errors */ \ + FUNC(BAD_HEADER_CONTINUATION, ERROR) \ FUNC(BAD_DATE, ERROR) \ FUNC(BAD_DATE_OVERFLOW, ERROR) \ FUNC(BAD_EMAIL, ERROR) \ + FUNC(BAD_GPGSIG, ERROR) \ FUNC(BAD_NAME, ERROR) \ FUNC(BAD_OBJECT_SHA1, ERROR) \ FUNC(BAD_PACKED_REF_ENTRY, ERROR) \ diff --git a/generate-perl.sh b/generate-perl.sh index 65f122ebfc..796d835932 100755 --- a/generate-perl.sh +++ b/generate-perl.sh @@ -30,7 +30,7 @@ sed -e '1{' \ "$INPUT" >"$OUTPUT" case "$INPUT" in -*.perl) +*.perl|*git-contacts) chmod a+x "$OUTPUT";; *) ;; @@ -586,6 +586,7 @@ static struct cmd_struct commands[] = { { "grep", cmd_grep, RUN_SETUP_GENTLY }, { "hash-object", cmd_hash_object }, { "help", cmd_help }, + { "history", cmd_history, RUN_SETUP }, { "hook", cmd_hook, RUN_SETUP }, { "index-pack", cmd_index_pack, RUN_SETUP_GENTLY | NO_PARSEOPT }, { "init", cmd_init_db }, diff --git a/gpg-interface.c b/gpg-interface.c index 2f4f0e32cb..f680ed38c0 100644 --- a/gpg-interface.c +++ b/gpg-interface.c @@ -443,7 +443,7 @@ static void parse_ssh_output(struct signature_check *sigc) key = strstr(line, "key "); if (key) { - sigc->fingerprint = xstrdup(strstr(line, "key ") + 4); + sigc->fingerprint = xstrdup(key + 4); sigc->key = xstrdup(sigc->fingerprint); } else { /* @@ -821,8 +821,7 @@ static char *get_ssh_key_fingerprint(const char *signing_key) struct child_process ssh_keygen = CHILD_PROCESS_INIT; int ret = -1; struct strbuf fingerprint_stdout = STRBUF_INIT; - struct strbuf **fingerprint; - char *fingerprint_ret; + char *fingerprint_ret, *begin, *delim; const char *literal_key = NULL; /* @@ -845,13 +844,17 @@ static char *get_ssh_key_fingerprint(const char *signing_key) die_errno(_("failed to get the ssh fingerprint for key '%s'"), signing_key); - fingerprint = strbuf_split_max(&fingerprint_stdout, ' ', 3); - if (!fingerprint[1]) - die_errno(_("failed to get the ssh fingerprint for key '%s'"), + begin = fingerprint_stdout.buf; + delim = strchr(begin, ' '); + if (!delim) + die(_("failed to get the ssh fingerprint for key %s"), signing_key); - - fingerprint_ret = strbuf_detach(fingerprint[1], NULL); - strbuf_list_free(fingerprint); + begin = delim + 1; + delim = strchr(begin, ' '); + if (!delim) + die(_("failed to get the ssh fingerprint for key %s"), + signing_key); + fingerprint_ret = xmemdupz(begin, delim - begin); strbuf_release(&fingerprint_stdout); return fingerprint_ret; } @@ -862,12 +865,12 @@ static char *get_default_ssh_signing_key(void) struct child_process ssh_default_key = CHILD_PROCESS_INIT; int ret = -1; struct strbuf key_stdout = STRBUF_INIT, key_stderr = STRBUF_INIT; - struct strbuf **keys; char *key_command = NULL; const char **argv; int n; char *default_key = NULL; const char *literal_key = NULL; + char *begin, *new_line, *first_line; if (!ssh_default_key_command) die(_("either user.signingkey or gpg.ssh.defaultKeyCommand needs to be configured")); @@ -876,7 +879,7 @@ static char *get_default_ssh_signing_key(void) n = split_cmdline(key_command, &argv); if (n < 0) - die("malformed build-time gpg.ssh.defaultKeyCommand: %s", + die(_("malformed build-time gpg.ssh.defaultKeyCommand: %s"), split_cmdline_strerror(n)); strvec_pushv(&ssh_default_key.args, argv); @@ -884,19 +887,24 @@ static char *get_default_ssh_signing_key(void) &key_stderr, 0); if (!ret) { - keys = strbuf_split_max(&key_stdout, '\n', 2); - if (keys[0] && is_literal_ssh_key(keys[0]->buf, &literal_key)) { + begin = key_stdout.buf; + new_line = strchr(begin, '\n'); + if (new_line) + first_line = xmemdupz(begin, new_line - begin); + else + first_line = xstrdup(begin); + if (is_literal_ssh_key(first_line, &literal_key)) { /* * We only use `is_literal_ssh_key` here to check validity * The prefix will be stripped when the key is used. */ - default_key = strbuf_detach(keys[0], NULL); + default_key = first_line; } else { + free(first_line); warning(_("gpg.ssh.defaultKeyCommand succeeded but returned no keys: %s %s"), key_stderr.buf, key_stdout.buf); } - strbuf_list_free(keys); } else { warning(_("gpg.ssh.defaultKeyCommand failed: %s %s"), key_stderr.buf, key_stdout.buf); diff --git a/gpg-interface.h b/gpg-interface.h index 50487aa148..ead1ed6967 100644 --- a/gpg-interface.h +++ b/gpg-interface.h @@ -3,9 +3,9 @@ struct strbuf; -#define GPG_VERIFY_VERBOSE 1 -#define GPG_VERIFY_RAW 2 -#define GPG_VERIFY_OMIT_STATUS 4 +#define GPG_VERIFY_VERBOSE (1<<0) +#define GPG_VERIFY_RAW (1<<1) +#define GPG_VERIFY_OMIT_STATUS (1<<2) enum signature_trust_level { TRUST_UNDEFINED, @@ -241,7 +241,47 @@ const char *empty_tree_oid_hex(const struct git_hash_algo *algop) return oid_to_hex_r(buf, algop->empty_tree); } -int hash_algo_by_name(const char *name) +const struct git_hash_algo *hash_algo_ptr_by_offset(uint32_t algo) +{ + return &hash_algos[algo]; +} + +struct git_hash_ctx *git_hash_alloc(void) +{ + return malloc(sizeof(struct git_hash_ctx)); +} + +void git_hash_free(struct git_hash_ctx *ctx) +{ + free(ctx); +} + +void git_hash_init(struct git_hash_ctx *ctx, const struct git_hash_algo *algop) +{ + algop->init_fn(ctx); +} + +void git_hash_clone(struct git_hash_ctx *dst, const struct git_hash_ctx *src) +{ + src->algop->clone_fn(dst, src); +} + +void git_hash_update(struct git_hash_ctx *ctx, const void *in, size_t len) +{ + ctx->algop->update_fn(ctx, in, len); +} + +void git_hash_final(unsigned char *hash, struct git_hash_ctx *ctx) +{ + ctx->algop->final_fn(hash, ctx); +} + +void git_hash_final_oid(struct object_id *oid, struct git_hash_ctx *ctx) +{ + ctx->algop->final_oid_fn(oid, ctx); +} + +uint32_t hash_algo_by_name(const char *name) { if (!name) return GIT_HASH_UNKNOWN; @@ -251,7 +291,7 @@ int hash_algo_by_name(const char *name) return GIT_HASH_UNKNOWN; } -int hash_algo_by_id(uint32_t format_id) +uint32_t hash_algo_by_id(uint32_t format_id) { for (size_t i = 1; i < GIT_HASH_NALGOS; i++) if (format_id == hash_algos[i].format_id) @@ -259,7 +299,7 @@ int hash_algo_by_id(uint32_t format_id) return GIT_HASH_UNKNOWN; } -int hash_algo_by_length(size_t len) +uint32_t hash_algo_by_length(size_t len) { for (size_t i = 1; i < GIT_HASH_NALGOS; i++) if (len == hash_algos[i].rawsz) @@ -211,7 +211,7 @@ static inline void git_SHA256_Clone(git_SHA256_CTX *dst, const git_SHA256_CTX *s struct object_id { unsigned char hash[GIT_MAX_RAWSZ]; - int algo; /* XXX requires 4-byte alignment */ + uint32_t algo; /* XXX requires 4-byte alignment */ }; #define GET_OID_QUIETLY 01 @@ -320,37 +320,25 @@ struct git_hash_algo { }; extern const struct git_hash_algo hash_algos[GIT_HASH_NALGOS]; -static inline void git_hash_clone(struct git_hash_ctx *dst, const struct git_hash_ctx *src) -{ - src->algop->clone_fn(dst, src); -} - -static inline void git_hash_update(struct git_hash_ctx *ctx, const void *in, size_t len) -{ - ctx->algop->update_fn(ctx, in, len); -} - -static inline void git_hash_final(unsigned char *hash, struct git_hash_ctx *ctx) -{ - ctx->algop->final_fn(hash, ctx); -} - -static inline void git_hash_final_oid(struct object_id *oid, struct git_hash_ctx *ctx) -{ - ctx->algop->final_oid_fn(oid, ctx); -} - +void git_hash_init(struct git_hash_ctx *ctx, const struct git_hash_algo *algop); +void git_hash_clone(struct git_hash_ctx *dst, const struct git_hash_ctx *src); +void git_hash_update(struct git_hash_ctx *ctx, const void *in, size_t len); +void git_hash_final(unsigned char *hash, struct git_hash_ctx *ctx); +void git_hash_final_oid(struct object_id *oid, struct git_hash_ctx *ctx); +const struct git_hash_algo *hash_algo_ptr_by_offset(uint32_t algo); +struct git_hash_ctx *git_hash_alloc(void); +void git_hash_free(struct git_hash_ctx *ctx); /* * Return a GIT_HASH_* constant based on the name. Returns GIT_HASH_UNKNOWN if * the name doesn't match a known algorithm. */ -int hash_algo_by_name(const char *name); +uint32_t hash_algo_by_name(const char *name); /* Identical, except based on the format ID. */ -int hash_algo_by_id(uint32_t format_id); +uint32_t hash_algo_by_id(uint32_t format_id); /* Identical, except based on the length. */ -int hash_algo_by_length(size_t len); +uint32_t hash_algo_by_length(size_t len); /* Identical, except for a pointer to struct git_hash_algo. */ -static inline int hash_algo_by_ptr(const struct git_hash_algo *p) +static inline uint32_t hash_algo_by_ptr(const struct git_hash_algo *p) { size_t i; for (i = 0; i < GIT_HASH_NALGOS; i++) { @@ -851,18 +851,16 @@ struct similar_ref_cb { struct string_list *similar_refs; }; -static int append_similar_ref(const char *refname, const char *referent UNUSED, - const struct object_id *oid UNUSED, - int flags UNUSED, void *cb_data) +static int append_similar_ref(const struct reference *ref, void *cb_data) { struct similar_ref_cb *cb = (struct similar_ref_cb *)(cb_data); - char *branch = strrchr(refname, '/') + 1; + char *branch = strrchr(ref->name, '/') + 1; /* A remote branch of the same name is deemed similar */ - if (starts_with(refname, "refs/remotes/") && + if (starts_with(ref->name, "refs/remotes/") && !strcmp(branch, cb->base_ref)) string_list_append_nodup(cb->similar_refs, - refs_shorten_unambiguous_ref(get_main_ref_store(the_repository), refname, 1)); + refs_shorten_unambiguous_ref(get_main_ref_store(the_repository), ref->name, 1)); return 0; } @@ -65,11 +65,22 @@ static int pick_next_hook(struct child_process *cp, cp->no_stdin = 1; strvec_pushv(&cp->env, hook_cb->options->env.v); + + if (hook_cb->options->path_to_stdin && hook_cb->options->feed_pipe) + BUG("options path_to_stdin and feed_pipe are mutually exclusive"); + /* reopen the file for stdin; run_command closes it. */ if (hook_cb->options->path_to_stdin) { cp->no_stdin = 0; cp->in = xopen(hook_cb->options->path_to_stdin, O_RDONLY); } + + if (hook_cb->options->feed_pipe) { + cp->no_stdin = 0; + /* start_command() will allocate a pipe / stdin fd for us */ + cp->in = -1; + } + cp->stdout_to_stderr = 1; cp->trace2_hook_name = hook_cb->hook_name; cp->dir = hook_cb->options->dir; @@ -136,10 +147,12 @@ int run_hooks_opt(struct repository *r, const char *hook_name, .tr2_label = hook_name, .processes = 1, - .ungroup = 1, + .ungroup = options->ungroup, .get_next_task = pick_next_hook, .start_failure = notify_start_failure, + .feed_pipe = options->feed_pipe, + .consume_sideband = options->consume_sideband, .task_finished = notify_hook_finished, .data = &cb_data, @@ -148,6 +161,9 @@ int run_hooks_opt(struct repository *r, const char *hook_name, if (!options) BUG("a struct run_hooks_opt must be provided to run_hooks"); + if (options->path_to_stdin && options->feed_pipe) + BUG("choose only one method to populate hook stdin"); + if (options->invoked_hook) *options->invoked_hook = 0; @@ -177,6 +193,9 @@ int run_hooks(struct repository *r, const char *hook_name) { struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; + /* All use-cases of this API require ungrouping. */ + opt.ungroup = 1; + return run_hooks_opt(r, hook_name, &opt); } @@ -1,6 +1,7 @@ #ifndef HOOK_H #define HOOK_H #include "strvec.h" +#include "run-command.h" struct repository; @@ -34,9 +35,44 @@ struct run_hooks_opt int *invoked_hook; /** + * Allow hooks to set run_processes_parallel() 'ungroup' behavior. + */ + unsigned int ungroup:1; + + /** * Path to file which should be piped to stdin for each hook. */ const char *path_to_stdin; + + /** + * Callback to ask for more content to pipe to each hook stdin. + * + * If a hook needs to consume large quantities of data (e.g. a + * list of all refs received in a client push), feeding data via + * in-memory strings or slurping to/from files via path_to_stdin + * is inefficient, so this callback allows for piecemeal writes. + * + * Add initalization context to hook.feed_pipe_ctx. + * + * The caller owns hook.feed_pipe_ctx and has to release any + * resources after hooks finish execution. + */ + feed_pipe_fn feed_pipe; + void *feed_pipe_ctx; + + /** + * Use this to keep internal state for your feed_pipe_fn callback. + * Only useful when using run_hooks_opt.feed_pipe, otherwise ignore it. + */ + void *feed_pipe_cb_data; + + /* + * Populate this to capture output and prevent it from being printed to + * stderr. This will be passed directly through to + * run_command:run_parallel_processes(). See t/helper/test-run-command.c + * for an example. + */ + consume_sideband_fn consume_sideband; }; #define RUN_HOOKS_OPT_INIT { \ diff --git a/http-backend.c b/http-backend.c index 9084058f1e..273ed7266f 100644 --- a/http-backend.c +++ b/http-backend.c @@ -513,18 +513,17 @@ static void run_service(const char **argv, int buffer_input) exit(1); } -static int show_text_ref(const char *name, const char *referent UNUSED, const struct object_id *oid, - int flag UNUSED, void *cb_data) +static int show_text_ref(const struct reference *ref, void *cb_data) { - const char *name_nons = strip_namespace(name); + const char *name_nons = strip_namespace(ref->name); struct strbuf *buf = cb_data; - struct object *o = parse_object(the_repository, oid); + struct object *o = parse_object(the_repository, ref->oid); if (!o) return 0; - strbuf_addf(buf, "%s\t%s\n", oid_to_hex(oid), name_nons); + strbuf_addf(buf, "%s\t%s\n", oid_to_hex(ref->oid), name_nons); if (o->type == OBJ_TAG) { - o = deref_tag(the_repository, o, name, 0); + o = deref_tag(the_repository, o, ref->name, 0); if (!o) return 0; strbuf_addf(buf, "%s\t%s^{}\n", oid_to_hex(&o->oid), @@ -569,21 +568,20 @@ static void get_info_refs(struct strbuf *hdr, char *arg UNUSED) strbuf_release(&buf); } -static int show_head_ref(const char *refname, const char *referent UNUSED, const struct object_id *oid, - int flag, void *cb_data) +static int show_head_ref(const struct reference *ref, void *cb_data) { struct strbuf *buf = cb_data; - if (flag & REF_ISSYMREF) { + if (ref->flags & REF_ISSYMREF) { const char *target = refs_resolve_ref_unsafe(get_main_ref_store(the_repository), - refname, + ref->name, RESOLVE_REF_READING, NULL, NULL); if (target) strbuf_addf(buf, "ref: %s\n", strip_namespace(target)); } else { - strbuf_addf(buf, "%s\n", oid_to_hex(oid)); + strbuf_addf(buf, "%s\n", oid_to_hex(ref->oid)); } return 0; @@ -603,19 +601,18 @@ static void get_head(struct strbuf *hdr, char *arg UNUSED) static void get_info_packs(struct strbuf *hdr, char *arg UNUSED) { size_t objdirlen = strlen(repo_get_object_directory(the_repository)); - struct packfile_store *packs = the_repository->objects->packfiles; struct strbuf buf = STRBUF_INIT; struct packed_git *p; size_t cnt = 0; select_getanyfile(hdr); - for (p = packfile_store_get_all_packs(packs); p; p = p->next) { + repo_for_each_pack(the_repository, p) { if (p->pack_local) cnt++; } strbuf_grow(&buf, cnt * 53 + 2); - for (p = packfile_store_get_all_packs(packs); p; p = p->next) { + repo_for_each_pack(the_repository, p) { if (p->pack_local) strbuf_addf(&buf, "P %s\n", p->pack_name + objdirlen + 6); } diff --git a/http-push.c b/http-push.c index a1c01e3b9b..d86ce77119 100644 --- a/http-push.c +++ b/http-push.c @@ -104,7 +104,7 @@ struct repo { int has_info_refs; int can_update_info_refs; int has_info_packs; - struct packed_git *packs; + struct packfile_list packs; struct remote_lock *locks; }; @@ -311,7 +311,7 @@ static void start_fetch_packed(struct transfer_request *request) struct transfer_request *check_request = request_queue_head; struct http_pack_request *preq; - target = find_oid_pack(&request->obj->oid, repo->packs); + target = packfile_list_find_oid(repo->packs.head, &request->obj->oid); if (!target) { fprintf(stderr, "Unable to fetch %s, will not be able to update server info refs\n", oid_to_hex(&request->obj->oid)); repo->can_update_info_refs = 0; @@ -683,7 +683,7 @@ static int add_send_request(struct object *obj, struct remote_lock *lock) get_remote_object_list(obj->oid.hash[0]); if (obj->flags & (REMOTE | PUSHING)) return 0; - target = find_oid_pack(&obj->oid, repo->packs); + target = packfile_list_find_oid(repo->packs.head, &obj->oid); if (target) { obj->flags |= REMOTE; return 0; diff --git a/http-walker.c b/http-walker.c index 0f7ae46d7f..e886e64866 100644 --- a/http-walker.c +++ b/http-walker.c @@ -15,7 +15,7 @@ struct alt_base { char *base; int got_indices; - struct packed_git *packs; + struct packfile_list packs; struct alt_base *next; }; @@ -324,11 +324,8 @@ static void process_alternates_response(void *callback_data) } else if (is_alternate_allowed(target.buf)) { warning("adding alternate object store: %s", target.buf); - newalt = xmalloc(sizeof(*newalt)); - newalt->next = NULL; + CALLOC_ARRAY(newalt, 1); newalt->base = strbuf_detach(&target, NULL); - newalt->got_indices = 0; - newalt->packs = NULL; while (tail->next != NULL) tail = tail->next; @@ -435,7 +432,7 @@ static int http_fetch_pack(struct walker *walker, struct alt_base *repo, if (fetch_indices(walker, repo)) return -1; - target = find_oid_pack(oid, repo->packs); + target = packfile_list_find_oid(repo->packs.head, oid); if (!target) return -1; close_pack_index(target); @@ -584,17 +581,15 @@ static void cleanup(struct walker *walker) if (data) { alt = data->alt; while (alt) { - struct packed_git *pack; + struct packfile_list_entry *e; alt_next = alt->next; - pack = alt->packs; - while (pack) { - struct packed_git *pack_next = pack->next; - close_pack(pack); - free(pack); - pack = pack_next; + for (e = alt->packs.head; e; e = e->next) { + close_pack(e->pack); + free(e->pack); } + packfile_list_clear(&alt->packs); free(alt->base); free(alt); @@ -612,14 +607,11 @@ struct walker *get_http_walker(const char *url) struct walker_data *data = xmalloc(sizeof(struct walker_data)); struct walker *walker = xmalloc(sizeof(struct walker)); - data->alt = xmalloc(sizeof(*data->alt)); + CALLOC_ARRAY(data->alt, 1); data->alt->base = xstrdup(url); for (s = data->alt->base + strlen(data->alt->base) - 1; *s == '/'; --s) *s = 0; - data->alt->got_indices = 0; - data->alt->packs = NULL; - data->alt->next = NULL; data->got_alternates = -1; walker->corrupt_object_found = 0; @@ -2413,10 +2413,10 @@ static char *fetch_pack_index(unsigned char *hash, const char *base_url) return tmp; } -static int fetch_and_setup_pack_index(struct packed_git **packs_head, - unsigned char *sha1, const char *base_url) +static int fetch_and_setup_pack_index(struct packfile_list *packs, + unsigned char *sha1, + const char *base_url) { - struct packfile_store *packs = the_repository->objects->packfiles; struct packed_git *new_pack, *p; char *tmp_idx = NULL; int ret; @@ -2425,7 +2425,7 @@ static int fetch_and_setup_pack_index(struct packed_git **packs_head, * If we already have the pack locally, no need to fetch its index or * even add it to list; we already have all of its objects. */ - for (p = packfile_store_get_all_packs(packs); p; p = p->next) { + repo_for_each_pack(the_repository, p) { if (hasheq(p->hash, sha1, the_repository->hash_algo)) return 0; } @@ -2449,12 +2449,11 @@ static int fetch_and_setup_pack_index(struct packed_git **packs_head, if (ret) return -1; - new_pack->next = *packs_head; - *packs_head = new_pack; + packfile_list_prepend(packs, new_pack); return 0; } -int http_get_info_packs(const char *base_url, struct packed_git **packs_head) +int http_get_info_packs(const char *base_url, struct packfile_list *packs) { struct http_get_options options = {0}; int ret = 0; @@ -2478,7 +2477,7 @@ int http_get_info_packs(const char *base_url, struct packed_git **packs_head) !parse_oid_hex(data, &oid, &data) && skip_prefix(data, ".pack", &data) && (*data == '\n' || *data == '\0')) { - fetch_and_setup_pack_index(packs_head, oid.hash, base_url); + fetch_and_setup_pack_index(packs, oid.hash, base_url); } else { data = strchrnul(data, '\n'); } @@ -2542,14 +2541,9 @@ cleanup: } void http_install_packfile(struct packed_git *p, - struct packed_git **list_to_remove_from) + struct packfile_list *list_to_remove_from) { - struct packed_git **lst = list_to_remove_from; - - while (*lst != p) - lst = &((*lst)->next); - *lst = (*lst)->next; - + packfile_list_remove(list_to_remove_from, p); packfile_store_add_pack(the_repository->objects->packfiles, p); } @@ -2,6 +2,7 @@ #define HTTP_H struct packed_git; +struct packfile_list; #include "git-zlib.h" @@ -190,7 +191,7 @@ struct curl_slist *http_append_auth_header(const struct credential *c, /* Helpers for fetching packs */ int http_get_info_packs(const char *base_url, - struct packed_git **packs_head); + struct packfile_list *packs); /* Helper for getting Accept-Language header */ const char *http_get_accept_language_header(void); @@ -226,7 +227,7 @@ void release_http_pack_request(struct http_pack_request *preq); * from http_get_info_packs() and have chosen a specific pack to fetch. */ void http_install_packfile(struct packed_git *p, - struct packed_git **list_to_remove_from); + struct packfile_list *list_to_remove_from); /* Helpers for fetching object */ struct http_object_request { diff --git a/log-tree.c b/log-tree.c index 7d917f2a83..1729b0c201 100644 --- a/log-tree.c +++ b/log-tree.c @@ -147,9 +147,7 @@ static int ref_filter_match(const char *refname, return 1; } -static int add_ref_decoration(const char *refname, const char *referent UNUSED, const struct object_id *oid, - int flags UNUSED, - void *cb_data) +static int add_ref_decoration(const struct reference *ref, void *cb_data) { int i; struct object *obj; @@ -158,16 +156,16 @@ static int add_ref_decoration(const char *refname, const char *referent UNUSED, struct decoration_filter *filter = (struct decoration_filter *)cb_data; const char *git_replace_ref_base = ref_namespace[NAMESPACE_REPLACE].ref; - if (filter && !ref_filter_match(refname, filter)) + if (filter && !ref_filter_match(ref->name, filter)) return 0; - if (starts_with(refname, git_replace_ref_base)) { + if (starts_with(ref->name, git_replace_ref_base)) { struct object_id original_oid; if (!replace_refs_enabled(the_repository)) return 0; - if (get_oid_hex(refname + strlen(git_replace_ref_base), + if (get_oid_hex(ref->name + strlen(git_replace_ref_base), &original_oid)) { - warning("invalid replace ref %s", refname); + warning("invalid replace ref %s", ref->name); return 0; } obj = parse_object(the_repository, &original_oid); @@ -176,10 +174,10 @@ static int add_ref_decoration(const char *refname, const char *referent UNUSED, return 0; } - objtype = odb_read_object_info(the_repository->objects, oid, NULL); + objtype = odb_read_object_info(the_repository->objects, ref->oid, NULL); if (objtype < 0) return 0; - obj = lookup_object_by_type(the_repository, oid, objtype); + obj = lookup_object_by_type(the_repository, ref->oid, objtype); for (i = 0; i < ARRAY_SIZE(ref_namespace); i++) { struct ref_namespace_info *info = &ref_namespace[i]; @@ -187,24 +185,24 @@ static int add_ref_decoration(const char *refname, const char *referent UNUSED, if (!info->decoration) continue; if (info->exact) { - if (!strcmp(refname, info->ref)) { + if (!strcmp(ref->name, info->ref)) { deco_type = info->decoration; break; } - } else if (starts_with(refname, info->ref)) { + } else if (starts_with(ref->name, info->ref)) { deco_type = info->decoration; break; } } - add_name_decoration(deco_type, refname, obj); + add_name_decoration(deco_type, ref->name, obj); while (obj->type == OBJ_TAG) { if (!obj->parsed) parse_object(the_repository, &obj->oid); obj = ((struct tag *)obj)->tagged; if (!obj) break; - add_name_decoration(DECORATION_REF_TAG, refname, obj); + add_name_decoration(DECORATION_REF_TAG, ref->name, obj); } return 0; } @@ -1,6 +1,7 @@ #include "git-compat-util.h" #include "hash.h" #include "path.h" +#include "object-file.h" #include "odb.h" #include "hex.h" #include "repository.h" @@ -48,13 +49,13 @@ static int insert_loose_map(struct odb_source *source, const struct object_id *oid, const struct object_id *compat_oid) { - struct loose_object_map *map = source->loose_map; + struct loose_object_map *map = source->loose->map; int inserted = 0; inserted |= insert_oid_pair(map->to_compat, oid, compat_oid); inserted |= insert_oid_pair(map->to_storage, compat_oid, oid); if (inserted) - oidtree_insert(source->loose_objects_cache, compat_oid); + oidtree_insert(source->loose->cache, compat_oid); return inserted; } @@ -64,11 +65,11 @@ static int load_one_loose_object_map(struct repository *repo, struct odb_source struct strbuf buf = STRBUF_INIT, path = STRBUF_INIT; FILE *fp; - if (!source->loose_map) - loose_object_map_init(&source->loose_map); - if (!source->loose_objects_cache) { - ALLOC_ARRAY(source->loose_objects_cache, 1); - oidtree_init(source->loose_objects_cache); + if (!source->loose->map) + loose_object_map_init(&source->loose->map); + if (!source->loose->cache) { + ALLOC_ARRAY(source->loose->cache, 1); + oidtree_init(source->loose->cache); } insert_loose_map(source, repo->hash_algo->empty_tree, repo->compat_hash_algo->empty_tree); @@ -124,7 +125,7 @@ int repo_read_loose_object_map(struct repository *repo) int repo_write_loose_object_map(struct repository *repo) { - kh_oid_map_t *map = repo->objects->sources->loose_map->to_compat; + kh_oid_map_t *map = repo->objects->sources->loose->map->to_compat; struct lock_file lock; int fd; khiter_t iter; @@ -230,7 +231,7 @@ int repo_loose_object_map_oid(struct repository *repo, khiter_t pos; for (source = repo->objects->sources; source; source = source->next) { - struct loose_object_map *loose_map = source->loose_map; + struct loose_object_map *loose_map = source->loose->map; if (!loose_map) continue; map = (to == repo->compat_hash_algo) ? @@ -75,42 +75,42 @@ struct ls_refs_data { unsigned unborn : 1; }; -static int send_ref(const char *refname, const char *referent UNUSED, const struct object_id *oid, - int flag, void *cb_data) +static int send_ref(const struct reference *ref, void *cb_data) { struct ls_refs_data *data = cb_data; - const char *refname_nons = strip_namespace(refname); + const char *refname_nons = strip_namespace(ref->name); strbuf_reset(&data->buf); - if (ref_is_hidden(refname_nons, refname, &data->hidden_refs)) + if (ref_is_hidden(refname_nons, ref->name, &data->hidden_refs)) return 0; if (!ref_match(&data->prefixes, refname_nons)) return 0; - if (oid) - strbuf_addf(&data->buf, "%s %s", oid_to_hex(oid), refname_nons); + if (ref->oid) + strbuf_addf(&data->buf, "%s %s", oid_to_hex(ref->oid), refname_nons); else strbuf_addf(&data->buf, "unborn %s", refname_nons); - if (data->symrefs && flag & REF_ISSYMREF) { + if (data->symrefs && ref->flags & REF_ISSYMREF) { + int unused_flag; struct object_id unused; const char *symref_target = refs_resolve_ref_unsafe(get_main_ref_store(the_repository), - refname, + ref->name, 0, &unused, - &flag); + &unused_flag); if (!symref_target) - die("'%s' is a symref but it is not?", refname); + die("'%s' is a symref but it is not?", ref->name); strbuf_addf(&data->buf, " symref-target:%s", strip_namespace(symref_target)); } - if (data->peel && oid) { + if (data->peel && ref->oid) { struct object_id peeled; - if (!peel_iterated_oid(the_repository, oid, &peeled)) + if (!reference_get_peeled_oid(the_repository, ref, &peeled)) strbuf_addf(&data->buf, " peeled:%s", oid_to_hex(&peeled)); } @@ -131,9 +131,17 @@ static void send_possibly_unborn_head(struct ls_refs_data *data) if (!refs_resolve_ref_unsafe(get_main_ref_store(the_repository), namespaced.buf, 0, &oid, &flag)) return; /* bad ref */ oid_is_null = is_null_oid(&oid); + if (!oid_is_null || - (data->unborn && data->symrefs && (flag & REF_ISSYMREF))) - send_ref(namespaced.buf, NULL, oid_is_null ? NULL : &oid, flag, data); + (data->unborn && data->symrefs && (flag & REF_ISSYMREF))) { + struct reference ref = { + .name = namespaced.buf, + .oid = oid_is_null ? NULL : &oid, + .flags = flag, + }; + + send_ref(&ref, data); + } strbuf_release(&namespaced); } diff --git a/merge-ort.c b/merge-ort.c index 29858074f9..8bda80d7b7 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -2913,6 +2913,32 @@ static int process_renames(struct merge_options *opt, continue; /* + * Rename caching from a previous commit might give us an + * irrelevant rename for the current commit. + * + * Imagine: + * foo/A -> bar/A + * was a cached rename for the upstream side from the + * previous commit (without the directories being renamed), + * but the next commit being replayed + * * does NOT add or delete files + * * does NOT have directory renames + * * does NOT modify any files under bar/ + * * does NOT modify foo/A + * * DOES modify other files under foo/ (otherwise the + * !oldinfo check above would have already exited for + * us) + * In such a case, our trivial directory resolution will + * have already merged bar/, and our attempt to process + * the cached + * foo/A -> bar/A + * would be counterproductive, and lack the necessary + * information anyway. Skip such renames. + */ + if (!newinfo) + continue; + + /* * diff_filepairs have copies of pathnames, thus we have to * use standard 'strcmp()' (negated) instead of '=='. */ @@ -3438,7 +3464,7 @@ static int collect_renames(struct merge_options *opt, continue; } if (opt->detect_directory_renames == MERGE_DIRECTORY_RENAMES_NONE && - p->status == 'R' && 1) { + p->status == 'R') { possibly_cache_new_pair(renames, p, side_index, NULL); goto skip_directory_renames; } @@ -5118,7 +5144,8 @@ static void merge_check_renames_reusable(struct merge_options *opt, * optimization" comment near that case). * * This could be revisited in the future; see the commit message - * where this comment was added for some possible pointers. + * where this comment was added for some possible pointers, or the + * later commit where this comment was added. */ if (opt->detect_directory_renames == MERGE_DIRECTORY_RENAMES_NONE) { renames->cached_pairs_valid_side = 0; /* neither side valid */ @@ -5495,8 +5522,6 @@ int parse_merge_opt(struct merge_options *opt, const char *s) long value = parse_algorithm_value(arg); if (value < 0) return -1; - /* clear out previous settings */ - DIFF_XDL_CLR(opt, NEED_MINIMAL); opt->xdl_opts &= ~XDF_DIFF_ALGORITHM_MASK; opt->xdl_opts |= value; } diff --git a/meson.build b/meson.build index cee9424475..c109da5b46 100644 --- a/meson.build +++ b/meson.build @@ -463,7 +463,14 @@ libgit_sources = [ 'reftable/tree.c', 'reftable/writer.c', 'remote.c', + 'repack.c', + 'repack-cruft.c', + 'repack-filtered.c', + 'repack-geometry.c', + 'repack-midx.c', + 'repack-promisor.c', 'replace-object.c', + 'replay.c', 'repo-settings.c', 'repository.c', 'rerere.c', @@ -603,6 +610,7 @@ builtin_sources = [ 'builtin/grep.c', 'builtin/hash-object.c', 'builtin/help.c', + 'builtin/history.c', 'builtin/hook.c', 'builtin/index-pack.c', 'builtin/init-db.c', @@ -762,13 +770,18 @@ if test_output_directory == '' test_output_directory = meson.project_build_root() / 'test-output' endif +htmldir = get_option('htmldir') +if htmldir == '' + htmldir = get_option('datadir') / 'doc/git-doc' +endif + # These variables are used for building libgit.a. libgit_c_args = [ '-DBINDIR="' + get_option('bindir') + '"', '-DDEFAULT_GIT_TEMPLATE_DIR="' + get_option('datadir') / 'git-core/templates' + '"', '-DFALLBACK_RUNTIME_PREFIX="' + get_option('prefix') + '"', '-DGIT_HOST_CPU="' + host_machine.cpu_family() + '"', - '-DGIT_HTML_PATH="' + get_option('datadir') / 'doc/git-doc"', + '-DGIT_HTML_PATH="' + htmldir + '"', '-DGIT_INFO_PATH="' + get_option('infodir') + '"', '-DGIT_LOCALE_PATH="' + get_option('localedir') + '"', '-DGIT_MAN_PATH="' + get_option('mandir') + '"', @@ -1708,6 +1721,10 @@ rust_option = get_option('rust').disable_auto_if(not cargo.found()) if rust_option.allowed() subdir('src') libgit_c_args += '-DWITH_RUST' + + if host_machine.system() == 'windows' + libgit_dependencies += compiler.find_library('userenv') + endif else libgit_sources += [ 'varint.c', diff --git a/meson_options.txt b/meson_options.txt index 143dee9237..e0be260ae1 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -1,4 +1,6 @@ # Configuration for Git installation +option('htmldir', type: 'string', value: '', + description: 'Directory to install HTML docs to. Defaults to <datadir>/doc/git-doc') option('perllibdir', type: 'string', value: '', description: 'Directory to install perl lib to. Defaults to <datadir>/perl5') diff --git a/midx-write.c b/midx-write.c index c73010df6d..23e61cb000 100644 --- a/midx-write.c +++ b/midx-write.c @@ -697,28 +697,27 @@ static void prepare_midx_packing_data(struct packing_data *pdata, trace2_region_leave("midx", "prepare_midx_packing_data", ctx->repo); } -static int add_ref_to_pending(const char *refname, const char *referent UNUSED, - const struct object_id *oid, - int flag, void *cb_data) +static int add_ref_to_pending(const struct reference *ref, void *cb_data) { struct rev_info *revs = (struct rev_info*)cb_data; + const struct object_id *maybe_peeled = ref->oid; struct object_id peeled; struct object *object; - if ((flag & REF_ISSYMREF) && (flag & REF_ISBROKEN)) { - warning("symbolic ref is dangling: %s", refname); + if ((ref->flags & REF_ISSYMREF) && (ref->flags & REF_ISBROKEN)) { + warning("symbolic ref is dangling: %s", ref->name); return 0; } - if (!peel_iterated_oid(revs->repo, oid, &peeled)) - oid = &peeled; + if (!reference_get_peeled_oid(revs->repo, ref, &peeled)) + maybe_peeled = &peeled; - object = parse_object_or_die(revs->repo, oid, refname); + object = parse_object_or_die(revs->repo, maybe_peeled, ref->name); if (object->type != OBJ_COMMIT) return 0; add_pending_object(revs, object, ""); - if (bitmap_is_preferred_refname(revs->repo, refname)) + if (bitmap_is_preferred_refname(revs->repo, ref->name)) object->flags |= NEEDS_BITMAP; return 0; } @@ -462,8 +462,6 @@ int prepare_midx_pack(struct multi_pack_index *m, m->pack_names[pack_int_id]); p = packfile_store_load_pack(r->objects->packfiles, pack_name.buf, m->source->local); - if (p) - list_add_tail(&p->mru, &r->objects->packfiles->mru); strbuf_release(&pack_name); if (!p) { diff --git a/negotiator/default.c b/negotiator/default.c index c479da9b09..116dedcf83 100644 --- a/negotiator/default.c +++ b/negotiator/default.c @@ -38,11 +38,10 @@ static void rev_list_push(struct negotiation_state *ns, } } -static int clear_marks(const char *refname, const char *referent UNUSED, const struct object_id *oid, - int flag UNUSED, - void *cb_data UNUSED) +static int clear_marks(const struct reference *ref, void *cb_data UNUSED) { - struct object *o = deref_tag(the_repository, parse_object(the_repository, oid), refname, 0); + struct object *o = deref_tag(the_repository, parse_object(the_repository, ref->oid), + ref->name, 0); if (o && o->type == OBJ_COMMIT) clear_commit_marks((struct commit *)o, diff --git a/negotiator/skipping.c b/negotiator/skipping.c index 616df6bf3a..0a272130fb 100644 --- a/negotiator/skipping.c +++ b/negotiator/skipping.c @@ -75,11 +75,10 @@ static struct entry *rev_list_push(struct data *data, struct commit *commit, int return entry; } -static int clear_marks(const char *refname, const char *referent UNUSED, const struct object_id *oid, - int flag UNUSED, - void *cb_data UNUSED) +static int clear_marks(const struct reference *ref, void *cb_data UNUSED) { - struct object *o = deref_tag(the_repository, parse_object(the_repository, oid), refname, 0); + struct object *o = deref_tag(the_repository, parse_object(the_repository, ref->oid), + ref->name, 0); if (o && o->type == OBJ_COMMIT) clear_commit_marks((struct commit *)o, @@ -938,13 +938,11 @@ out: return ret; } -static int string_list_add_one_ref(const char *refname, const char *referent UNUSED, - const struct object_id *oid UNUSED, - int flag UNUSED, void *cb) +static int string_list_add_one_ref(const struct reference *ref, void *cb) { struct string_list *refs = cb; - if (!unsorted_string_list_has_string(refs, refname)) - string_list_append(refs, refname); + if (!unsorted_string_list_has_string(refs, ref->name)) + string_list_append(refs, ref->name); return 0; } diff --git a/object-file-convert.c b/object-file-convert.c index 7ab875afe6..f8dce94811 100644 --- a/object-file-convert.c +++ b/object-file-convert.c @@ -13,7 +13,7 @@ #include "gpg-interface.h" #include "object-file-convert.h" -int repo_oid_to_algop(struct repository *repo, const struct object_id *src, +int repo_oid_to_algop(struct repository *repo, const struct object_id *srcoid, const struct git_hash_algo *to, struct object_id *dest) { /* @@ -21,9 +21,17 @@ int repo_oid_to_algop(struct repository *repo, const struct object_id *src, * default hash algorithm for that object. */ const struct git_hash_algo *from = - src->algo ? &hash_algos[src->algo] : repo->hash_algo; + srcoid->algo ? &hash_algos[srcoid->algo] : repo->hash_algo; + struct object_id temp; + const struct object_id *src = srcoid; - if (from == to) { + if (!srcoid->algo) { + oidcpy(&temp, srcoid); + temp.algo = hash_algo_by_ptr(repo->hash_algo); + src = &temp; + } + + if (from == to || !to) { if (src != dest) oidcpy(dest, src); return 0; diff --git a/object-file.c b/object-file.c index 4675c8ed6b..811c569ed3 100644 --- a/object-file.c +++ b/object-file.c @@ -99,8 +99,8 @@ static int check_and_freshen_source(struct odb_source *source, return check_and_freshen_file(path.buf, freshen); } -int has_loose_object(struct odb_source *source, - const struct object_id *oid) +int odb_source_loose_has_object(struct odb_source *source, + const struct object_id *oid) { return check_and_freshen_source(source, oid, 0); } @@ -167,25 +167,22 @@ int stream_object_signature(struct repository *r, const struct object_id *oid) } /* - * Find "oid" as a loose object in the local repository or in an alternate. + * Find "oid" as a loose object in given source. * Returns 0 on success, negative on failure. * * The "path" out-parameter will give the path of the object we found (if any). * Note that it may point to static storage and is only valid until another * call to stat_loose_object(). */ -static int stat_loose_object(struct repository *r, const struct object_id *oid, +static int stat_loose_object(struct odb_source_loose *loose, + const struct object_id *oid, struct stat *st, const char **path) { - struct odb_source *source; static struct strbuf buf = STRBUF_INIT; - odb_prepare_alternates(r->objects); - for (source = r->objects->sources; source; source = source->next) { - *path = odb_loose_path(source, &buf, oid); - if (!lstat(*path, st)) - return 0; - } + *path = odb_loose_path(loose->source, &buf, oid); + if (!lstat(*path, st)) + return 0; return -1; } @@ -194,39 +191,24 @@ static int stat_loose_object(struct repository *r, const struct object_id *oid, * Like stat_loose_object(), but actually open the object and return the * descriptor. See the caveats on the "path" parameter above. */ -static int open_loose_object(struct repository *r, +static int open_loose_object(struct odb_source_loose *loose, const struct object_id *oid, const char **path) { - int fd; - struct odb_source *source; - int most_interesting_errno = ENOENT; static struct strbuf buf = STRBUF_INIT; + int fd; - odb_prepare_alternates(r->objects); - for (source = r->objects->sources; source; source = source->next) { - *path = odb_loose_path(source, &buf, oid); - fd = git_open(*path); - if (fd >= 0) - return fd; + *path = odb_loose_path(loose->source, &buf, oid); + fd = git_open(*path); + if (fd >= 0) + return fd; - if (most_interesting_errno == ENOENT) - most_interesting_errno = errno; - } - errno = most_interesting_errno; return -1; } -static int quick_has_loose(struct repository *r, +static int quick_has_loose(struct odb_source_loose *loose, const struct object_id *oid) { - struct odb_source *source; - - odb_prepare_alternates(r->objects); - for (source = r->objects->sources; source; source = source->next) { - if (oidtree_contains(odb_loose_cache(source, oid), oid)) - return 1; - } - return 0; + return !!oidtree_contains(odb_source_loose_cache(loose->source, oid), oid); } /* @@ -252,12 +234,12 @@ static void *map_fd(int fd, const char *path, unsigned long *size) return map; } -void *map_loose_object(struct repository *r, - const struct object_id *oid, - unsigned long *size) +void *odb_source_loose_map_object(struct odb_source *source, + const struct object_id *oid, + unsigned long *size) { const char *p; - int fd = open_loose_object(r, oid, &p); + int fd = open_loose_object(source->loose, oid, &p); if (fd < 0) return NULL; @@ -407,9 +389,9 @@ int parse_loose_header(const char *hdr, struct object_info *oi) return 0; } -int loose_object_info(struct repository *r, - const struct object_id *oid, - struct object_info *oi, int flags) +int odb_source_loose_read_object_info(struct odb_source *source, + const struct object_id *oid, + struct object_info *oi, int flags) { int status = 0; int fd; @@ -422,7 +404,7 @@ int loose_object_info(struct repository *r, enum object_type type_scratch; if (oi->delta_base_oid) - oidclr(oi->delta_base_oid, r->hash_algo); + oidclr(oi->delta_base_oid, source->odb->repo->hash_algo); /* * If we don't care about type or size, then we don't @@ -435,15 +417,15 @@ int loose_object_info(struct repository *r, if (!oi->typep && !oi->sizep && !oi->contentp) { struct stat st; if (!oi->disk_sizep && (flags & OBJECT_INFO_QUICK)) - return quick_has_loose(r, oid) ? 0 : -1; - if (stat_loose_object(r, oid, &st, &path) < 0) + return quick_has_loose(source->loose, oid) ? 0 : -1; + if (stat_loose_object(source->loose, oid, &st, &path) < 0) return -1; if (oi->disk_sizep) *oi->disk_sizep = st.st_size; return 0; } - fd = open_loose_object(r, oid, &path); + fd = open_loose_object(source->loose, oid, &path); if (fd < 0) { if (errno != ENOENT) error_errno(_("unable to open loose object %s"), oid_to_hex(oid)); @@ -986,35 +968,15 @@ static int write_loose_object(struct odb_source *source, FOF_SKIP_COLLISION_CHECK); } -static int freshen_loose_object(struct object_database *odb, - const struct object_id *oid) +int odb_source_loose_freshen_object(struct odb_source *source, + const struct object_id *oid) { - odb_prepare_alternates(odb); - for (struct odb_source *source = odb->sources; source; source = source->next) - if (check_and_freshen_source(source, oid, 1)) - return 1; - return 0; + return !!check_and_freshen_source(source, oid, 1); } -static int freshen_packed_object(struct object_database *odb, - const struct object_id *oid) -{ - struct pack_entry e; - if (!find_pack_entry(odb->repo, oid, &e)) - return 0; - if (e.p->is_cruft) - return 0; - if (e.p->freshened) - return 1; - if (!freshen_file(e.p->pack_name)) - return 0; - e.p->freshened = 1; - return 1; -} - -int stream_loose_object(struct odb_source *source, - struct input_stream *in_stream, size_t len, - struct object_id *oid) +int odb_source_loose_write_stream(struct odb_source *source, + struct odb_write_stream *in_stream, size_t len, + struct object_id *oid) { const struct git_hash_algo *compat = source->odb->repo->compat_hash_algo; struct object_id compat_oid; @@ -1091,12 +1053,10 @@ int stream_loose_object(struct odb_source *source, die(_("deflateEnd on stream object failed (%d)"), ret); close_loose_object(source, fd, tmp_file.buf); - if (freshen_packed_object(source->odb, oid) || - freshen_loose_object(source->odb, oid)) { + if (odb_freshen_object(source->odb, oid)) { unlink_or_warn(tmp_file.buf); goto cleanup; } - odb_loose_path(source, &filename, oid); /* We finally know the object path, and create the missing dir. */ @@ -1124,10 +1084,10 @@ cleanup: return err; } -int write_object_file(struct odb_source *source, - const void *buf, unsigned long len, - enum object_type type, struct object_id *oid, - struct object_id *compat_oid_in, unsigned flags) +int odb_source_loose_write_object(struct odb_source *source, + const void *buf, unsigned long len, + enum object_type type, struct object_id *oid, + struct object_id *compat_oid_in, unsigned flags) { const struct git_hash_algo *algo = source->odb->repo->hash_algo; const struct git_hash_algo *compat = source->odb->repo->compat_hash_algo; @@ -1155,8 +1115,7 @@ int write_object_file(struct odb_source *source, * it out into .git/objects/??/?{38} file. */ write_object_file_prepare(algo, buf, len, type, oid, hdr, &hdrlen); - if (freshen_packed_object(source->odb, oid) || - freshen_loose_object(source->odb, oid)) + if (odb_freshen_object(source->odb, oid)) return 0; if (write_loose_object(source, oid, hdr, hdrlen, buf, len, 0, flags)) return -1; @@ -1179,7 +1138,7 @@ int force_object_loose(struct odb_source *source, int ret; for (struct odb_source *s = source->odb->sources; s; s = s->next) - if (has_loose_object(s, oid)) + if (odb_source_loose_has_object(s, oid)) return 0; oi.typep = &type; @@ -1802,44 +1761,49 @@ static int append_loose_object(const struct object_id *oid, return 0; } -struct oidtree *odb_loose_cache(struct odb_source *source, - const struct object_id *oid) +struct oidtree *odb_source_loose_cache(struct odb_source *source, + const struct object_id *oid) { int subdir_nr = oid->hash[0]; struct strbuf buf = STRBUF_INIT; - size_t word_bits = bitsizeof(source->loose_objects_subdir_seen[0]); + size_t word_bits = bitsizeof(source->loose->subdir_seen[0]); size_t word_index = subdir_nr / word_bits; size_t mask = (size_t)1u << (subdir_nr % word_bits); uint32_t *bitmap; if (subdir_nr < 0 || - (size_t) subdir_nr >= bitsizeof(source->loose_objects_subdir_seen)) + (size_t) subdir_nr >= bitsizeof(source->loose->subdir_seen)) BUG("subdir_nr out of range"); - bitmap = &source->loose_objects_subdir_seen[word_index]; + bitmap = &source->loose->subdir_seen[word_index]; if (*bitmap & mask) - return source->loose_objects_cache; - if (!source->loose_objects_cache) { - ALLOC_ARRAY(source->loose_objects_cache, 1); - oidtree_init(source->loose_objects_cache); + return source->loose->cache; + if (!source->loose->cache) { + ALLOC_ARRAY(source->loose->cache, 1); + oidtree_init(source->loose->cache); } strbuf_addstr(&buf, source->path); for_each_file_in_obj_subdir(subdir_nr, &buf, source->odb->repo->hash_algo, append_loose_object, NULL, NULL, - source->loose_objects_cache); + source->loose->cache); *bitmap |= mask; strbuf_release(&buf); - return source->loose_objects_cache; + return source->loose->cache; } -void odb_clear_loose_cache(struct odb_source *source) +static void odb_source_loose_clear_cache(struct odb_source_loose *loose) { - oidtree_clear(source->loose_objects_cache); - FREE_AND_NULL(source->loose_objects_cache); - memset(&source->loose_objects_subdir_seen, 0, - sizeof(source->loose_objects_subdir_seen)); + oidtree_clear(loose->cache); + FREE_AND_NULL(loose->cache); + memset(&loose->subdir_seen, 0, + sizeof(loose->subdir_seen)); +} + +void odb_source_loose_reprepare(struct odb_source *source) +{ + odb_source_loose_clear_cache(source->loose); } static int check_stream_oid(git_zstream *stream, @@ -1995,3 +1959,20 @@ void object_file_transaction_commit(struct odb_transaction *transaction) transaction->odb->transaction = NULL; free(transaction); } + +struct odb_source_loose *odb_source_loose_new(struct odb_source *source) +{ + struct odb_source_loose *loose; + CALLOC_ARRAY(loose, 1); + loose->source = source; + return loose; +} + +void odb_source_loose_free(struct odb_source_loose *loose) +{ + if (!loose) + return; + odb_source_loose_clear_cache(loose); + loose_object_map_clear(&loose->map); + free(loose); +} diff --git a/object-file.h b/object-file.h index 3fd48dcafb..eeffa67bbd 100644 --- a/object-file.h +++ b/object-file.h @@ -7,14 +7,6 @@ struct index_state; -/* - * Set this to 0 to prevent odb_read_object_info_extended() from fetching missing - * blobs. This has a difference only if extensions.partialClone is set. - * - * Its default value is 1. - */ -extern int fetch_if_missing; - enum { INDEX_WRITE_OBJECT = (1 << 0), INDEX_FORMAT_CHECK = (1 << 1), @@ -26,15 +18,65 @@ int index_path(struct index_state *istate, struct object_id *oid, const char *pa struct odb_source; +struct odb_source_loose { + struct odb_source *source; + + /* + * Used to store the results of readdir(3) calls when we are OK + * sacrificing accuracy due to races for speed. That includes + * object existence with OBJECT_INFO_QUICK, as well as + * our search for unique abbreviated hashes. Don't use it for tasks + * requiring greater accuracy! + * + * Be sure to call odb_load_loose_cache() before using. + */ + uint32_t subdir_seen[8]; /* 256 bits */ + struct oidtree *cache; + + /* Map between object IDs for loose objects. */ + struct loose_object_map *map; +}; + +struct odb_source_loose *odb_source_loose_new(struct odb_source *source); +void odb_source_loose_free(struct odb_source_loose *loose); + +/* Reprepare the loose source by emptying the loose object cache. */ +void odb_source_loose_reprepare(struct odb_source *source); + +int odb_source_loose_read_object_info(struct odb_source *source, + const struct object_id *oid, + struct object_info *oi, int flags); + +void *odb_source_loose_map_object(struct odb_source *source, + const struct object_id *oid, + unsigned long *size); + /* - * Populate and return the loose object cache array corresponding to the - * given object ID. + * Return true iff an object database source has a loose object + * with the specified name. This function does not respect replace + * references. */ -struct oidtree *odb_loose_cache(struct odb_source *source, +int odb_source_loose_has_object(struct odb_source *source, const struct object_id *oid); -/* Empty the loose object cache for the specified object directory. */ -void odb_clear_loose_cache(struct odb_source *source); +int odb_source_loose_freshen_object(struct odb_source *source, + const struct object_id *oid); + +int odb_source_loose_write_object(struct odb_source *source, + const void *buf, unsigned long len, + enum object_type type, struct object_id *oid, + struct object_id *compat_oid_in, unsigned flags); + +int odb_source_loose_write_stream(struct odb_source *source, + struct odb_write_stream *stream, size_t len, + struct object_id *oid); + +/* + * Populate and return the loose object cache array corresponding to the + * given object ID. + */ +struct oidtree *odb_source_loose_cache(struct odb_source *source, + const struct object_id *oid); /* * Put in `buf` the name of the file in the local object database that @@ -45,17 +87,6 @@ const char *odb_loose_path(struct odb_source *source, const struct object_id *oid); /* - * Return true iff an object database source has a loose object - * with the specified name. This function does not respect replace - * references. - */ -int has_loose_object(struct odb_source *source, - const struct object_id *oid); - -void *map_loose_object(struct repository *r, const struct object_id *oid, - unsigned long *size); - -/* * Iterate over the files in the loose-object parts of the object * directory "path", triggering the following callbacks: * @@ -146,21 +177,6 @@ enum unpack_loose_header_result unpack_loose_header(git_zstream *stream, struct object_info; int parse_loose_header(const char *hdr, struct object_info *oi); -int write_object_file(struct odb_source *source, - const void *buf, unsigned long len, - enum object_type type, struct object_id *oid, - struct object_id *compat_oid_in, unsigned flags); - -struct input_stream { - const void *(*read)(struct input_stream *, unsigned long *len); - void *data; - int is_finished; -}; - -int stream_loose_object(struct odb_source *source, - struct input_stream *in_stream, size_t len, - struct object_id *oid); - int force_object_loose(struct odb_source *source, const struct object_id *oid, time_t mtime); @@ -182,10 +198,6 @@ int check_object_signature(struct repository *r, const struct object_id *oid, */ int stream_object_signature(struct repository *r, const struct object_id *oid); -int loose_object_info(struct repository *r, - const struct object_id *oid, - struct object_info *oi, int flags); - enum finalize_object_file_flags { FOF_SKIP_COLLISION_CHECK = 1, }; diff --git a/object-name.c b/object-name.c index f6902e140d..fed5de5153 100644 --- a/object-name.c +++ b/object-name.c @@ -116,7 +116,7 @@ static void find_short_object_filename(struct disambiguate_state *ds) struct odb_source *source; for (source = ds->repo->objects->sources; source && !ds->ambiguous; source = source->next) - oidtree_each(odb_loose_cache(source, &ds->bin_pfx), + oidtree_each(odb_source_loose_cache(source, &ds->bin_pfx), &ds->bin_pfx, ds->len, match_prefix, ds); } @@ -213,9 +213,11 @@ static void find_short_packed_object(struct disambiguate_state *ds) unique_in_midx(m, ds); } - for (p = packfile_store_get_packs(ds->repo->objects->packfiles); p && !ds->ambiguous; - p = p->next) + repo_for_each_pack(ds->repo, p) { + if (ds->ambiguous) + break; unique_in_pack(p, ds); + } } static int finish_object_disambiguation(struct disambiguate_state *ds, @@ -805,7 +807,7 @@ static void find_abbrev_len_packed(struct min_abbrev_data *mad) find_abbrev_len_for_midx(m, mad); } - for (p = packfile_store_get_packs(mad->repo->objects->packfiles); p; p = p->next) + repo_for_each_pack(mad->repo, p) find_abbrev_len_for_pack(p, mad); } @@ -1444,18 +1446,16 @@ struct handle_one_ref_cb { struct commit_list **list; }; -static int handle_one_ref(const char *path, const char *referent UNUSED, const struct object_id *oid, - int flag UNUSED, - void *cb_data) +static int handle_one_ref(const struct reference *ref, void *cb_data) { struct handle_one_ref_cb *cb = cb_data; struct commit_list **list = cb->list; - struct object *object = parse_object(cb->repo, oid); + struct object *object = parse_object(cb->repo, ref->oid); if (!object) return 0; if (object->type == OBJ_TAG) { - object = deref_tag(cb->repo, object, path, - strlen(path)); + object = deref_tag(cb->repo, object, ref->name, + strlen(ref->name)); if (!object) return 0; } @@ -209,7 +209,8 @@ struct object *lookup_object_by_type(struct repository *r, enum peel_status peel_object(struct repository *r, const struct object_id *name, - struct object_id *oid) + struct object_id *oid, + unsigned flags) { struct object *o = lookup_unknown_object(r, name); @@ -222,7 +223,20 @@ enum peel_status peel_object(struct repository *r, if (o->type != OBJ_TAG) return PEEL_NON_TAG; - o = deref_tag_noverify(r, o); + while (o && o->type == OBJ_TAG) { + o = parse_object(r, &o->oid); + if (o && o->type == OBJ_TAG && ((struct tag *)o)->tagged) { + o = ((struct tag *)o)->tagged; + + if (flags & PEEL_OBJECT_VERIFY_TAGGED_OBJECT_TYPE) { + int type = odb_read_object_info(r->objects, &o->oid, NULL); + if (type < 0 || !object_as_type(o, type, 0)) + return PEEL_INVALID; + } + } else { + o = NULL; + } + } if (!o) return PEEL_INVALID; @@ -75,11 +75,11 @@ void object_array_init(struct object_array *array); * http-push.c: 11-----14 * commit-graph.c: 15 * commit-reach.c: 16-----19 + * builtin/last-modified.c: 1617 * sha1-name.c: 20 * list-objects-filter.c: 21 * bloom.c: 2122 * builtin/fsck.c: 0--3 - * builtin/gc.c: 0 * builtin/index-pack.c: 2021 * reflog.c: 10--12 * builtin/show-branch.c: 0-------------------------------------------26 @@ -287,6 +287,17 @@ enum peel_status { PEEL_BROKEN = -4 }; +enum peel_object_flags { + /* + * Always verify the object type of the tagged object, even in the case + * where the looked-up object already has an object type. This can be + * useful when the tagged object type may be invalid. One such case is + * when looking up objects via tags, where we blindly trust the object + * type declared by the tag. + */ + PEEL_OBJECT_VERIFY_TAGGED_OBJECT_TYPE = (1 << 0), +}; + /* * Peel the named object; i.e., if the object is a tag, resolve the * tag recursively until a non-tag is found. If successful, store the @@ -295,7 +306,9 @@ enum peel_status { * and leave oid unchanged. */ enum peel_status peel_object(struct repository *r, - const struct object_id *name, struct object_id *oid); + const struct object_id *name, + struct object_id *oid, + unsigned flags); struct object_list *object_list_insert(struct object *item, struct object_list **list_p); @@ -86,17 +86,16 @@ int odb_mkstemp(struct object_database *odb, /* * Return non-zero iff the path is usable as an alternate object database. */ -static int alt_odb_usable(struct object_database *o, - struct strbuf *path, - const char *normalized_objdir, khiter_t *pos) +static int alt_odb_usable(struct object_database *o, const char *path, + const char *normalized_objdir) { int r; /* Detect cases where alternate disappeared */ - if (!is_directory(path->buf)) { + if (!is_directory(path)) { error(_("object directory %s does not exist; " "check .git/objects/info/alternates"), - path->buf); + path); return 0; } @@ -113,11 +112,14 @@ static int alt_odb_usable(struct object_database *o, assert(r == 1); /* never used */ kh_value(o->source_by_path, p) = o->sources; } - if (fspatheq(path->buf, normalized_objdir)) + + if (fspatheq(path, normalized_objdir)) + return 0; + + if (kh_get_odb_path_map(o->source_by_path, path) < kh_end(o->source_by_path)) return 0; - *pos = kh_put_odb_path_map(o->source_by_path, path->buf, &r); - /* r: 0 = exists, 1 = never used, 2 = deleted */ - return r == 0 ? 0 : 1; + + return 1; } /* @@ -139,6 +141,21 @@ static void read_info_alternates(struct object_database *odb, const char *relative_base, int depth); +struct odb_source *odb_source_new(struct object_database *odb, + const char *path, + bool local) +{ + struct odb_source *source; + + CALLOC_ARRAY(source, 1); + source->odb = odb; + source->local = local; + source->path = xstrdup(path); + source->loose = odb_source_loose_new(source); + + return source; +} + static struct odb_source *link_alt_odb_entry(struct object_database *odb, const char *dir, const char *relative_base, @@ -148,6 +165,7 @@ static struct odb_source *link_alt_odb_entry(struct object_database *odb, struct strbuf pathbuf = STRBUF_INIT; struct strbuf tmp = STRBUF_INIT; khiter_t pos; + int ret; if (!is_absolute_path(dir) && relative_base) { strbuf_realpath(&pathbuf, relative_base, 1); @@ -172,20 +190,18 @@ static struct odb_source *link_alt_odb_entry(struct object_database *odb, strbuf_reset(&tmp); strbuf_realpath(&tmp, odb->sources->path, 1); - if (!alt_odb_usable(odb, &pathbuf, tmp.buf, &pos)) + if (!alt_odb_usable(odb, pathbuf.buf, tmp.buf)) goto error; - CALLOC_ARRAY(alternate, 1); - alternate->odb = odb; - alternate->local = false; - /* pathbuf.buf is already in r->objects->source_by_path */ - alternate->path = strbuf_detach(&pathbuf, NULL); + alternate = odb_source_new(odb, pathbuf.buf, false); /* add the alternate entry */ *odb->sources_tail = alternate; odb->sources_tail = &(alternate->next); - alternate->next = NULL; - assert(odb->source_by_path); + + pos = kh_put_odb_path_map(odb->source_by_path, alternate->path, &ret); + if (!ret) + BUG("source must not yet exist"); kh_value(odb->source_by_path, pos) = alternate; /* recursively add alternates */ @@ -337,9 +353,7 @@ struct odb_source *odb_set_temporary_primary_source(struct object_database *odb, * Make a new primary odb and link the old primary ODB in as an * alternate */ - source = xcalloc(1, sizeof(*source)); - source->odb = odb; - source->path = xstrdup(dir); + source = odb_source_new(odb, dir, false); /* * Disable ref updates while a temporary odb is active, since @@ -352,11 +366,10 @@ struct odb_source *odb_set_temporary_primary_source(struct object_database *odb, return source->next; } -static void free_object_directory(struct odb_source *source) +static void odb_source_free(struct odb_source *source) { free(source->path); - odb_clear_loose_cache(source); - loose_object_map_clear(&source->loose_map); + odb_source_loose_free(source->loose); free(source); } @@ -374,7 +387,7 @@ void odb_restore_primary_source(struct object_database *odb, BUG("we expect the old primary object store to be the first alternate"); odb->sources = restore_source; - free_object_directory(cur_source); + odb_source_free(cur_source); } char *compute_alternate_path(const char *path, struct strbuf *err) @@ -684,13 +697,18 @@ static int do_oid_object_info_extended(struct object_database *odb, return 0; } + odb_prepare_alternates(odb); + while (1) { + struct odb_source *source; + if (find_pack_entry(odb->repo, real, &e)) break; /* Most likely it's a loose object. */ - if (!loose_object_info(odb->repo, real, oi, flags)) - return 0; + for (source = odb->sources; source; source = source->next) + if (!odb_source_loose_read_object_info(source, real, oi, flags)) + return 0; /* Not a loose object; someone else may have just packed it. */ if (!(flags & OBJECT_INFO_QUICK)) { @@ -969,6 +987,22 @@ int odb_has_object(struct object_database *odb, const struct object_id *oid, return odb_read_object_info_extended(odb, oid, NULL, object_info_flags) >= 0; } +int odb_freshen_object(struct object_database *odb, + const struct object_id *oid) +{ + struct odb_source *source; + + if (packfile_store_freshen_object(odb->packfiles, oid)) + return 1; + + odb_prepare_alternates(odb); + for (source = odb->sources; source; source = source->next) + if (odb_source_loose_freshen_object(source, oid)) + return 1; + + return 0; +} + void odb_assert_oid_type(struct object_database *odb, const struct object_id *oid, enum object_type expect) { @@ -987,7 +1021,15 @@ int odb_write_object_ext(struct object_database *odb, struct object_id *compat_oid, unsigned flags) { - return write_object_file(odb->sources, buf, len, type, oid, compat_oid, flags); + return odb_source_loose_write_object(odb->sources, buf, len, type, + oid, compat_oid, flags); +} + +int odb_write_object_stream(struct object_database *odb, + struct odb_write_stream *stream, size_t len, + struct object_id *oid) +{ + return odb_source_loose_write_stream(odb->sources, stream, len, oid); } struct object_database *odb_new(struct repository *repo) @@ -1002,13 +1044,13 @@ struct object_database *odb_new(struct repository *repo) return o; } -static void free_object_directories(struct object_database *o) +static void odb_free_sources(struct object_database *o) { while (o->sources) { struct odb_source *next; next = o->sources->next; - free_object_directory(o->sources); + odb_source_free(o->sources); o->sources = next; } kh_destroy_odb_path_map(o->source_by_path); @@ -1026,7 +1068,7 @@ void odb_clear(struct object_database *o) o->commit_graph = NULL; o->commit_graph_attempted = 0; - free_object_directories(o); + odb_free_sources(o); o->sources_tail = NULL; o->loaded_alternates = 0; @@ -1057,7 +1099,7 @@ void odb_reprepare(struct object_database *o) odb_prepare_alternates(o); for (source = o->sources; source; source = source->next) - odb_clear_loose_cache(source); + odb_source_loose_reprepare(source); o->approximate_object_count_valid = 0; @@ -15,6 +15,14 @@ struct repository; struct multi_pack_index; /* + * Set this to 0 to prevent odb_read_object_info_extended() from fetching missing + * blobs. This has a difference only if extensions.partialClone is set. + * + * Its default value is 1. + */ +extern int fetch_if_missing; + +/* * Compute the exact path an alternate is at and returns it. In case of * error NULL is returned and the human readable error is added to `err` * `path` may be relative and should point to $GIT_DIR. @@ -40,20 +48,8 @@ struct odb_source { /* Object database that owns this object source. */ struct object_database *odb; - /* - * Used to store the results of readdir(3) calls when we are OK - * sacrificing accuracy due to races for speed. That includes - * object existence with OBJECT_INFO_QUICK, as well as - * our search for unique abbreviated hashes. Don't use it for tasks - * requiring greater accuracy! - * - * Be sure to call odb_load_loose_cache() before using. - */ - uint32_t loose_objects_subdir_seen[8]; /* 256 bits */ - struct oidtree *loose_objects_cache; - - /* Map between object IDs for loose objects. */ - struct loose_object_map *loose_map; + /* Private state for loose objects. */ + struct odb_source_loose *loose; /* * private data @@ -89,6 +85,10 @@ struct odb_source { char *path; }; +struct odb_source *odb_source_new(struct object_database *odb, + const char *path, + bool local); + struct packed_git; struct packfile_store; struct cached_object_entry; @@ -396,6 +396,9 @@ int odb_has_object(struct object_database *odb, const struct object_id *oid, unsigned flags); +int odb_freshen_object(struct object_database *odb, + const struct object_id *oid); + void odb_assert_oid_type(struct object_database *odb, const struct object_id *oid, enum object_type expect); @@ -489,4 +492,14 @@ static inline int odb_write_object(struct object_database *odb, return odb_write_object_ext(odb, buf, len, type, oid, NULL, 0); } +struct odb_write_stream { + const void *(*read)(struct odb_write_stream *, unsigned long *len); + void *data; + int is_finished; +}; + +int odb_write_object_stream(struct object_database *odb, + struct odb_write_stream *stream, size_t len, + struct object_id *oid); + #endif /* ODB_H */ @@ -10,7 +10,7 @@ struct oidtree_iter_data { oidtree_iter fn; void *arg; size_t *last_nibble_at; - int algo; + uint32_t algo; uint8_t last_byte; }; diff --git a/pack-bitmap.c b/pack-bitmap.c index ac71035d77..291e1a9cf4 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -664,7 +664,7 @@ static int open_pack_bitmap(struct repository *r, struct packed_git *p; int ret = -1; - for (p = packfile_store_get_all_packs(r->objects->packfiles); p; p = p->next) { + repo_for_each_pack(r, p) { if (open_pack_bitmap_1(bitmap_git, p) == 0) { ret = 0; /* @@ -3347,6 +3347,7 @@ static int verify_bitmap_file(const struct git_hash_algo *algop, int verify_bitmap_files(struct repository *r) { struct odb_source *source; + struct packed_git *p; int res = 0; odb_prepare_alternates(r->objects); @@ -3362,8 +3363,7 @@ int verify_bitmap_files(struct repository *r) free(midx_bitmap_name); } - for (struct packed_git *p = packfile_store_get_all_packs(r->objects->packfiles); - p; p = p->next) { + repo_for_each_pack(r, p) { char *pack_bitmap_name = pack_bitmap_filename(p); res |= verify_bitmap_file(r->hash_algo, pack_bitmap_name); free(pack_bitmap_name); diff --git a/pack-objects.c b/pack-objects.c index 9d6ee72569..48510dd343 100644 --- a/pack-objects.c +++ b/pack-objects.c @@ -87,7 +87,6 @@ struct object_entry *packlist_find(struct packing_data *pdata, static void prepare_in_pack_by_idx(struct packing_data *pdata) { - struct packfile_store *packs = pdata->repo->objects->packfiles; struct packed_git **mapping, *p; int cnt = 0, nr = 1U << OE_IN_PACK_BITS; @@ -97,13 +96,13 @@ static void prepare_in_pack_by_idx(struct packing_data *pdata) * (i.e. in_pack_idx also zero) should return NULL. */ mapping[cnt++] = NULL; - for (p = packfile_store_get_all_packs(packs); p; p = p->next, cnt++) { + repo_for_each_pack(pdata->repo, p) { if (cnt == nr) { free(mapping); return; } p->index = cnt; - mapping[cnt] = p; + mapping[cnt++] = p; } pdata->in_pack_by_idx = mapping; } diff --git a/pack-refs.c b/pack-refs.c index 1a5e07d8b8..eb6b2ba2c2 100644 --- a/pack-refs.c +++ b/pack-refs.c @@ -14,10 +14,10 @@ int pack_refs_core(int argc, { struct ref_exclusions excludes = REF_EXCLUSIONS_INIT; struct string_list included_refs = STRING_LIST_INIT_NODUP; - struct pack_refs_opts pack_refs_opts = { + struct refs_optimize_opts optimize_opts = { .exclusions = &excludes, .includes = &included_refs, - .flags = PACK_REFS_PRUNE, + .flags = REFS_OPTIMIZE_PRUNE, }; struct string_list option_excluded_refs = STRING_LIST_INIT_NODUP; struct string_list_item *item; @@ -26,9 +26,9 @@ int pack_refs_core(int argc, struct option opts[] = { OPT_BOOL(0, "all", &pack_all, N_("pack everything")), - OPT_BIT(0, "prune", &pack_refs_opts.flags, N_("prune loose refs (default)"), PACK_REFS_PRUNE), - OPT_BIT(0, "auto", &pack_refs_opts.flags, N_("auto-pack refs as needed"), PACK_REFS_AUTO), - OPT_STRING_LIST(0, "include", pack_refs_opts.includes, N_("pattern"), + OPT_BIT(0, "prune", &optimize_opts.flags, N_("prune loose refs (default)"), REFS_OPTIMIZE_PRUNE), + OPT_BIT(0, "auto", &optimize_opts.flags, N_("auto-pack refs as needed"), REFS_OPTIMIZE_AUTO), + OPT_STRING_LIST(0, "include", optimize_opts.includes, N_("pattern"), N_("references to include")), OPT_STRING_LIST(0, "exclude", &option_excluded_refs, N_("pattern"), N_("references to exclude")), @@ -39,15 +39,15 @@ int pack_refs_core(int argc, usage_with_options(usage_opts, opts); for_each_string_list_item(item, &option_excluded_refs) - add_ref_exclusion(pack_refs_opts.exclusions, item->string); + add_ref_exclusion(optimize_opts.exclusions, item->string); if (pack_all) - string_list_append(pack_refs_opts.includes, "*"); + string_list_append(optimize_opts.includes, "*"); - if (!pack_refs_opts.includes->nr) - string_list_append(pack_refs_opts.includes, "refs/tags/*"); + if (!optimize_opts.includes->nr) + string_list_append(optimize_opts.includes, "refs/tags/*"); - ret = refs_optimize(get_main_ref_store(repo), &pack_refs_opts); + ret = refs_optimize(get_main_ref_store(repo), &optimize_opts); clear_ref_exclusions(&excludes); string_list_clear(&included_refs, 0); diff --git a/packfile.c b/packfile.c index 5a7caec292..9cc11b6dc5 100644 --- a/packfile.c +++ b/packfile.c @@ -47,6 +47,89 @@ static size_t pack_mapped; #define SZ_FMT PRIuMAX static inline uintmax_t sz_fmt(size_t s) { return s; } +void packfile_list_clear(struct packfile_list *list) +{ + struct packfile_list_entry *e, *next; + + for (e = list->head; e; e = next) { + next = e->next; + free(e); + } + + list->head = list->tail = NULL; +} + +static struct packfile_list_entry *packfile_list_remove_internal(struct packfile_list *list, + struct packed_git *pack) +{ + struct packfile_list_entry *e, *prev; + + for (e = list->head, prev = NULL; e; prev = e, e = e->next) { + if (e->pack != pack) + continue; + + if (prev) + prev->next = e->next; + if (list->head == e) + list->head = e->next; + if (list->tail == e) + list->tail = prev; + + return e; + } + + return NULL; +} + +void packfile_list_remove(struct packfile_list *list, struct packed_git *pack) +{ + free(packfile_list_remove_internal(list, pack)); +} + +void packfile_list_prepend(struct packfile_list *list, struct packed_git *pack) +{ + struct packfile_list_entry *entry; + + entry = packfile_list_remove_internal(list, pack); + if (!entry) { + entry = xmalloc(sizeof(*entry)); + entry->pack = pack; + } + entry->next = list->head; + + list->head = entry; + if (!list->tail) + list->tail = entry; +} + +void packfile_list_append(struct packfile_list *list, struct packed_git *pack) +{ + struct packfile_list_entry *entry; + + entry = packfile_list_remove_internal(list, pack); + if (!entry) { + entry = xmalloc(sizeof(*entry)); + entry->pack = pack; + } + entry->next = NULL; + + if (list->tail) { + list->tail->next = entry; + list->tail = entry; + } else { + list->head = list->tail = entry; + } +} + +struct packed_git *packfile_list_find_oid(struct packfile_list_entry *packs, + const struct object_id *oid) +{ + for (; packs; packs = packs->next) + if (find_pack_entry_one(oid, packs->pack)) + return packs->pack; + return NULL; +} + void pack_report(struct repository *repo) { fprintf(stderr, @@ -273,13 +356,14 @@ static void scan_windows(struct packed_git *p, static int unuse_one_window(struct packed_git *current) { - struct packed_git *p, *lru_p = NULL; + struct packfile_list_entry *e; + struct packed_git *lru_p = NULL; struct pack_window *lru_w = NULL, *lru_l = NULL; if (current) scan_windows(current, &lru_p, &lru_w, &lru_l); - for (p = current->repo->objects->packfiles->packs; p; p = p->next) - scan_windows(p, &lru_p, &lru_w, &lru_l); + for (e = current->repo->objects->packfiles->packs.head; e; e = e->next) + scan_windows(e->pack, &lru_p, &lru_w, &lru_l); if (lru_p) { munmap(lru_w->base, lru_w->len); pack_mapped -= lru_w->len; @@ -459,14 +543,15 @@ static void find_lru_pack(struct packed_git *p, struct packed_git **lru_p, struc static int close_one_pack(struct repository *r) { - struct packed_git *p, *lru_p = NULL; + struct packfile_list_entry *e; + struct packed_git *lru_p = NULL; struct pack_window *mru_w = NULL; int accept_windows_inuse = 1; - for (p = r->objects->packfiles->packs; p; p = p->next) { - if (p->pack_fd == -1) + for (e = r->objects->packfiles->packs.head; e; e = e->next) { + if (e->pack->pack_fd == -1) continue; - find_lru_pack(p, &lru_p, &mru_w, &accept_windows_inuse); + find_lru_pack(e->pack, &lru_p, &mru_w, &accept_windows_inuse); } if (lru_p) @@ -785,11 +870,8 @@ void packfile_store_add_pack(struct packfile_store *store, if (pack->pack_fd != -1) pack_open_fds++; - pack->next = store->packs; - store->packs = pack; - - hashmap_entry_init(&pack->packmap_ent, strhash(pack->pack_name)); - hashmap_add(&store->map, &pack->packmap_ent); + packfile_list_append(&store->packs, pack); + strmap_put(&store->packs_by_path, pack->pack_name, pack); } struct packed_git *packfile_store_load_pack(struct packfile_store *store, @@ -806,8 +888,7 @@ struct packed_git *packfile_store_load_pack(struct packfile_store *store, strbuf_strip_suffix(&key, ".idx"); strbuf_addstr(&key, ".pack"); - p = hashmap_get_entry_from_hash(&store->map, strhash(key.buf), key.buf, - struct packed_git, packmap_ent); + p = strmap_get(&store->packs_by_path, key.buf); if (!p) { p = add_packed_git(store->odb->repo, idx_path, strlen(idx_path), local); @@ -819,6 +900,22 @@ struct packed_git *packfile_store_load_pack(struct packfile_store *store, return p; } +int packfile_store_freshen_object(struct packfile_store *store, + const struct object_id *oid) +{ + struct pack_entry e; + if (!find_pack_entry(store->odb->repo, oid, &e)) + return 0; + if (e.p->is_cruft) + return 0; + if (e.p->freshened) + return 1; + if (utime(e.p->pack_name, NULL)) + return 0; + e.p->freshened = 1; + return 1; +} + void (*report_garbage)(unsigned seen_bits, const char *path); static void report_helper(const struct string_list *list, @@ -965,9 +1062,10 @@ static void prepare_packed_git_one(struct odb_source *source) string_list_clear(data.garbage, 0); } -DEFINE_LIST_SORT(static, sort_packs, struct packed_git, next); +DEFINE_LIST_SORT(static, sort_packs, struct packfile_list_entry, next); -static int sort_pack(const struct packed_git *a, const struct packed_git *b) +static int sort_pack(const struct packfile_list_entry *a, + const struct packfile_list_entry *b) { int st; @@ -977,7 +1075,7 @@ static int sort_pack(const struct packed_git *a, const struct packed_git *b) * remote ones could be on a network mounted filesystem. * Favor local ones for these reasons. */ - st = a->pack_local - b->pack_local; + st = a->pack->pack_local - b->pack->pack_local; if (st) return -st; @@ -986,23 +1084,13 @@ static int sort_pack(const struct packed_git *a, const struct packed_git *b) * and more recent objects tend to get accessed more * often. */ - if (a->mtime < b->mtime) + if (a->pack->mtime < b->pack->mtime) return 1; - else if (a->mtime == b->mtime) + else if (a->pack->mtime == b->pack->mtime) return 0; return -1; } -static void packfile_store_prepare_mru(struct packfile_store *store) -{ - struct packed_git *p; - - INIT_LIST_HEAD(&store->mru); - - for (p = store->packs; p; p = p->next) - list_add_tail(&p->mru, &store->mru); -} - void packfile_store_prepare(struct packfile_store *store) { struct odb_source *source; @@ -1015,9 +1103,12 @@ void packfile_store_prepare(struct packfile_store *store) prepare_multi_pack_index_one(source); prepare_packed_git_one(source); } - sort_packs(&store->packs, sort_pack); - packfile_store_prepare_mru(store); + sort_packs(&store->packs.head, sort_pack); + for (struct packfile_list_entry *e = store->packs.head; e; e = e->next) + if (!e->next) + store->packs.tail = e; + store->initialized = true; } @@ -1027,13 +1118,7 @@ void packfile_store_reprepare(struct packfile_store *store) packfile_store_prepare(store); } -struct packed_git *packfile_store_get_packs(struct packfile_store *store) -{ - packfile_store_prepare(store); - return store->packs; -} - -struct packed_git *packfile_store_get_all_packs(struct packfile_store *store) +struct packfile_list_entry *packfile_store_get_packs(struct packfile_store *store) { packfile_store_prepare(store); @@ -1045,13 +1130,7 @@ struct packed_git *packfile_store_get_all_packs(struct packfile_store *store) prepare_midx_pack(m, i); } - return store->packs; -} - -struct list_head *packfile_store_get_packs_mru(struct packfile_store *store) -{ - packfile_store_prepare(store); - return &store->mru; + return store->packs.head; } /* @@ -1068,16 +1147,16 @@ unsigned long repo_approximate_object_count(struct repository *r) unsigned long count = 0; struct packed_git *p; - packfile_store_prepare(r->objects->packfiles); + odb_prepare_alternates(r->objects); for (source = r->objects->sources; source; source = source->next) { struct multi_pack_index *m = get_multi_pack_index(source); if (m) - count += m->num_objects; + count += m->num_objects + m->num_objects_in_base; } - for (p = r->objects->packfiles->packs; p; p = p->next) { - if (open_pack_index(p)) + repo_for_each_pack(r, p) { + if (p->multi_pack_index || open_pack_index(p)) continue; count += p->num_objects; } @@ -1201,11 +1280,11 @@ void mark_bad_packed_object(struct packed_git *p, const struct object_id *oid) const struct packed_git *has_packed_and_bad(struct repository *r, const struct object_id *oid) { - struct packed_git *p; + struct packfile_list_entry *e; - for (p = r->objects->packfiles->packs; p; p = p->next) - if (oidset_contains(&p->bad_objects, oid)) - return p; + for (e = r->objects->packfiles->packs.head; e; e = e->next) + if (oidset_contains(&e->pack->bad_objects, oid)) + return e->pack; return NULL; } @@ -2013,19 +2092,6 @@ int is_pack_valid(struct packed_git *p) return !open_packed_git(p); } -struct packed_git *find_oid_pack(const struct object_id *oid, - struct packed_git *packs) -{ - struct packed_git *p; - - for (p = packs; p; p = p->next) { - if (find_pack_entry_one(oid, p)) - return p; - } - return NULL; - -} - static int fill_pack_entry(const struct object_id *oid, struct pack_entry *e, struct packed_git *p) @@ -2056,7 +2122,7 @@ static int fill_pack_entry(const struct object_id *oid, int find_pack_entry(struct repository *r, const struct object_id *oid, struct pack_entry *e) { - struct list_head *pos; + struct packfile_list_entry *l; packfile_store_prepare(r->objects->packfiles); @@ -2064,13 +2130,15 @@ int find_pack_entry(struct repository *r, const struct object_id *oid, struct pa if (source->midx && fill_midx_entry(source->midx, oid, e)) return 1; - if (!r->objects->packfiles->packs) + if (!r->objects->packfiles->packs.head) return 0; - list_for_each(pos, &r->objects->packfiles->mru) { - struct packed_git *p = list_entry(pos, struct packed_git, mru); + for (l = r->objects->packfiles->packs.head; l; l = l->next) { + struct packed_git *p = l->pack; + if (!p->multi_pack_index && fill_pack_entry(oid, e, p)) { - list_move(&p->mru, &r->objects->packfiles->mru); + if (!r->objects->packfiles->skip_mru_updates) + packfile_list_prepend(&r->objects->packfiles->packs, p); return 1; } } @@ -2105,7 +2173,7 @@ struct packed_git **kept_pack_cache(struct repository *r, unsigned flags) * covers, one kept and one not kept, but the midx returns only * the non-kept version. */ - for (p = packfile_store_get_all_packs(r->objects->packfiles); p; p = p->next) { + repo_for_each_pack(r, p) { if ((p->pack_keep && (flags & ON_DISK_KEEP_PACKS)) || (p->pack_keep_in_core && (flags & IN_CORE_KEEP_PACKS))) { ALLOC_GROW(packs, nr + 1, alloc); @@ -2202,7 +2270,8 @@ int for_each_packed_object(struct repository *repo, each_packed_object_fn cb, int r = 0; int pack_errors = 0; - for (p = packfile_store_get_all_packs(repo->objects->packfiles); p; p = p->next) { + repo->objects->packfiles->skip_mru_updates = true; + repo_for_each_pack(repo, p) { if ((flags & FOR_EACH_OBJECT_LOCAL_ONLY) && !p->pack_local) continue; if ((flags & FOR_EACH_OBJECT_PROMISOR_ONLY) && @@ -2222,6 +2291,8 @@ int for_each_packed_object(struct repository *repo, each_packed_object_fn cb, if (r) break; } + repo->objects->packfiles->skip_mru_updates = false; + return r ? r : pack_errors; } @@ -2317,45 +2388,30 @@ int parse_pack_header_option(const char *in, unsigned char *out, unsigned int *l return 0; } -static int pack_map_entry_cmp(const void *cmp_data UNUSED, - const struct hashmap_entry *entry, - const struct hashmap_entry *entry2, - const void *keydata) -{ - const char *key = keydata; - const struct packed_git *pg1, *pg2; - - pg1 = container_of(entry, const struct packed_git, packmap_ent); - pg2 = container_of(entry2, const struct packed_git, packmap_ent); - - return strcmp(pg1->pack_name, key ? key : pg2->pack_name); -} - struct packfile_store *packfile_store_new(struct object_database *odb) { struct packfile_store *store; CALLOC_ARRAY(store, 1); store->odb = odb; - INIT_LIST_HEAD(&store->mru); - hashmap_init(&store->map, pack_map_entry_cmp, NULL, 0); + strmap_init(&store->packs_by_path); return store; } void packfile_store_free(struct packfile_store *store) { - for (struct packed_git *p = store->packs, *next; p; p = next) { - next = p->next; - free(p); - } - hashmap_clear(&store->map); + for (struct packfile_list_entry *e = store->packs.head; e; e = e->next) + free(e->pack); + packfile_list_clear(&store->packs); + + strmap_clear(&store->packs_by_path, 0); free(store); } void packfile_store_close(struct packfile_store *store) { - for (struct packed_git *p = store->packs; p; p = p->next) { - if (p->do_not_close) + for (struct packfile_list_entry *e = store->packs.head; e; e = e->next) { + if (e->pack->do_not_close) BUG("want to close pack marked 'do-not-close'"); - close_pack(p); + close_pack(e->pack); } } diff --git a/packfile.h b/packfile.h index e7a5792b6c..0e178fb279 100644 --- a/packfile.h +++ b/packfile.h @@ -5,14 +5,12 @@ #include "object.h" #include "odb.h" #include "oidset.h" +#include "strmap.h" /* in odb.h */ struct object_info; struct packed_git { - struct hashmap_entry packmap_ent; - struct packed_git *next; - struct list_head mru; struct pack_window *windows; off_t pack_size; const void *index_data; @@ -52,6 +50,28 @@ struct packed_git { char pack_name[FLEX_ARRAY]; /* more */ }; +struct packfile_list { + struct packfile_list_entry *head, *tail; +}; + +struct packfile_list_entry { + struct packfile_list_entry *next; + struct packed_git *pack; +}; + +void packfile_list_clear(struct packfile_list *list); +void packfile_list_remove(struct packfile_list *list, struct packed_git *pack); +void packfile_list_prepend(struct packfile_list *list, struct packed_git *pack); +void packfile_list_append(struct packfile_list *list, struct packed_git *pack); + +/* + * Find the pack within the "packs" list whose index contains the object + * "oid". For general object lookups, you probably don't want this; use + * find_pack_entry() instead. + */ +struct packed_git *packfile_list_find_oid(struct packfile_list_entry *packs, + const struct object_id *oid); + /* * A store that manages packfiles for a given object database. */ @@ -59,10 +79,10 @@ struct packfile_store { struct object_database *odb; /* - * The list of packfiles in the order in which they are being added to - * the store. + * The list of packfiles in the order in which they have been most + * recently used. */ - struct packed_git *packs; + struct packfile_list packs; /* * Cache of packfiles which are marked as "kept", either because there @@ -78,20 +98,32 @@ struct packfile_store { unsigned flags; } kept_cache; - /* A most-recently-used ordered version of the packs list. */ - struct list_head mru; - /* * A map of packfile names to packed_git structs for tracking which * packs have been loaded already. */ - struct hashmap map; + struct strmap packs_by_path; /* * Whether packfiles have already been populated with this store's * packs. */ bool initialized; + + /* + * Usually, packfiles will be reordered to the front of the `packs` + * list whenever an object is looked up via them. This has the effect + * that packs that contain a lot of accessed objects will be located + * towards the front. + * + * This is usually desireable, but there are exceptions. One exception + * is when the looking up multiple objects in a loop for each packfile. + * In that case, we may easily end up with an infinite loop as the + * packfiles get reordered to the front repeatedly. + * + * Setting this field to `true` thus disables these reorderings. + */ + bool skip_mru_updates; }; /* @@ -137,21 +169,19 @@ void packfile_store_add_pack(struct packfile_store *store, struct packed_git *pack); /* - * Get packs managed by the given store. Does not load the MIDX or any packs - * referenced by it. + * Load and iterate through all packs of the given repository. This helper + * function will yield packfiles from all object sources connected to the + * repository. */ -struct packed_git *packfile_store_get_packs(struct packfile_store *store); +#define repo_for_each_pack(repo, p) \ + for (struct packfile_list_entry *e = packfile_store_get_packs(repo->objects->packfiles); \ + ((p) = (e ? e->pack : NULL)); e = e->next) /* * Get all packs managed by the given store, including packfiles that are * referenced by multi-pack indices. */ -struct packed_git *packfile_store_get_all_packs(struct packfile_store *store); - -/* - * Get all packs in most-recently-used order. - */ -struct list_head *packfile_store_get_packs_mru(struct packfile_store *store); +struct packfile_list_entry *packfile_store_get_packs(struct packfile_store *store); /* * Open the packfile and add it to the store if it isn't yet known. Returns @@ -161,6 +191,9 @@ struct list_head *packfile_store_get_packs_mru(struct packfile_store *store); struct packed_git *packfile_store_load_pack(struct packfile_store *store, const char *idx_path, int local); +int packfile_store_freshen_object(struct packfile_store *store, + const struct object_id *oid); + struct pack_window { struct pack_window *next; unsigned char *base; @@ -243,14 +276,6 @@ extern void (*report_garbage)(unsigned seen_bits, const char *path); */ unsigned long repo_approximate_object_count(struct repository *r); -/* - * Find the pack within the "packs" list whose index contains the object "oid". - * For general object lookups, you probably don't want this; use - * find_pack_entry() instead. - */ -struct packed_git *find_oid_pack(const struct object_id *oid, - struct packed_git *packs); - void pack_report(struct repository *repo); /* diff --git a/parse-options.c b/parse-options.c index 5933468c19..c9cafc21b9 100644 --- a/parse-options.c +++ b/parse-options.c @@ -208,12 +208,12 @@ static enum parse_opt_result do_get_value(struct parse_opt_ctx_t *p, case OPTION_FILENAME: { const char *value; - int is_optional; + bool is_optional; if (unset) value = NULL; else if (opt->flags & PARSE_OPT_OPTARG && !p->opt) - value = (char *)opt->defval; + value = (const char *)opt->defval; else { int err = get_arg(p, opt, flags, &value); if (err) @@ -223,10 +223,8 @@ static enum parse_opt_result do_get_value(struct parse_opt_ctx_t *p, return 0; is_optional = skip_prefix(value, ":(optional)", &value); - if (!value) - is_optional = 0; value = fix_filename(p->prefix, value); - if (is_optional && is_empty_or_missing_file(value)) { + if (is_optional && is_missing_file(value)) { free((char *)value); } else { FREE_AND_NULL(*(char **)opt->value); diff --git a/pseudo-merge.c b/pseudo-merge.c index 893b763fe4..a2d5bd85f9 100644 --- a/pseudo-merge.c +++ b/pseudo-merge.c @@ -221,28 +221,25 @@ void load_pseudo_merges_from_config(struct repository *r, } } -static int find_pseudo_merge_group_for_ref(const char *refname, - const char *referent UNUSED, - const struct object_id *oid, - int flags UNUSED, - void *_data) +static int find_pseudo_merge_group_for_ref(const struct reference *ref, void *_data) { struct bitmap_writer *writer = _data; + const struct object_id *maybe_peeled = ref->oid; struct object_id peeled; struct commit *c; uint32_t i; int has_bitmap; - if (!peel_iterated_oid(the_repository, oid, &peeled)) - oid = &peeled; + if (!reference_get_peeled_oid(the_repository, ref, &peeled)) + maybe_peeled = &peeled; - c = lookup_commit(the_repository, oid); + c = lookup_commit(the_repository, maybe_peeled); if (!c) return 0; - if (!packlist_find(writer->to_pack, oid)) + if (!packlist_find(writer->to_pack, maybe_peeled)) return 0; - has_bitmap = bitmap_writer_has_bitmapped_object_id(writer, oid); + has_bitmap = bitmap_writer_has_bitmapped_object_id(writer, maybe_peeled); for (i = 0; i < writer->pseudo_merge_groups.nr; i++) { struct pseudo_merge_group *group; @@ -252,7 +249,7 @@ static int find_pseudo_merge_group_for_ref(const char *refname, size_t j; group = writer->pseudo_merge_groups.items[i].util; - if (regexec(group->pattern, refname, ARRAY_SIZE(captures), + if (regexec(group->pattern, ref->name, ARRAY_SIZE(captures), captures, 0)) continue; @@ -269,7 +266,7 @@ static int find_pseudo_merge_group_for_ref(const char *refname, if (group_name.len) strbuf_addch(&group_name, '-'); - strbuf_add(&group_name, refname + match->rm_so, + strbuf_add(&group_name, ref->name + match->rm_so, match->rm_eo - match->rm_so); } diff --git a/reachable.c b/reachable.c index 22266db523..b753c39553 100644 --- a/reachable.c +++ b/reachable.c @@ -83,18 +83,17 @@ static void add_rebase_files(struct rev_info *revs) free_worktrees(worktrees); } -static int add_one_ref(const char *path, const char *referent UNUSED, const struct object_id *oid, - int flag, void *cb_data) +static int add_one_ref(const struct reference *ref, void *cb_data) { struct rev_info *revs = (struct rev_info *)cb_data; struct object *object; - if ((flag & REF_ISSYMREF) && (flag & REF_ISBROKEN)) { - warning("symbolic ref is dangling: %s", path); + if ((ref->flags & REF_ISSYMREF) && (ref->flags & REF_ISBROKEN)) { + warning("symbolic ref is dangling: %s", ref->name); return 0; } - object = parse_object_or_die(the_repository, oid, path); + object = parse_object_or_die(the_repository, ref->oid, ref->name); add_pending_object(revs, object, ""); return 0; diff --git a/ref-filter.c b/ref-filter.c index 520d2539c9..d7454269e8 100644 --- a/ref-filter.c +++ b/ref-filter.c @@ -91,6 +91,7 @@ static struct expand_data { struct object_id delta_base_oid; void *content; + struct object *maybe_object; struct object_info info; } oi, oi_deref; @@ -1475,11 +1476,29 @@ static void grab_common_values(struct atom_value *val, int deref, struct expand_ } } +static struct object *get_or_parse_object(struct expand_data *data, const char *refname, + struct strbuf *err, int *eaten) +{ + if (!data->maybe_object) { + data->maybe_object = parse_object_buffer(the_repository, &data->oid, data->type, + data->size, data->content, eaten); + if (!data->maybe_object) { + strbuf_addf(err, _("parse_object_buffer failed on %s for %s"), + oid_to_hex(&data->oid), refname); + return NULL; + } + } + + return data->maybe_object; +} + /* See grab_values */ -static void grab_tag_values(struct atom_value *val, int deref, struct object *obj) +static int grab_tag_values(struct atom_value *val, int deref, + struct expand_data *data, const char *refname, + struct strbuf *err, int *eaten) { + struct tag *tag = NULL; int i; - struct tag *tag = (struct tag *) obj; for (i = 0; i < used_atom_cnt; i++) { const char *name = used_atom[i].name; @@ -1487,6 +1506,14 @@ static void grab_tag_values(struct atom_value *val, int deref, struct object *ob struct atom_value *v = &val[i]; if (!!deref != (*name == '*')) continue; + + if (!tag) { + tag = (struct tag *) get_or_parse_object(data, refname, + err, eaten); + if (!tag) + return -1; + } + if (deref) name++; if (atom_type == ATOM_TAG) @@ -1496,22 +1523,35 @@ static void grab_tag_values(struct atom_value *val, int deref, struct object *ob else if (atom_type == ATOM_OBJECT && tag->tagged) v->s = xstrdup(oid_to_hex(&tag->tagged->oid)); } + + return 0; } /* See grab_values */ -static void grab_commit_values(struct atom_value *val, int deref, struct object *obj) +static int grab_commit_values(struct atom_value *val, int deref, + struct expand_data *data, const char *refname, + struct strbuf *err, int *eaten) { int i; - struct commit *commit = (struct commit *) obj; + struct commit *commit = NULL; for (i = 0; i < used_atom_cnt; i++) { const char *name = used_atom[i].name; enum atom_type atom_type = used_atom[i].atom_type; struct atom_value *v = &val[i]; + if (!!deref != (*name == '*')) continue; if (deref) name++; + + if (!commit) { + commit = (struct commit *) get_or_parse_object(data, refname, + err, eaten); + if (!commit) + return -1; + } + if (atom_type == ATOM_TREE && grab_oid(name, "tree", get_commit_tree_oid(commit), v, &used_atom[i])) continue; @@ -1531,6 +1571,8 @@ static void grab_commit_values(struct atom_value *val, int deref, struct object v->s = strbuf_detach(&s, NULL); } } + + return 0; } static const char *find_wholine(const char *who, int wholen, const char *buf) @@ -1759,10 +1801,12 @@ static void grab_person(const char *who, struct atom_value *val, int deref, void } } -static void grab_signature(struct atom_value *val, int deref, struct object *obj) +static int grab_signature(struct atom_value *val, int deref, + struct expand_data *data, const char *refname, + struct strbuf *err, int *eaten) { int i; - struct commit *commit = (struct commit *) obj; + struct commit *commit = NULL; struct signature_check sigc = { 0 }; int signature_checked = 0; @@ -1790,6 +1834,13 @@ static void grab_signature(struct atom_value *val, int deref, struct object *obj continue; if (!signature_checked) { + if (!commit) { + commit = (struct commit *) get_or_parse_object(data, refname, + err, eaten); + if (!commit) + return -1; + } + check_commit_signature(commit, &sigc); signature_checked = 1; } @@ -1843,6 +1894,8 @@ static void grab_signature(struct atom_value *val, int deref, struct object *obj if (signature_checked) signature_check_clear(&sigc); + + return 0; } static void find_subpos(const char *buf, @@ -1920,9 +1973,8 @@ static void append_lines(struct strbuf *out, const char *buf, unsigned long size } static void grab_describe_values(struct atom_value *val, int deref, - struct object *obj) + struct expand_data *data) { - struct commit *commit = (struct commit *)obj; int i; for (i = 0; i < used_atom_cnt; i++) { @@ -1944,7 +1996,7 @@ static void grab_describe_values(struct atom_value *val, int deref, cmd.git_cmd = 1; strvec_push(&cmd.args, "describe"); strvec_pushv(&cmd.args, atom->u.describe_args.v); - strvec_push(&cmd.args, oid_to_hex(&commit->object.oid)); + strvec_push(&cmd.args, oid_to_hex(&data->oid)); if (pipe_command(&cmd, NULL, 0, &out, 0, &err, 0) < 0) { error(_("failed to run 'describe'")); v->s = xstrdup(""); @@ -2066,24 +2118,36 @@ static void fill_missing_values(struct atom_value *val) * pointed at by the ref itself; otherwise it is the object the * ref (which is a tag) refers to. */ -static void grab_values(struct atom_value *val, int deref, struct object *obj, struct expand_data *data) +static int grab_values(struct atom_value *val, int deref, struct expand_data *data, + const char *refname, struct strbuf *err, int *eaten) { void *buf = data->content; + int ret; - switch (obj->type) { + switch (data->type) { case OBJ_TAG: - grab_tag_values(val, deref, obj); + ret = grab_tag_values(val, deref, data, refname, err, eaten); + if (ret < 0) + goto out; + grab_sub_body_contents(val, deref, data); grab_person("tagger", val, deref, buf); - grab_describe_values(val, deref, obj); + grab_describe_values(val, deref, data); break; case OBJ_COMMIT: - grab_commit_values(val, deref, obj); + ret = grab_commit_values(val, deref, data, refname, err, eaten); + if (ret < 0) + goto out; + grab_sub_body_contents(val, deref, data); grab_person("author", val, deref, buf); grab_person("committer", val, deref, buf); - grab_signature(val, deref, obj); - grab_describe_values(val, deref, obj); + + ret = grab_signature(val, deref, data, refname, err, eaten); + if (ret < 0) + goto out; + + grab_describe_values(val, deref, data); break; case OBJ_TREE: /* grab_tree_values(val, deref, obj, buf, sz); */ @@ -2094,8 +2158,12 @@ static void grab_values(struct atom_value *val, int deref, struct object *obj, s grab_sub_body_contents(val, deref, data); break; default: - die("Eh? Object of type %d?", obj->type); + die("Eh? Object of type %d?", data->type); } + + ret = 0; +out: + return ret; } static inline char *copy_advance(char *dst, const char *src) @@ -2292,38 +2360,43 @@ static const char *get_refname(struct used_atom *atom, struct ref_array_item *re return show_ref(&atom->u.refname, ref->refname); } -static int get_object(struct ref_array_item *ref, int deref, struct object **obj, +static int get_object(struct ref_array_item *ref, int deref, struct expand_data *oi, struct strbuf *err) { - /* parse_object_buffer() will set eaten to 0 if free() will be needed */ - int eaten = 1; + /* parse_object_buffer() will set eaten to 1 if free() will be needed */ + int eaten = 0; + int ret; + + oi->maybe_object = NULL; + if (oi->info.contentp) { /* We need to know that to use parse_object_buffer properly */ oi->info.sizep = &oi->size; oi->info.typep = &oi->type; } + if (odb_read_object_info_extended(the_repository->objects, &oi->oid, &oi->info, - OBJECT_INFO_LOOKUP_REPLACE)) - return strbuf_addf_ret(err, -1, _("missing object %s for %s"), - oid_to_hex(&oi->oid), ref->refname); + OBJECT_INFO_LOOKUP_REPLACE)) { + ret = strbuf_addf_ret(err, -1, _("missing object %s for %s"), + oid_to_hex(&oi->oid), ref->refname); + goto out; + } if (oi->info.disk_sizep && oi->disk_size < 0) BUG("Object size is less than zero."); if (oi->info.contentp) { - *obj = parse_object_buffer(the_repository, &oi->oid, oi->type, oi->size, oi->content, &eaten); - if (!*obj) { - if (!eaten) - free(oi->content); - return strbuf_addf_ret(err, -1, _("parse_object_buffer failed on %s for %s"), - oid_to_hex(&oi->oid), ref->refname); - } - grab_values(ref->value, deref, *obj, oi); + ret = grab_values(ref->value, deref, oi, ref->refname, err, &eaten); + if (ret < 0) + goto out; } grab_common_values(ref->value, deref, oi); + ret = 0; + +out: if (!eaten) free(oi->content); - return 0; + return ret; } static void populate_worktree_map(struct hashmap *map, struct worktree **worktrees) @@ -2376,7 +2449,6 @@ static char *get_worktree_path(const struct ref_array_item *ref) */ static int populate_value(struct ref_array_item *ref, struct strbuf *err) { - struct object *obj; int i; struct object_info empty = OBJECT_INFO_INIT; int ahead_behind_atoms = 0; @@ -2564,24 +2636,32 @@ static int populate_value(struct ref_array_item *ref, struct strbuf *err) oi.oid = ref->objectname; - if (get_object(ref, 0, &obj, &oi, err)) + if (get_object(ref, 0, &oi, err)) return -1; /* * If there is no atom that wants to know about tagged * object, we are done. */ - if (!need_tagged || (obj->type != OBJ_TAG)) + if (!need_tagged || (oi.type != OBJ_TAG)) return 0; /* * If it is a tag object, see if we use the peeled value. If we do, * grab the peeled OID. */ - if (need_tagged && peel_iterated_oid(the_repository, &obj->oid, &oi_deref.oid)) - die("bad tag"); + if (need_tagged) { + if (!is_null_oid(&ref->peeled_oid)) { + oidcpy(&oi_deref.oid, &ref->peeled_oid); + } else if (!peel_object(the_repository, &oi.oid, &oi_deref.oid, + PEEL_OBJECT_VERIFY_TAGGED_OBJECT_TYPE)) { + /* We managed to peel the object ourselves. */ + } else { + die("bad tag"); + } + } - return get_object(ref, 1, &obj, &oi_deref, err); + return get_object(ref, 1, &oi_deref, err); } /* @@ -2664,7 +2744,7 @@ static int match_name_as_path(const char **pattern, const char *refname, /* Return 1 if the refname matches one of the patterns, otherwise 0. */ static int filter_pattern_match(struct ref_filter *filter, const char *refname) { - if (!*filter->name_patterns) + if (!filter->name_patterns || !*filter->name_patterns) return 1; /* No pattern always matches */ if (filter->match_as_path) return match_name_as_path(filter->name_patterns, refname, @@ -2751,7 +2831,7 @@ static int for_each_fullref_in_pattern(struct ref_filter *filter, return for_each_fullref_with_seek(filter, cb, cb_data, 0); } - if (!filter->name_patterns[0]) { + if (!filter->name_patterns || !filter->name_patterns[0]) { /* no patterns; we have to look at everything */ return for_each_fullref_with_seek(filter, cb, cb_data, 0); } @@ -2807,12 +2887,15 @@ static int match_points_at(struct oid_array *points_at, * Callers can then fill in other struct members at their leisure. */ static struct ref_array_item *new_ref_array_item(const char *refname, - const struct object_id *oid) + const struct object_id *oid, + const struct object_id *peeled_oid) { struct ref_array_item *ref; FLEX_ALLOC_STR(ref, refname, refname); oidcpy(&ref->objectname, oid); + if (peeled_oid) + oidcpy(&ref->peeled_oid, peeled_oid); ref->rest = NULL; return ref; @@ -2826,14 +2909,15 @@ static void ref_array_append(struct ref_array *array, struct ref_array_item *ref struct ref_array_item *ref_array_push(struct ref_array *array, const char *refname, - const struct object_id *oid) + const struct object_id *oid, + const struct object_id *peeled_oid) { - struct ref_array_item *ref = new_ref_array_item(refname, oid); + struct ref_array_item *ref = new_ref_array_item(refname, oid, peeled_oid); ref_array_append(array, ref); return ref; } -static int ref_kind_from_refname(const char *refname) +int ref_kind_from_refname(const char *refname) { unsigned int i; @@ -2871,25 +2955,25 @@ static int filter_ref_kind(struct ref_filter *filter, const char *refname) return ref_kind_from_refname(refname); } -static struct ref_array_item *apply_ref_filter(const char *refname, const char *referent, const struct object_id *oid, - int flag, struct ref_filter *filter) +static struct ref_array_item *apply_ref_filter(const struct reference *ref, + struct ref_filter *filter) { - struct ref_array_item *ref; + struct ref_array_item *item; struct commit *commit = NULL; unsigned int kind; - if (flag & REF_BAD_NAME) { - warning(_("ignoring ref with broken name %s"), refname); + if (ref->flags & REF_BAD_NAME) { + warning(_("ignoring ref with broken name %s"), ref->name); return NULL; } - if (flag & REF_ISBROKEN) { - warning(_("ignoring broken ref %s"), refname); + if (ref->flags & REF_ISBROKEN) { + warning(_("ignoring broken ref %s"), ref->name); return NULL; } /* Obtain the current ref kind from filter_ref_kind() and ignore unwanted refs. */ - kind = filter_ref_kind(filter, refname); + kind = filter_ref_kind(filter, ref->name); /* * Generally HEAD refs are printed with special description denoting a rebase, @@ -2902,13 +2986,13 @@ static struct ref_array_item *apply_ref_filter(const char *refname, const char * else if (!(kind & filter->kind)) return NULL; - if (!filter_pattern_match(filter, refname)) + if (!filter_pattern_match(filter, ref->name)) return NULL; - if (filter_exclude_match(filter, refname)) + if (filter_exclude_match(filter, ref->name)) return NULL; - if (filter->points_at.nr && !match_points_at(&filter->points_at, oid, refname)) + if (filter->points_at.nr && !match_points_at(&filter->points_at, ref->oid, ref->name)) return NULL; /* @@ -2918,7 +3002,7 @@ static struct ref_array_item *apply_ref_filter(const char *refname, const char * */ if (filter->reachable_from || filter->unreachable_from || filter->with_commit || filter->no_commit || filter->verbose) { - commit = lookup_commit_reference_gently(the_repository, oid, 1); + commit = lookup_commit_reference_gently(the_repository, ref->oid, 1); if (!commit) return NULL; /* We perform the filtering for the '--contains' option... */ @@ -2936,13 +3020,13 @@ static struct ref_array_item *apply_ref_filter(const char *refname, const char * * to do its job and the resulting list may yet to be pruned * by maxcount logic. */ - ref = new_ref_array_item(refname, oid); - ref->commit = commit; - ref->flag = flag; - ref->kind = kind; - ref->symref = xstrdup_or_null(referent); + item = new_ref_array_item(ref->name, ref->oid, ref->peeled_oid); + item->commit = commit; + item->flag = ref->flags; + item->kind = kind; + item->symref = xstrdup_or_null(ref->target); - return ref; + return item; } struct ref_filter_cbdata { @@ -2954,14 +3038,14 @@ struct ref_filter_cbdata { * A call-back given to for_each_ref(). Filter refs and keep them for * later object processing. */ -static int filter_one(const char *refname, const char *referent, const struct object_id *oid, int flag, void *cb_data) +static int filter_one(const struct reference *ref, void *cb_data) { struct ref_filter_cbdata *ref_cbdata = cb_data; - struct ref_array_item *ref; + struct ref_array_item *item; - ref = apply_ref_filter(refname, referent, oid, flag, ref_cbdata->filter); - if (ref) - ref_array_append(ref_cbdata->array, ref); + item = apply_ref_filter(ref, ref_cbdata->filter); + if (item) + ref_array_append(ref_cbdata->array, item); return 0; } @@ -2990,17 +3074,17 @@ struct ref_filter_and_format_cbdata { } internal; }; -static int filter_and_format_one(const char *refname, const char *referent, const struct object_id *oid, int flag, void *cb_data) +static int filter_and_format_one(const struct reference *ref, void *cb_data) { struct ref_filter_and_format_cbdata *ref_cbdata = cb_data; - struct ref_array_item *ref; + struct ref_array_item *item; struct strbuf output = STRBUF_INIT, err = STRBUF_INIT; - ref = apply_ref_filter(refname, referent, oid, flag, ref_cbdata->filter); - if (!ref) + item = apply_ref_filter(ref, ref_cbdata->filter); + if (!item) return 0; - if (format_ref_array_item(ref, ref_cbdata->format, &output, &err)) + if (format_ref_array_item(item, ref_cbdata->format, &output, &err)) die("%s", err.buf); if (output.len || !ref_cbdata->format->array_opts.omit_empty) { @@ -3010,7 +3094,7 @@ static int filter_and_format_one(const char *refname, const char *referent, cons strbuf_release(&output); strbuf_release(&err); - free_array_item(ref); + free_array_item(item); /* * Increment the running count of refs that match the filter. If @@ -3583,13 +3667,14 @@ void print_formatted_ref_array(struct ref_array *array, struct ref_format *forma } void pretty_print_ref(const char *name, const struct object_id *oid, + const struct object_id *peeled_oid, struct ref_format *format) { struct ref_array_item *ref_item; struct strbuf output = STRBUF_INIT; struct strbuf err = STRBUF_INIT; - ref_item = new_ref_array_item(name, oid); + ref_item = new_ref_array_item(name, oid, peeled_oid); ref_item->kind = ref_kind_from_refname(name); if (format_ref_array_item(ref_item, format, &output, &err)) die("%s", err.buf); diff --git a/ref-filter.h b/ref-filter.h index 81f2c229a9..120221b47f 100644 --- a/ref-filter.h +++ b/ref-filter.h @@ -41,6 +41,7 @@ enum ref_sorting_order { struct ref_array_item { struct object_id objectname; + struct object_id peeled_oid; const char *rest; int flag; unsigned int kind; @@ -135,6 +136,8 @@ struct ref_format { OPT_STRVEC(0, "exclude", &(var)->exclude, \ N_("pattern"), N_("exclude refs which match pattern")) +/* Get the reference kind from the provided reference name. */ +int ref_kind_from_refname(const char *refname); /* * API for filtering a set of refs. Based on the type of refs the user * has requested, we iterate through those refs and apply filters @@ -185,6 +188,7 @@ void print_formatted_ref_array(struct ref_array *array, struct ref_format *forma * name must be a fully qualified refname. */ void pretty_print_ref(const char *name, const struct object_id *oid, + const struct object_id *peeled_oid, struct ref_format *format); /* @@ -193,7 +197,8 @@ void pretty_print_ref(const char *name, const struct object_id *oid, */ struct ref_array_item *ref_array_push(struct ref_array *array, const char *refname, - const struct object_id *oid); + const struct object_id *oid, + const struct object_id *peeled_oid); /* * If the provided format includes ahead-behind atoms, then compute the @@ -423,16 +423,13 @@ int should_expire_reflog_ent_verbose(struct object_id *ooid, return expire; } -static int push_tip_to_list(const char *refname UNUSED, - const char *referent UNUSED, - const struct object_id *oid, - int flags, void *cb_data) +static int push_tip_to_list(const struct reference *ref, void *cb_data) { struct commit_list **list = cb_data; struct commit *tip_commit; - if (flags & REF_ISSYMREF) + if (ref->flags & REF_ISSYMREF) return 0; - tip_commit = lookup_commit_reference_gently(the_repository, oid, 1); + tip_commit = lookup_commit_reference_gently(the_repository, ref->oid, 1); if (!tip_commit) return 0; commit_list_insert(tip_commit, list); @@ -426,17 +426,19 @@ int refs_ref_exists(struct ref_store *refs, const char *refname) NULL, NULL); } -static int for_each_filter_refs(const char *refname, const char *referent, - const struct object_id *oid, - int flags, void *data) +static int for_each_filter_refs(const struct reference *ref, void *data) { struct for_each_ref_filter *filter = data; - if (wildmatch(filter->pattern, refname, 0)) + if (wildmatch(filter->pattern, ref->name, 0)) return 0; - if (filter->prefix) - skip_prefix(refname, filter->prefix, &refname); - return filter->fn(refname, referent, oid, flags, filter->cb_data); + if (filter->prefix) { + struct reference skipped = *ref; + skip_prefix(skipped.name, filter->prefix, &skipped.name); + return filter->fn(&skipped, filter->cb_data); + } else { + return filter->fn(ref, filter->cb_data); + } } struct warn_if_dangling_data { @@ -447,17 +449,15 @@ struct warn_if_dangling_data { int dry_run; }; -static int warn_if_dangling_symref(const char *refname, const char *referent UNUSED, - const struct object_id *oid UNUSED, - int flags, void *cb_data) +static int warn_if_dangling_symref(const struct reference *ref, void *cb_data) { struct warn_if_dangling_data *d = cb_data; const char *resolves_to, *msg; - if (!(flags & REF_ISSYMREF)) + if (!(ref->flags & REF_ISSYMREF)) return 0; - resolves_to = refs_resolve_ref_unsafe(d->refs, refname, 0, NULL, NULL); + resolves_to = refs_resolve_ref_unsafe(d->refs, ref->name, 0, NULL, NULL); if (!resolves_to || !string_list_has_string(d->refnames, resolves_to)) { return 0; @@ -466,7 +466,7 @@ static int warn_if_dangling_symref(const char *refname, const char *referent UNU msg = d->dry_run ? _("%s%s will become dangling after %s is deleted\n") : _("%s%s has become dangling after %s was deleted\n"); - fprintf(d->fp, msg, d->indent, refname, resolves_to); + fprintf(d->fp, msg, d->indent, ref->name, resolves_to); return 0; } @@ -507,8 +507,15 @@ int refs_head_ref_namespaced(struct ref_store *refs, each_ref_fn fn, void *cb_da int flag; strbuf_addf(&buf, "%sHEAD", get_git_namespace()); - if (!refs_read_ref_full(refs, buf.buf, RESOLVE_REF_READING, &oid, &flag)) - ret = fn(buf.buf, NULL, &oid, flag, cb_data); + if (!refs_read_ref_full(refs, buf.buf, RESOLVE_REF_READING, &oid, &flag)) { + struct reference ref = { + .name = buf.buf, + .oid = &oid, + .flags = flag, + }; + + ret = fn(&ref, cb_data); + } strbuf_release(&buf); return ret; @@ -1741,8 +1748,15 @@ int refs_head_ref(struct ref_store *refs, each_ref_fn fn, void *cb_data) int flag; if (refs_resolve_ref_unsafe(refs, "HEAD", RESOLVE_REF_READING, - &oid, &flag)) - return fn("HEAD", NULL, &oid, flag, cb_data); + &oid, &flag)) { + struct reference ref = { + .name = "HEAD", + .oid = &oid, + .flags = flag, + }; + + return fn(&ref, cb_data); + } return 0; } @@ -2299,25 +2313,28 @@ void base_ref_store_init(struct ref_store *refs, struct repository *repo, refs->gitdir = xstrdup(path); } -/* backend functions */ -int refs_pack_refs(struct ref_store *refs, struct pack_refs_opts *opts) +int refs_optimize(struct ref_store *refs, struct refs_optimize_opts *opts) { - return refs->be->pack_refs(refs, opts); + return refs->be->optimize(refs, opts); } -int refs_optimize(struct ref_store *refs, struct pack_refs_opts *opts) +int refs_optimize_required(struct ref_store *refs, + struct refs_optimize_opts *opts, + bool *required) { - return refs->be->optimize(refs, opts); + return refs->be->optimize_required(refs, opts, required); } -int peel_iterated_oid(struct repository *r, const struct object_id *base, struct object_id *peeled) +int reference_get_peeled_oid(struct repository *repo, + const struct reference *ref, + struct object_id *peeled_oid) { - if (current_ref_iter && - (current_ref_iter->oid == base || - oideq(current_ref_iter->oid, base))) - return ref_iterator_peel(current_ref_iter, peeled); + if (ref->peeled_oid) { + oidcpy(peeled_oid, ref->peeled_oid); + return 0; + } - return peel_object(r, base, peeled) ? -1 : 0; + return peel_object(repo, ref->oid, peeled_oid, 0) ? -1 : 0; } int refs_update_symref(struct ref_store *refs, const char *ref, @@ -2405,68 +2422,73 @@ static int ref_update_reject_duplicates(struct string_list *refnames, return 0; } -static int run_transaction_hook(struct ref_transaction *transaction, - const char *state) +struct transaction_feed_cb_data { + size_t index; + struct strbuf buf; +}; + +static int transaction_hook_feed_stdin(int hook_stdin_fd, void *pp_cb, void *pp_task_cb UNUSED) { - struct child_process proc = CHILD_PROCESS_INIT; - struct strbuf buf = STRBUF_INIT; - const char *hook; - int ret = 0; + struct hook_cb_data *hook_cb = pp_cb; + struct run_hooks_opt *opt = hook_cb->options; + struct ref_transaction *transaction = opt->feed_pipe_ctx; + struct transaction_feed_cb_data *feed_cb_data = opt->feed_pipe_cb_data; + struct strbuf *buf = &feed_cb_data->buf; + struct ref_update *update; + size_t i = feed_cb_data->index++; + int ret; - hook = find_hook(transaction->ref_store->repo, "reference-transaction"); - if (!hook) - return ret; + if (i >= transaction->nr) + return 1; /* No more refs to process */ - strvec_pushl(&proc.args, hook, state, NULL); - proc.in = -1; - proc.stdout_to_stderr = 1; - proc.trace2_hook_name = "reference-transaction"; + update = transaction->updates[i]; - ret = start_command(&proc); - if (ret) - return ret; + if (update->flags & REF_LOG_ONLY) + return 0; - sigchain_push(SIGPIPE, SIG_IGN); + strbuf_reset(buf); - for (size_t i = 0; i < transaction->nr; i++) { - struct ref_update *update = transaction->updates[i]; + if (!(update->flags & REF_HAVE_OLD)) + strbuf_addf(buf, "%s ", oid_to_hex(null_oid(the_hash_algo))); + else if (update->old_target) + strbuf_addf(buf, "ref:%s ", update->old_target); + else + strbuf_addf(buf, "%s ", oid_to_hex(&update->old_oid)); - if (update->flags & REF_LOG_ONLY) - continue; + if (!(update->flags & REF_HAVE_NEW)) + strbuf_addf(buf, "%s ", oid_to_hex(null_oid(the_hash_algo))); + else if (update->new_target) + strbuf_addf(buf, "ref:%s ", update->new_target); + else + strbuf_addf(buf, "%s ", oid_to_hex(&update->new_oid)); - strbuf_reset(&buf); + strbuf_addf(buf, "%s\n", update->refname); - if (!(update->flags & REF_HAVE_OLD)) - strbuf_addf(&buf, "%s ", oid_to_hex(null_oid(the_hash_algo))); - else if (update->old_target) - strbuf_addf(&buf, "ref:%s ", update->old_target); - else - strbuf_addf(&buf, "%s ", oid_to_hex(&update->old_oid)); + ret = write_in_full(hook_stdin_fd, buf->buf, buf->len); + if (ret < 0 && errno != EPIPE) + return ret; - if (!(update->flags & REF_HAVE_NEW)) - strbuf_addf(&buf, "%s ", oid_to_hex(null_oid(the_hash_algo))); - else if (update->new_target) - strbuf_addf(&buf, "ref:%s ", update->new_target); - else - strbuf_addf(&buf, "%s ", oid_to_hex(&update->new_oid)); + return 0; /* no more input to feed */ +} + +static int run_transaction_hook(struct ref_transaction *transaction, + const char *state) +{ + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; + struct transaction_feed_cb_data feed_ctx = { 0 }; + int ret = 0; - strbuf_addf(&buf, "%s\n", update->refname); + strvec_push(&opt.args, state); - if (write_in_full(proc.in, buf.buf, buf.len) < 0) { - if (errno != EPIPE) { - /* Don't leak errno outside this API */ - errno = 0; - ret = -1; - } - break; - } - } + opt.feed_pipe = transaction_hook_feed_stdin; + opt.feed_pipe_ctx = transaction; + opt.feed_pipe_cb_data = &feed_ctx; - close(proc.in); - sigchain_pop(SIGPIPE); - strbuf_release(&buf); + strbuf_init(&feed_ctx.buf, 0); - ret |= finish_command(&proc); + ret = run_hooks_opt(transaction->ref_store->repo, "reference-transaction", &opt); + + strbuf_release(&feed_ctx.buf); return ret; } @@ -2689,7 +2711,7 @@ enum ref_transaction_error refs_verify_refnames_available(struct ref_store *refs while ((ok = ref_iterator_advance(iter)) == ITER_OK) { if (skip && - string_list_has_string(skip, iter->refname)) + string_list_has_string(skip, iter->ref.name)) continue; if (transaction && ref_transaction_maybe_set_rejected( @@ -2698,7 +2720,7 @@ enum ref_transaction_error refs_verify_refnames_available(struct ref_store *refs continue; strbuf_addf(err, _("'%s' exists; cannot create '%s'"), - iter->refname, refname); + iter->ref.name, refname); goto cleanup; } @@ -2753,14 +2775,10 @@ struct do_for_each_reflog_help { void *cb_data; }; -static int do_for_each_reflog_helper(const char *refname, - const char *referent UNUSED, - const struct object_id *oid UNUSED, - int flags UNUSED, - void *cb_data) +static int do_for_each_reflog_helper(const struct reference *ref, void *cb_data) { struct do_for_each_reflog_help *hp = cb_data; - return hp->fn(refname, hp->cb_data); + return hp->fn(ref->name, hp->cb_data); } int refs_for_each_reflog(struct ref_store *refs, each_reflog_fn fn, void *cb_data) @@ -2976,25 +2994,24 @@ struct migration_data { uint64_t index; }; -static int migrate_one_ref(const char *refname, const char *referent UNUSED, const struct object_id *oid, - int flags, void *cb_data) +static int migrate_one_ref(const struct reference *ref, void *cb_data) { struct migration_data *data = cb_data; struct strbuf symref_target = STRBUF_INIT; int ret; - if (flags & REF_ISSYMREF) { - ret = refs_read_symbolic_ref(data->old_refs, refname, &symref_target); + if (ref->flags & REF_ISSYMREF) { + ret = refs_read_symbolic_ref(data->old_refs, ref->name, &symref_target); if (ret < 0) goto done; - ret = ref_transaction_update(data->transaction, refname, NULL, null_oid(the_hash_algo), + ret = ref_transaction_update(data->transaction, ref->name, NULL, null_oid(the_hash_algo), symref_target.buf, NULL, REF_SKIP_CREATE_REFLOG | REF_NO_DEREF, NULL, data->errbuf); if (ret < 0) goto done; } else { - ret = ref_transaction_create(data->transaction, refname, oid, NULL, + ret = ref_transaction_create(data->transaction, ref->name, ref->oid, NULL, REF_SKIP_CREATE_REFLOG | REF_SKIP_OID_VERIFICATION, NULL, data->errbuf); if (ret < 0) @@ -333,36 +333,74 @@ struct ref_transaction; * stored in ref_iterator::flags. Other bits are for internal use * only: */ +enum reference_status { + /* Reference is a symbolic reference. */ + REF_ISSYMREF = (1 << 0), -/* Reference is a symbolic reference. */ -#define REF_ISSYMREF 0x01 + /* Reference is a packed reference. */ + REF_ISPACKED = (1 << 1), -/* Reference is a packed reference. */ -#define REF_ISPACKED 0x02 + /* + * Reference cannot be resolved to an object name: dangling symbolic + * reference (directly or indirectly), corrupt reference file, + * reference exists but name is bad, or symbolic reference refers to + * ill-formatted reference name. + */ + REF_ISBROKEN = (1 << 2), -/* - * Reference cannot be resolved to an object name: dangling symbolic - * reference (directly or indirectly), corrupt reference file, - * reference exists but name is bad, or symbolic reference refers to - * ill-formatted reference name. - */ -#define REF_ISBROKEN 0x04 + /* + * Reference name is not well formed. + * + * See git-check-ref-format(1) for the definition of well formed ref names. + */ + REF_BAD_NAME = (1 << 3), +}; + +/* A reference passed to `for_each_ref()`-style callbacks. */ +struct reference { + /* The fully-qualified name of the reference. */ + const char *name; + + /* The target of a symbolic ref. `NULL` for direct references. */ + const char *target; + + /* + * The object ID of a reference. Either the direct object ID or the + * resolved object ID in the case of a symbolic ref. May be the zero + * object ID in case the symbolic ref cannot be resolved. + */ + const struct object_id *oid; + + /* + * An optional peeled object ID. This field _may_ be set for tags in + * case the peeled value is present in the backend. Please refer to + * `reference_get_peeled_oid()`. + */ + const struct object_id *peeled_oid; + + /* A bitfield of `enum reference_status` flags. */ + unsigned flags; +}; /* - * Reference name is not well formed. + * Peel the tag to a non-tag commit. If present, this uses the peeled object ID + * exposed by the reference backend. Otherwise, the object is peeled via the + * object database, which is less efficient. * - * See git-check-ref-format(1) for the definition of well formed ref names. + * Return `0` if the reference could be peeled, a negative error code + * otherwise. */ -#define REF_BAD_NAME 0x08 +int reference_get_peeled_oid(struct repository *repo, + const struct reference *ref, + struct object_id *peeled_oid); /* * The signature for the callback function for the for_each_*() - * functions below. The memory pointed to by the refname and oid - * arguments is only guaranteed to be valid for the duration of a + * functions below. The memory pointed to by the `struct reference` + * argument is only guaranteed to be valid for the duration of a * single callback invocation. */ -typedef int each_ref_fn(const char *refname, const char *referent, - const struct object_id *oid, int flags, void *cb_data); +typedef int each_ref_fn(const struct reference *ref, void *cb_data); /* * The following functions invoke the specified callback function for @@ -461,32 +499,33 @@ void refs_warn_dangling_symrefs(struct ref_store *refs, FILE *fp, const struct string_list *refnames); /* - * Flags for controlling behaviour of pack_refs() - * PACK_REFS_PRUNE: Prune loose refs after packing - * PACK_REFS_AUTO: Pack refs on a best effort basis. The heuristics and end - * result are decided by the ref backend. Backends may ignore - * this flag and fall back to a normal repack. + * Flags for controlling behaviour of refs_optimize() + * REFS_OPTIMIZE_PRUNE: Prune loose refs after packing + * REFS_OPTIMIZE_AUTO: Pack refs on a best effort basis. The heuristics and end + * result are decided by the ref backend. Backends may ignore + * this flag and fall back to a normal repack. */ -#define PACK_REFS_PRUNE (1 << 0) -#define PACK_REFS_AUTO (1 << 1) +#define REFS_OPTIMIZE_PRUNE (1 << 0) +#define REFS_OPTIMIZE_AUTO (1 << 1) -struct pack_refs_opts { +struct refs_optimize_opts { unsigned int flags; struct ref_exclusions *exclusions; struct string_list *includes; }; /* - * Write a packed-refs file for the current repository. - * flags: Combination of the above PACK_REFS_* flags. + * Optimize the ref store. The exact behavior is up to the backend. + * For the files backend, this is equivalent to packing refs. */ -int refs_pack_refs(struct ref_store *refs, struct pack_refs_opts *opts); +int refs_optimize(struct ref_store *refs, struct refs_optimize_opts *opts); /* - * Optimize the ref store. The exact behavior is up to the backend. - * For the files backend, this is equivalent to packing refs. + * Check if refs backend can be optimized by calling 'refs_optimize'. */ -int refs_optimize(struct ref_store *refs, struct pack_refs_opts *opts); +int refs_optimize_required(struct ref_store *ref_store, + struct refs_optimize_opts *opts, + bool *required); /* * Setup reflog before using. Fill in err and return -1 on failure. @@ -1251,10 +1290,6 @@ int repo_migrate_ref_storage_format(struct repository *repo, * to the next entry, ref_iterator_advance() aborts the iteration, * frees the ref_iterator, and returns ITER_ERROR. * - * The reference currently being looked at can be peeled by calling - * ref_iterator_peel(). This function is often faster than peel_ref(), - * so it should be preferred when iterating over references. - * * Putting it all together, a typical iteration looks like this: * * int ok; @@ -1269,9 +1304,6 @@ int repo_migrate_ref_storage_format(struct repository *repo, * // Access information about the current reference: * if (!(iter->flags & REF_ISSYMREF)) * printf("%s is %s\n", iter->refname, oid_to_hex(iter->oid)); - * - * // If you need to peel the reference: - * ref_iterator_peel(iter, &oid); * } * * if (ok != ITER_DONE) @@ -1362,13 +1394,6 @@ enum ref_iterator_seek_flag { int ref_iterator_seek(struct ref_iterator *ref_iterator, const char *refname, unsigned int flags); -/* - * If possible, peel the reference currently being viewed by the - * iterator. Return 0 on success. - */ -int ref_iterator_peel(struct ref_iterator *ref_iterator, - struct object_id *peeled); - /* Free the reference iterator and any associated resources. */ void ref_iterator_free(struct ref_iterator *ref_iterator); diff --git a/refs/debug.c b/refs/debug.c index 697adbd0dc..3e31228c9a 100644 --- a/refs/debug.c +++ b/refs/debug.c @@ -47,6 +47,14 @@ static int debug_create_on_disk(struct ref_store *refs, int flags, struct strbuf return res; } +static int debug_remove_on_disk(struct ref_store *refs, struct strbuf *err) +{ + struct debug_ref_store *drefs = (struct debug_ref_store *)refs; + int res = drefs->refs->be->remove_on_disk(drefs->refs, err); + trace_printf_key(&trace_refs, "remove_on_disk: %d\n", res); + return res; +} + static int debug_transaction_prepare(struct ref_store *refs, struct ref_transaction *transaction, struct strbuf *err) @@ -116,11 +124,22 @@ static int debug_transaction_abort(struct ref_store *refs, return res; } -static int debug_pack_refs(struct ref_store *ref_store, struct pack_refs_opts *opts) +static int debug_optimize(struct ref_store *ref_store, struct refs_optimize_opts *opts) { struct debug_ref_store *drefs = (struct debug_ref_store *)ref_store; - int res = drefs->refs->be->pack_refs(drefs->refs, opts); - trace_printf_key(&trace_refs, "pack_refs: %d\n", res); + int res = drefs->refs->be->optimize(drefs->refs, opts); + trace_printf_key(&trace_refs, "optimize: %d\n", res); + return res; +} + +static int debug_optimize_required(struct ref_store *ref_store, + struct refs_optimize_opts *opts, + bool *required) +{ + struct debug_ref_store *drefs = (struct debug_ref_store *)ref_store; + int res = drefs->refs->be->optimize_required(drefs->refs, opts, required); + trace_printf_key(&trace_refs, "optimize_required: %s, res: %d\n", + required ? "yes" : "no", res); return res; } @@ -160,11 +179,9 @@ static int debug_ref_iterator_advance(struct ref_iterator *ref_iterator) trace_printf_key(&trace_refs, "iterator_advance: (%d)\n", res); else trace_printf_key(&trace_refs, "iterator_advance: %s (0)\n", - diter->iter->refname); + diter->iter->ref.name); - diter->base.refname = diter->iter->refname; - diter->base.oid = diter->iter->oid; - diter->base.flags = diter->iter->flags; + diter->base.ref = diter->iter->ref; return res; } @@ -179,16 +196,6 @@ static int debug_ref_iterator_seek(struct ref_iterator *ref_iterator, return res; } -static int debug_ref_iterator_peel(struct ref_iterator *ref_iterator, - struct object_id *peeled) -{ - struct debug_ref_iterator *diter = - (struct debug_ref_iterator *)ref_iterator; - int res = diter->iter->vtable->peel(diter->iter, peeled); - trace_printf_key(&trace_refs, "iterator_peel: %s: %d\n", diter->iter->refname, res); - return res; -} - static void debug_ref_iterator_release(struct ref_iterator *ref_iterator) { struct debug_ref_iterator *diter = @@ -200,7 +207,6 @@ static void debug_ref_iterator_release(struct ref_iterator *ref_iterator) static struct ref_iterator_vtable debug_ref_iterator_vtable = { .advance = debug_ref_iterator_advance, .seek = debug_ref_iterator_seek, - .peel = debug_ref_iterator_peel, .release = debug_ref_iterator_release, }; @@ -432,6 +438,7 @@ struct ref_storage_be refs_be_debug = { .init = NULL, .release = debug_release, .create_on_disk = debug_create_on_disk, + .remove_on_disk = debug_remove_on_disk, /* * None of these should be NULL. If the "files" backend (in @@ -443,7 +450,9 @@ struct ref_storage_be refs_be_debug = { .transaction_finish = debug_transaction_finish, .transaction_abort = debug_transaction_abort, - .pack_refs = debug_pack_refs, + .optimize = debug_optimize, + .optimize_required = debug_optimize_required, + .rename_ref = debug_rename_ref, .copy_ref = debug_copy_ref, diff --git a/refs/files-backend.c b/refs/files-backend.c index 5ddf418b18..6f6f76a8d8 100644 --- a/refs/files-backend.c +++ b/refs/files-backend.c @@ -961,26 +961,23 @@ static int files_ref_iterator_advance(struct ref_iterator *ref_iterator) while ((ok = ref_iterator_advance(iter->iter0)) == ITER_OK) { if (iter->flags & DO_FOR_EACH_PER_WORKTREE_ONLY && - parse_worktree_ref(iter->iter0->refname, NULL, NULL, + parse_worktree_ref(iter->iter0->ref.name, NULL, NULL, NULL) != REF_WORKTREE_CURRENT) continue; if ((iter->flags & DO_FOR_EACH_OMIT_DANGLING_SYMREFS) && - (iter->iter0->flags & REF_ISSYMREF) && - (iter->iter0->flags & REF_ISBROKEN)) + (iter->iter0->ref.flags & REF_ISSYMREF) && + (iter->iter0->ref.flags & REF_ISBROKEN)) continue; if (!(iter->flags & DO_FOR_EACH_INCLUDE_BROKEN) && - !ref_resolves_to_object(iter->iter0->refname, + !ref_resolves_to_object(iter->iter0->ref.name, iter->repo, - iter->iter0->oid, - iter->iter0->flags)) + iter->iter0->ref.oid, + iter->iter0->ref.flags)) continue; - iter->base.refname = iter->iter0->refname; - iter->base.oid = iter->iter0->oid; - iter->base.flags = iter->iter0->flags; - iter->base.referent = iter->iter0->referent; + iter->base.ref = iter->iter0->ref; return ITER_OK; } @@ -996,15 +993,6 @@ static int files_ref_iterator_seek(struct ref_iterator *ref_iterator, return ref_iterator_seek(iter->iter0, refname, flags); } -static int files_ref_iterator_peel(struct ref_iterator *ref_iterator, - struct object_id *peeled) -{ - struct files_ref_iterator *iter = - (struct files_ref_iterator *)ref_iterator; - - return ref_iterator_peel(iter->iter0, peeled); -} - static void files_ref_iterator_release(struct ref_iterator *ref_iterator) { struct files_ref_iterator *iter = @@ -1015,7 +1003,6 @@ static void files_ref_iterator_release(struct ref_iterator *ref_iterator) static struct ref_iterator_vtable files_ref_iterator_vtable = { .advance = files_ref_iterator_advance, .seek = files_ref_iterator_seek, - .peel = files_ref_iterator_peel, .release = files_ref_iterator_release, }; @@ -1367,37 +1354,36 @@ static void prune_refs(struct files_ref_store *refs, struct ref_to_prune **refs_ * Return true if the specified reference should be packed. */ static int should_pack_ref(struct files_ref_store *refs, - const char *refname, - const struct object_id *oid, unsigned int ref_flags, - struct pack_refs_opts *opts) + const struct reference *ref, + struct refs_optimize_opts *opts) { struct string_list_item *item; /* Do not pack per-worktree refs: */ - if (parse_worktree_ref(refname, NULL, NULL, NULL) != + if (parse_worktree_ref(ref->name, NULL, NULL, NULL) != REF_WORKTREE_SHARED) return 0; /* Do not pack symbolic refs: */ - if (ref_flags & REF_ISSYMREF) + if (ref->flags & REF_ISSYMREF) return 0; /* Do not pack broken refs: */ - if (!ref_resolves_to_object(refname, refs->base.repo, oid, ref_flags)) + if (!ref_resolves_to_object(ref->name, refs->base.repo, ref->oid, ref->flags)) return 0; - if (ref_excluded(opts->exclusions, refname)) + if (ref_excluded(opts->exclusions, ref->name)) return 0; for_each_string_list_item(item, opts->includes) - if (!wildmatch(item->string, refname, 0)) + if (!wildmatch(item->string, ref->name, 0)) return 1; return 0; } static int should_pack_refs(struct files_ref_store *refs, - struct pack_refs_opts *opts) + struct refs_optimize_opts *opts) { struct ref_iterator *iter; size_t packed_size; @@ -1405,7 +1391,7 @@ static int should_pack_refs(struct files_ref_store *refs, size_t limit; int ret; - if (!(opts->flags & PACK_REFS_AUTO)) + if (!(opts->flags & REFS_OPTIMIZE_AUTO)) return 1; ret = packed_refs_size(refs->packed_ref_store, &packed_size); @@ -1443,8 +1429,7 @@ static int should_pack_refs(struct files_ref_store *refs, iter = cache_ref_iterator_begin(get_loose_ref_cache(refs, 0), NULL, refs->base.repo, 0); while ((ret = ref_iterator_advance(iter)) == ITER_OK) { - if (should_pack_ref(refs, iter->refname, iter->oid, - iter->flags, opts)) + if (should_pack_ref(refs, &iter->ref, opts)) refcount++; if (refcount >= limit) { ref_iterator_free(iter); @@ -1459,8 +1444,8 @@ static int should_pack_refs(struct files_ref_store *refs, return 0; } -static int files_pack_refs(struct ref_store *ref_store, - struct pack_refs_opts *opts) +static int files_optimize(struct ref_store *ref_store, + struct refs_optimize_opts *opts) { struct files_ref_store *refs = files_downcast(ref_store, REF_STORE_WRITE | REF_STORE_ODB, @@ -1489,24 +1474,24 @@ static int files_pack_refs(struct ref_store *ref_store, * in the packed ref cache. If the reference should be * pruned, also add it to refs_to_prune. */ - if (!should_pack_ref(refs, iter->refname, iter->oid, iter->flags, opts)) + if (!should_pack_ref(refs, &iter->ref, opts)) continue; /* * Add a reference creation for this reference to the * packed-refs transaction: */ - if (ref_transaction_update(transaction, iter->refname, - iter->oid, NULL, NULL, NULL, + if (ref_transaction_update(transaction, iter->ref.name, + iter->ref.oid, NULL, NULL, NULL, REF_NO_DEREF, NULL, &err)) die("failure preparing to create packed reference %s: %s", - iter->refname, err.buf); + iter->ref.name, err.buf); /* Schedule the loose reference for pruning if requested. */ - if ((opts->flags & PACK_REFS_PRUNE)) { + if ((opts->flags & REFS_OPTIMIZE_PRUNE)) { struct ref_to_prune *n; - FLEX_ALLOC_STR(n, name, iter->refname); - oidcpy(&n->oid, iter->oid); + FLEX_ALLOC_STR(n, name, iter->ref.name); + oidcpy(&n->oid, iter->ref.oid); n->next = refs_to_prune; refs_to_prune = n; } @@ -1527,13 +1512,14 @@ static int files_pack_refs(struct ref_store *ref_store, return 0; } -static int files_optimize(struct ref_store *ref_store, struct pack_refs_opts *opts) +static int files_optimize_required(struct ref_store *ref_store, + struct refs_optimize_opts *opts, + bool *required) { - /* - * For the "files" backend, "optimizing" is the same as "packing". - * So, we just call the existing worker function for packing. - */ - return files_pack_refs(ref_store, opts); + struct files_ref_store *refs = files_downcast(ref_store, REF_STORE_READ, + "optimize_required"); + *required = should_pack_refs(refs, opts); + return 0; } /* @@ -2113,20 +2099,35 @@ static int commit_ref_update(struct files_ref_store *refs, return 0; } -#ifdef NO_SYMLINK_HEAD +#if defined(NO_SYMLINK_HEAD) || defined(WITH_BREAKING_CHANGES) #define create_ref_symlink(a, b) (-1) #else static int create_ref_symlink(struct ref_lock *lock, const char *target) { + static int warn_once = 1; + char *ref_path; int ret = -1; - char *ref_path = get_locked_file_path(&lock->lk); + ref_path = get_locked_file_path(&lock->lk); unlink(ref_path); ret = symlink(target, ref_path); free(ref_path); if (ret) fprintf(stderr, "no symlink - falling back to symbolic ref\n"); + + if (warn_once) + warning(_("'core.preferSymlinkRefs=true' is nominated for removal.\n" + "hint: The use of symbolic links for symbolic refs is deprecated\n" + "hint: and will be removed in Git 3.0. The configuration that\n" + "hint: tells Git to use them is thus going away. You can unset\n" + "hint: it with:\n" + "hint:\n" + "hint:\tgit config unset core.preferSymlinkRefs\n" + "hint:\n" + "hint: Git will then use the textual symref format instead.")); + warn_once = 0; + return ret; } #endif @@ -2379,7 +2380,7 @@ static int files_reflog_iterator_advance(struct ref_iterator *ref_iterator) REFNAME_ALLOW_ONELEVEL)) continue; - iter->base.refname = diter->relative_path; + iter->base.ref.name = diter->relative_path; return ITER_OK; } @@ -2393,12 +2394,6 @@ static int files_reflog_iterator_seek(struct ref_iterator *ref_iterator UNUSED, BUG("ref_iterator_seek() called for reflog_iterator"); } -static int files_reflog_iterator_peel(struct ref_iterator *ref_iterator UNUSED, - struct object_id *peeled UNUSED) -{ - BUG("ref_iterator_peel() called for reflog_iterator"); -} - static void files_reflog_iterator_release(struct ref_iterator *ref_iterator) { struct files_reflog_iterator *iter = @@ -2409,7 +2404,6 @@ static void files_reflog_iterator_release(struct ref_iterator *ref_iterator) static struct ref_iterator_vtable files_reflog_iterator_vtable = { .advance = files_reflog_iterator_advance, .seek = files_reflog_iterator_seek, - .peel = files_reflog_iterator_peel, .release = files_reflog_iterator_release, }; @@ -3109,7 +3103,7 @@ static int parse_and_write_reflog(struct files_ref_store *refs, if (!(update->flags & REF_HAVE_OLD) || !(update->flags & REF_HAVE_NEW) || !(update->flags & REF_LOG_ONLY)) { - strbuf_addf(err, _("trying to write reflog for '%s'" + strbuf_addf(err, _("trying to write reflog for '%s' " "with incomplete values"), update->refname); return REF_TRANSACTION_ERROR_GENERIC; } @@ -3150,14 +3144,11 @@ static int parse_and_write_reflog(struct files_ref_store *refs, return 0; } -static int ref_present(const char *refname, const char *referent UNUSED, - const struct object_id *oid UNUSED, - int flags UNUSED, - void *cb_data) +static int ref_present(const struct reference *ref, void *cb_data) { struct string_list *affected_refnames = cb_data; - return string_list_has_string(affected_refnames, refname); + return string_list_has_string(affected_refnames, ref->name); } static int files_transaction_finish_initial(struct files_ref_store *refs, @@ -3327,7 +3318,13 @@ static int files_transaction_finish(struct ref_store *ref_store, * next update. If not, we try and create a regular symref. */ if (update->new_target && refs->prefer_symlink_refs) - if (!create_ref_symlink(lock, update->new_target)) + /* + * By using the `NOT_CONSTANT()` trick, we can avoid + * errors by `clang`'s `-Wunreachable` logic that would + * report that the `continue` statement is not reachable + * when `NO_SYMLINK_HEAD` is `#define`d. + */ + if (NOT_CONSTANT(!create_ref_symlink(lock, update->new_target))) continue; if (update->flags & REF_NEEDS_COMMIT) { @@ -3994,8 +3991,8 @@ struct ref_storage_be refs_be_files = { .transaction_finish = files_transaction_finish, .transaction_abort = files_transaction_abort, - .pack_refs = files_pack_refs, .optimize = files_optimize, + .optimize_required = files_optimize_required, .rename_ref = files_rename_ref, .copy_ref = files_copy_ref, diff --git a/refs/iterator.c b/refs/iterator.c index 17ef841d8a..d79aa5ec82 100644 --- a/refs/iterator.c +++ b/refs/iterator.c @@ -21,12 +21,6 @@ int ref_iterator_seek(struct ref_iterator *ref_iterator, const char *refname, return ref_iterator->vtable->seek(ref_iterator, refname, flags); } -int ref_iterator_peel(struct ref_iterator *ref_iterator, - struct object_id *peeled) -{ - return ref_iterator->vtable->peel(ref_iterator, peeled); -} - void ref_iterator_free(struct ref_iterator *ref_iterator) { if (ref_iterator) { @@ -41,10 +35,7 @@ void base_ref_iterator_init(struct ref_iterator *iter, struct ref_iterator_vtable *vtable) { iter->vtable = vtable; - iter->refname = NULL; - iter->referent = NULL; - iter->oid = NULL; - iter->flags = 0; + memset(&iter->ref, 0, sizeof(iter->ref)); } struct empty_ref_iterator { @@ -63,12 +54,6 @@ static int empty_ref_iterator_seek(struct ref_iterator *ref_iterator UNUSED, return 0; } -static int empty_ref_iterator_peel(struct ref_iterator *ref_iterator UNUSED, - struct object_id *peeled UNUSED) -{ - BUG("peel called for empty iterator"); -} - static void empty_ref_iterator_release(struct ref_iterator *ref_iterator UNUSED) { } @@ -76,7 +61,6 @@ static void empty_ref_iterator_release(struct ref_iterator *ref_iterator UNUSED) static struct ref_iterator_vtable empty_ref_iterator_vtable = { .advance = empty_ref_iterator_advance, .seek = empty_ref_iterator_seek, - .peel = empty_ref_iterator_peel, .release = empty_ref_iterator_release, }; @@ -127,8 +111,8 @@ enum iterator_selection ref_iterator_select(struct ref_iterator *iter_worktree, * latter. */ if (iter_worktree) { - int cmp = strcmp(iter_worktree->refname, - iter_common->refname); + int cmp = strcmp(iter_worktree->ref.name, + iter_common->ref.name); if (cmp < 0) return ITER_SELECT_0; else if (!cmp) @@ -139,7 +123,7 @@ enum iterator_selection ref_iterator_select(struct ref_iterator *iter_worktree, * We now know that the lexicographically-next ref is a common * ref. When the common ref is a shared one we return it. */ - if (parse_worktree_ref(iter_common->refname, NULL, NULL, + if (parse_worktree_ref(iter_common->ref.name, NULL, NULL, NULL) == REF_WORKTREE_SHARED) return ITER_SELECT_1; @@ -212,10 +196,7 @@ static int merge_ref_iterator_advance(struct ref_iterator *ref_iterator) } if (selection & ITER_YIELD_CURRENT) { - iter->base.referent = (*iter->current)->referent; - iter->base.refname = (*iter->current)->refname; - iter->base.oid = (*iter->current)->oid; - iter->base.flags = (*iter->current)->flags; + iter->base.ref = (*iter->current)->ref; return ITER_OK; } } @@ -246,18 +227,6 @@ static int merge_ref_iterator_seek(struct ref_iterator *ref_iterator, return 0; } -static int merge_ref_iterator_peel(struct ref_iterator *ref_iterator, - struct object_id *peeled) -{ - struct merge_ref_iterator *iter = - (struct merge_ref_iterator *)ref_iterator; - - if (!iter->current) { - BUG("peel called before advance for merge iterator"); - } - return ref_iterator_peel(*iter->current, peeled); -} - static void merge_ref_iterator_release(struct ref_iterator *ref_iterator) { struct merge_ref_iterator *iter = @@ -269,7 +238,6 @@ static void merge_ref_iterator_release(struct ref_iterator *ref_iterator) static struct ref_iterator_vtable merge_ref_iterator_vtable = { .advance = merge_ref_iterator_advance, .seek = merge_ref_iterator_seek, - .peel = merge_ref_iterator_peel, .release = merge_ref_iterator_release, }; @@ -313,7 +281,7 @@ static enum iterator_selection overlay_iterator_select( else if (!front) return ITER_SELECT_1; - cmp = strcmp(front->refname, back->refname); + cmp = strcmp(front->ref.name, back->ref.name); if (cmp < 0) return ITER_SELECT_0; @@ -371,7 +339,7 @@ static int prefix_ref_iterator_advance(struct ref_iterator *ref_iterator) int ok; while ((ok = ref_iterator_advance(iter->iter0)) == ITER_OK) { - int cmp = compare_prefix(iter->iter0->refname, iter->prefix); + int cmp = compare_prefix(iter->iter0->ref.name, iter->prefix); if (cmp < 0) continue; /* @@ -382,6 +350,8 @@ static int prefix_ref_iterator_advance(struct ref_iterator *ref_iterator) if (cmp > 0) return ITER_DONE; + iter->base.ref = iter->iter0->ref; + if (iter->trim) { /* * It is nonsense to trim off characters that @@ -392,15 +362,11 @@ static int prefix_ref_iterator_advance(struct ref_iterator *ref_iterator) * one character left in the refname after * trimming, report it as a bug: */ - if (strlen(iter->iter0->refname) <= iter->trim) + if (strlen(iter->base.ref.name) <= iter->trim) BUG("attempt to trim too many characters"); - iter->base.refname = iter->iter0->refname + iter->trim; - } else { - iter->base.refname = iter->iter0->refname; + iter->base.ref.name += iter->trim; } - iter->base.oid = iter->iter0->oid; - iter->base.flags = iter->iter0->flags; return ITER_OK; } @@ -420,15 +386,6 @@ static int prefix_ref_iterator_seek(struct ref_iterator *ref_iterator, return ref_iterator_seek(iter->iter0, refname, flags); } -static int prefix_ref_iterator_peel(struct ref_iterator *ref_iterator, - struct object_id *peeled) -{ - struct prefix_ref_iterator *iter = - (struct prefix_ref_iterator *)ref_iterator; - - return ref_iterator_peel(iter->iter0, peeled); -} - static void prefix_ref_iterator_release(struct ref_iterator *ref_iterator) { struct prefix_ref_iterator *iter = @@ -440,7 +397,6 @@ static void prefix_ref_iterator_release(struct ref_iterator *ref_iterator) static struct ref_iterator_vtable prefix_ref_iterator_vtable = { .advance = prefix_ref_iterator_advance, .seek = prefix_ref_iterator_seek, - .peel = prefix_ref_iterator_peel, .release = prefix_ref_iterator_release, }; @@ -466,23 +422,18 @@ struct ref_iterator *prefix_ref_iterator_begin(struct ref_iterator *iter0, return ref_iterator; } -struct ref_iterator *current_ref_iter = NULL; - int do_for_each_ref_iterator(struct ref_iterator *iter, each_ref_fn fn, void *cb_data) { int retval = 0, ok; - struct ref_iterator *old_ref_iter = current_ref_iter; - current_ref_iter = iter; while ((ok = ref_iterator_advance(iter)) == ITER_OK) { - retval = fn(iter->refname, iter->referent, iter->oid, iter->flags, cb_data); + retval = fn(&iter->ref, cb_data); if (retval) goto out; } out: - current_ref_iter = old_ref_iter; if (ok == ITER_ERROR) retval = -1; ref_iterator_free(iter); diff --git a/refs/packed-backend.c b/refs/packed-backend.c index a8c22a0a7f..4ea0c12299 100644 --- a/refs/packed-backend.c +++ b/refs/packed-backend.c @@ -882,6 +882,7 @@ static int next_record(struct packed_ref_iterator *iter) { const char *p, *eol; + memset(&iter->base.ref, 0, sizeof(iter->base.ref)); strbuf_reset(&iter->refname_buf); /* @@ -908,7 +909,7 @@ static int next_record(struct packed_ref_iterator *iter) if (iter->pos == iter->eof) return ITER_DONE; - iter->base.flags = REF_ISPACKED; + iter->base.ref.flags = REF_ISPACKED; p = iter->pos; if (iter->eof - p < snapshot_hexsz(iter->snapshot) + 2 || @@ -916,6 +917,7 @@ static int next_record(struct packed_ref_iterator *iter) !isspace(*p++)) die_invalid_line(iter->snapshot->refs->path, iter->pos, iter->eof - iter->pos); + iter->base.ref.oid = &iter->oid; eol = memchr(p, '\n', iter->eof - p); if (!eol) @@ -923,22 +925,22 @@ static int next_record(struct packed_ref_iterator *iter) iter->pos, iter->eof - iter->pos); strbuf_add(&iter->refname_buf, p, eol - p); - iter->base.refname = iter->refname_buf.buf; + iter->base.ref.name = iter->refname_buf.buf; if (refname_contains_nul(&iter->refname_buf)) - die("packed refname contains embedded NULL: %s", iter->base.refname); + die("packed refname contains embedded NULL: %s", iter->base.ref.name); - if (check_refname_format(iter->base.refname, REFNAME_ALLOW_ONELEVEL)) { - if (!refname_is_safe(iter->base.refname)) + if (check_refname_format(iter->base.ref.name, REFNAME_ALLOW_ONELEVEL)) { + if (!refname_is_safe(iter->base.ref.name)) die("packed refname is dangerous: %s", - iter->base.refname); + iter->base.ref.name); oidclr(&iter->oid, iter->repo->hash_algo); - iter->base.flags |= REF_BAD_NAME | REF_ISBROKEN; + iter->base.ref.flags |= REF_BAD_NAME | REF_ISBROKEN; } if (iter->snapshot->peeled == PEELED_FULLY || (iter->snapshot->peeled == PEELED_TAGS && - starts_with(iter->base.refname, "refs/tags/"))) - iter->base.flags |= REF_KNOWS_PEELED; + starts_with(iter->base.ref.name, "refs/tags/"))) + iter->base.ref.flags |= REF_KNOWS_PEELED; iter->pos = eol + 1; @@ -956,11 +958,12 @@ static int next_record(struct packed_ref_iterator *iter) * definitely know the value of *this* reference. But * we suppress it if the reference is broken: */ - if ((iter->base.flags & REF_ISBROKEN)) { + if ((iter->base.ref.flags & REF_ISBROKEN)) { oidclr(&iter->peeled, iter->repo->hash_algo); - iter->base.flags &= ~REF_KNOWS_PEELED; + iter->base.ref.flags &= ~REF_KNOWS_PEELED; } else { - iter->base.flags |= REF_KNOWS_PEELED; + iter->base.ref.flags |= REF_KNOWS_PEELED; + iter->base.ref.peeled_oid = &iter->peeled; } } else { oidclr(&iter->peeled, iter->repo->hash_algo); @@ -976,15 +979,15 @@ static int packed_ref_iterator_advance(struct ref_iterator *ref_iterator) int ok; while ((ok = next_record(iter)) == ITER_OK) { - const char *refname = iter->base.refname; + const char *refname = iter->base.ref.name; const char *prefix = iter->prefix; if (iter->flags & DO_FOR_EACH_PER_WORKTREE_ONLY && - !is_per_worktree_ref(iter->base.refname)) + !is_per_worktree_ref(iter->base.ref.name)) continue; if (!(iter->flags & DO_FOR_EACH_INCLUDE_BROKEN) && - !ref_resolves_to_object(iter->base.refname, iter->repo, + !ref_resolves_to_object(iter->base.ref.name, iter->repo, &iter->oid, iter->flags)) continue; @@ -1027,22 +1030,6 @@ static int packed_ref_iterator_seek(struct ref_iterator *ref_iterator, return 0; } -static int packed_ref_iterator_peel(struct ref_iterator *ref_iterator, - struct object_id *peeled) -{ - struct packed_ref_iterator *iter = - (struct packed_ref_iterator *)ref_iterator; - - if ((iter->base.flags & REF_KNOWS_PEELED)) { - oidcpy(peeled, &iter->peeled); - return is_null_oid(&iter->peeled) ? -1 : 0; - } else if ((iter->base.flags & (REF_ISBROKEN | REF_ISSYMREF))) { - return -1; - } else { - return peel_object(iter->repo, &iter->oid, peeled) ? -1 : 0; - } -} - static void packed_ref_iterator_release(struct ref_iterator *ref_iterator) { struct packed_ref_iterator *iter = @@ -1056,7 +1043,6 @@ static void packed_ref_iterator_release(struct ref_iterator *ref_iterator) static struct ref_iterator_vtable packed_ref_iterator_vtable = { .advance = packed_ref_iterator_advance, .seek = packed_ref_iterator_seek, - .peel = packed_ref_iterator_peel, .release = packed_ref_iterator_release, }; @@ -1194,7 +1180,6 @@ static struct ref_iterator *packed_ref_iterator_begin( iter->snapshot = snapshot; acquire_snapshot(snapshot); strbuf_init(&iter->refname_buf, 0); - iter->base.oid = &iter->oid; iter->repo = ref_store->repo; iter->flags = flags; @@ -1436,7 +1421,7 @@ static enum ref_transaction_error write_with_updates(struct packed_ref_store *re if (!iter) cmp = +1; else - cmp = strcmp(iter->refname, update->refname); + cmp = strcmp(iter->ref.name, update->refname); } if (!cmp) { @@ -1459,11 +1444,11 @@ static enum ref_transaction_error write_with_updates(struct packed_ref_store *re } goto error; - } else if (!oideq(&update->old_oid, iter->oid)) { + } else if (!oideq(&update->old_oid, iter->ref.oid)) { strbuf_addf(err, "cannot update ref '%s': " "is at %s but expected %s", update->refname, - oid_to_hex(iter->oid), + oid_to_hex(iter->ref.oid), oid_to_hex(&update->old_oid)); ret = REF_TRANSACTION_ERROR_INCORRECT_OLD_VALUE; @@ -1523,13 +1508,8 @@ static enum ref_transaction_error write_with_updates(struct packed_ref_store *re if (cmp < 0) { /* Pass the old reference through. */ - - struct object_id peeled; - int peel_error = ref_iterator_peel(iter, &peeled); - - if (write_packed_entry(out, iter->refname, - iter->oid, - peel_error ? NULL : &peeled)) + if (write_packed_entry(out, iter->ref.name, + iter->ref.oid, iter->ref.peeled_oid)) goto write_error; if ((ok = ref_iterator_advance(iter)) != ITER_OK) { @@ -1547,9 +1527,8 @@ static enum ref_transaction_error write_with_updates(struct packed_ref_store *re i++; } else { struct object_id peeled; - int peel_error = peel_object(refs->base.repo, - &update->new_oid, - &peeled); + int peel_error = peel_object(refs->base.repo, &update->new_oid, + &peeled, PEEL_OBJECT_VERIFY_TAGGED_OBJECT_TYPE); if (write_packed_entry(out, update->refname, &update->new_oid, @@ -1794,8 +1773,8 @@ cleanup: return ret; } -static int packed_pack_refs(struct ref_store *ref_store UNUSED, - struct pack_refs_opts *pack_opts UNUSED) +static int packed_optimize(struct ref_store *ref_store UNUSED, + struct refs_optimize_opts *opts UNUSED) { /* * Packed refs are already packed. It might be that loose refs @@ -1805,6 +1784,17 @@ static int packed_pack_refs(struct ref_store *ref_store UNUSED, return 0; } +static int packed_optimize_required(struct ref_store *ref_store UNUSED, + struct refs_optimize_opts *opts UNUSED, + bool *required) +{ + /* + * Packed refs are already optimized. + */ + *required = false; + return 0; +} + static struct ref_iterator *packed_reflog_iterator_begin(struct ref_store *ref_store UNUSED) { return empty_ref_iterator_begin(); @@ -2150,7 +2140,9 @@ struct ref_storage_be refs_be_packed = { .transaction_finish = packed_transaction_finish, .transaction_abort = packed_transaction_abort, - .pack_refs = packed_pack_refs, + .optimize = packed_optimize, + .optimize_required = packed_optimize_required, + .rename_ref = NULL, .copy_ref = NULL, diff --git a/refs/ref-cache.c b/refs/ref-cache.c index e5e5df16d8..ffef01a597 100644 --- a/refs/ref-cache.c +++ b/refs/ref-cache.c @@ -425,10 +425,11 @@ static int cache_ref_iterator_advance(struct ref_iterator *ref_iterator) level->prefix_state = entry_prefix_state; level->index = -1; } else { - iter->base.refname = entry->name; - iter->base.referent = entry->u.value.referent; - iter->base.oid = &entry->u.value.oid; - iter->base.flags = entry->flag; + memset(&iter->base.ref, 0, sizeof(iter->base.ref)); + iter->base.ref.name = entry->name; + iter->base.ref.target = entry->u.value.referent; + iter->base.ref.oid = &entry->u.value.oid; + iter->base.ref.flags = entry->flag; return ITER_OK; } } @@ -545,14 +546,6 @@ static int cache_ref_iterator_seek(struct ref_iterator *ref_iterator, return 0; } -static int cache_ref_iterator_peel(struct ref_iterator *ref_iterator, - struct object_id *peeled) -{ - struct cache_ref_iterator *iter = - (struct cache_ref_iterator *)ref_iterator; - return peel_object(iter->repo, ref_iterator->oid, peeled) ? -1 : 0; -} - static void cache_ref_iterator_release(struct ref_iterator *ref_iterator) { struct cache_ref_iterator *iter = @@ -564,7 +557,6 @@ static void cache_ref_iterator_release(struct ref_iterator *ref_iterator) static struct ref_iterator_vtable cache_ref_iterator_vtable = { .advance = cache_ref_iterator_advance, .seek = cache_ref_iterator_seek, - .peel = cache_ref_iterator_peel, .release = cache_ref_iterator_release, }; diff --git a/refs/refs-internal.h b/refs/refs-internal.h index 4ef3bd75c6..c7d2a6e50b 100644 --- a/refs/refs-internal.h +++ b/refs/refs-internal.h @@ -249,10 +249,7 @@ const char *find_descendant_ref(const char *dirname, */ struct ref_iterator { struct ref_iterator_vtable *vtable; - const char *refname; - const char *referent; - const struct object_id *oid; - unsigned int flags; + struct reference ref; }; /* @@ -361,12 +358,6 @@ typedef int ref_iterator_seek_fn(struct ref_iterator *ref_iterator, const char *refname, unsigned int flags); /* - * Peels the current ref, returning 0 for success or -1 for failure. - */ -typedef int ref_iterator_peel_fn(struct ref_iterator *ref_iterator, - struct object_id *peeled); - -/* * Implementations of this function should free any resources specific * to the derived class. */ @@ -375,23 +366,9 @@ typedef void ref_iterator_release_fn(struct ref_iterator *ref_iterator); struct ref_iterator_vtable { ref_iterator_advance_fn *advance; ref_iterator_seek_fn *seek; - ref_iterator_peel_fn *peel; ref_iterator_release_fn *release; }; -/* - * current_ref_iter is a performance hack: when iterating over - * references using the for_each_ref*() functions, current_ref_iter is - * set to the reference iterator before calling the callback function. - * If the callback function calls peel_ref(), then peel_ref() first - * checks whether the reference to be peeled is the one referred to by - * the iterator (it usually is) and if so, asks the iterator for the - * peeled version of the reference if it is available. This avoids a - * refname lookup in a common case. current_ref_iter is set to NULL - * when the iteration is over. - */ -extern struct ref_iterator *current_ref_iter; - struct ref_store; /* refs backends */ @@ -445,10 +422,13 @@ typedef int ref_transaction_commit_fn(struct ref_store *refs, struct ref_transaction *transaction, struct strbuf *err); -typedef int pack_refs_fn(struct ref_store *ref_store, - struct pack_refs_opts *opts); typedef int optimize_fn(struct ref_store *ref_store, - struct pack_refs_opts *opts); + struct refs_optimize_opts *opts); + +typedef int optimize_required_fn(struct ref_store *ref_store, + struct refs_optimize_opts *opts, + bool *required); + typedef int rename_ref_fn(struct ref_store *ref_store, const char *oldref, const char *newref, const char *logmsg); @@ -573,8 +553,8 @@ struct ref_storage_be { ref_transaction_finish_fn *transaction_finish; ref_transaction_abort_fn *transaction_abort; - pack_refs_fn *pack_refs; optimize_fn *optimize; + optimize_required_fn *optimize_required; rename_ref_fn *rename_ref; copy_ref_fn *copy_ref; diff --git a/refs/reftable-backend.c b/refs/reftable-backend.c index d4b7928620..4319a4eacb 100644 --- a/refs/reftable-backend.c +++ b/refs/reftable-backend.c @@ -547,6 +547,7 @@ struct reftable_ref_iterator { struct reftable_iterator iter; struct reftable_ref_record ref; struct object_id oid; + struct object_id peeled_oid; char *prefix; size_t prefix_len; @@ -671,6 +672,8 @@ static int reftable_ref_iterator_advance(struct ref_iterator *ref_iterator) case REFTABLE_REF_VAL2: oidread(&iter->oid, iter->ref.value.val2.value, refs->base.repo->hash_algo); + oidread(&iter->peeled_oid, iter->ref.value.val2.target_value, + refs->base.repo->hash_algo); break; case REFTABLE_REF_SYMREF: referent = refs_resolve_ref_unsafe(&iter->refs->base, @@ -704,10 +707,13 @@ static int reftable_ref_iterator_advance(struct ref_iterator *ref_iterator) &iter->oid, flags)) continue; - iter->base.refname = iter->ref.refname; - iter->base.referent = referent; - iter->base.oid = &iter->oid; - iter->base.flags = flags; + memset(&iter->base.ref, 0, sizeof(iter->base.ref)); + iter->base.ref.name = iter->ref.refname; + iter->base.ref.target = referent; + iter->base.ref.oid = &iter->oid; + if (iter->ref.value_type == REFTABLE_REF_VAL2) + iter->base.ref.peeled_oid = &iter->peeled_oid; + iter->base.ref.flags = flags; break; } @@ -738,21 +744,6 @@ static int reftable_ref_iterator_seek(struct ref_iterator *ref_iterator, return iter->err; } -static int reftable_ref_iterator_peel(struct ref_iterator *ref_iterator, - struct object_id *peeled) -{ - struct reftable_ref_iterator *iter = - (struct reftable_ref_iterator *)ref_iterator; - - if (iter->ref.value_type == REFTABLE_REF_VAL2) { - oidread(peeled, iter->ref.value.val2.target_value, - iter->refs->base.repo->hash_algo); - return 0; - } - - return -1; -} - static void reftable_ref_iterator_release(struct ref_iterator *ref_iterator) { struct reftable_ref_iterator *iter = @@ -770,7 +761,6 @@ static void reftable_ref_iterator_release(struct ref_iterator *ref_iterator) static struct ref_iterator_vtable reftable_ref_iterator_vtable = { .advance = reftable_ref_iterator_advance, .seek = reftable_ref_iterator_seek, - .peel = reftable_ref_iterator_peel, .release = reftable_ref_iterator_release, }; @@ -828,7 +818,7 @@ static struct reftable_ref_iterator *ref_iterator_for_stack(struct reftable_ref_ iter = xcalloc(1, sizeof(*iter)); base_ref_iterator_init(&iter->base, &reftable_ref_iterator_vtable); - iter->base.oid = &iter->oid; + iter->base.ref.oid = &iter->oid; iter->flags = flags; iter->refs = refs; iter->exclude_patterns = filter_exclude_patterns(exclude_patterns); @@ -1103,7 +1093,7 @@ static enum ref_transaction_error prepare_single_update(struct reftable_ref_stor if (!(u->flags & REF_HAVE_OLD) || !(u->flags & REF_HAVE_NEW) || !(u->flags & REF_LOG_ONLY)) { - strbuf_addf(err, _("trying to write reflog for '%s'" + strbuf_addf(err, _("trying to write reflog for '%s' " "with incomplete values"), u->refname); return REF_TRANSACTION_ERROR_GENERIC; } @@ -1642,7 +1632,8 @@ static int write_transaction_table(struct reftable_writer *writer, void *cb_data ref.refname = (char *)u->refname; ref.update_index = ts; - peel_error = peel_object(arg->refs->base.repo, &u->new_oid, &peeled); + peel_error = peel_object(arg->refs->base.repo, &u->new_oid, &peeled, + PEEL_OBJECT_VERIFY_TAGGED_OBJECT_TYPE); if (!peel_error) { ref.value_type = REFTABLE_REF_VAL2; memcpy(ref.value.val2.target_value, peeled.hash, GIT_MAX_RAWSZ); @@ -1709,11 +1700,11 @@ done: return ret; } -static int reftable_be_pack_refs(struct ref_store *ref_store, - struct pack_refs_opts *opts) +static int reftable_be_optimize(struct ref_store *ref_store, + struct refs_optimize_opts *opts) { struct reftable_ref_store *refs = - reftable_be_downcast(ref_store, REF_STORE_WRITE | REF_STORE_ODB, "pack_refs"); + reftable_be_downcast(ref_store, REF_STORE_WRITE | REF_STORE_ODB, "optimize_refs"); struct reftable_stack *stack; int ret; @@ -1724,7 +1715,7 @@ static int reftable_be_pack_refs(struct ref_store *ref_store, if (!stack) stack = refs->main_backend.stack; - if (opts->flags & PACK_REFS_AUTO) + if (opts->flags & REFS_OPTIMIZE_AUTO) ret = reftable_stack_auto_compact(stack); else ret = reftable_stack_compact_all(stack, NULL); @@ -1742,10 +1733,27 @@ out: return ret; } -static int reftable_be_optimize(struct ref_store *ref_store, - struct pack_refs_opts *opts) +static int reftable_be_optimize_required(struct ref_store *ref_store, + struct refs_optimize_opts *opts, + bool *required) { - return reftable_be_pack_refs(ref_store, opts); + struct reftable_ref_store *refs = reftable_be_downcast(ref_store, REF_STORE_READ, + "optimize_refs_required"); + struct reftable_stack *stack; + bool use_heuristics = false; + + if (refs->err) + return refs->err; + + stack = refs->worktree_backend.stack; + if (!stack) + stack = refs->main_backend.stack; + + if (opts->flags & REFS_OPTIMIZE_AUTO) + use_heuristics = true; + + return reftable_stack_compaction_required(stack, use_heuristics, + required); } struct write_create_symref_arg { @@ -2072,7 +2080,7 @@ static int reftable_reflog_iterator_advance(struct ref_iterator *ref_iterator) strbuf_reset(&iter->last_name); strbuf_addstr(&iter->last_name, iter->log.refname); - iter->base.refname = iter->log.refname; + iter->base.ref.name = iter->log.refname; break; } @@ -2092,13 +2100,6 @@ static int reftable_reflog_iterator_seek(struct ref_iterator *ref_iterator UNUSE return -1; } -static int reftable_reflog_iterator_peel(struct ref_iterator *ref_iterator UNUSED, - struct object_id *peeled UNUSED) -{ - BUG("reftable reflog iterator cannot be peeled"); - return -1; -} - static void reftable_reflog_iterator_release(struct ref_iterator *ref_iterator) { struct reftable_reflog_iterator *iter = @@ -2111,7 +2112,6 @@ static void reftable_reflog_iterator_release(struct ref_iterator *ref_iterator) static struct ref_iterator_vtable reftable_reflog_iterator_vtable = { .advance = reftable_reflog_iterator_advance, .seek = reftable_reflog_iterator_seek, - .peel = reftable_reflog_iterator_peel, .release = reftable_reflog_iterator_release, }; @@ -2515,7 +2515,7 @@ static int write_reflog_expiry_table(struct reftable_writer *writer, void *cb_da ref.refname = (char *)arg->refname; ref.update_index = ts; - if (!peel_object(arg->refs->base.repo, &arg->update_oid, &peeled)) { + if (!peel_object(arg->refs->base.repo, &arg->update_oid, &peeled, 0)) { ref.value_type = REFTABLE_REF_VAL2; memcpy(ref.value.val2.target_value, peeled.hash, GIT_MAX_RAWSZ); memcpy(ref.value.val2.value, arg->update_oid.hash, GIT_MAX_RAWSZ); @@ -2778,8 +2778,9 @@ struct ref_storage_be refs_be_reftable = { .transaction_finish = reftable_be_transaction_finish, .transaction_abort = reftable_be_transaction_abort, - .pack_refs = reftable_be_pack_refs, .optimize = reftable_be_optimize, + .optimize_required = reftable_be_optimize_required, + .rename_ref = reftable_be_rename_ref, .copy_ref = reftable_be_copy_ref, diff --git a/reftable/reftable-stack.h b/reftable/reftable-stack.h index d70fcb705d..c2415cbc6e 100644 --- a/reftable/reftable-stack.h +++ b/reftable/reftable-stack.h @@ -123,6 +123,17 @@ struct reftable_log_expiry_config { int reftable_stack_compact_all(struct reftable_stack *st, struct reftable_log_expiry_config *config); +/* + * Check if compaction is required. + * + * When `use_heuristics` is false, check if all tables can be compacted to a + * single table. If true, use heuristics to determine if the tables need to be + * compacted to maintain geometric progression. + */ +int reftable_stack_compaction_required(struct reftable_stack *st, + bool use_heuristics, + bool *required); + /* heuristically compact unbalanced table stack. */ int reftable_stack_auto_compact(struct reftable_stack *st); diff --git a/reftable/stack.c b/reftable/stack.c index 65d89820bd..826500abed 100644 --- a/reftable/stack.c +++ b/reftable/stack.c @@ -1626,7 +1626,8 @@ struct segment suggest_compaction_segment(uint64_t *sizes, size_t n, return seg; } -static uint64_t *stack_table_sizes_for_compaction(struct reftable_stack *st) +static int stack_segments_for_compaction(struct reftable_stack *st, + struct segment *seg) { int version = (st->opts.hash_id == REFTABLE_HASH_SHA1) ? 1 : 2; int overhead = header_size(version) - 1; @@ -1634,31 +1635,63 @@ static uint64_t *stack_table_sizes_for_compaction(struct reftable_stack *st) REFTABLE_CALLOC_ARRAY(sizes, st->merged->tables_len); if (!sizes) - return NULL; + return REFTABLE_OUT_OF_MEMORY_ERROR; for (size_t i = 0; i < st->merged->tables_len; i++) sizes[i] = st->tables[i]->size - overhead; - return sizes; + *seg = suggest_compaction_segment(sizes, st->merged->tables_len, + st->opts.auto_compaction_factor); + reftable_free(sizes); + + return 0; } -int reftable_stack_auto_compact(struct reftable_stack *st) +static int update_segment_if_compaction_required(struct reftable_stack *st, + struct segment *seg, + bool use_heuristics, + bool *required) { - struct segment seg; - uint64_t *sizes; + int err; - if (st->merged->tables_len < 2) + if (st->merged->tables_len < 2) { + *required = false; return 0; + } - sizes = stack_table_sizes_for_compaction(st); - if (!sizes) - return REFTABLE_OUT_OF_MEMORY_ERROR; + if (!use_heuristics) { + *required = true; + return 0; + } - seg = suggest_compaction_segment(sizes, st->merged->tables_len, - st->opts.auto_compaction_factor); - reftable_free(sizes); + err = stack_segments_for_compaction(st, seg); + if (err) + return err; + + *required = segment_size(seg) > 0; + return 0; +} + +int reftable_stack_compaction_required(struct reftable_stack *st, + bool use_heuristics, + bool *required) +{ + struct segment seg; + return update_segment_if_compaction_required(st, &seg, use_heuristics, + required); +} + +int reftable_stack_auto_compact(struct reftable_stack *st) +{ + struct segment seg; + bool required; + int err; + + err = update_segment_if_compaction_required(st, &seg, true, &required); + if (err) + return err; - if (segment_size(&seg) > 0) + if (required) return stack_compact_range(st, seg.start, seg.end - 1, NULL, STACK_COMPACT_RANGE_BEST_EFFORT); @@ -2315,21 +2315,19 @@ int format_tracking_info(struct branch *branch, struct strbuf *sb, return 1; } -static int one_local_ref(const char *refname, const char *referent UNUSED, const struct object_id *oid, - int flag UNUSED, - void *cb_data) +static int one_local_ref(const struct reference *ref, void *cb_data) { struct ref ***local_tail = cb_data; - struct ref *ref; + struct ref *local_ref; /* we already know it starts with refs/ to get here */ - if (check_refname_format(refname + 5, 0)) + if (check_refname_format(ref->name + 5, 0)) return 0; - ref = alloc_ref(refname); - oidcpy(&ref->new_oid, oid); - **local_tail = ref; - *local_tail = &ref->next; + local_ref = alloc_ref(ref->name); + oidcpy(&local_ref->new_oid, ref->oid); + **local_tail = local_ref; + *local_tail = &local_ref->next; return 0; } @@ -2402,15 +2400,14 @@ struct stale_heads_info { struct refspec *rs; }; -static int get_stale_heads_cb(const char *refname, const char *referent UNUSED, const struct object_id *oid, - int flags, void *cb_data) +static int get_stale_heads_cb(const struct reference *ref, void *cb_data) { struct stale_heads_info *info = cb_data; struct string_list matches = STRING_LIST_INIT_DUP; struct refspec_item query; int i, stale = 1; memset(&query, 0, sizeof(struct refspec_item)); - query.dst = (char *)refname; + query.dst = (char *)ref->name; refspec_find_all_matches(info->rs, &query, &matches); if (matches.nr == 0) @@ -2423,7 +2420,7 @@ static int get_stale_heads_cb(const char *refname, const char *referent UNUSED, * overlapping refspecs, we need to go over all of the * matching refs. */ - if (flags & REF_ISSYMREF) + if (ref->flags & REF_ISSYMREF) goto clean_exit; for (i = 0; stale && i < matches.nr; i++) @@ -2431,8 +2428,8 @@ static int get_stale_heads_cb(const char *refname, const char *referent UNUSED, stale = 0; if (stale) { - struct ref *ref = make_linked_ref(refname, &info->stale_refs_tail); - oidcpy(&ref->new_oid, oid); + struct ref *linked_ref = make_linked_ref(ref->name, &info->stale_refs_tail); + oidcpy(&linked_ref->new_oid, ref->oid); } clean_exit: diff --git a/repack-cruft.c b/repack-cruft.c new file mode 100644 index 0000000000..0653e88792 --- /dev/null +++ b/repack-cruft.c @@ -0,0 +1,98 @@ +#include "git-compat-util.h" +#include "repack.h" +#include "packfile.h" +#include "repository.h" +#include "run-command.h" + +static void combine_small_cruft_packs(FILE *in, off_t combine_cruft_below_size, + struct existing_packs *existing) +{ + struct packed_git *p; + struct strbuf buf = STRBUF_INIT; + size_t i; + + repo_for_each_pack(existing->repo, p) { + if (!(p->is_cruft && p->pack_local)) + continue; + + strbuf_reset(&buf); + strbuf_addstr(&buf, pack_basename(p)); + strbuf_strip_suffix(&buf, ".pack"); + + if (!string_list_has_string(&existing->cruft_packs, buf.buf)) + continue; + + if (p->pack_size < combine_cruft_below_size) { + fprintf(in, "-%s\n", pack_basename(p)); + } else { + existing_packs_retain_cruft(existing, p); + fprintf(in, "%s\n", pack_basename(p)); + } + } + + for (i = 0; i < existing->non_kept_packs.nr; i++) + fprintf(in, "-%s.pack\n", + existing->non_kept_packs.items[i].string); + + strbuf_release(&buf); +} + +int write_cruft_pack(const struct write_pack_opts *opts, + const char *cruft_expiration, + unsigned long combine_cruft_below_size, + struct string_list *names, + struct existing_packs *existing) +{ + struct child_process cmd = CHILD_PROCESS_INIT; + struct string_list_item *item; + FILE *in; + int ret; + const char *pack_prefix = write_pack_opts_pack_prefix(opts); + + prepare_pack_objects(&cmd, opts->po_args, opts->destination); + + strvec_push(&cmd.args, "--cruft"); + if (cruft_expiration) + strvec_pushf(&cmd.args, "--cruft-expiration=%s", + cruft_expiration); + + strvec_push(&cmd.args, "--non-empty"); + + cmd.in = -1; + + ret = start_command(&cmd); + if (ret) + return ret; + + /* + * names has a confusing double use: it both provides the list + * of just-written new packs, and accepts the name of the cruft + * pack we are writing. + * + * By the time it is read here, it contains only the pack(s) + * that were just written, which is exactly the set of packs we + * want to consider kept. + * + * If `--expire-to` is given, the double-use served by `names` + * ensures that the pack written to `--expire-to` excludes any + * objects contained in the cruft pack. + */ + in = xfdopen(cmd.in, "w"); + for_each_string_list_item(item, names) + fprintf(in, "%s-%s.pack\n", pack_prefix, item->string); + if (combine_cruft_below_size && !cruft_expiration) { + combine_small_cruft_packs(in, combine_cruft_below_size, + existing); + } else { + for_each_string_list_item(item, &existing->non_kept_packs) + fprintf(in, "-%s.pack\n", item->string); + for_each_string_list_item(item, &existing->cruft_packs) + fprintf(in, "-%s.pack\n", item->string); + } + for_each_string_list_item(item, &existing->kept_packs) + fprintf(in, "%s.pack\n", item->string); + fclose(in); + + return finish_pack_objects_cmd(existing->repo->hash_algo, opts, &cmd, + names); +} diff --git a/repack-filtered.c b/repack-filtered.c new file mode 100644 index 0000000000..edcf7667c5 --- /dev/null +++ b/repack-filtered.c @@ -0,0 +1,51 @@ +#include "git-compat-util.h" +#include "repack.h" +#include "repository.h" +#include "run-command.h" +#include "string-list.h" + +int write_filtered_pack(const struct write_pack_opts *opts, + struct existing_packs *existing, + struct string_list *names) +{ + struct child_process cmd = CHILD_PROCESS_INIT; + struct string_list_item *item; + FILE *in; + int ret; + const char *caret; + const char *pack_prefix = write_pack_opts_pack_prefix(opts); + + prepare_pack_objects(&cmd, opts->po_args, opts->destination); + + strvec_push(&cmd.args, "--stdin-packs"); + + for_each_string_list_item(item, &existing->kept_packs) + strvec_pushf(&cmd.args, "--keep-pack=%s", item->string); + + cmd.in = -1; + + ret = start_command(&cmd); + if (ret) + return ret; + + /* + * Here 'names' contains only the pack(s) that were just + * written, which is exactly the packs we want to keep. Also + * 'existing_kept_packs' already contains the packs in + * 'keep_pack_list'. + */ + in = xfdopen(cmd.in, "w"); + for_each_string_list_item(item, names) + fprintf(in, "^%s-%s.pack\n", pack_prefix, item->string); + for_each_string_list_item(item, &existing->non_kept_packs) + fprintf(in, "%s.pack\n", item->string); + for_each_string_list_item(item, &existing->cruft_packs) + fprintf(in, "%s.pack\n", item->string); + caret = opts->po_args->pack_kept_objects ? "" : "^"; + for_each_string_list_item(item, &existing->kept_packs) + fprintf(in, "%s%s.pack\n", caret, item->string); + fclose(in); + + return finish_pack_objects_cmd(existing->repo->hash_algo, opts, &cmd, + names); +} diff --git a/repack-geometry.c b/repack-geometry.c new file mode 100644 index 0000000000..b3e32cd07e --- /dev/null +++ b/repack-geometry.c @@ -0,0 +1,232 @@ +#define DISABLE_SIGN_COMPARE_WARNINGS + +#include "git-compat-util.h" +#include "repack.h" +#include "repository.h" +#include "hex.h" +#include "packfile.h" + +static uint32_t pack_geometry_weight(struct packed_git *p) +{ + if (open_pack_index(p)) + die(_("cannot open index for %s"), p->pack_name); + return p->num_objects; +} + +static int pack_geometry_cmp(const void *va, const void *vb) +{ + uint32_t aw = pack_geometry_weight(*(struct packed_git **)va), + bw = pack_geometry_weight(*(struct packed_git **)vb); + + if (aw < bw) + return -1; + if (aw > bw) + return 1; + return 0; +} + +void pack_geometry_init(struct pack_geometry *geometry, + struct existing_packs *existing, + const struct pack_objects_args *args) +{ + struct packed_git *p; + struct strbuf buf = STRBUF_INIT; + + repo_for_each_pack(existing->repo, p) { + if (args->local && !p->pack_local) + /* + * When asked to only repack local packfiles we skip + * over any packfiles that are borrowed from alternate + * object directories. + */ + continue; + + if (!args->pack_kept_objects) { + /* + * Any pack that has its pack_keep bit set will + * appear in existing->kept_packs below, but + * this saves us from doing a more expensive + * check. + */ + if (p->pack_keep) + continue; + + /* + * The pack may be kept via the --keep-pack + * option; check 'existing->kept_packs' to + * determine whether to ignore it. + */ + strbuf_reset(&buf); + strbuf_addstr(&buf, pack_basename(p)); + strbuf_strip_suffix(&buf, ".pack"); + + if (string_list_has_string(&existing->kept_packs, buf.buf)) + continue; + } + if (p->is_cruft) + continue; + + ALLOC_GROW(geometry->pack, + geometry->pack_nr + 1, + geometry->pack_alloc); + + geometry->pack[geometry->pack_nr] = p; + geometry->pack_nr++; + } + + QSORT(geometry->pack, geometry->pack_nr, pack_geometry_cmp); + strbuf_release(&buf); +} + +void pack_geometry_split(struct pack_geometry *geometry) +{ + uint32_t i; + uint32_t split; + off_t total_size = 0; + + if (!geometry->pack_nr) { + geometry->split = geometry->pack_nr; + return; + } + + /* + * First, count the number of packs (in descending order of size) which + * already form a geometric progression. + */ + for (i = geometry->pack_nr - 1; i > 0; i--) { + struct packed_git *ours = geometry->pack[i]; + struct packed_git *prev = geometry->pack[i - 1]; + + if (unsigned_mult_overflows(geometry->split_factor, + pack_geometry_weight(prev))) + die(_("pack %s too large to consider in geometric " + "progression"), + prev->pack_name); + + if (pack_geometry_weight(ours) < + geometry->split_factor * pack_geometry_weight(prev)) + break; + } + + split = i; + + if (split) { + /* + * Move the split one to the right, since the top element in the + * last-compared pair can't be in the progression. Only do this + * when we split in the middle of the array (otherwise if we got + * to the end, then the split is in the right place). + */ + split++; + } + + /* + * Then, anything to the left of 'split' must be in a new pack. But, + * creating that new pack may cause packs in the heavy half to no longer + * form a geometric progression. + * + * Compute an expected size of the new pack, and then determine how many + * packs in the heavy half need to be joined into it (if any) to restore + * the geometric progression. + */ + for (i = 0; i < split; i++) { + struct packed_git *p = geometry->pack[i]; + + if (unsigned_add_overflows(total_size, pack_geometry_weight(p))) + die(_("pack %s too large to roll up"), p->pack_name); + total_size += pack_geometry_weight(p); + } + for (i = split; i < geometry->pack_nr; i++) { + struct packed_git *ours = geometry->pack[i]; + + if (unsigned_mult_overflows(geometry->split_factor, + total_size)) + die(_("pack %s too large to roll up"), ours->pack_name); + + if (pack_geometry_weight(ours) < + geometry->split_factor * total_size) { + if (unsigned_add_overflows(total_size, + pack_geometry_weight(ours))) + die(_("pack %s too large to roll up"), + ours->pack_name); + + split++; + total_size += pack_geometry_weight(ours); + } else + break; + } + + geometry->split = split; +} + +struct packed_git *pack_geometry_preferred_pack(struct pack_geometry *geometry) +{ + uint32_t i; + + if (!geometry) { + /* + * No geometry means either an all-into-one repack (in which + * case there is only one pack left and it is the largest) or an + * incremental one. + * + * If repacking incrementally, then we could check the size of + * all packs to determine which should be preferred, but leave + * this for later. + */ + return NULL; + } + if (geometry->split == geometry->pack_nr) + return NULL; + + /* + * The preferred pack is the largest pack above the split line. In + * other words, it is the largest pack that does not get rolled up in + * the geometric repack. + */ + for (i = geometry->pack_nr; i > geometry->split; i--) + /* + * A pack that is not local would never be included in a + * multi-pack index. We thus skip over any non-local packs. + */ + if (geometry->pack[i - 1]->pack_local) + return geometry->pack[i - 1]; + + return NULL; +} + +void pack_geometry_remove_redundant(struct pack_geometry *geometry, + struct string_list *names, + struct existing_packs *existing, + const char *packdir) +{ + const struct git_hash_algo *algop = existing->repo->hash_algo; + struct strbuf buf = STRBUF_INIT; + uint32_t i; + + for (i = 0; i < geometry->split; i++) { + struct packed_git *p = geometry->pack[i]; + if (string_list_has_string(names, hash_to_hex_algop(p->hash, + algop))) + continue; + + strbuf_reset(&buf); + strbuf_addstr(&buf, pack_basename(p)); + strbuf_strip_suffix(&buf, ".pack"); + + if ((p->pack_keep) || + (string_list_has_string(&existing->kept_packs, buf.buf))) + continue; + + repack_remove_redundant_pack(existing->repo, packdir, buf.buf); + } + + strbuf_release(&buf); +} + +void pack_geometry_release(struct pack_geometry *geometry) +{ + if (!geometry) + return; + + free(geometry->pack); +} diff --git a/repack-midx.c b/repack-midx.c new file mode 100644 index 0000000000..74bdfa3a6e --- /dev/null +++ b/repack-midx.c @@ -0,0 +1,370 @@ +#include "git-compat-util.h" +#include "repack.h" +#include "hash.h" +#include "hex.h" +#include "odb.h" +#include "oidset.h" +#include "pack-bitmap.h" +#include "refs.h" +#include "run-command.h" +#include "tempfile.h" + +struct midx_snapshot_ref_data { + struct repository *repo; + struct tempfile *f; + struct oidset seen; + int preferred; +}; + +static int midx_snapshot_ref_one(const struct reference *ref, void *_data) +{ + struct midx_snapshot_ref_data *data = _data; + const struct object_id *maybe_peeled = ref->oid; + struct object_id peeled; + + if (!reference_get_peeled_oid(data->repo, ref, &peeled)) + maybe_peeled = &peeled; + + if (oidset_insert(&data->seen, maybe_peeled)) + return 0; /* already seen */ + + if (odb_read_object_info(data->repo->objects, maybe_peeled, NULL) != OBJ_COMMIT) + return 0; + + fprintf(data->f->fp, "%s%s\n", data->preferred ? "+" : "", + oid_to_hex(maybe_peeled)); + + return 0; +} + +void midx_snapshot_refs(struct repository *repo, struct tempfile *f) +{ + struct midx_snapshot_ref_data data; + const struct string_list *preferred = bitmap_preferred_tips(repo); + + data.repo = repo; + data.f = f; + data.preferred = 0; + oidset_init(&data.seen, 0); + + if (!fdopen_tempfile(f, "w")) + die(_("could not open tempfile %s for writing"), + get_tempfile_path(f)); + + if (preferred) { + struct string_list_item *item; + + data.preferred = 1; + for_each_string_list_item(item, preferred) + refs_for_each_ref_in(get_main_ref_store(repo), + item->string, + midx_snapshot_ref_one, &data); + data.preferred = 0; + } + + refs_for_each_ref(get_main_ref_store(repo), + midx_snapshot_ref_one, &data); + + if (close_tempfile_gently(f)) { + int save_errno = errno; + delete_tempfile(&f); + errno = save_errno; + die_errno(_("could not close refs snapshot tempfile")); + } + + oidset_clear(&data.seen); +} + +static int midx_has_unknown_packs(struct string_list *include, + struct pack_geometry *geometry, + struct existing_packs *existing) +{ + struct string_list_item *item; + + string_list_sort(include); + + for_each_string_list_item(item, &existing->midx_packs) { + const char *pack_name = item->string; + + /* + * Determine whether or not each MIDX'd pack from the existing + * MIDX (if any) is represented in the new MIDX. For each pack + * in the MIDX, it must either be: + * + * - In the "include" list of packs to be included in the new + * MIDX. Note this function is called before the include + * list is populated with any cruft pack(s). + * + * - Below the geometric split line (if using pack geometry), + * indicating that the pack won't be included in the new + * MIDX, but its contents were rolled up as part of the + * geometric repack. + * + * - In the existing non-kept packs list (if not using pack + * geometry), and marked as non-deleted. + */ + if (string_list_has_string(include, pack_name)) { + continue; + } else if (geometry) { + struct strbuf buf = STRBUF_INIT; + uint32_t j; + + for (j = 0; j < geometry->split; j++) { + strbuf_reset(&buf); + strbuf_addstr(&buf, pack_basename(geometry->pack[j])); + strbuf_strip_suffix(&buf, ".pack"); + strbuf_addstr(&buf, ".idx"); + + if (!strcmp(pack_name, buf.buf)) { + strbuf_release(&buf); + break; + } + } + + strbuf_release(&buf); + + if (j < geometry->split) + continue; + } else { + struct string_list_item *item; + + item = string_list_lookup(&existing->non_kept_packs, + pack_name); + if (item && !existing_pack_is_marked_for_deletion(item)) + continue; + } + + /* + * If we got to this point, the MIDX includes some pack that we + * don't know about. + */ + return 1; + } + + return 0; +} + +static void midx_included_packs(struct string_list *include, + struct repack_write_midx_opts *opts) +{ + struct existing_packs *existing = opts->existing; + struct pack_geometry *geometry = opts->geometry; + struct string_list *names = opts->names; + struct string_list_item *item; + struct strbuf buf = STRBUF_INIT; + + for_each_string_list_item(item, &existing->kept_packs) { + strbuf_reset(&buf); + strbuf_addf(&buf, "%s.idx", item->string); + string_list_insert(include, buf.buf); + } + + for_each_string_list_item(item, names) { + strbuf_reset(&buf); + strbuf_addf(&buf, "pack-%s.idx", item->string); + string_list_insert(include, buf.buf); + } + + if (geometry->split_factor) { + uint32_t i; + + for (i = geometry->split; i < geometry->pack_nr; i++) { + struct packed_git *p = geometry->pack[i]; + + /* + * The multi-pack index never refers to packfiles part + * of an alternate object database, so we skip these. + * While git-multi-pack-index(1) would silently ignore + * them anyway, this allows us to skip executing the + * command completely when we have only non-local + * packfiles. + */ + if (!p->pack_local) + continue; + + strbuf_reset(&buf); + strbuf_addstr(&buf, pack_basename(p)); + strbuf_strip_suffix(&buf, ".pack"); + strbuf_addstr(&buf, ".idx"); + + string_list_insert(include, buf.buf); + } + } else { + for_each_string_list_item(item, &existing->non_kept_packs) { + if (existing_pack_is_marked_for_deletion(item)) + continue; + + strbuf_reset(&buf); + strbuf_addf(&buf, "%s.idx", item->string); + string_list_insert(include, buf.buf); + } + } + + if (opts->midx_must_contain_cruft || + midx_has_unknown_packs(include, geometry, existing)) { + /* + * If there are one or more unknown pack(s) present (see + * midx_has_unknown_packs() for what makes a pack + * "unknown") in the MIDX before the repack, keep them + * as they may be required to form a reachability + * closure if the MIDX is bitmapped. + * + * For example, a cruft pack can be required to form a + * reachability closure if the MIDX is bitmapped and one + * or more of the bitmap's selected commits reaches a + * once-cruft object that was later made reachable. + */ + for_each_string_list_item(item, &existing->cruft_packs) { + /* + * When doing a --geometric repack, there is no + * need to check for deleted packs, since we're + * by definition not doing an ALL_INTO_ONE + * repack (hence no packs will be deleted). + * Otherwise we must check for and exclude any + * packs which are enqueued for deletion. + * + * So we could omit the conditional below in the + * --geometric case, but doing so is unnecessary + * since no packs are marked as pending + * deletion (since we only call + * `existing_packs_mark_for_deletion()` when + * doing an all-into-one repack). + */ + if (existing_pack_is_marked_for_deletion(item)) + continue; + + strbuf_reset(&buf); + strbuf_addf(&buf, "%s.idx", item->string); + string_list_insert(include, buf.buf); + } + } else { + /* + * Modern versions of Git (with the appropriate + * configuration setting) will write new copies of + * once-cruft objects when doing a --geometric repack. + * + * If the MIDX has no cruft pack, new packs written + * during a --geometric repack will not rely on the + * cruft pack to form a reachability closure, so we can + * avoid including them in the MIDX in that case. + */ + ; + } + + strbuf_release(&buf); +} + +static void remove_redundant_bitmaps(struct string_list *include, + const char *packdir) +{ + struct strbuf path = STRBUF_INIT; + struct string_list_item *item; + size_t packdir_len; + + strbuf_addstr(&path, packdir); + strbuf_addch(&path, '/'); + packdir_len = path.len; + + /* + * Remove any pack bitmaps corresponding to packs which are now + * included in the MIDX. + */ + for_each_string_list_item(item, include) { + strbuf_addstr(&path, item->string); + strbuf_strip_suffix(&path, ".idx"); + strbuf_addstr(&path, ".bitmap"); + + if (unlink(path.buf) && errno != ENOENT) + warning_errno(_("could not remove stale bitmap: %s"), + path.buf); + + strbuf_setlen(&path, packdir_len); + } + strbuf_release(&path); +} + +int write_midx_included_packs(struct repack_write_midx_opts *opts) +{ + struct child_process cmd = CHILD_PROCESS_INIT; + struct string_list include = STRING_LIST_INIT_DUP; + struct string_list_item *item; + struct packed_git *preferred = pack_geometry_preferred_pack(opts->geometry); + FILE *in; + int ret = 0; + + midx_included_packs(&include, opts); + if (!include.nr) + goto done; + + cmd.in = -1; + cmd.git_cmd = 1; + + strvec_push(&cmd.args, "multi-pack-index"); + strvec_pushl(&cmd.args, "write", "--stdin-packs", NULL); + + if (opts->show_progress) + strvec_push(&cmd.args, "--progress"); + else + strvec_push(&cmd.args, "--no-progress"); + + if (opts->write_bitmaps) + strvec_push(&cmd.args, "--bitmap"); + + if (preferred) + strvec_pushf(&cmd.args, "--preferred-pack=%s", + pack_basename(preferred)); + else if (opts->names->nr) { + /* The largest pack was repacked, meaning that either + * one or two packs exist depending on whether the + * repository has a cruft pack or not. + * + * Select the non-cruft one as preferred to encourage + * pack-reuse among packs containing reachable objects + * over unreachable ones. + * + * (Note we could write multiple packs here if + * `--max-pack-size` was given, but any one of them + * will suffice, so pick the first one.) + */ + for_each_string_list_item(item, opts->names) { + struct generated_pack *pack = item->util; + if (generated_pack_has_ext(pack, ".mtimes")) + continue; + + strvec_pushf(&cmd.args, "--preferred-pack=pack-%s.pack", + item->string); + break; + } + } else { + /* + * No packs were kept, and no packs were written. The + * only thing remaining are .keep packs (unless + * --pack-kept-objects was given). + * + * Set the `--preferred-pack` arbitrarily here. + */ + ; + } + + if (opts->refs_snapshot) + strvec_pushf(&cmd.args, "--refs-snapshot=%s", + opts->refs_snapshot); + + ret = start_command(&cmd); + if (ret) + goto done; + + in = xfdopen(cmd.in, "w"); + for_each_string_list_item(item, &include) + fprintf(in, "%s\n", item->string); + fclose(in); + + ret = finish_command(&cmd); +done: + if (!ret && opts->write_bitmaps) + remove_redundant_bitmaps(&include, opts->packdir); + + string_list_clear(&include, 0); + + return ret; +} diff --git a/repack-promisor.c b/repack-promisor.c new file mode 100644 index 0000000000..ee6e0669f6 --- /dev/null +++ b/repack-promisor.c @@ -0,0 +1,102 @@ +#include "git-compat-util.h" +#include "repack.h" +#include "hex.h" +#include "pack.h" +#include "packfile.h" +#include "path.h" +#include "repository.h" +#include "run-command.h" + +struct write_oid_context { + struct child_process *cmd; + const struct git_hash_algo *algop; +}; + +/* + * Write oid to the given struct child_process's stdin, starting it first if + * necessary. + */ +static int write_oid(const struct object_id *oid, + struct packed_git *pack UNUSED, + uint32_t pos UNUSED, void *data) +{ + struct write_oid_context *ctx = data; + struct child_process *cmd = ctx->cmd; + + if (cmd->in == -1) { + if (start_command(cmd)) + die(_("could not start pack-objects to repack promisor objects")); + } + + if (write_in_full(cmd->in, oid_to_hex(oid), ctx->algop->hexsz) < 0 || + write_in_full(cmd->in, "\n", 1) < 0) + die(_("failed to feed promisor objects to pack-objects")); + return 0; +} + +void repack_promisor_objects(struct repository *repo, + const struct pack_objects_args *args, + struct string_list *names, const char *packtmp) +{ + struct write_oid_context ctx; + struct child_process cmd = CHILD_PROCESS_INIT; + FILE *out; + struct strbuf line = STRBUF_INIT; + + prepare_pack_objects(&cmd, args, packtmp); + cmd.in = -1; + + /* + * NEEDSWORK: Giving pack-objects only the OIDs without any ordering + * hints may result in suboptimal deltas in the resulting pack. See if + * the OIDs can be sent with fake paths such that pack-objects can use a + * {type -> existing pack order} ordering when computing deltas instead + * of a {type -> size} ordering, which may produce better deltas. + */ + ctx.cmd = &cmd; + ctx.algop = repo->hash_algo; + for_each_packed_object(repo, write_oid, &ctx, + FOR_EACH_OBJECT_PROMISOR_ONLY); + + if (cmd.in == -1) { + /* No packed objects; cmd was never started */ + child_process_clear(&cmd); + return; + } + + close(cmd.in); + + out = xfdopen(cmd.out, "r"); + while (strbuf_getline_lf(&line, out) != EOF) { + struct string_list_item *item; + char *promisor_name; + + if (line.len != repo->hash_algo->hexsz) + die(_("repack: Expecting full hex object ID lines only from pack-objects.")); + item = string_list_append(names, line.buf); + + /* + * pack-objects creates the .pack and .idx files, but not the + * .promisor file. Create the .promisor file, which is empty. + * + * NEEDSWORK: fetch-pack sometimes generates non-empty + * .promisor files containing the ref names and associated + * hashes at the point of generation of the corresponding + * packfile, but this would not preserve their contents. Maybe + * concatenate the contents of all .promisor files instead of + * just creating a new empty file. + */ + promisor_name = mkpathdup("%s-%s.promisor", packtmp, + line.buf); + write_promisor_file(promisor_name, NULL, 0); + + item->util = generated_pack_populate(item->string, packtmp); + + free(promisor_name); + } + + fclose(out); + if (finish_command(&cmd)) + die(_("could not finish pack-objects to repack promisor objects")); + strbuf_release(&line); +} diff --git a/repack.c b/repack.c new file mode 100644 index 0000000000..596841027a --- /dev/null +++ b/repack.c @@ -0,0 +1,359 @@ +#include "git-compat-util.h" +#include "dir.h" +#include "midx.h" +#include "odb.h" +#include "packfile.h" +#include "path.h" +#include "repack.h" +#include "repository.h" +#include "run-command.h" +#include "tempfile.h" + +void prepare_pack_objects(struct child_process *cmd, + const struct pack_objects_args *args, + const char *out) +{ + strvec_push(&cmd->args, "pack-objects"); + if (args->window) + strvec_pushf(&cmd->args, "--window=%s", args->window); + if (args->window_memory) + strvec_pushf(&cmd->args, "--window-memory=%s", args->window_memory); + if (args->depth) + strvec_pushf(&cmd->args, "--depth=%s", args->depth); + if (args->threads) + strvec_pushf(&cmd->args, "--threads=%s", args->threads); + if (args->max_pack_size) + strvec_pushf(&cmd->args, "--max-pack-size=%lu", args->max_pack_size); + if (args->no_reuse_delta) + strvec_pushf(&cmd->args, "--no-reuse-delta"); + if (args->no_reuse_object) + strvec_pushf(&cmd->args, "--no-reuse-object"); + if (args->name_hash_version) + strvec_pushf(&cmd->args, "--name-hash-version=%d", args->name_hash_version); + if (args->path_walk) + strvec_pushf(&cmd->args, "--path-walk"); + if (args->local) + strvec_push(&cmd->args, "--local"); + if (args->quiet) + strvec_push(&cmd->args, "--quiet"); + if (args->delta_base_offset) + strvec_push(&cmd->args, "--delta-base-offset"); + if (!args->pack_kept_objects) + strvec_push(&cmd->args, "--honor-pack-keep"); + strvec_push(&cmd->args, out); + cmd->git_cmd = 1; + cmd->out = -1; +} + +void pack_objects_args_release(struct pack_objects_args *args) +{ + free(args->window); + free(args->window_memory); + free(args->depth); + free(args->threads); + list_objects_filter_release(&args->filter_options); +} + +void repack_remove_redundant_pack(struct repository *repo, const char *dir_name, + const char *base_name) +{ + struct strbuf buf = STRBUF_INIT; + struct odb_source *source = repo->objects->sources; + struct multi_pack_index *m = get_multi_pack_index(source); + strbuf_addf(&buf, "%s.pack", base_name); + if (m && source->local && midx_contains_pack(m, buf.buf)) + clear_midx_file(repo); + strbuf_insertf(&buf, 0, "%s/", dir_name); + unlink_pack_path(buf.buf, 1); + strbuf_release(&buf); +} + +const char *write_pack_opts_pack_prefix(const struct write_pack_opts *opts) +{ + const char *pack_prefix; + if (!skip_prefix(opts->packtmp, opts->packdir, &pack_prefix)) + die(_("pack prefix %s does not begin with objdir %s"), + opts->packtmp, opts->packdir); + if (*pack_prefix == '/') + pack_prefix++; + return pack_prefix; +} + +bool write_pack_opts_is_local(const struct write_pack_opts *opts) +{ + return starts_with(opts->destination, opts->packdir); +} + +int finish_pack_objects_cmd(const struct git_hash_algo *algop, + const struct write_pack_opts *opts, + struct child_process *cmd, + struct string_list *names) +{ + FILE *out; + bool local = write_pack_opts_is_local(opts); + struct strbuf line = STRBUF_INIT; + + out = xfdopen(cmd->out, "r"); + while (strbuf_getline_lf(&line, out) != EOF) { + struct string_list_item *item; + + if (line.len != algop->hexsz) + die(_("repack: Expecting full hex object ID lines only " + "from pack-objects.")); + /* + * Avoid putting packs written outside of the repository in the + * list of names. + */ + if (local) { + item = string_list_append(names, line.buf); + item->util = generated_pack_populate(line.buf, + opts->packtmp); + } + } + fclose(out); + + strbuf_release(&line); + + return finish_command(cmd); +} + +#define DELETE_PACK 1 +#define RETAIN_PACK 2 + +void existing_packs_collect(struct existing_packs *existing, + const struct string_list *extra_keep) +{ + struct packed_git *p; + struct strbuf buf = STRBUF_INIT; + + repo_for_each_pack(existing->repo, p) { + size_t i; + const char *base; + + if (p->multi_pack_index) + string_list_append(&existing->midx_packs, + pack_basename(p)); + if (!p->pack_local) + continue; + + base = pack_basename(p); + + for (i = 0; i < extra_keep->nr; i++) + if (!fspathcmp(base, extra_keep->items[i].string)) + break; + + strbuf_reset(&buf); + strbuf_addstr(&buf, base); + strbuf_strip_suffix(&buf, ".pack"); + + if ((extra_keep->nr > 0 && i < extra_keep->nr) || p->pack_keep) + string_list_append(&existing->kept_packs, buf.buf); + else if (p->is_cruft) + string_list_append(&existing->cruft_packs, buf.buf); + else + string_list_append(&existing->non_kept_packs, buf.buf); + } + + string_list_sort(&existing->kept_packs); + string_list_sort(&existing->non_kept_packs); + string_list_sort(&existing->cruft_packs); + string_list_sort(&existing->midx_packs); + strbuf_release(&buf); +} + +int existing_packs_has_non_kept(const struct existing_packs *existing) +{ + return existing->non_kept_packs.nr || existing->cruft_packs.nr; +} + +static void existing_pack_mark_for_deletion(struct string_list_item *item) +{ + item->util = (void*)((uintptr_t)item->util | DELETE_PACK); +} + +static void existing_pack_unmark_for_deletion(struct string_list_item *item) +{ + item->util = (void*)((uintptr_t)item->util & ~DELETE_PACK); +} + +int existing_pack_is_marked_for_deletion(struct string_list_item *item) +{ + return (uintptr_t)item->util & DELETE_PACK; +} + +static void existing_packs_mark_retained(struct string_list_item *item) +{ + item->util = (void*)((uintptr_t)item->util | RETAIN_PACK); +} + +static int existing_pack_is_retained(struct string_list_item *item) +{ + return (uintptr_t)item->util & RETAIN_PACK; +} + +static void existing_packs_mark_for_deletion_1(const struct git_hash_algo *algop, + struct string_list *names, + struct string_list *list) +{ + struct string_list_item *item; + const size_t hexsz = algop->hexsz; + + for_each_string_list_item(item, list) { + char *sha1; + size_t len = strlen(item->string); + if (len < hexsz) + continue; + sha1 = item->string + len - hexsz; + + if (existing_pack_is_retained(item)) { + existing_pack_unmark_for_deletion(item); + } else if (!string_list_has_string(names, sha1)) { + /* + * Mark this pack for deletion, which ensures + * that this pack won't be included in a MIDX + * (if `--write-midx` was given) and that we + * will actually delete this pack (if `-d` was + * given). + */ + existing_pack_mark_for_deletion(item); + } + } +} + +void existing_packs_retain_cruft(struct existing_packs *existing, + struct packed_git *cruft) +{ + struct strbuf buf = STRBUF_INIT; + struct string_list_item *item; + + strbuf_addstr(&buf, pack_basename(cruft)); + strbuf_strip_suffix(&buf, ".pack"); + + item = string_list_lookup(&existing->cruft_packs, buf.buf); + if (!item) + BUG("could not find cruft pack '%s'", pack_basename(cruft)); + + existing_packs_mark_retained(item); + strbuf_release(&buf); +} + +void existing_packs_mark_for_deletion(struct existing_packs *existing, + struct string_list *names) + +{ + const struct git_hash_algo *algop = existing->repo->hash_algo; + existing_packs_mark_for_deletion_1(algop, names, + &existing->non_kept_packs); + existing_packs_mark_for_deletion_1(algop, names, + &existing->cruft_packs); +} + +static void remove_redundant_packs_1(struct repository *repo, + struct string_list *packs, + const char *packdir) +{ + struct string_list_item *item; + for_each_string_list_item(item, packs) { + if (!existing_pack_is_marked_for_deletion(item)) + continue; + repack_remove_redundant_pack(repo, packdir, item->string); + } +} + +void existing_packs_remove_redundant(struct existing_packs *existing, + const char *packdir) +{ + remove_redundant_packs_1(existing->repo, &existing->non_kept_packs, + packdir); + remove_redundant_packs_1(existing->repo, &existing->cruft_packs, + packdir); +} + +void existing_packs_release(struct existing_packs *existing) +{ + string_list_clear(&existing->kept_packs, 0); + string_list_clear(&existing->non_kept_packs, 0); + string_list_clear(&existing->cruft_packs, 0); + string_list_clear(&existing->midx_packs, 0); +} + +static struct { + const char *name; + unsigned optional:1; +} exts[] = { + {".pack"}, + {".rev", 1}, + {".mtimes", 1}, + {".bitmap", 1}, + {".promisor", 1}, + {".idx"}, +}; + +struct generated_pack { + struct tempfile *tempfiles[ARRAY_SIZE(exts)]; +}; + +struct generated_pack *generated_pack_populate(const char *name, + const char *packtmp) +{ + struct stat statbuf; + struct strbuf path = STRBUF_INIT; + struct generated_pack *pack = xcalloc(1, sizeof(*pack)); + size_t i; + + for (i = 0; i < ARRAY_SIZE(exts); i++) { + strbuf_reset(&path); + strbuf_addf(&path, "%s-%s%s", packtmp, name, exts[i].name); + + if (stat(path.buf, &statbuf)) + continue; + + pack->tempfiles[i] = register_tempfile(path.buf); + } + + strbuf_release(&path); + return pack; +} + +int generated_pack_has_ext(const struct generated_pack *pack, const char *ext) +{ + size_t i; + for (i = 0; i < ARRAY_SIZE(exts); i++) { + if (strcmp(exts[i].name, ext)) + continue; + return !!pack->tempfiles[i]; + } + BUG("unknown pack extension: '%s'", ext); +} + +void generated_pack_install(struct generated_pack *pack, const char *name, + const char *packdir, const char *packtmp) +{ + size_t ext; + for (ext = 0; ext < ARRAY_SIZE(exts); ext++) { + char *fname; + + fname = mkpathdup("%s/pack-%s%s", packdir, name, + exts[ext].name); + + if (pack->tempfiles[ext]) { + const char *fname_old = get_tempfile_path(pack->tempfiles[ext]); + struct stat statbuffer; + + if (!stat(fname_old, &statbuffer)) { + statbuffer.st_mode &= ~(S_IWUSR | S_IWGRP | S_IWOTH); + chmod(fname_old, statbuffer.st_mode); + } + + if (rename_tempfile(&pack->tempfiles[ext], fname)) + die_errno(_("renaming pack to '%s' failed"), + fname); + } else if (!exts[ext].optional) + die(_("pack-objects did not write a '%s' file for pack %s-%s"), + exts[ext].name, packtmp, name); + else if (unlink(fname) < 0 && errno != ENOENT) + die_errno(_("could not unlink: %s"), fname); + + free(fname); + } +} diff --git a/repack.h b/repack.h new file mode 100644 index 0000000000..3a688a12ee --- /dev/null +++ b/repack.h @@ -0,0 +1,146 @@ +#ifndef REPACK_H +#define REPACK_H + +#include "list-objects-filter-options.h" +#include "string-list.h" + +struct pack_objects_args { + char *window; + char *window_memory; + char *depth; + char *threads; + unsigned long max_pack_size; + int no_reuse_delta; + int no_reuse_object; + int quiet; + int local; + int name_hash_version; + int path_walk; + int delta_base_offset; + int pack_kept_objects; + struct list_objects_filter_options filter_options; +}; + +#define PACK_OBJECTS_ARGS_INIT { \ + .delta_base_offset = 1, \ + .pack_kept_objects = -1, \ +} + +struct child_process; + +void prepare_pack_objects(struct child_process *cmd, + const struct pack_objects_args *args, + const char *out); +void pack_objects_args_release(struct pack_objects_args *args); + +void repack_remove_redundant_pack(struct repository *repo, const char *dir_name, + const char *base_name); + +struct write_pack_opts { + struct pack_objects_args *po_args; + const char *destination; + const char *packdir; + const char *packtmp; +}; + +const char *write_pack_opts_pack_prefix(const struct write_pack_opts *opts); +bool write_pack_opts_is_local(const struct write_pack_opts *opts); + +int finish_pack_objects_cmd(const struct git_hash_algo *algop, + const struct write_pack_opts *opts, + struct child_process *cmd, + struct string_list *names); + +struct repository; +struct packed_git; + +struct existing_packs { + struct repository *repo; + struct string_list kept_packs; + struct string_list non_kept_packs; + struct string_list cruft_packs; + struct string_list midx_packs; +}; + +#define EXISTING_PACKS_INIT { \ + .kept_packs = STRING_LIST_INIT_DUP, \ + .non_kept_packs = STRING_LIST_INIT_DUP, \ + .cruft_packs = STRING_LIST_INIT_DUP, \ +} + +/* + * Adds all packs hex strings (pack-$HASH) to either packs->non_kept + * or packs->kept based on whether each pack has a corresponding + * .keep file or not. Packs without a .keep file are not to be kept + * if we are going to pack everything into one file. + */ +void existing_packs_collect(struct existing_packs *existing, + const struct string_list *extra_keep); +int existing_packs_has_non_kept(const struct existing_packs *existing); +int existing_pack_is_marked_for_deletion(struct string_list_item *item); +void existing_packs_retain_cruft(struct existing_packs *existing, + struct packed_git *cruft); +void existing_packs_mark_for_deletion(struct existing_packs *existing, + struct string_list *names); +void existing_packs_remove_redundant(struct existing_packs *existing, + const char *packdir); +void existing_packs_release(struct existing_packs *existing); + +struct generated_pack; + +struct generated_pack *generated_pack_populate(const char *name, + const char *packtmp); +int generated_pack_has_ext(const struct generated_pack *pack, const char *ext); +void generated_pack_install(struct generated_pack *pack, const char *name, + const char *packdir, const char *packtmp); + +void repack_promisor_objects(struct repository *repo, + const struct pack_objects_args *args, + struct string_list *names, const char *packtmp); + +struct pack_geometry { + struct packed_git **pack; + uint32_t pack_nr, pack_alloc; + uint32_t split; + + int split_factor; +}; + +void pack_geometry_init(struct pack_geometry *geometry, + struct existing_packs *existing, + const struct pack_objects_args *args); +void pack_geometry_split(struct pack_geometry *geometry); +struct packed_git *pack_geometry_preferred_pack(struct pack_geometry *geometry); +void pack_geometry_remove_redundant(struct pack_geometry *geometry, + struct string_list *names, + struct existing_packs *existing, + const char *packdir); +void pack_geometry_release(struct pack_geometry *geometry); + +struct tempfile; + +struct repack_write_midx_opts { + struct existing_packs *existing; + struct pack_geometry *geometry; + struct string_list *names; + const char *refs_snapshot; + const char *packdir; + int show_progress; + int write_bitmaps; + int midx_must_contain_cruft; +}; + +void midx_snapshot_refs(struct repository *repo, struct tempfile *f); +int write_midx_included_packs(struct repack_write_midx_opts *opts); + +int write_filtered_pack(const struct write_pack_opts *opts, + struct existing_packs *existing, + struct string_list *names); + +int write_cruft_pack(const struct write_pack_opts *opts, + const char *cruft_expiration, + unsigned long combine_cruft_below_size, + struct string_list *names, + struct existing_packs *existing); + +#endif /* REPACK_H */ diff --git a/replace-object.c b/replace-object.c index 3eae051074..03d0f1f083 100644 --- a/replace-object.c +++ b/replace-object.c @@ -8,31 +8,27 @@ #include "repository.h" #include "commit.h" -static int register_replace_ref(const char *refname, - const char *referent UNUSED, - const struct object_id *oid, - int flag UNUSED, - void *cb_data) +static int register_replace_ref(const struct reference *ref, void *cb_data) { struct repository *r = cb_data; /* Get sha1 from refname */ - const char *slash = strrchr(refname, '/'); - const char *hash = slash ? slash + 1 : refname; + const char *slash = strrchr(ref->name, '/'); + const char *hash = slash ? slash + 1 : ref->name; struct replace_object *repl_obj = xmalloc(sizeof(*repl_obj)); if (get_oid_hex_algop(hash, &repl_obj->original.oid, r->hash_algo)) { free(repl_obj); - warning(_("bad replace ref name: %s"), refname); + warning(_("bad replace ref name: %s"), ref->name); return 0; } /* Copy sha1 from the read ref */ - oidcpy(&repl_obj->replacement, oid); + oidcpy(&repl_obj->replacement, ref->oid); /* Register new object */ if (oidmap_put(&r->objects->replace_map, repl_obj)) - die(_("duplicate replace ref: %s"), refname); + die(_("duplicate replace ref: %s"), ref->name); return 0; } diff --git a/replay.c b/replay.c new file mode 100644 index 0000000000..58fdc20140 --- /dev/null +++ b/replay.c @@ -0,0 +1,115 @@ +#define USE_THE_REPOSITORY_VARIABLE + +#include "git-compat-util.h" +#include "commit.h" +#include "environment.h" +#include "gettext.h" +#include "ident.h" +#include "object.h" +#include "object-name.h" +#include "replay.h" +#include "tree.h" + +static const char *short_commit_name(struct repository *repo, + struct commit *commit) +{ + return repo_find_unique_abbrev(repo, &commit->object.oid, + DEFAULT_ABBREV); +} + +static char *get_author(const char *message) +{ + size_t len; + const char *a; + + a = find_commit_header(message, "author", &len); + if (a) + return xmemdupz(a, len); + + return NULL; +} + +struct commit *replay_create_commit(struct repository *repo, + struct tree *tree, + struct commit *based_on, + struct commit *parent) +{ + struct object_id ret; + struct object *obj = NULL; + struct commit_list *parents = NULL; + char *author; + char *sign_commit = NULL; /* FIXME: cli users might want to sign again */ + struct commit_extra_header *extra = NULL; + struct strbuf msg = STRBUF_INIT; + const char *out_enc = get_commit_output_encoding(); + const char *message = repo_logmsg_reencode(repo, based_on, + NULL, out_enc); + const char *orig_message = NULL; + const char *exclude_gpgsig[] = { "gpgsig", NULL }; + + commit_list_insert(parent, &parents); + extra = read_commit_extra_headers(based_on, exclude_gpgsig); + find_commit_subject(message, &orig_message); + strbuf_addstr(&msg, orig_message); + author = get_author(message); + reset_ident_date(); + if (commit_tree_extended(msg.buf, msg.len, &tree->object.oid, parents, + &ret, author, NULL, sign_commit, extra)) { + error(_("failed to write commit object")); + goto out; + } + + obj = parse_object(repo, &ret); + +out: + repo_unuse_commit_buffer(repo, based_on, message); + free_commit_extra_headers(extra); + free_commit_list(parents); + strbuf_release(&msg); + free(author); + return (struct commit *)obj; +} + +static struct commit *mapped_commit(kh_oid_map_t *replayed_commits, + struct commit *commit, + struct commit *fallback) +{ + khint_t pos = kh_get_oid_map(replayed_commits, commit->object.oid); + if (pos == kh_end(replayed_commits)) + return fallback; + return kh_value(replayed_commits, pos); +} + +struct commit *replay_pick_regular_commit(struct repository *repo, + struct commit *pickme, + kh_oid_map_t *replayed_commits, + struct commit *onto, + struct merge_options *merge_opt, + struct merge_result *result) +{ + struct commit *base, *replayed_base; + struct tree *pickme_tree, *base_tree; + + base = pickme->parents->item; + replayed_base = mapped_commit(replayed_commits, base, onto); + + result->tree = repo_get_commit_tree(repo, replayed_base); + pickme_tree = repo_get_commit_tree(repo, pickme); + base_tree = repo_get_commit_tree(repo, base); + + merge_opt->branch1 = short_commit_name(repo, replayed_base); + merge_opt->branch2 = short_commit_name(repo, pickme); + merge_opt->ancestor = xstrfmt("parent of %s", merge_opt->branch2); + + merge_incore_nonrecursive(merge_opt, + base_tree, + result->tree, + pickme_tree, + result); + + free((char*)merge_opt->ancestor); + merge_opt->ancestor = NULL; + if (!result->clean) + return NULL; + return replay_create_commit(repo, result->tree, pickme, replayed_base); +} diff --git a/replay.h b/replay.h new file mode 100644 index 0000000000..d6535ee56c --- /dev/null +++ b/replay.h @@ -0,0 +1,23 @@ +#ifndef REPLAY_H +#define REPLAY_H + +#include "khash.h" +#include "merge-ort.h" +#include "repository.h" + +struct commit; +struct tree; + +struct commit *replay_create_commit(struct repository *repo, + struct tree *tree, + struct commit *based_on, + struct commit *parent); + +struct commit *replay_pick_regular_commit(struct repository *repo, + struct commit *pickme, + kh_oid_map_t *replayed_commits, + struct commit *onto, + struct merge_options *merge_opt, + struct merge_result *result); + +#endif diff --git a/repository.c b/repository.c index 6faf5c7398..a9d9d4dc95 100644 --- a/repository.c +++ b/repository.c @@ -3,6 +3,7 @@ #include "repository.h" #include "odb.h" #include "config.h" +#include "gettext.h" #include "object.h" #include "lockfile.h" #include "path.h" @@ -38,7 +39,7 @@ struct repository *the_repository = &the_repo; static void set_default_hash_algo(struct repository *repo) { const char *hash_name; - int algo; + uint32_t algo; hash_name = getenv("GIT_TEST_DEFAULT_HASH_ALGO"); if (!hash_name) @@ -160,20 +161,24 @@ void repo_set_gitdir(struct repository *repo, * until after xstrdup(root). Then we can free it. */ char *old_gitdir = repo->gitdir; + char *objects_path = NULL; repo->gitdir = xstrdup(gitfile ? gitfile : root); free(old_gitdir); repo_set_commondir(repo, o->commondir); + expand_base_dir(&objects_path, o->object_dir, + repo->commondir, "objects"); if (!repo->objects->sources) { - CALLOC_ARRAY(repo->objects->sources, 1); - repo->objects->sources->odb = repo->objects; - repo->objects->sources->local = true; + repo->objects->sources = odb_source_new(repo->objects, + objects_path, true); repo->objects->sources_tail = &repo->objects->sources->next; + free(objects_path); + } else { + free(repo->objects->sources->path); + repo->objects->sources->path = objects_path; } - expand_base_dir(&repo->objects->sources->path, o->object_dir, - repo->commondir, "objects"); repo->objects->sources->disable_ref_updates = o->disable_ref_updates; @@ -185,18 +190,24 @@ void repo_set_gitdir(struct repository *repo, repo->gitdir, "index"); } -void repo_set_hash_algo(struct repository *repo, int hash_algo) +void repo_set_hash_algo(struct repository *repo, uint32_t hash_algo) { repo->hash_algo = &hash_algos[hash_algo]; } -void repo_set_compat_hash_algo(struct repository *repo, int algo) +void repo_set_compat_hash_algo(struct repository *repo, uint32_t algo) { +#ifdef WITH_RUST if (hash_algo_by_ptr(repo->hash_algo) == algo) BUG("hash_algo and compat_hash_algo match"); repo->compat_hash_algo = algo ? &hash_algos[algo] : NULL; if (repo->compat_hash_algo) repo_read_loose_object_map(repo); +#else + (void)repo; + if (algo) + die(_("compatibility hash algorithm support requires Rust")); +#endif } void repo_set_ref_storage_format(struct repository *repo, @@ -288,6 +299,7 @@ int repo_init(struct repository *repo, repo->repository_format_worktree_config = format.worktree_config; repo->repository_format_relative_worktrees = format.relative_worktrees; repo->repository_format_precious_objects = format.precious_objects; + repo->repository_format_submodule_encoding = format.submodule_encoding; /* take ownership of format.partial_clone */ repo->repository_format_partial_clone = format.partial_clone; diff --git a/repository.h b/repository.h index 5808a5d610..890aa872e1 100644 --- a/repository.h +++ b/repository.h @@ -158,6 +158,7 @@ struct repository { int repository_format_worktree_config; int repository_format_relative_worktrees; int repository_format_precious_objects; + int repository_format_submodule_encoding; /* Indicate if a repository has a different 'commondir' from 'gitdir' */ unsigned different_commondir:1; @@ -193,8 +194,8 @@ struct set_gitdir_args { void repo_set_gitdir(struct repository *repo, const char *root, const struct set_gitdir_args *extra_args); void repo_set_worktree(struct repository *repo, const char *path); -void repo_set_hash_algo(struct repository *repo, int algo); -void repo_set_compat_hash_algo(struct repository *repo, int compat_algo); +void repo_set_hash_algo(struct repository *repo, uint32_t algo); +void repo_set_compat_hash_algo(struct repository *repo, uint32_t compat_algo); void repo_set_ref_storage_format(struct repository *repo, enum ref_storage_format format); void initialize_repository(struct repository *repo); diff --git a/revision.c b/revision.c index cf5e6c1ec9..5f0850ae5c 100644 --- a/revision.c +++ b/revision.c @@ -1644,19 +1644,17 @@ struct all_refs_cb { struct worktree *wt; }; -static int handle_one_ref(const char *path, const char *referent UNUSED, const struct object_id *oid, - int flag UNUSED, - void *cb_data) +static int handle_one_ref(const struct reference *ref, void *cb_data) { struct all_refs_cb *cb = cb_data; struct object *object; - if (ref_excluded(&cb->all_revs->ref_excludes, path)) + if (ref_excluded(&cb->all_revs->ref_excludes, ref->name)) return 0; - object = get_reference(cb->all_revs, path, oid, cb->all_flags); - add_rev_cmdline(cb->all_revs, object, path, REV_CMD_REF, cb->all_flags); - add_pending_object(cb->all_revs, object, path); + object = get_reference(cb->all_revs, ref->name, ref->oid, cb->all_flags); + add_rev_cmdline(cb->all_revs, object, ref->name, REV_CMD_REF, cb->all_flags); + add_pending_object(cb->all_revs, object, ref->name); return 0; } diff --git a/run-command.c b/run-command.c index ed9575bd6a..f217adcad6 100644 --- a/run-command.c +++ b/run-command.c @@ -1578,7 +1578,10 @@ static void pp_cleanup(struct parallel_processes *pp, * When get_next_task added messages to the buffer in its last * iteration, the buffered output is non empty. */ - strbuf_write(&pp->buffered_output, stderr); + if (opts->consume_sideband) + opts->consume_sideband(&pp->buffered_output, opts->data); + else + strbuf_write(&pp->buffered_output, stderr); strbuf_release(&pp->buffered_output); sigchain_pop_common(); @@ -1652,6 +1655,44 @@ static int pp_start_one(struct parallel_processes *pp, return 0; } +static void pp_buffer_stdin(struct parallel_processes *pp, + const struct run_process_parallel_opts *opts) +{ + /* Buffer stdin for each pipe. */ + for (ssize_t i = 0; i < opts->processes; i++) { + struct child_process *proc = &pp->children[i].process; + int ret; + + if (pp->children[i].state != GIT_CP_WORKING || proc->in <= 0) + continue; + + /* + * child input is provided via path_to_stdin when the feed_pipe cb is + * missing, so we just signal an EOF. + */ + if (!opts->feed_pipe) { + close(proc->in); + proc->in = 0; + continue; + } + + /** + * Feed the pipe: + * ret < 0 means error + * ret == 0 means there is more data to be fed + * ret > 0 means feeding finished + */ + ret = opts->feed_pipe(proc->in, opts->data, pp->children[i].data); + if (ret < 0) + die_errno("feed_pipe"); + + if (ret) { + close(proc->in); + proc->in = 0; + } + } +} + static void pp_buffer_stderr(struct parallel_processes *pp, const struct run_process_parallel_opts *opts, int output_timeout) @@ -1679,13 +1720,17 @@ static void pp_buffer_stderr(struct parallel_processes *pp, } } -static void pp_output(const struct parallel_processes *pp) +static void pp_output(const struct parallel_processes *pp, + const struct run_process_parallel_opts *opts) { size_t i = pp->output_owner; if (pp->children[i].state == GIT_CP_WORKING && pp->children[i].err.len) { - strbuf_write(&pp->children[i].err, stderr); + if (opts->consume_sideband) + opts->consume_sideband(&pp->children[i].err, opts->data); + else + strbuf_write(&pp->children[i].err, stderr); strbuf_reset(&pp->children[i].err); } } @@ -1722,6 +1767,7 @@ static int pp_collect_finished(struct parallel_processes *pp, pp->children[i].state = GIT_CP_FREE; if (pp->pfd) pp->pfd[i].fd = -1; + pp->children[i].process.in = 0; child_process_init(&pp->children[i].process); if (opts->ungroup) { @@ -1732,11 +1778,15 @@ static int pp_collect_finished(struct parallel_processes *pp, } else { const size_t n = opts->processes; - strbuf_write(&pp->children[i].err, stderr); + /* Output errors, then all other finished child processes */ + if (opts->consume_sideband) { + opts->consume_sideband(&pp->children[i].err, opts->data); + opts->consume_sideband(&pp->buffered_output, opts->data); + } else { + strbuf_write(&pp->children[i].err, stderr); + strbuf_write(&pp->buffered_output, stderr); + } strbuf_reset(&pp->children[i].err); - - /* Output all other finished child processes */ - strbuf_write(&pp->buffered_output, stderr); strbuf_reset(&pp->buffered_output); /* @@ -1756,6 +1806,32 @@ static int pp_collect_finished(struct parallel_processes *pp, return result; } +static void pp_handle_child_IO(struct parallel_processes *pp, + const struct run_process_parallel_opts *opts, + int output_timeout) +{ + /* + * First push input, if any (it might no-op), to child tasks to avoid them blocking + * after input. This also prevents deadlocks when ungrouping below, if a child blocks + * while the parent also waits for them to finish. + */ + pp_buffer_stdin(pp, opts); + + if (opts->ungroup) { + for (size_t i = 0; i < opts->processes; i++) { + int child_ready_for_cleanup = + pp->children[i].state == GIT_CP_WORKING && + pp->children[i].process.in == 0; + + if (child_ready_for_cleanup) + pp->children[i].state = GIT_CP_WAIT_CLEANUP; + } + } else { + pp_buffer_stderr(pp, opts, output_timeout); + pp_output(pp, opts); + } +} + void run_processes_parallel(const struct run_process_parallel_opts *opts) { int i, code; @@ -1775,6 +1851,16 @@ void run_processes_parallel(const struct run_process_parallel_opts *opts) "max:%"PRIuMAX, (uintmax_t)opts->processes); + if (opts->ungroup && opts->consume_sideband) + BUG("ungroup and reading sideband are mutualy exclusive"); + + /* + * Child tasks might receive input via stdin, terminating early (or not), so + * ignore the default SIGPIPE which gets handled by each feed_pipe_fn which + * actually writes the data to children stdin fds. + */ + sigchain_push(SIGPIPE, SIG_IGN); + pp_init(&pp, opts, &pp_sig); while (1) { for (i = 0; @@ -1792,13 +1878,7 @@ void run_processes_parallel(const struct run_process_parallel_opts *opts) } if (!pp.nr_processes) break; - if (opts->ungroup) { - for (size_t i = 0; i < opts->processes; i++) - pp.children[i].state = GIT_CP_WAIT_CLEANUP; - } else { - pp_buffer_stderr(&pp, opts, output_timeout); - pp_output(&pp); - } + pp_handle_child_IO(&pp, opts, output_timeout); code = pp_collect_finished(&pp, opts); if (code) { pp.shutdown = 1; @@ -1809,6 +1889,8 @@ void run_processes_parallel(const struct run_process_parallel_opts *opts) pp_cleanup(&pp, opts); + sigchain_pop(SIGPIPE); + if (do_trace2) trace2_region_leave(tr2_category, tr2_label, NULL); } diff --git a/run-command.h b/run-command.h index 0df25e445f..2c2484478b 100644 --- a/run-command.h +++ b/run-command.h @@ -421,6 +421,36 @@ typedef int (*start_failure_fn)(struct strbuf *out, void *pp_task_cb); /** + * This callback is repeatedly called on every child process who requests + * start_command() to create a pipe by setting child_process.in < 0. + * + * pp_cb is the callback cookie as passed into run_processes_parallel, and + * pp_task_cb is the callback cookie as passed into get_next_task_fn. + * The contents of 'send' will be read into the pipe and passed to the pipe. + * + * Returns < 0 for error + * Returns == 0 when there is more data to be fed (will be called again) + * Returns > 0 when finished (child closed fd or no more data to be fed) + */ +typedef int (*feed_pipe_fn)(int child_in, + void *pp_cb, + void *pp_task_cb); + +/** + * If this callback is provided, instead of collating process output to stderr, + * they will be collated into a new pipe. consume_sideband_fn will be called + * repeatedly. When output is available on that pipe, it will be contained in + * 'output'. But it will be called with an empty 'output' too, to allow for + * keepalives or similar operations if necessary. + * + * pp_cb is the callback cookie as passed into run_processes_parallel. + * + * Since this callback is provided with the collated output, no task cookie is + * provided. + */ +typedef void (*consume_sideband_fn)(struct strbuf *output, void *pp_cb); + +/** * This callback is called on every child process that finished processing. * * See run_processes_parallel() below for a discussion of the "struct @@ -473,6 +503,18 @@ struct run_process_parallel_opts */ start_failure_fn start_failure; + /* + * feed_pipe: see feed_pipe_fn() above. This can be NULL to omit any + * special handling. + */ + feed_pipe_fn feed_pipe; + + /* + * consume_sideband: see consume_sideband_fn() above. This can be NULL + * to omit any special handling. + */ + consume_sideband_fn consume_sideband; + /** * task_finished: See task_finished_fn() above. This can be * NULL to omit any special handling. @@ -166,6 +166,7 @@ static int set_recommended_config(int reconfigure) #endif /* Optional */ { "status.aheadBehind", "false" }, + { "commitGraph.changedPaths", "true" }, { "commitGraph.generationVersion", "1" }, { "core.autoCRLF", "false" }, { "core.safeCRLF", "false" }, diff --git a/sequencer.c b/sequencer.c index 5476d39ba9..3f99605038 100644 --- a/sequencer.c +++ b/sequencer.c @@ -209,6 +209,7 @@ static GIT_PATH_FUNC(rebase_path_reschedule_failed_exec, "rebase-merge/reschedul static GIT_PATH_FUNC(rebase_path_no_reschedule_failed_exec, "rebase-merge/no-reschedule-failed-exec") static GIT_PATH_FUNC(rebase_path_drop_redundant_commits, "rebase-merge/drop_redundant_commits") static GIT_PATH_FUNC(rebase_path_keep_redundant_commits, "rebase-merge/keep_redundant_commits") +static GIT_PATH_FUNC(rebase_path_trailer, "rebase-merge/trailer") /* * A 'struct replay_ctx' represents the private state of the sequencer. @@ -420,6 +421,7 @@ void replay_opts_release(struct replay_opts *opts) if (opts->revs) release_revisions(opts->revs); free(opts->revs); + strvec_clear(&opts->trailer_args); replay_ctx_release(ctx); free(opts->ctx); } @@ -1292,32 +1294,40 @@ int update_head_with_reflog(const struct commit *old_head, return ret; } +static int pipe_from_strbuf(int hook_stdin_fd, void *pp_cb, void *pp_task_cb UNUSED) +{ + struct hook_cb_data *hook_cb = pp_cb; + struct strbuf *to_pipe = hook_cb->options->feed_pipe_ctx; + int ret; + + if (!to_pipe) + BUG("pipe_from_strbuf called without feed_pipe_ctx"); + + ret = write_in_full(hook_stdin_fd, to_pipe->buf, to_pipe->len); + if (ret < 0 && errno != EPIPE) + return ret; + + return 1; /* done writing */ +} + static int run_rewrite_hook(const struct object_id *oldoid, const struct object_id *newoid) { - struct child_process proc = CHILD_PROCESS_INIT; + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; int code; struct strbuf sb = STRBUF_INIT; - const char *hook_path = find_hook(the_repository, "post-rewrite"); - if (!hook_path) - return 0; + strbuf_addf(&sb, "%s %s\n", oid_to_hex(oldoid), oid_to_hex(newoid)); - strvec_pushl(&proc.args, hook_path, "amend", NULL); - proc.in = -1; - proc.stdout_to_stderr = 1; - proc.trace2_hook_name = "post-rewrite"; + opt.feed_pipe_ctx = &sb; + opt.feed_pipe = pipe_from_strbuf; + + strvec_push(&opt.args, "amend"); + + code = run_hooks_opt(the_repository, "post-rewrite", &opt); - code = start_command(&proc); - if (code) - return code; - strbuf_addf(&sb, "%s %s\n", oid_to_hex(oldoid), oid_to_hex(newoid)); - sigchain_push(SIGPIPE, SIG_IGN); - write_in_full(proc.in, sb.buf, sb.len); - close(proc.in); strbuf_release(&sb); - sigchain_pop(SIGPIPE); - return finish_command(&proc); + return code; } void commit_post_rewrite(struct repository *r, @@ -2025,6 +2035,10 @@ static int append_squash_message(struct strbuf *buf, const char *body, if (opts->signoff) append_signoff(buf, 0, 0); + if (opts->trailer_args.nr && + amend_strbuf_with_trailers(buf, &opts->trailer_args)) + return error(_("unable to add trailers to commit message")); + if ((command == TODO_FIXUP) && (flag & TODO_REPLACE_FIXUP_MSG) && (file_exists(rebase_path_fixup_msg()) || @@ -2443,6 +2457,14 @@ static int do_pick_commit(struct repository *r, if (opts->signoff && !is_fixup(command)) append_signoff(&ctx->message, 0, 0); + if (opts->trailer_args.nr && !is_fixup(command)) { + if (amend_strbuf_with_trailers(&ctx->message, + &opts->trailer_args)) { + res = error(_("unable to add trailers to commit message")); + goto leave; + } + } + if (is_rebase_i(opts) && write_author_script(msg.message) < 0) res = -1; else if (!opts->strategy || @@ -2517,6 +2539,7 @@ static int do_pick_commit(struct repository *r, _("dropping %s %s -- patch contents already upstream\n"), oid_to_hex(&commit->object.oid), msg.subject); } /* else allow == 0 and there's nothing special to do */ + if (!opts->no_commit && !drop_commit) { if (author || command == TODO_REVERT || (flags & AMEND_MSG)) res = do_commit(r, msg_file, author, reflog_action, @@ -3234,6 +3257,17 @@ static int read_populate_opts(struct replay_opts *opts) read_strategy_opts(opts, &buf); strbuf_reset(&buf); + if (strbuf_read_file(&buf, rebase_path_trailer(), 0) >= 0) { + char *p = buf.buf, *nl; + + while ((nl = strchr(p, '\n'))) { + *nl = '\0'; + if (*p) + strvec_push(&opts->trailer_args, p); + p = nl + 1; + } + strbuf_reset(&buf); + } if (read_oneliner(&ctx->current_fixups, rebase_path_current_fixups(), @@ -3328,6 +3362,14 @@ int write_basic_state(struct replay_opts *opts, const char *head_name, write_file(rebase_path_reschedule_failed_exec(), "%s", ""); else write_file(rebase_path_no_reschedule_failed_exec(), "%s", ""); + if (opts->trailer_args.nr) { + struct strbuf buf = STRBUF_INIT; + + for (size_t i = 0; i < opts->trailer_args.nr; i++) + strbuf_addf(&buf, "%s\n", opts->trailer_args.v[i]); + write_file(rebase_path_trailer(), "%s", buf.buf); + strbuf_release(&buf); + } return 0; } diff --git a/sequencer.h b/sequencer.h index 719684c8a9..e21835c5a0 100644 --- a/sequencer.h +++ b/sequencer.h @@ -44,6 +44,7 @@ struct replay_opts { int record_origin; int no_commit; int signoff; + struct strvec trailer_args; int allow_ff; int allow_rerere_auto; int allow_empty; @@ -82,8 +83,9 @@ struct replay_opts { struct replay_ctx *ctx; }; #define REPLAY_OPTS_INIT { \ - .edit = -1, \ .action = -1, \ + .edit = -1, \ + .trailer_args = STRVEC_INIT, \ .xopts = STRVEC_INIT, \ .ctx = replay_ctx_new(), \ } @@ -14,7 +14,7 @@ static int advertise_sid = -1; static int advertise_object_info = -1; -static int client_hash_algo = GIT_HASH_SHA1_LEGACY; +static uint32_t client_hash_algo = GIT_HASH_SHA1_LEGACY; static int always_advertise(struct repository *r UNUSED, struct strbuf *value UNUSED) diff --git a/server-info.c b/server-info.c index 1d33de821e..4243e24edc 100644 --- a/server-info.c +++ b/server-info.c @@ -148,23 +148,21 @@ out: return ret; } -static int add_info_ref(const char *path, const char *referent UNUSED, const struct object_id *oid, - int flag UNUSED, - void *cb_data) +static int add_info_ref(const struct reference *ref, void *cb_data) { struct update_info_ctx *uic = cb_data; - struct object *o = parse_object(uic->repo, oid); + struct object *o = parse_object(uic->repo, ref->oid); if (!o) return -1; - if (uic_printf(uic, "%s %s\n", oid_to_hex(oid), path) < 0) + if (uic_printf(uic, "%s %s\n", oid_to_hex(ref->oid), ref->name) < 0) return -1; if (o->type == OBJ_TAG) { - o = deref_tag(uic->repo, o, path, 0); + o = deref_tag(uic->repo, o, ref->name, 0); if (o) if (uic_printf(uic, "%s %s^{}\n", - oid_to_hex(&o->oid), path) < 0) + oid_to_hex(&o->oid), ref->name) < 0) return -1; } return 0; @@ -287,13 +285,12 @@ static int compare_info(const void *a_, const void *b_) static void init_pack_info(struct repository *r, const char *infofile, int force) { - struct packfile_store *packs = r->objects->packfiles; struct packed_git *p; int stale; int i; size_t alloc = 0; - for (p = packfile_store_get_all_packs(packs); p; p = p->next) { + repo_for_each_pack(r, p) { /* we ignore things on alternate path since they are * not available to the pullers in general. */ @@ -687,6 +687,9 @@ static enum extension_result handle_extension(const char *var, } else if (!strcmp(ext, "relativeworktrees")) { data->relative_worktrees = git_config_bool(var, value); return EXTENSION_OK; + } else if (!strcmp(ext, "submoduleencoding")) { + data->submodule_encoding = git_config_bool(var, value); + return EXTENSION_OK; } return EXTENSION_UNKNOWN; } @@ -1865,6 +1868,8 @@ const char *setup_git_directory_gently(int *nongit_ok) repo_fmt.worktree_config; the_repository->repository_format_relative_worktrees = repo_fmt.relative_worktrees; + the_repository->repository_format_submodule_encoding = + repo_fmt.submodule_encoding; /* take ownership of repo_fmt.partial_clone */ the_repository->repository_format_partial_clone = repo_fmt.partial_clone; @@ -1963,6 +1968,8 @@ void check_repository_format(struct repository_format *fmt) fmt->ref_storage_format); the_repository->repository_format_worktree_config = fmt->worktree_config; + the_repository->repository_format_submodule_encoding = + fmt->submodule_encoding; the_repository->repository_format_relative_worktrees = fmt->relative_worktrees; the_repository->repository_format_partial_clone = @@ -130,6 +130,7 @@ struct repository_format { char *partial_clone; /* value of extensions.partialclone */ int worktree_config; int relative_worktrees; + int submodule_encoding; int is_bare; int hash_algo; int compat_hash_algo; @@ -626,14 +626,10 @@ static void paint_down(struct paint_info *info, const struct object_id *oid, free(tmp); } -static int mark_uninteresting(const char *refname UNUSED, - const char *referent UNUSED, - const struct object_id *oid, - int flags UNUSED, - void *cb_data UNUSED) +static int mark_uninteresting(const struct reference *ref, void *cb_data UNUSED) { struct commit *commit = lookup_commit_reference_gently(the_repository, - oid, 1); + ref->oid, 1); if (!commit) return 0; commit->object.flags |= UNINTERESTING; @@ -742,16 +738,12 @@ struct commit_array { size_t nr, alloc; }; -static int add_ref(const char *refname UNUSED, - const char *referent UNUSED, - const struct object_id *oid, - int flags UNUSED, - void *cb_data) +static int add_ref(const struct reference *ref, void *cb_data) { struct commit_array *ca = cb_data; ALLOC_GROW(ca->commits, ca->nr + 1, ca->alloc); ca->commits[ca->nr] = lookup_commit_reference_gently(the_repository, - oid, 1); + ref->oid, 1); if (ca->commits[ca->nr]) ca->nr++; return 0; diff --git a/sparse-index.c b/sparse-index.c index 5634abafaa..76f90da5f5 100644 --- a/sparse-index.c +++ b/sparse-index.c @@ -32,7 +32,9 @@ int give_advice_on_expansion = 1; "Your working directory likely has contents that are outside of\n" \ "your sparse-checkout patterns. Use 'git sparse-checkout list' to\n" \ "see your sparse-checkout definition and compare it to your working\n" \ - "directory contents. Running 'git clean' may assist in this cleanup." + "directory contents. Cleaning up any merge conflicts or staged\n" \ + "changes before running 'git sparse-checkout clean' or 'git\n" \ + "sparse-checkout reapply' may assist in this cleanup." struct modify_index_context { struct index_state *write; diff --git a/src/cargo-meson.sh b/src/cargo-meson.sh index 99400986d9..3998db0435 100755 --- a/src/cargo-meson.sh +++ b/src/cargo-meson.sh @@ -26,7 +26,14 @@ then exit $RET fi -if ! cmp "$BUILD_DIR/$BUILD_TYPE/libgitcore.a" "$BUILD_DIR/libgitcore.a" >/dev/null 2>&1 +case "$(cargo -vV | sed -s 's/^host: \(.*\)$/\1/')" in + *-windows-*) + LIBNAME=gitcore.lib;; + *) + LIBNAME=libgitcore.a;; +esac + +if ! cmp "$BUILD_DIR/$BUILD_TYPE/$LIBNAME" "$BUILD_DIR/libgitcore.a" >/dev/null 2>&1 then - cp "$BUILD_DIR/$BUILD_TYPE/libgitcore.a" "$BUILD_DIR/libgitcore.a" + cp "$BUILD_DIR/$BUILD_TYPE/$LIBNAME" "$BUILD_DIR/libgitcore.a" fi diff --git a/src/csum_file.rs b/src/csum_file.rs new file mode 100644 index 0000000000..7f2c6c4fcb --- /dev/null +++ b/src/csum_file.rs @@ -0,0 +1,81 @@ +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation: version 2 of the License, dated June 1991. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, see <https://www.gnu.org/licenses/>. + +use crate::hash::{HashAlgorithm, GIT_MAX_RAWSZ}; +use std::ffi::CStr; +use std::io::{self, Write}; +use std::os::raw::c_void; + +/// A writer that can write files identified by their hash or containing a trailing hash. +pub struct HashFile { + ptr: *mut c_void, + algo: HashAlgorithm, +} + +impl HashFile { + /// Create a new HashFile. + /// + /// The hash used will be `algo`, its name should be in `name`, and an open file descriptor + /// pointing to that file should be in `fd`. + pub fn new(algo: HashAlgorithm, fd: i32, name: &CStr) -> HashFile { + HashFile { + ptr: unsafe { c::hashfd(algo.hash_algo_ptr(), fd, name.as_ptr()) }, + algo, + } + } + + /// Finalize this HashFile instance. + /// + /// Returns the hash computed over the data. + pub fn finalize(self, component: u32, flags: u32) -> Vec<u8> { + let mut result = vec![0u8; GIT_MAX_RAWSZ]; + unsafe { c::finalize_hashfile(self.ptr, result.as_mut_ptr(), component, flags) }; + result.truncate(self.algo.raw_len()); + result + } +} + +impl Write for HashFile { + fn write(&mut self, data: &[u8]) -> io::Result<usize> { + for chunk in data.chunks(u32::MAX as usize) { + unsafe { + c::hashwrite( + self.ptr, + chunk.as_ptr() as *const c_void, + chunk.len() as u32, + ) + }; + } + Ok(data.len()) + } + + fn flush(&mut self) -> io::Result<()> { + unsafe { c::hashflush(self.ptr) }; + Ok(()) + } +} + +pub mod c { + use std::os::raw::{c_char, c_int, c_void}; + + extern "C" { + pub fn hashfd(algop: *const c_void, fd: i32, name: *const c_char) -> *mut c_void; + pub fn hashwrite(f: *mut c_void, data: *const c_void, len: u32); + pub fn hashflush(f: *mut c_void); + pub fn finalize_hashfile( + f: *mut c_void, + data: *mut u8, + component: u32, + flags: u32, + ) -> c_int; + } +} diff --git a/src/hash.rs b/src/hash.rs new file mode 100644 index 0000000000..8798a50aef --- /dev/null +++ b/src/hash.rs @@ -0,0 +1,335 @@ +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation: version 2 of the License, dated June 1991. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, see <https://www.gnu.org/licenses/>. + +use std::io::{self, Write}; +use std::os::raw::c_void; + +pub const GIT_MAX_RAWSZ: usize = 32; + +/// A binary object ID. +#[repr(C)] +#[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq)] +pub struct ObjectID { + pub hash: [u8; GIT_MAX_RAWSZ], + pub algo: u32, +} + +#[allow(dead_code)] +impl ObjectID { + pub fn as_slice(&self) -> &[u8] { + match HashAlgorithm::from_u32(self.algo) { + Some(algo) => &self.hash[0..algo.raw_len()], + None => &self.hash, + } + } + + pub fn as_mut_slice(&mut self) -> &mut [u8] { + match HashAlgorithm::from_u32(self.algo) { + Some(algo) => &mut self.hash[0..algo.raw_len()], + None => &mut self.hash, + } + } +} + +pub struct Hasher { + algo: HashAlgorithm, + safe: bool, + ctx: *mut c_void, +} + +impl Hasher { + /// Create a new safe hasher. + pub fn new(algo: HashAlgorithm) -> Hasher { + let ctx = unsafe { c::git_hash_alloc() }; + unsafe { c::git_hash_init(ctx, algo.hash_algo_ptr()) }; + Hasher { + algo, + safe: true, + ctx, + } + } + + /// Return whether this is a safe hasher. + pub fn is_safe(&self) -> bool { + self.safe + } + + /// Update the hasher with the specified data. + pub fn update(&mut self, data: &[u8]) { + unsafe { c::git_hash_update(self.ctx, data.as_ptr() as *const c_void, data.len()) }; + } + + /// Return an object ID, consuming the hasher. + pub fn into_oid(self) -> ObjectID { + let mut oid = ObjectID { + hash: [0u8; 32], + algo: self.algo as u32, + }; + unsafe { c::git_hash_final_oid(&mut oid as *mut ObjectID as *mut c_void, self.ctx) }; + oid + } + + /// Return a hash as a `Vec`, consuming the hasher. + pub fn into_vec(self) -> Vec<u8> { + let mut v = vec![0u8; self.algo.raw_len()]; + unsafe { c::git_hash_final(v.as_mut_ptr(), self.ctx) }; + v + } +} + +impl Write for Hasher { + fn write(&mut self, data: &[u8]) -> io::Result<usize> { + self.update(data); + Ok(data.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl Clone for Hasher { + fn clone(&self) -> Hasher { + let ctx = unsafe { c::git_hash_alloc() }; + unsafe { c::git_hash_clone(ctx, self.ctx) }; + Hasher { + algo: self.algo, + safe: self.safe, + ctx, + } + } +} + +impl Drop for Hasher { + fn drop(&mut self) { + unsafe { c::git_hash_free(self.ctx) }; + } +} + +/// A hash algorithm, +#[repr(C)] +#[derive(Debug, Copy, Clone, Ord, PartialOrd, Eq, PartialEq)] +pub enum HashAlgorithm { + SHA1 = 1, + SHA256 = 2, +} + +#[allow(dead_code)] +impl HashAlgorithm { + const SHA1_NULL_OID: ObjectID = ObjectID { + hash: [0u8; 32], + algo: Self::SHA1 as u32, + }; + const SHA256_NULL_OID: ObjectID = ObjectID { + hash: [0u8; 32], + algo: Self::SHA256 as u32, + }; + + const SHA1_EMPTY_TREE: ObjectID = ObjectID { + hash: *b"\x4b\x82\x5d\xc6\x42\xcb\x6e\xb9\xa0\x60\xe5\x4b\xf8\xd6\x92\x88\xfb\xee\x49\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + algo: Self::SHA1 as u32, + }; + const SHA256_EMPTY_TREE: ObjectID = ObjectID { + hash: *b"\x6e\xf1\x9b\x41\x22\x5c\x53\x69\xf1\xc1\x04\xd4\x5d\x8d\x85\xef\xa9\xb0\x57\xb5\x3b\x14\xb4\xb9\xb9\x39\xdd\x74\xde\xcc\x53\x21", + algo: Self::SHA256 as u32, + }; + + const SHA1_EMPTY_BLOB: ObjectID = ObjectID { + hash: *b"\xe6\x9d\xe2\x9b\xb2\xd1\xd6\x43\x4b\x8b\x29\xae\x77\x5a\xd8\xc2\xe4\x8c\x53\x91\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", + algo: Self::SHA1 as u32, + }; + const SHA256_EMPTY_BLOB: ObjectID = ObjectID { + hash: *b"\x47\x3a\x0f\x4c\x3b\xe8\xa9\x36\x81\xa2\x67\xe3\xb1\xe9\xa7\xdc\xda\x11\x85\x43\x6f\xe1\x41\xf7\x74\x91\x20\xa3\x03\x72\x18\x13", + algo: Self::SHA256 as u32, + }; + + /// Return a hash algorithm based on the internal integer ID used by Git. + /// + /// Returns `None` if the algorithm doesn't indicate a valid algorithm. + pub const fn from_u32(algo: u32) -> Option<HashAlgorithm> { + match algo { + 1 => Some(HashAlgorithm::SHA1), + 2 => Some(HashAlgorithm::SHA256), + _ => None, + } + } + + /// Return a hash algorithm based on the internal integer ID used by Git. + /// + /// Returns `None` if the algorithm doesn't indicate a valid algorithm. + pub const fn from_format_id(algo: u32) -> Option<HashAlgorithm> { + match algo { + 0x73686131 => Some(HashAlgorithm::SHA1), + 0x73323536 => Some(HashAlgorithm::SHA256), + _ => None, + } + } + + /// The name of this hash algorithm as a string suitable for the configuration file. + pub const fn name(self) -> &'static str { + match self { + HashAlgorithm::SHA1 => "sha1", + HashAlgorithm::SHA256 => "sha256", + } + } + + /// The format ID of this algorithm for binary formats. + /// + /// Note that when writing this to a data format, it should be written in big-endian format + /// explicitly. + pub const fn format_id(self) -> u32 { + match self { + HashAlgorithm::SHA1 => 0x73686131, + HashAlgorithm::SHA256 => 0x73323536, + } + } + + /// The length of binary object IDs in this algorithm in bytes. + pub const fn raw_len(self) -> usize { + match self { + HashAlgorithm::SHA1 => 20, + HashAlgorithm::SHA256 => 32, + } + } + + /// The length of object IDs in this algorithm in hexadecimal characters. + pub const fn hex_len(self) -> usize { + self.raw_len() * 2 + } + + /// The number of bytes which is processed by one iteration of this algorithm's compression + /// function. + pub const fn block_size(self) -> usize { + match self { + HashAlgorithm::SHA1 => 64, + HashAlgorithm::SHA256 => 64, + } + } + + /// The object ID representing the empty blob. + pub const fn empty_blob(self) -> &'static ObjectID { + match self { + HashAlgorithm::SHA1 => &Self::SHA1_EMPTY_BLOB, + HashAlgorithm::SHA256 => &Self::SHA256_EMPTY_BLOB, + } + } + + /// The object ID representing the empty tree. + pub const fn empty_tree(self) -> &'static ObjectID { + match self { + HashAlgorithm::SHA1 => &Self::SHA1_EMPTY_TREE, + HashAlgorithm::SHA256 => &Self::SHA256_EMPTY_TREE, + } + } + + /// The object ID which is all zeros. + pub const fn null_oid(self) -> &'static ObjectID { + match self { + HashAlgorithm::SHA1 => &Self::SHA1_NULL_OID, + HashAlgorithm::SHA256 => &Self::SHA256_NULL_OID, + } + } + + /// A pointer to the C `struct git_hash_algo` for interoperability with C. + pub fn hash_algo_ptr(self) -> *const c_void { + unsafe { c::hash_algo_ptr_by_offset(self as u32) } + } + + /// Create a hasher for this algorithm. + pub fn hasher(self) -> Hasher { + Hasher::new(self) + } +} + +pub mod c { + use std::os::raw::c_void; + + extern "C" { + pub fn hash_algo_ptr_by_offset(n: u32) -> *const c_void; + pub fn unsafe_hash_algo(algop: *const c_void) -> *const c_void; + pub fn git_hash_alloc() -> *mut c_void; + pub fn git_hash_free(ctx: *mut c_void); + pub fn git_hash_init(dst: *mut c_void, algop: *const c_void); + pub fn git_hash_clone(dst: *mut c_void, src: *const c_void); + pub fn git_hash_update(ctx: *mut c_void, inp: *const c_void, len: usize); + pub fn git_hash_final(hash: *mut u8, ctx: *mut c_void); + pub fn git_hash_final_oid(hash: *mut c_void, ctx: *mut c_void); + } +} + +#[cfg(test)] +mod tests { + use super::{HashAlgorithm, ObjectID}; + use std::io::Write; + + fn all_algos() -> &'static [HashAlgorithm] { + &[HashAlgorithm::SHA1, HashAlgorithm::SHA256] + } + + #[test] + fn format_id_round_trips() { + for algo in all_algos() { + assert_eq!( + *algo, + HashAlgorithm::from_format_id(algo.format_id()).unwrap() + ); + } + } + + #[test] + fn offset_round_trips() { + for algo in all_algos() { + assert_eq!(*algo, HashAlgorithm::from_u32(*algo as u32).unwrap()); + } + } + + #[test] + fn slices_have_correct_length() { + for algo in all_algos() { + for oid in [algo.null_oid(), algo.empty_blob(), algo.empty_tree()] { + assert_eq!(oid.as_slice().len(), algo.raw_len()); + } + } + } + + #[test] + fn hasher_works_correctly() { + for algo in all_algos() { + let tests: &[(&[u8], &ObjectID)] = &[ + (b"blob 0\0", algo.empty_blob()), + (b"tree 0\0", algo.empty_tree()), + ]; + for (data, oid) in tests { + let mut h = algo.hasher(); + assert_eq!(h.is_safe(), true); + // Test that this works incrementally. + h.update(&data[0..2]); + h.update(&data[2..]); + + let h2 = h.clone(); + + let actual_oid = h.into_oid(); + assert_eq!(**oid, actual_oid); + + let v = h2.into_vec(); + assert_eq!((*oid).as_slice(), &v); + + let mut h = algo.hasher(); + h.write_all(&data[0..2]).unwrap(); + h.write_all(&data[2..]).unwrap(); + + let actual_oid = h.into_oid(); + assert_eq!(**oid, actual_oid); + } + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 9da70d8b57..0c598298b1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1 +1,4 @@ +pub mod csum_file; +pub mod hash; +pub mod loose; pub mod varint; diff --git a/src/loose.rs b/src/loose.rs new file mode 100644 index 0000000000..a4e7d2fa48 --- /dev/null +++ b/src/loose.rs @@ -0,0 +1,912 @@ +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation: version 2 of the License, dated June 1991. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, see <https://www.gnu.org/licenses/>. + +use crate::hash::{HashAlgorithm, ObjectID, GIT_MAX_RAWSZ}; +use std::collections::BTreeMap; +use std::convert::TryInto; +use std::io::{self, Write}; + +/// The type of object stored in the map. +/// +/// If this value is `Reserved`, then it is never written to disk and is used primarily to store +/// certain hard-coded objects, like the empty tree, empty blob, or null object ID. +/// +/// If this value is `LooseObject`, then this represents a loose object. `Shallow` represents a +/// shallow commit, its parent, or its tree. `Submodule` represents a submodule commit. +#[repr(C)] +#[derive(Debug, Clone, Copy, Ord, PartialOrd, Eq, PartialEq)] +pub enum MapType { + Reserved = 0, + LooseObject = 1, + Shallow = 2, + Submodule = 3, +} + +impl MapType { + pub fn from_u32(n: u32) -> Option<MapType> { + match n { + 0 => Some(Self::Reserved), + 1 => Some(Self::LooseObject), + 2 => Some(Self::Shallow), + 3 => Some(Self::Submodule), + _ => None, + } + } +} + +/// The value of an object stored in a `LooseObjectMemoryMap`. +/// +/// This keeps the object ID to which the key is mapped and its kind together. +struct MappedObject { + oid: ObjectID, + kind: MapType, +} + +/// Memory storage for a loose object. +struct LooseObjectMemoryMap { + to_compat: BTreeMap<ObjectID, MappedObject>, + to_storage: BTreeMap<ObjectID, MappedObject>, + compat: HashAlgorithm, + storage: HashAlgorithm, +} + +impl LooseObjectMemoryMap { + /// Create a new `LooseObjectMemoryMap`. + /// + /// The storage and compatibility `HashAlgorithm` instances are used to store the object IDs in + /// the correct map. + fn new(storage: HashAlgorithm, compat: HashAlgorithm) -> LooseObjectMemoryMap { + LooseObjectMemoryMap { + to_compat: BTreeMap::new(), + to_storage: BTreeMap::new(), + compat, + storage, + } + } + + fn len(&self) -> usize { + self.to_compat.len() + } + + /// Write this map to an interface implementing `std::io::Write`. + fn write<W: Write>(&self, wrtr: W) -> io::Result<()> { + const VERSION_NUMBER: u32 = 1; + const NUM_OBJECT_FORMATS: u32 = 2; + const PADDING: [u8; 4] = [0u8; 4]; + + let mut wrtr = wrtr; + let header_size: u32 = 4 + 4 + 4 + 4 + 4 + (4 + 4 + 8) * 2 + 8; + + wrtr.write_all(b"LMAP")?; + wrtr.write_all(&VERSION_NUMBER.to_be_bytes())?; + wrtr.write_all(&header_size.to_be_bytes())?; + wrtr.write_all(&(self.to_compat.len() as u32).to_be_bytes())?; + wrtr.write_all(&NUM_OBJECT_FORMATS.to_be_bytes())?; + + let storage_short_len = self.find_short_name_len(&self.to_compat, self.storage); + let compat_short_len = self.find_short_name_len(&self.to_storage, self.compat); + + let storage_npadding = Self::required_nul_padding(self.to_compat.len(), storage_short_len); + let compat_npadding = Self::required_nul_padding(self.to_compat.len(), compat_short_len); + + let mut offset: u64 = header_size as u64; + + for (algo, len, npadding) in &[ + (self.storage, storage_short_len, storage_npadding), + (self.compat, compat_short_len, compat_npadding), + ] { + wrtr.write_all(&algo.format_id().to_be_bytes())?; + wrtr.write_all(&(*len as u32).to_be_bytes())?; + + offset += *npadding; + wrtr.write_all(&offset.to_be_bytes())?; + + offset += self.to_compat.len() as u64 * (*len as u64 + algo.raw_len() as u64 + 4); + } + + wrtr.write_all(&offset.to_be_bytes())?; + + let order_map: BTreeMap<&ObjectID, usize> = self + .to_compat + .keys() + .enumerate() + .map(|(i, oid)| (oid, i)) + .collect(); + + wrtr.write_all(&PADDING[0..storage_npadding as usize])?; + for oid in self.to_compat.keys() { + wrtr.write_all(&oid.as_slice()[0..storage_short_len])?; + } + for oid in self.to_compat.keys() { + wrtr.write_all(oid.as_slice())?; + } + for meta in self.to_compat.values() { + wrtr.write_all(&(meta.kind as u32).to_be_bytes())?; + } + + wrtr.write_all(&PADDING[0..compat_npadding as usize])?; + for oid in self.to_storage.keys() { + wrtr.write_all(&oid.as_slice()[0..compat_short_len])?; + } + for meta in self.to_compat.values() { + wrtr.write_all(meta.oid.as_slice())?; + } + for meta in self.to_storage.values() { + wrtr.write_all(&(order_map[&meta.oid] as u32).to_be_bytes())?; + } + + Ok(()) + } + + fn required_nul_padding(nitems: usize, short_len: usize) -> u64 { + let shortened_table_len = nitems as u64 * short_len as u64; + let misalignment = shortened_table_len & 3; + // If the value is 0, return 0; otherwise, return the difference from 4. + (4 - misalignment) & 3 + } + + fn last_matching_offset(a: &ObjectID, b: &ObjectID, algop: HashAlgorithm) -> usize { + for i in 0..=algop.raw_len() { + if a.hash[i] != b.hash[i] { + return i; + } + } + algop.raw_len() + } + + fn find_short_name_len( + &self, + map: &BTreeMap<ObjectID, MappedObject>, + algop: HashAlgorithm, + ) -> usize { + if map.len() <= 1 { + return 1; + } + let mut len = 1; + let mut iter = map.keys(); + let mut cur = match iter.next() { + Some(cur) => cur, + None => return len, + }; + for item in iter { + let offset = Self::last_matching_offset(cur, item, algop); + if offset >= len { + len = offset + 1; + } + cur = item; + } + if len > algop.raw_len() { + algop.raw_len() + } else { + len + } + } +} + +struct ObjectFormatData { + data_off: usize, + shortened_len: usize, + full_off: usize, + mapping_off: Option<usize>, +} + +pub struct MmapedLooseObjectMapIter<'a> { + offset: usize, + algos: Vec<HashAlgorithm>, + source: &'a MmapedLooseObjectMap<'a>, +} + +impl<'a> Iterator for MmapedLooseObjectMapIter<'a> { + type Item = Vec<ObjectID>; + + fn next(&mut self) -> Option<Self::Item> { + if self.offset >= self.source.nitems { + return None; + } + let offset = self.offset; + self.offset += 1; + let v: Vec<ObjectID> = self + .algos + .iter() + .cloned() + .filter_map(|algo| self.source.oid_from_offset(offset, algo)) + .collect(); + if v.len() != self.algos.len() { + return None; + } + Some(v) + } +} + +#[allow(dead_code)] +pub struct MmapedLooseObjectMap<'a> { + memory: &'a [u8], + nitems: usize, + meta_off: usize, + obj_formats: BTreeMap<HashAlgorithm, ObjectFormatData>, + main_algo: HashAlgorithm, +} + +#[derive(Debug)] +#[allow(dead_code)] +enum MmapedParseError { + HeaderTooSmall, + InvalidSignature, + InvalidVersion, + UnknownAlgorithm, + OffsetTooLarge, + TooFewObjectFormats, + UnalignedData, + InvalidTrailerOffset, +} + +#[allow(dead_code)] +impl<'a> MmapedLooseObjectMap<'a> { + fn new( + slice: &'a [u8], + hash_algo: HashAlgorithm, + ) -> Result<MmapedLooseObjectMap<'a>, MmapedParseError> { + let object_format_header_size = 4 + 4 + 8; + let trailer_offset_size = 8; + let header_size: usize = + 4 + 4 + 4 + 4 + 4 + object_format_header_size * 2 + trailer_offset_size; + if slice.len() < header_size { + return Err(MmapedParseError::HeaderTooSmall); + } + if slice[0..4] != *b"LMAP" { + return Err(MmapedParseError::InvalidSignature); + } + if Self::u32_at_offset(slice, 4) != 1 { + return Err(MmapedParseError::InvalidVersion); + } + let _ = Self::u32_at_offset(slice, 8) as usize; + let nitems = Self::u32_at_offset(slice, 12) as usize; + let nobj_formats = Self::u32_at_offset(slice, 16) as usize; + if nobj_formats < 2 { + return Err(MmapedParseError::TooFewObjectFormats); + } + let mut offset = 20; + let mut meta_off = None; + let mut data = BTreeMap::new(); + for i in 0..nobj_formats { + if offset + object_format_header_size + trailer_offset_size > slice.len() { + return Err(MmapedParseError::HeaderTooSmall); + } + let format_id = Self::u32_at_offset(slice, offset); + let shortened_len = Self::u32_at_offset(slice, offset + 4) as usize; + let data_off = Self::u64_at_offset(slice, offset + 8); + + let algo = HashAlgorithm::from_format_id(format_id) + .ok_or(MmapedParseError::UnknownAlgorithm)?; + let data_off: usize = data_off + .try_into() + .map_err(|_| MmapedParseError::OffsetTooLarge)?; + + // Every object format must have these entries. + let shortened_table_len = shortened_len + .checked_mul(nitems) + .ok_or(MmapedParseError::OffsetTooLarge)?; + let full_off = data_off + .checked_add(shortened_table_len) + .ok_or(MmapedParseError::OffsetTooLarge)?; + Self::verify_aligned(full_off)?; + Self::verify_valid(slice, full_off as u64)?; + + let full_length = algo + .raw_len() + .checked_mul(nitems) + .ok_or(MmapedParseError::OffsetTooLarge)?; + let off = full_length + .checked_add(full_off) + .ok_or(MmapedParseError::OffsetTooLarge)?; + Self::verify_aligned(off)?; + Self::verify_valid(slice, off as u64)?; + + // This is for the metadata for the first object format and for the order mapping for + // other object formats. + let meta_size = nitems + .checked_mul(4) + .ok_or(MmapedParseError::OffsetTooLarge)?; + let meta_end = off + .checked_add(meta_size) + .ok_or(MmapedParseError::OffsetTooLarge)?; + Self::verify_valid(slice, meta_end as u64)?; + + let mut mapping_off = None; + if i == 0 { + meta_off = Some(off); + } else { + mapping_off = Some(off); + } + + data.insert( + algo, + ObjectFormatData { + data_off, + shortened_len, + full_off, + mapping_off, + }, + ); + offset += object_format_header_size; + } + let trailer = Self::u64_at_offset(slice, offset); + Self::verify_aligned(trailer as usize)?; + Self::verify_valid(slice, trailer)?; + let end = trailer + .checked_add(hash_algo.raw_len() as u64) + .ok_or(MmapedParseError::OffsetTooLarge)?; + if end != slice.len() as u64 { + return Err(MmapedParseError::InvalidTrailerOffset); + } + match meta_off { + Some(meta_off) => Ok(MmapedLooseObjectMap { + memory: slice, + nitems, + meta_off, + obj_formats: data, + main_algo: hash_algo, + }), + None => Err(MmapedParseError::TooFewObjectFormats), + } + } + + fn iter(&self) -> MmapedLooseObjectMapIter<'_> { + let mut algos = Vec::with_capacity(self.obj_formats.len()); + algos.push(self.main_algo); + for algo in self.obj_formats.keys().cloned() { + if algo != self.main_algo { + algos.push(algo); + } + } + MmapedLooseObjectMapIter { + offset: 0, + algos, + source: self, + } + } + + /// Treats `sl` as if it were a set of slices of `wanted.len()` bytes, and searches for + /// `wanted` within it. + /// + /// If found, returns the offset of the subslice in `sl`. + /// + /// ``` + /// let sl = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; + /// + /// assert_eq!(MmapedLooseObjectMap::binary_search_slice(sl, &[2, 3]), Some(1)); + /// assert_eq!(MmapedLooseObjectMap::binary_search_slice(sl, &[6, 7]), Some(4)); + /// assert_eq!(MmapedLooseObjectMap::binary_search_slice(sl, &[1, 2]), None); + /// assert_eq!(MmapedLooseObjectMap::binary_search_slice(sl, &[10, 20]), None); + /// ``` + fn binary_search_slice(sl: &[u8], wanted: &[u8]) -> Option<usize> { + let len = wanted.len(); + let res = sl.binary_search_by(|item| { + // We would like element_offset, but that is currently nightly only. Instead, do a + // pointer subtraction to find the index. + let index = unsafe { (item as *const u8).offset_from(sl.as_ptr()) } as usize; + // Now we have the index of this object. Round it down to the nearest full-sized + // chunk to find the actual offset where this starts. + let index = index - (index % len); + // Compute the comparison of that value instead, which will provide the expected + // result. + sl[index..index + wanted.len()].cmp(wanted) + }); + res.ok().map(|offset| offset / len) + } + + /// Look up `oid` in the map in order to convert it to `algo`. + /// + /// If this object is in the map, return the offset in the table for the main algorithm. + fn look_up_object(&self, oid: &ObjectID) -> Option<usize> { + let oid_algo = HashAlgorithm::from_u32(oid.algo)?; + let params = self.obj_formats.get(&oid_algo)?; + let short_table = + &self.memory[params.data_off..params.data_off + (params.shortened_len * self.nitems)]; + let index = + Self::binary_search_slice(short_table, &oid.as_slice()[0..params.shortened_len])?; + match params.mapping_off { + Some(from_off) => { + // oid is in a compatibility algorithm. Find the mapping index. + let mapped = Self::u32_at_offset(self.memory, from_off + index * 4) as usize; + if mapped >= self.nitems { + return None; + } + let oid_offset = params.full_off + mapped * oid_algo.raw_len(); + if self.memory[oid_offset..oid_offset + oid_algo.raw_len()] != *oid.as_slice() { + return None; + } + Some(mapped) + } + None => { + // oid is in the main algorithm. Find the object ID in the main map to confirm + // it's correct. + let oid_offset = params.full_off + index * oid_algo.raw_len(); + if self.memory[oid_offset..oid_offset + oid_algo.raw_len()] != *oid.as_slice() { + return None; + } + Some(index) + } + } + } + + #[allow(dead_code)] + fn map_object(&self, oid: &ObjectID, algo: HashAlgorithm) -> Option<MappedObject> { + let main = self.look_up_object(oid)?; + let meta = MapType::from_u32(Self::u32_at_offset(self.memory, self.meta_off + (main * 4)))?; + Some(MappedObject { + oid: self.oid_from_offset(main, algo)?, + kind: meta, + }) + } + + fn map_oid(&self, oid: &ObjectID, algo: HashAlgorithm) -> Option<ObjectID> { + if algo as u32 == oid.algo { + return Some(oid.clone()); + } + + let main = self.look_up_object(oid)?; + self.oid_from_offset(main, algo) + } + + fn oid_from_offset(&self, offset: usize, algo: HashAlgorithm) -> Option<ObjectID> { + let aparams = self.obj_formats.get(&algo)?; + + let mut hash = [0u8; GIT_MAX_RAWSZ]; + let len = algo.raw_len(); + let oid_off = aparams.full_off + (offset * len); + hash[0..len].copy_from_slice(&self.memory[oid_off..oid_off + len]); + Some(ObjectID { + hash, + algo: algo as u32, + }) + } + + fn u32_at_offset(slice: &[u8], offset: usize) -> u32 { + u32::from_be_bytes(slice[offset..offset + 4].try_into().unwrap()) + } + + fn u64_at_offset(slice: &[u8], offset: usize) -> u64 { + u64::from_be_bytes(slice[offset..offset + 8].try_into().unwrap()) + } + + fn verify_aligned(offset: usize) -> Result<(), MmapedParseError> { + if (offset & 3) != 0 { + return Err(MmapedParseError::UnalignedData); + } + Ok(()) + } + + fn verify_valid(slice: &[u8], offset: u64) -> Result<(), MmapedParseError> { + if offset >= slice.len() as u64 { + return Err(MmapedParseError::OffsetTooLarge); + } + Ok(()) + } +} + +/// A map for loose and other non-packed object IDs that maps between a storage and compatibility +/// mapping. +/// +/// In addition to the in-memory option, there is an optional batched storage, which can be used to +/// write objects to disk in an efficient way. +pub struct LooseObjectMap { + mem: LooseObjectMemoryMap, + batch: Option<LooseObjectMemoryMap>, +} + +impl LooseObjectMap { + /// Create a new `LooseObjectMap` with the given hash algorithms. + /// + /// This initializes the memory map to automatically map the empty tree, empty blob, and null + /// object ID. + pub fn new(storage: HashAlgorithm, compat: HashAlgorithm) -> LooseObjectMap { + let mut map = LooseObjectMemoryMap::new(storage, compat); + for (main, compat) in &[ + (storage.empty_tree(), compat.empty_tree()), + (storage.empty_blob(), compat.empty_blob()), + (storage.null_oid(), compat.null_oid()), + ] { + map.to_storage.insert( + (*compat).clone(), + MappedObject { + oid: (*main).clone(), + kind: MapType::Reserved, + }, + ); + map.to_compat.insert( + (*main).clone(), + MappedObject { + oid: (*compat).clone(), + kind: MapType::Reserved, + }, + ); + } + LooseObjectMap { + mem: map, + batch: None, + } + } + + pub fn hash_algo(&self) -> HashAlgorithm { + self.mem.storage + } + + /// Start a batch for efficient writing. + /// + /// If there is already a batch started, this does nothing and the existing batch is retained. + pub fn start_batch(&mut self) { + if self.batch.is_none() { + self.batch = Some(LooseObjectMemoryMap::new(self.mem.storage, self.mem.compat)); + } + } + + pub fn batch_len(&self) -> Option<usize> { + self.batch.as_ref().map(|b| b.len()) + } + + /// If a batch exists, write it to the writer. + pub fn finish_batch<W: Write>(&mut self, w: W) -> io::Result<()> { + if let Some(txn) = self.batch.take() { + txn.write(w)?; + } + Ok(()) + } + + /// If a batch exists, write it to the writer. + pub fn abort_batch(&mut self) { + self.batch = None; + } + + /// Return whether there is a batch already started. + /// + /// If you just want a batch to exist and don't care whether one has already been started, you + /// may simply call `start_batch` unconditionally. + pub fn has_batch(&self) -> bool { + self.batch.is_some() + } + + /// Insert an object into the map. + /// + /// If `write` is true and there is a batch started, write the object into the batch as well as + /// into the memory map. + pub fn insert(&mut self, oid1: &ObjectID, oid2: &ObjectID, kind: MapType, write: bool) { + let (compat_oid, storage_oid) = + if HashAlgorithm::from_u32(oid1.algo) == Some(self.mem.compat) { + (oid1, oid2) + } else { + (oid2, oid1) + }; + Self::insert_into(&mut self.mem, storage_oid, compat_oid, kind); + if write { + if let Some(ref mut batch) = self.batch { + Self::insert_into(batch, storage_oid, compat_oid, kind); + } + } + } + + fn insert_into( + map: &mut LooseObjectMemoryMap, + storage: &ObjectID, + compat: &ObjectID, + kind: MapType, + ) { + map.to_compat.insert( + storage.clone(), + MappedObject { + oid: compat.clone(), + kind, + }, + ); + map.to_storage.insert( + compat.clone(), + MappedObject { + oid: storage.clone(), + kind, + }, + ); + } + + #[allow(dead_code)] + fn map_object(&self, oid: &ObjectID, algo: HashAlgorithm) -> Option<&MappedObject> { + let map = if algo == self.mem.storage { + &self.mem.to_storage + } else { + &self.mem.to_compat + }; + map.get(oid) + } + + #[allow(dead_code)] + fn map_oid<'a, 'b: 'a>( + &'b self, + oid: &'a ObjectID, + algo: HashAlgorithm, + ) -> Option<&'a ObjectID> { + if algo as u32 == oid.algo { + return Some(oid); + } + let entry = self.map_object(oid, algo); + entry.map(|obj| &obj.oid) + } +} + +#[cfg(test)] +mod tests { + use super::{LooseObjectMap, LooseObjectMemoryMap, MapType, MmapedLooseObjectMap}; + use crate::hash::{HashAlgorithm, Hasher, ObjectID}; + use std::convert::TryInto; + use std::io::{self, Cursor, Write}; + + struct TrailingWriter { + curs: Cursor<Vec<u8>>, + hasher: Hasher, + } + + impl TrailingWriter { + fn new() -> TrailingWriter { + TrailingWriter { + curs: Cursor::new(Vec::new()), + hasher: Hasher::new(HashAlgorithm::SHA256), + } + } + + fn finalize(mut self) -> Vec<u8> { + let _ = self.hasher.flush(); + let mut v = self.curs.into_inner(); + v.extend(self.hasher.into_vec()); + v + } + } + + impl Write for TrailingWriter { + fn write(&mut self, data: &[u8]) -> io::Result<usize> { + self.hasher.write_all(data)?; + self.curs.write_all(data)?; + Ok(data.len()) + } + + fn flush(&mut self) -> io::Result<()> { + self.hasher.flush()?; + self.curs.flush()?; + Ok(()) + } + } + + fn sha1_oid(b: &[u8]) -> ObjectID { + assert_eq!(b.len(), 20); + let mut data = [0u8; 32]; + data[0..20].copy_from_slice(b); + ObjectID { + hash: data, + algo: HashAlgorithm::SHA1 as u32, + } + } + + fn sha256_oid(b: &[u8]) -> ObjectID { + assert_eq!(b.len(), 32); + ObjectID { + hash: b.try_into().unwrap(), + algo: HashAlgorithm::SHA256 as u32, + } + } + + fn test_entries() -> &'static [(&'static str, &'static [u8], &'static [u8], MapType, bool)] { + // These are all example blobs containing the content in the first argument. + &[ + ("abc", b"\xf2\xba\x8f\x84\xab\x5c\x1b\xce\x84\xa7\xb4\x41\xcb\x19\x59\xcf\xc7\x09\x3b\x7f", b"\xc1\xcf\x6e\x46\x50\x77\x93\x0e\x88\xdc\x51\x36\x64\x1d\x40\x2f\x72\xa2\x29\xdd\xd9\x96\xf6\x27\xd6\x0e\x96\x39\xea\xba\x35\xa6", MapType::LooseObject, false), + ("def", b"\x0c\x00\x38\x32\xe7\xbf\xa9\xca\x8b\x5c\x20\x35\xc9\xbd\x68\x4a\x5f\x26\x23\xbc", b"\x8a\x90\x17\x26\x48\x4d\xb0\xf2\x27\x9f\x30\x8d\x58\x96\xd9\x6b\xf6\x3a\xd6\xde\x95\x7c\xa3\x8a\xdc\x33\x61\x68\x03\x6e\xf6\x63", MapType::Shallow, true), + ("ghi", b"\x45\xa8\x2e\x29\x5c\x52\x47\x31\x14\xc5\x7c\x18\xf4\xf5\x23\x68\xdf\x2a\x3c\xfd", b"\x6e\x47\x4c\x74\xf5\xd7\x78\x14\xc7\xf7\xf0\x7c\x37\x80\x07\x90\x53\x42\xaf\x42\x81\xe6\x86\x8d\x33\x46\x45\x4b\xb8\x63\xab\xc3", MapType::Submodule, false), + ("jkl", b"\x45\x32\x8c\x36\xff\x2e\x9b\x9b\x4e\x59\x2c\x84\x7d\x3f\x9a\x7f\xd9\xb3\xe7\x16", b"\xc3\xee\xf7\x54\xa2\x1e\xc6\x9d\x43\x75\xbe\x6f\x18\x47\x89\xa8\x11\x6f\xd9\x66\xfc\x67\xdc\x31\xd2\x11\x15\x42\xc8\xd5\xa0\xaf", MapType::LooseObject, true), + ] + } + + fn test_map(write_all: bool) -> Box<LooseObjectMap> { + let mut map = Box::new(LooseObjectMap::new( + HashAlgorithm::SHA256, + HashAlgorithm::SHA1, + )); + + map.start_batch(); + + for (_blob_content, sha1, sha256, kind, swap) in test_entries() { + let s256 = sha256_oid(sha256); + let s1 = sha1_oid(sha1); + let write = write_all || (*kind as u32 & 2) == 0; + if *swap { + // Insert the item into the batch arbitrarily based on the type. This tests that + // we can specify either order and we'll do the right thing. + map.insert(&s256, &s1, *kind, write); + } else { + map.insert(&s1, &s256, *kind, write); + } + } + + map + } + + #[test] + fn can_read_and_write_format() { + for full in &[true, false] { + let mut map = test_map(*full); + let mut wrtr = TrailingWriter::new(); + map.finish_batch(&mut wrtr).unwrap(); + + assert_eq!(map.has_batch(), false); + + let data = wrtr.finalize(); + MmapedLooseObjectMap::new(&data, HashAlgorithm::SHA256).unwrap(); + } + } + + #[test] + fn looks_up_from_mmaped() { + let mut map = test_map(true); + let mut wrtr = TrailingWriter::new(); + map.finish_batch(&mut wrtr).unwrap(); + + assert_eq!(map.has_batch(), false); + + let data = wrtr.finalize(); + let entries = test_entries(); + let map = MmapedLooseObjectMap::new(&data, HashAlgorithm::SHA256).unwrap(); + + for (_, sha1, sha256, kind, _) in entries { + let s256 = sha256_oid(sha256); + let s1 = sha1_oid(sha1); + + let res = map.map_object(&s256, HashAlgorithm::SHA1).unwrap(); + assert_eq!(res.oid, s1); + assert_eq!(res.kind, *kind); + let res = map.map_oid(&s256, HashAlgorithm::SHA1).unwrap(); + assert_eq!(res, s1); + + let res = map.map_object(&s256, HashAlgorithm::SHA256).unwrap(); + assert_eq!(res.oid, s256); + assert_eq!(res.kind, *kind); + let res = map.map_oid(&s256, HashAlgorithm::SHA256).unwrap(); + assert_eq!(res, s256); + + let res = map.map_object(&s1, HashAlgorithm::SHA256).unwrap(); + assert_eq!(res.oid, s256); + assert_eq!(res.kind, *kind); + let res = map.map_oid(&s1, HashAlgorithm::SHA256).unwrap(); + assert_eq!(res, s256); + + let res = map.map_object(&s1, HashAlgorithm::SHA1).unwrap(); + assert_eq!(res.oid, s1); + assert_eq!(res.kind, *kind); + let res = map.map_oid(&s1, HashAlgorithm::SHA1).unwrap(); + assert_eq!(res, s1); + } + + for octet in &[0x00u8, 0x6d, 0x6e, 0x8a, 0xff] { + let missing_oid = ObjectID { + hash: [*octet; 32], + algo: HashAlgorithm::SHA256 as u32, + }; + + assert!(map.map_object(&missing_oid, HashAlgorithm::SHA1).is_none()); + assert!(map.map_oid(&missing_oid, HashAlgorithm::SHA1).is_none()); + + assert_eq!( + map.map_oid(&missing_oid, HashAlgorithm::SHA256).unwrap(), + missing_oid + ); + } + } + + #[test] + fn binary_searches_slices_correctly() { + let sl = &[ + 0, 1, 2, 15, 14, 13, 18, 10, 2, 20, 20, 20, 21, 21, 0, 21, 21, 1, 21, 21, 21, 21, 21, + 22, 22, 23, 24, + ]; + + let expected: &[(&[u8], Option<usize>)] = &[ + (&[0, 1, 2], Some(0)), + (&[15, 14, 13], Some(1)), + (&[18, 10, 2], Some(2)), + (&[20, 20, 20], Some(3)), + (&[21, 21, 0], Some(4)), + (&[21, 21, 1], Some(5)), + (&[21, 21, 21], Some(6)), + (&[21, 21, 22], Some(7)), + (&[22, 23, 24], Some(8)), + (&[2, 15, 14], None), + (&[0, 21, 21], None), + (&[21, 21, 23], None), + (&[22, 22, 23], None), + (&[0xff, 0xff, 0xff], None), + (&[0, 0, 0], None), + ]; + + for (wanted, value) in expected { + assert_eq!( + MmapedLooseObjectMap::binary_search_slice(sl, wanted), + *value + ); + } + } + + #[test] + fn looks_up_oid_correctly() { + let map = test_map(false); + let entries = test_entries(); + + let s256 = sha256_oid(entries[0].2); + let s1 = sha1_oid(entries[0].1); + + let missing_oid = ObjectID { + hash: [0xffu8; 32], + algo: HashAlgorithm::SHA256 as u32, + }; + + let res = map.map_object(&s256, HashAlgorithm::SHA1).unwrap(); + assert_eq!(res.oid, s1); + assert_eq!(res.kind, MapType::LooseObject); + let res = map.map_oid(&s256, HashAlgorithm::SHA1).unwrap(); + assert_eq!(*res, s1); + + let res = map.map_object(&s1, HashAlgorithm::SHA256).unwrap(); + assert_eq!(res.oid, s256); + assert_eq!(res.kind, MapType::LooseObject); + let res = map.map_oid(&s1, HashAlgorithm::SHA256).unwrap(); + assert_eq!(*res, s256); + + assert!(map.map_object(&missing_oid, HashAlgorithm::SHA1).is_none()); + assert!(map.map_oid(&missing_oid, HashAlgorithm::SHA1).is_none()); + + assert_eq!( + *map.map_oid(&missing_oid, HashAlgorithm::SHA256).unwrap(), + missing_oid + ); + } + + #[test] + fn looks_up_known_oids_correctly() { + let map = test_map(false); + + let funcs: &[&dyn Fn(HashAlgorithm) -> &'static ObjectID] = &[ + &|h: HashAlgorithm| h.empty_tree(), + &|h: HashAlgorithm| h.empty_blob(), + &|h: HashAlgorithm| h.null_oid(), + ]; + + for f in funcs { + let s256 = f(HashAlgorithm::SHA256); + let s1 = f(HashAlgorithm::SHA1); + + let res = map.map_object(&s256, HashAlgorithm::SHA1).unwrap(); + assert_eq!(res.oid, *s1); + assert_eq!(res.kind, MapType::Reserved); + let res = map.map_oid(&s256, HashAlgorithm::SHA1).unwrap(); + assert_eq!(*res, *s1); + + let res = map.map_object(&s1, HashAlgorithm::SHA256).unwrap(); + assert_eq!(res.oid, *s256); + assert_eq!(res.kind, MapType::Reserved); + let res = map.map_oid(&s1, HashAlgorithm::SHA256).unwrap(); + assert_eq!(*res, *s256); + } + } + + #[test] + fn nul_padding() { + assert_eq!(LooseObjectMemoryMap::required_nul_padding(1, 1), 3); + assert_eq!(LooseObjectMemoryMap::required_nul_padding(2, 1), 2); + assert_eq!(LooseObjectMemoryMap::required_nul_padding(3, 1), 1); + assert_eq!(LooseObjectMemoryMap::required_nul_padding(2, 2), 0); + + assert_eq!(LooseObjectMemoryMap::required_nul_padding(39, 3), 3); + } +} diff --git a/src/meson.build b/src/meson.build index 25b9ad5a14..45739957b4 100644 --- a/src/meson.build +++ b/src/meson.build @@ -1,5 +1,8 @@ libgit_rs_sources = [ + 'csum_file.rs', + 'hash.rs', 'lib.rs', + 'loose.rs', 'varint.rs', ] diff --git a/src/varint.rs b/src/varint.rs index 6e610bdd8e..06492dfc5e 100644 --- a/src/varint.rs +++ b/src/varint.rs @@ -1,3 +1,10 @@ +/// Decode the variable-length integer stored in `bufp` and return the decoded value. +/// +/// Returns 0 in case the decoded integer would overflow u64::MAX. +/// +/// # Safety +/// +/// The buffer must be NUL-terminated to ensure safety. #[no_mangle] pub unsafe extern "C" fn decode_varint(bufp: *mut *const u8) -> u64 { let mut buf = *bufp; @@ -22,6 +29,14 @@ pub unsafe extern "C" fn decode_varint(bufp: *mut *const u8) -> u64 { val } +/// Encode `value` into `buf` as a variable-length integer unless `buf` is null. +/// +/// Returns the number of bytes written, or, if `buf` is null, the number of bytes that would be +/// written to encode the integer. +/// +/// # Safety +/// +/// `buf` must either be null or point to at least 16 bytes of memory. #[no_mangle] pub unsafe extern "C" fn encode_varint(value: u64, buf: *mut u8) -> u8 { let mut varint: [u8; 16] = [0; 16]; diff --git a/streaming.c b/streaming.c index 4b13827668..00ad649ae3 100644 --- a/streaming.c +++ b/streaming.c @@ -230,12 +230,21 @@ static int open_istream_loose(struct git_istream *st, struct repository *r, enum object_type *type) { struct object_info oi = OBJECT_INFO_INIT; + struct odb_source *source; + oi.sizep = &st->size; oi.typep = type; - st->u.loose.mapped = map_loose_object(r, oid, &st->u.loose.mapsize); + odb_prepare_alternates(r->objects); + for (source = r->objects->sources; source; source = source->next) { + st->u.loose.mapped = odb_source_loose_map_object(source, oid, + &st->u.loose.mapsize); + if (st->u.loose.mapped) + break; + } if (!st->u.loose.mapped) return -1; + switch (unpack_loose_header(&st->z, st->u.loose.mapped, st->u.loose.mapsize, st->u.loose.hdr, sizeof(st->u.loose.hdr))) { diff --git a/submodule.c b/submodule.c index 35c55155f7..ff2f45457d 100644 --- a/submodule.c +++ b/submodule.c @@ -31,6 +31,7 @@ #include "commit-reach.h" #include "read-cache-ll.h" #include "setup.h" +#include "url.h" static int config_update_recurse_submodules = RECURSE_SUBMODULES_OFF; static int initialized_fetch_ref_tips; @@ -934,10 +935,7 @@ static void free_submodules_data(struct string_list *submodules) string_list_clear(submodules, 1); } -static int has_remote(const char *refname UNUSED, - const char *referent UNUSED, - const struct object_id *oid UNUSED, - int flags UNUSED, void *cb_data UNUSED) +static int has_remote(const struct reference *ref UNUSED, void *cb_data UNUSED) { return 1; } @@ -1255,13 +1253,10 @@ int push_unpushed_submodules(struct repository *r, return ret; } -static int append_oid_to_array(const char *ref UNUSED, - const char *referent UNUSED, - const struct object_id *oid, - int flags UNUSED, void *data) +static int append_oid_to_array(const struct reference *ref, void *data) { struct oid_array *array = data; - oid_array_append(array, oid); + oid_array_append(array, ref->oid); return 0; } @@ -2256,16 +2251,30 @@ out: return ret; } +/* + * Find the last submodule name in the gitdir path (modules can be nested). + * Returns a pointer into `path` to the beginning of the name or NULL if not found. + */ +static char *find_last_submodule_name(char *git_dir_path) +{ + const char *modules_marker = "/modules/"; + char *p = git_dir_path; + char *last = NULL; + + while ((p = strstr(p, modules_marker))) { + last = p + strlen(modules_marker); + p++; + } + + return last; +} + int validate_submodule_git_dir(char *git_dir, const char *submodule_name) { size_t len = strlen(git_dir), suffix_len = strlen(submodule_name); - char *p; - int ret = 0; - - if (len <= suffix_len || (p = git_dir + len - suffix_len)[-1] != '/' || - strcmp(p, submodule_name)) - BUG("submodule name '%s' not a suffix of git dir '%s'", - submodule_name, git_dir); + char *p = git_dir + len - suffix_len; + bool suffixes_match = !strcmp(p, submodule_name); + int ret = 0, config_ignorecase = 0; /* * We prevent the contents of sibling submodules' git directories to @@ -2277,7 +2286,7 @@ int validate_submodule_git_dir(char *git_dir, const char *submodule_name) * but the latter directory is already designated to contain the hooks * of the former. */ - for (; *p; p++) { + for (; *p && suffixes_match; p++) { if (is_dir_sep(*p)) { char c = *p; @@ -2294,6 +2303,51 @@ int validate_submodule_git_dir(char *git_dir, const char *submodule_name) } } + /* tests after this check are only for encoded names, when the extension is enabled */ + if (!the_repository->repository_format_submodule_encoding) + return 0; + + /* Prevent the use of '/' in names */ + p = find_last_submodule_name(git_dir); + if (p && strchr(p, '/') != NULL) + return error("submodule gitdir name '%s' contains unexpected '/'", p); + + /* Prevent conflicts on case-folding filesystems */ + repo_config_get_bool(the_repository, "core.ignorecase", &config_ignorecase); + if (ignore_case || config_ignorecase) { + char *lower_gitdir = xstrdup(git_dir); + char *module_name = find_last_submodule_name(lower_gitdir); + + if (module_name) { + for (p = module_name; *p; p++) + *p = tolower(*p); + + /* + * If lower path is different and already exists, check for collision. + * Intentionally double-check to eliminate false-positives. + */ + if (strcmp(lower_gitdir, git_dir) && is_git_directory(lower_gitdir)) { + char *canonical = real_pathdup(git_dir, 0); + if (canonical) { + struct strbuf norm_git_dir = STRBUF_INIT; + strbuf_addstr(&norm_git_dir, git_dir); + strbuf_normalize_path(&norm_git_dir); + + if (strcmp(canonical, norm_git_dir.buf)) + ret = error(_("submodule git dir '%s' " + "collides with '%s'"), + canonical, norm_git_dir.buf); + + strbuf_release(&norm_git_dir); + FREE_AND_NULL(canonical); + } + } + } + + FREE_AND_NULL(lower_gitdir); + return ret; + } + return 0; } @@ -2581,29 +2635,70 @@ cleanup: return ret; } +static int validate_and_set_submodule_gitdir(struct strbuf *gitdir_path, + const char *submodule_name) +{ + char *key; + + if (validate_submodule_git_dir(gitdir_path->buf, submodule_name)) + return -1; + + key = xstrfmt("submodule.%s.gitdir", submodule_name); + repo_config_set_gently(the_repository, key, gitdir_path->buf); + FREE_AND_NULL(key); + + return 0; + +} + void submodule_name_to_gitdir(struct strbuf *buf, struct repository *r, const char *submodule_name) { + const char *gitdir; + char *key; + + repo_git_path_append(r, buf, "modules/"); + strbuf_addstr(buf, submodule_name); + + /* If extensions.submoduleEncoding is disabled, use the plain path set above */ + if (!r->repository_format_submodule_encoding) + return; + + /* Extension is enabled: use the gitdir config if it exists */ + key = xstrfmt("submodule.%s.gitdir", submodule_name); + if (!repo_config_get_string_tmp(r, key, &gitdir)) { + strbuf_reset(buf); + strbuf_addstr(buf, gitdir); + FREE_AND_NULL(key); + return; + } + FREE_AND_NULL(key); + /* - * NEEDSWORK: The current way of mapping a submodule's name to - * its location in .git/modules/ has problems with some naming - * schemes. For example, if a submodule is named "foo" and - * another is named "foo/bar" (whether present in the same - * superproject commit or not - the problem will arise if both - * superproject commits have been checked out at any point in - * time), or if two submodule names only have different cases in - * a case-insensitive filesystem. - * - * There are several solutions, including encoding the path in - * some way, introducing a submodule.<name>.gitdir config in - * .git/config (not .gitmodules) that allows overriding what the - * gitdir of a submodule would be (and teach Git, upon noticing - * a clash, to automatically determine a non-clashing name and - * to write such a config), or introducing a - * submodule.<name>.gitdir config in .gitmodules that repo - * administrators can explicitly set. Nothing has been decided, - * so for now, just append the name at the end of the path. + * The gitdir config does not exist, even though the extension is enabled. + * Therefore we are in one of the following cases: */ + + /* Case 1: legacy migration of valid plain submodule names */ + if (!validate_and_set_submodule_gitdir(buf, submodule_name)) + return; + + /* Case 2.1: Try URI-safe (RFC3986) encoding first, this fixes nested gitdirs */ + strbuf_reset(buf); repo_git_path_append(r, buf, "modules/"); - strbuf_addstr(buf, submodule_name); + strbuf_addstr_urlencode(buf, submodule_name, is_rfc3986_unreserved); + if (!validate_and_set_submodule_gitdir(buf, submodule_name)) + return; + + /* Case 2.2: Try extended uppercase URI (RFC3986) encoding, to fix case-folding */ + strbuf_reset(buf); + repo_git_path_append(r, buf, "modules/"); + strbuf_addstr_urlencode(buf, submodule_name, is_casefolding_rfc3986_unreserved); + if (!validate_and_set_submodule_gitdir(buf, submodule_name)) + return; + + /* Case 3: error out */ + die(_("Cannot construct a valid gitdir path for submodule '%s': " + "please set a unique git config for 'submodule.%s.gitdir'."), + submodule_name, submodule_name); } diff --git a/t/for-each-ref-tests.sh b/t/for-each-ref-tests.sh index e3ad19298a..4593be5fd5 100644 --- a/t/for-each-ref-tests.sh +++ b/t/for-each-ref-tests.sh @@ -1809,7 +1809,9 @@ test_expect_success "${git_for_each_ref} reports broken tags" ' bad=$(git hash-object -w -t tag bad) && git update-ref refs/tags/broken-tag-bad $bad && test_must_fail ${git_for_each_ref} --format="%(*objectname)" \ - refs/tags/broken-tag-* + refs/tags/broken-tag-* && + test_must_fail ${git_for_each_ref} --format="%(*objectname)" \ + refs/tags/broken-tag-bad ' test_expect_success 'set up tag with signature and no blank lines' ' diff --git a/t/helper/test-delete-gpgsig.c b/t/helper/test-delete-gpgsig.c index e36831af03..658c7a37f7 100644 --- a/t/helper/test-delete-gpgsig.c +++ b/t/helper/test-delete-gpgsig.c @@ -23,8 +23,7 @@ int cmd__delete_gpgsig(int argc, const char **argv) if (!strcmp(pattern, "trailer")) { size_t payload_size = parse_signed_buffer(buf.buf, buf.len); fwrite(buf.buf, 1, payload_size, stdout); - fflush(stdout); - return 0; + goto out; } bufptr = buf.buf; @@ -56,7 +55,9 @@ int cmd__delete_gpgsig(int argc, const char **argv) fwrite(bufptr, 1, (eol - bufptr) + 1, stdout); bufptr = eol + 1; } - fflush(stdout); +out: + fflush(stdout); + strbuf_release(&buf); return 0; } diff --git a/t/helper/test-find-pack.c b/t/helper/test-find-pack.c index e001dc3066..fc4b8a77b3 100644 --- a/t/helper/test-find-pack.c +++ b/t/helper/test-find-pack.c @@ -39,11 +39,12 @@ int cmd__find_pack(int argc, const char **argv) if (repo_get_oid(the_repository, argv[0], &oid)) die("cannot parse %s as an object name", argv[0]); - for (p = packfile_store_get_all_packs(the_repository->objects->packfiles); p; p = p->next) + repo_for_each_pack(the_repository, p) { if (find_pack_entry_one(&oid, p)) { printf("%s\n", p->pack_name); actual_count++; } + } if (count > -1 && count != actual_count) die("bad packfile count %d instead of %d", actual_count, count); diff --git a/t/helper/test-pack-mtimes.c b/t/helper/test-pack-mtimes.c index 7c428c1601..7a8ee1de24 100644 --- a/t/helper/test-pack-mtimes.c +++ b/t/helper/test-pack-mtimes.c @@ -37,7 +37,7 @@ int cmd__pack_mtimes(int argc, const char **argv) if (argc != 2) usage(pack_mtimes_usage); - for (p = packfile_store_get_all_packs(the_repository->objects->packfiles); p; p = p->next) { + repo_for_each_pack(the_repository, p) { strbuf_addstr(&buf, basename(p->pack_name)); strbuf_strip_suffix(&buf, ".pack"); strbuf_addstr(&buf, ".mtimes"); diff --git a/t/helper/test-reach.c b/t/helper/test-reach.c index 028ec00306..c58c93800f 100644 --- a/t/helper/test-reach.c +++ b/t/helper/test-reach.c @@ -63,7 +63,7 @@ int cmd__reach(int ac, const char **av) die("failed to resolve %s", buf.buf + 2); orig = parse_object(r, &oid); - peeled = deref_tag_noverify(the_repository, orig); + peeled = deref_tag(the_repository, orig, NULL, 0); if (!peeled) die("failed to load commit for input %s resulting in oid %s", diff --git a/t/helper/test-ref-store.c b/t/helper/test-ref-store.c index 83b06d39a3..b1215947c5 100644 --- a/t/helper/test-ref-store.c +++ b/t/helper/test-ref-store.c @@ -154,10 +154,9 @@ static int cmd_rename_ref(struct ref_store *refs, const char **argv) return refs_rename_ref(refs, oldref, newref, logmsg); } -static int each_ref(const char *refname, const char *referent UNUSED, const struct object_id *oid, - int flags, void *cb_data UNUSED) +static int each_ref(const struct reference *ref, void *cb_data UNUSED) { - printf("%s %s 0x%x\n", oid_to_hex(oid), refname, flags); + printf("%s %s 0x%x\n", oid_to_hex(ref->oid), ref->name, ref->flags); return 0; } diff --git a/t/helper/test-run-command.c b/t/helper/test-run-command.c index 3719f23cc2..95152a0395 100644 --- a/t/helper/test-run-command.c +++ b/t/helper/test-run-command.c @@ -23,19 +23,26 @@ static int number_callbacks; static int parallel_next(struct child_process *cp, struct strbuf *err, void *cb, - void **task_cb UNUSED) + void **task_cb) { struct child_process *d = cb; if (number_callbacks >= 4) return 0; strvec_pushv(&cp->args, d->args.v); + cp->in = d->in; + cp->no_stdin = d->no_stdin; if (err) strbuf_addstr(err, "preloaded output of a child\n"); else fprintf(stderr, "preloaded output of a child\n"); number_callbacks++; + + /* test_stdin callback will use this to count remaining lines */ + *task_cb = xmalloc(sizeof(int)); + *(int*)(*task_cb) = 2; + return 1; } @@ -51,18 +58,61 @@ static int no_job(struct child_process *cp UNUSED, return 0; } +static void test_consume_sideband(struct strbuf *output, void *cb UNUSED) +{ + FILE *sideband; + + sideband = fopen("./sideband", "a"); + + strbuf_write(output, sideband); + fclose(sideband); +} + static int task_finished(int result UNUSED, struct strbuf *err, void *pp_cb UNUSED, - void *pp_task_cb UNUSED) + void *pp_task_cb) { if (err) strbuf_addstr(err, "asking for a quick stop\n"); else fprintf(stderr, "asking for a quick stop\n"); + if (pp_task_cb) + FREE_AND_NULL(pp_task_cb); return 1; } +static int task_finished_quiet(int result UNUSED, + struct strbuf *err UNUSED, + void *pp_cb UNUSED, + void *pp_task_cb) +{ + if (pp_task_cb) + FREE_AND_NULL(pp_task_cb); + return 0; +} + +static int test_stdin_pipe_feed(int hook_stdin_fd, void *cb UNUSED, void *task_cb) +{ + int *lines_remaining = task_cb; + + if (*lines_remaining) { + struct strbuf buf = STRBUF_INIT; + strbuf_addf(&buf, "sample stdin %d\n", --(*lines_remaining)); + if (write_in_full(hook_stdin_fd, buf.buf, buf.len) < 0) { + if (errno == EPIPE) { + /* child closed stdin, nothing more to do */ + strbuf_release(&buf); + return 1; + } + die_errno("write"); + } + strbuf_release(&buf); + } + + return !(*lines_remaining); +} + struct testsuite { struct string_list tests, failed; int next; @@ -157,6 +207,8 @@ static int testsuite(int argc, const char **argv) struct run_process_parallel_opts opts = { .get_next_task = next_test, .start_failure = test_failed, + .feed_pipe = test_stdin_pipe_feed, + .consume_sideband = test_consume_sideband, .task_finished = test_finished, .data = &suite, }; @@ -460,12 +512,23 @@ int cmd__run_command(int argc, const char **argv) if (!strcmp(argv[1], "run-command-parallel")) { opts.get_next_task = parallel_next; + opts.task_finished = task_finished_quiet; } else if (!strcmp(argv[1], "run-command-abort")) { opts.get_next_task = parallel_next; opts.task_finished = task_finished; } else if (!strcmp(argv[1], "run-command-no-jobs")) { opts.get_next_task = no_job; opts.task_finished = task_finished; + } else if (!strcmp(argv[1], "run-command-stdin")) { + proc.in = -1; + proc.no_stdin = 0; + opts.get_next_task = parallel_next; + opts.task_finished = task_finished_quiet; + opts.feed_pipe = test_stdin_pipe_feed; + } else if (!strcmp(argv[1], "run-command-sideband")) { + opts.get_next_task = parallel_next; + opts.consume_sideband = test_consume_sideband; + opts.task_finished = task_finished_quiet; } else { ret = 1; fprintf(stderr, "check usage\n"); diff --git a/t/lib-gpg.sh b/t/lib-gpg.sh index 937b876bd0..97268ae07c 100644 --- a/t/lib-gpg.sh +++ b/t/lib-gpg.sh @@ -9,6 +9,16 @@ GNUPGHOME="$(pwd)/gpghome" export GNUPGHOME +# All the "test_lazy_prereq GPG*" below should use +# `prepare_gnupghome()` either directly or through a call to +# `test_have_prereq GPG*`. That's because `gpg` and `gpgsm` +# only create the directory specified using "$GNUPGHOME" or +# `--homedir` if it's the default (usually "~/.gnupg"). +prepare_gnupghome() { + mkdir -p "$GNUPGHOME" && + chmod 0700 "$GNUPGHOME" +} + test_lazy_prereq GPG ' gpg_version=$(gpg --version 2>&1) test $? != 127 || exit 1 @@ -38,8 +48,7 @@ test_lazy_prereq GPG ' # To export ownertrust: # gpg --homedir /tmp/gpghome --export-ownertrust \ # > lib-gpg/ownertrust - mkdir "$GNUPGHOME" && - chmod 0700 "$GNUPGHOME" && + prepare_gnupghome && (gpgconf --kill all || : ) && gpg --homedir "${GNUPGHOME}" --import \ "$TEST_DIRECTORY"/lib-gpg/keyring.gpg && @@ -62,7 +71,16 @@ test_lazy_prereq GPG2 ' exit 1 ;; *) + prepare_gnupghome && (gpgconf --kill all || : ) && + + # NEEDSWORK: prepare_gnupghome() should definitely be + # called here, but it looks like it exposes a + # pre-existing, hidden bug by allowing some tests in + # t1016-compatObjectFormat.sh to run instead of being + # skipped. See: + # https://lore.kernel.org/git/ZoV8b2RvYxLOotSJ@teonanacatl.net/ + gpg --homedir "${GNUPGHOME}" --import \ "$TEST_DIRECTORY"/lib-gpg/keyring.gpg && gpg --homedir "${GNUPGHOME}" --import-ownertrust \ @@ -132,8 +150,7 @@ test_lazy_prereq GPGSSH ' test $? = 0 || exit 1; # Setup some keys and an allowed signers file - mkdir -p "${GNUPGHOME}" && - chmod 0700 "${GNUPGHOME}" && + prepare_gnupghome && (setfacl -k "${GNUPGHOME}" 2>/dev/null || true) && ssh-keygen -t ed25519 -N "" -C "git ed25519 key" -f "${GPGSSH_KEY_PRIMARY}" >/dev/null && ssh-keygen -t rsa -b 2048 -N "" -C "git rsa2048 key" -f "${GPGSSH_KEY_SECONDARY}" >/dev/null && diff --git a/t/lib-verify-submodule-gitdir-path.sh b/t/lib-verify-submodule-gitdir-path.sh new file mode 100644 index 0000000000..62794df976 --- /dev/null +++ b/t/lib-verify-submodule-gitdir-path.sh @@ -0,0 +1,24 @@ +# Helper to verify if repo $1 contains a submodule named $2 with gitdir path $3 + +# This does not check filesystem existence. That is done in submodule.c via the +# submodule_name_to_gitdir() API which this helper ends up calling. The gitdirs +# might or might not exist (e.g. when adding a new submodule), so this only +# checks the expected configuration path, which might be overridden by the user. + +verify_submodule_gitdir_path() { + repo="$1" && + name="$2" && + path="$3" && + ( + cd "$repo" && + # Compute expected absolute path + expected="$(git rev-parse --git-common-dir)/$path" && + expected="$(test-tool path-utils real_path "$expected")" && + # Compute actual absolute path + actual="$(git submodule--helper gitdir "$name")" && + actual="$(test-tool path-utils real_path "$actual")" && + echo "$expected" >expect && + echo "$actual" >actual && + test_cmp expect actual + ) +} diff --git a/t/meson.build b/t/meson.build index 401b24e50e..358477fbfc 100644 --- a/t/meson.build +++ b/t/meson.build @@ -238,6 +238,7 @@ integration_tests = [ 't1701-racy-split-index.sh', 't1800-hook.sh', 't1900-repo.sh', + 't1901-repo-structure.sh', 't2000-conflict-when-checking-files-out.sh', 't2002-checkout-cache-u.sh', 't2003-checkout-cache-mkdir.sh', @@ -384,6 +385,10 @@ integration_tests = [ 't3436-rebase-more-options.sh', 't3437-rebase-fixup-options.sh', 't3438-rebase-broken-files.sh', + 't3440-rebase-trailer.sh', + 't3450-history.sh', + 't3451-history-reword.sh', + 't3452-history-split.sh', 't3500-cherry.sh', 't3501-revert-cherry-pick.sh', 't3502-cherry-pick-merge.sh', @@ -883,6 +888,7 @@ integration_tests = [ 't7422-submodule-output.sh', 't7423-submodule-symlinks.sh', 't7424-submodule-mixed-ref-formats.sh', + 't7425-submodule-encoding.sh', 't7450-bad-git-dotfiles.sh', 't7500-commit-template-squash-signoff.sh', 't7501-commit-basic-functionality.sh', @@ -955,6 +961,7 @@ integration_tests = [ 't8012-blame-colors.sh', 't8013-blame-ignore-revs.sh', 't8014-blame-ignore-fuzzy.sh', + 't8015-blame-diff-algorithm.sh', 't8020-last-modified.sh', 't9001-send-email.sh', 't9002-column.sh', @@ -1037,6 +1044,7 @@ integration_tests = [ 't9303-fast-import-compression.sh', 't9304-fast-import-marks.sh', 't9305-fast-import-signatures.sh', + 't9306-fast-import-signed-tags.sh', 't9350-fast-export.sh', 't9351-fast-export-anonymize.sh', 't9400-git-cvsserver-server.sh', diff --git a/t/pack-refs-tests.sh b/t/pack-refs-tests.sh index 3dbcc01718..81086c3690 100644 --- a/t/pack-refs-tests.sh +++ b/t/pack-refs-tests.sh @@ -428,4 +428,34 @@ do ' done -test_done +test_expect_success 'pack-refs does not store invalid peeled tag value' ' + test_when_finished rm -rf repo && + git init repo && + ( + cd repo && + git commit --allow-empty --message initial && + + echo garbage >blob-content && + blob_id=$(git hash-object -w -t blob blob-content) && + + # Write an invalid tag into the object database. The tag itself + # is well-formed, but the tagged object is a blob while we + # claim that it is a commit. + cat >tag-content <<-EOF && + object $blob_id + type commit + tag bad-tag + tagger C O Mitter <committer@example.com> 1112354055 +0200 + + annotated + EOF + tag_id=$(git hash-object -w -t tag tag-content) && + git update-ref refs/tags/bad-tag "$tag_id" && + + # The packed-refs file should not contain the peeled object ID. + # If it did this would cause commands that use the peeled value + # to not notice this corrupted tag. + git pack-refs --all && + test_grep ! "^\^" .git/packed-refs + ) +' diff --git a/t/perf/p6010-merge-base.sh b/t/perf/p6010-merge-base.sh new file mode 100755 index 0000000000..54f52fa23e --- /dev/null +++ b/t/perf/p6010-merge-base.sh @@ -0,0 +1,101 @@ +#!/bin/sh + +test_description='Test git merge-base' + +. ./perf-lib.sh + +test_perf_fresh_repo + +# +# Creates lots of merges to make history traversal costly. In +# particular it creates 2^($max_level-1)-1 2-way merges on top of +# 2^($max_level-1) root commits. E.g., the commit history looks like +# this for a $max_level of 3: +# +# _1_ +# / \ +# 2 3 +# / \ / \ +# 4 5 6 7 +# +# The numbers are the fast-import marks, which also are the commit +# messages. 1 is the HEAD commit and a merge, 2 and 3 are also merges, +# 4-7 are the root commits. +# +build_history () { + local max_level="$1" && + local level="${2:-1}" && + local mark="${3:-1}" && + if test $level -eq $max_level + then + echo "reset refs/heads/master" && + echo "from $ZERO_OID" && + echo "commit refs/heads/master" && + echo "mark :$mark" && + echo "committer C <c@example.com> 1234567890 +0000" && + echo "data <<EOF" && + echo "$mark" && + echo "EOF" + else + local level1=$((level+1)) && + local mark1=$((2*mark)) && + local mark2=$((2*mark+1)) && + build_history $max_level $level1 $mark1 && + build_history $max_level $level1 $mark2 && + echo "commit refs/heads/master" && + echo "mark :$mark" && + echo "committer C <c@example.com> 1234567890 +0000" && + echo "data <<EOF" && + echo "$mark" && + echo "EOF" && + echo "from :$mark1" && + echo "merge :$mark2" + fi +} + +# +# Creates a new merge history in the same shape as build_history does, +# while reusing the same root commits. This way the two top commits +# have 2^($max_level-1) merge bases between them. +# +build_history2 () { + local max_level="$1" && + local level="${2:-1}" && + local mark="${3:-1}" && + if test $level -lt $max_level + then + local level1=$((level+1)) && + local mark1=$((2*mark)) && + local mark2=$((2*mark+1)) && + build_history2 $max_level $level1 $mark1 && + build_history2 $max_level $level1 $mark2 && + echo "commit refs/heads/master" && + echo "mark :$mark" && + echo "committer C <c@example.com> 1234567890 +0000" && + echo "data <<EOF" && + echo "$mark II" && + echo "EOF" && + echo "from :$mark1" && + echo "merge :$mark2" + fi +} + +test_expect_success 'setup' ' + max_level=15 && + build_history $max_level | git fast-import --export-marks=marks && + git tag one && + build_history2 $max_level | git fast-import --import-marks=marks --force && + git tag two && + git gc && + git log --format=%H --no-merges >expect +' + +test_perf 'git merge-base' ' + git merge-base --all one two >actual +' + +test_expect_success 'verify result' ' + test_cmp expect actual +' + +test_done diff --git a/t/t0008-ignores.sh b/t/t0008-ignores.sh index 273d71411f..db8bde280e 100755 --- a/t/t0008-ignores.sh +++ b/t/t0008-ignores.sh @@ -847,6 +847,17 @@ test_expect_success 'directories and ** matches' ' test_cmp expect actual ' +test_expect_success '** not confused by matching leading prefix' ' + cat >.gitignore <<-\EOF && + foo**/bar + EOF + git check-ignore foobar foo/bar >actual && + cat >expect <<-\EOF && + foo/bar + EOF + test_cmp expect actual +' + ############################################################################ # # test whitespace handling diff --git a/t/t0061-run-command.sh b/t/t0061-run-command.sh index 76d4936a87..f133d71783 100755 --- a/t/t0061-run-command.sh +++ b/t/t0061-run-command.sh @@ -164,6 +164,44 @@ test_expect_success 'run_command runs ungrouped in parallel with more tasks than test_line_count = 4 err ' +test_expect_success 'run_command can divert output' ' + test_when_finished rm sideband && + test-tool run-command run-command-sideband 3 sh -c "printf \"%s\n%s\n\" Hello World" 2>actual && + test_must_be_empty actual && + test_cmp expect sideband +' + +test_expect_success 'run_command listens to stdin' ' + cat >expect <<-\EOF && + preloaded output of a child + listening for stdin: + sample stdin 1 + sample stdin 0 + preloaded output of a child + listening for stdin: + sample stdin 1 + sample stdin 0 + preloaded output of a child + listening for stdin: + sample stdin 1 + sample stdin 0 + preloaded output of a child + listening for stdin: + sample stdin 1 + sample stdin 0 + EOF + + write_script stdin-script <<-\EOF && + echo "listening for stdin:" + while read line + do + echo "$line" + done + EOF + test-tool run-command run-command-stdin 2 ./stdin-script 2>actual && + test_cmp expect actual +' + cat >expect <<-EOF preloaded output of a child asking for a quick stop diff --git a/t/t0450/adoc-help-mismatches b/t/t0450/adoc-help-mismatches index 2c6ecd5fc8..8ee2d3f7c8 100644 --- a/t/t0450/adoc-help-mismatches +++ b/t/t0450/adoc-help-mismatches @@ -2,7 +2,6 @@ add am apply archive -bisect blame branch check-ref-format diff --git a/t/t0600-reffiles-backend.sh b/t/t0600-reffiles-backend.sh index 1e62c791d9..b11126ed47 100755 --- a/t/t0600-reffiles-backend.sh +++ b/t/t0600-reffiles-backend.sh @@ -477,9 +477,29 @@ test_expect_success SYMLINKS 'symref transaction supports symlinks' ' prepare commit EOF - git update-ref --no-deref --stdin <stdin && - test_path_is_symlink .git/TEST_SYMREF_HEAD && - test "$(test_readlink .git/TEST_SYMREF_HEAD)" = refs/heads/new + git update-ref --no-deref --stdin <stdin 2>err && + if test_have_prereq WITH_BREAKING_CHANGES + then + test_path_is_file .git/TEST_SYMREF_HEAD && + echo "ref: refs/heads/new" >expect && + test_cmp expect .git/TEST_SYMREF_HEAD && + test_must_be_empty err + else + test_path_is_symlink .git/TEST_SYMREF_HEAD && + test "$(test_readlink .git/TEST_SYMREF_HEAD)" = refs/heads/new && + cat >expect <<-EOF && + warning: ${SQ}core.preferSymlinkRefs=true${SQ} is nominated for removal. + hint: The use of symbolic links for symbolic refs is deprecated + hint: and will be removed in Git 3.0. The configuration that + hint: tells Git to use them is thus going away. You can unset + hint: it with: + hint: + hint: git config unset core.preferSymlinkRefs + hint: + hint: Git will then use the textual symref format instead. + EOF + test_cmp expect err + fi ' test_expect_success 'symref transaction supports false symlink config' ' diff --git a/t/t0601-reffiles-pack-refs.sh b/t/t0601-reffiles-pack-refs.sh index 12cf5d1dcb..3c706978ef 100755 --- a/t/t0601-reffiles-pack-refs.sh +++ b/t/t0601-reffiles-pack-refs.sh @@ -18,3 +18,5 @@ export GIT_TEST_DEFAULT_REF_FORMAT . ./test-lib.sh . "$TEST_DIRECTORY"/pack-refs-tests.sh + +test_done diff --git a/t/t0610-reftable-basics.sh b/t/t0610-reftable-basics.sh index 3ea5d51532..6575528f21 100755 --- a/t/t0610-reftable-basics.sh +++ b/t/t0610-reftable-basics.sh @@ -1135,4 +1135,32 @@ test_expect_success 'fetch: accessing FETCH_HEAD special ref works' ' test_cmp expect actual ' +test_expect_success 'writes do not persist peeled value for invalid tags' ' + test_when_finished rm -rf repo && + git init repo && + ( + cd repo && + git commit --allow-empty --message initial && + + # We cannot easily verify that the peeled value is not stored + # in the tables. Instead, we test this indirectly: we create + # two tags that both point to the same object, but they claim + # different object types. If we parse both tags we notice that + # the parsed tagged object has a mismatch between the two tags + # and bail out. + # + # If we instead use the persisted peeled value we would not + # even parse the tags. As such, we would not notice the + # discrepancy either and thus listing these tags would succeed. + git tag tag-1 -m "tag 1" && + git cat-file tag tag-1 >raw-tag && + sed "s/^type commit$/type blob/" <raw-tag >broken-tag && + broken_tag_id=$(git hash-object -w -t tag broken-tag) && + git update-ref refs/tags/tag-2 $broken_tag_id && + + test_must_fail git for-each-ref --format="%(*objectname)" refs/tags/ 2>err && + test_grep "bad tag pointer" err + ) +' + test_done diff --git a/t/t1006-cat-file.sh b/t/t1006-cat-file.sh index 1f61b666a7..29a9503523 100755 --- a/t/t1006-cat-file.sh +++ b/t/t1006-cat-file.sh @@ -241,10 +241,16 @@ hello_content="Hello World" hello_size=$(strlen "$hello_content") hello_oid=$(echo_without_newline "$hello_content" | git hash-object --stdin) -test_expect_success "setup" ' +test_expect_success "setup part 1" ' git config core.repositoryformatversion 1 && - git config extensions.objectformat $test_hash_algo && - git config extensions.compatobjectformat $test_compat_hash_algo && + git config extensions.objectformat $test_hash_algo +' + +test_expect_success RUST 'compat setup' ' + git config extensions.compatobjectformat $test_compat_hash_algo +' + +test_expect_success 'setup part 2' ' echo_without_newline "$hello_content" > hello && git update-index --add hello && echo_without_newline "$hello_content" > "path with spaces" && @@ -273,9 +279,13 @@ run_blob_tests () { ' } -hello_compat_oid=$(git rev-parse --output-object-format=$test_compat_hash_algo $hello_oid) run_blob_tests $hello_oid -run_blob_tests $hello_compat_oid + +if test_have_prereq RUST +then + hello_compat_oid=$(git rev-parse --output-object-format=$test_compat_hash_algo $hello_oid) + run_blob_tests $hello_compat_oid +fi test_expect_success '--batch-check without %(rest) considers whole line' ' echo "$hello_oid blob $hello_size" >expect && @@ -286,62 +296,76 @@ test_expect_success '--batch-check without %(rest) considers whole line' ' ' tree_oid=$(git write-tree) -tree_compat_oid=$(git rev-parse --output-object-format=$test_compat_hash_algo $tree_oid) tree_size=$((2 * $(test_oid rawsz) + 13 + 24)) -tree_compat_size=$((2 * $(test_oid --hash=compat rawsz) + 13 + 24)) tree_pretty_content="100644 blob $hello_oid hello${LF}100755 blob $hello_oid path with spaces${LF}" -tree_compat_pretty_content="100644 blob $hello_compat_oid hello${LF}100755 blob $hello_compat_oid path with spaces${LF}" run_tests 'tree' $tree_oid "" $tree_size "" "$tree_pretty_content" -run_tests 'tree' $tree_compat_oid "" $tree_compat_size "" "$tree_compat_pretty_content" run_tests 'blob' "$tree_oid:hello" "100644" $hello_size "" "$hello_content" $hello_oid -run_tests 'blob' "$tree_compat_oid:hello" "100644" $hello_size "" "$hello_content" $hello_compat_oid run_tests 'blob' "$tree_oid:path with spaces" "100755" $hello_size "" "$hello_content" $hello_oid -run_tests 'blob' "$tree_compat_oid:path with spaces" "100755" $hello_size "" "$hello_content" $hello_compat_oid + +if test_have_prereq RUST +then + tree_compat_oid=$(git rev-parse --output-object-format=$test_compat_hash_algo $tree_oid) + tree_compat_size=$((2 * $(test_oid --hash=compat rawsz) + 13 + 24)) + tree_compat_pretty_content="100644 blob $hello_compat_oid hello${LF}100755 blob $hello_compat_oid path with spaces${LF}" + + run_tests 'tree' $tree_compat_oid "" $tree_compat_size "" "$tree_compat_pretty_content" + run_tests 'blob' "$tree_compat_oid:hello" "100644" $hello_size "" "$hello_content" $hello_compat_oid + run_tests 'blob' "$tree_compat_oid:path with spaces" "100755" $hello_size "" "$hello_content" $hello_compat_oid +fi commit_message="Initial commit" commit_oid=$(echo_without_newline "$commit_message" | git commit-tree $tree_oid) -commit_compat_oid=$(git rev-parse --output-object-format=$test_compat_hash_algo $commit_oid) commit_size=$(($(test_oid hexsz) + 137)) -commit_compat_size=$(($(test_oid --hash=compat hexsz) + 137)) commit_content="tree $tree_oid author $GIT_AUTHOR_NAME <$GIT_AUTHOR_EMAIL> $GIT_AUTHOR_DATE committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE $commit_message" -commit_compat_content="tree $tree_compat_oid +run_tests 'commit' $commit_oid "" $commit_size "$commit_content" "$commit_content" + +if test_have_prereq RUST +then + commit_compat_oid=$(git rev-parse --output-object-format=$test_compat_hash_algo $commit_oid) + commit_compat_size=$(($(test_oid --hash=compat hexsz) + 137)) + commit_compat_content="tree $tree_compat_oid author $GIT_AUTHOR_NAME <$GIT_AUTHOR_EMAIL> $GIT_AUTHOR_DATE committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE $commit_message" -run_tests 'commit' $commit_oid "" $commit_size "$commit_content" "$commit_content" -run_tests 'commit' $commit_compat_oid "" $commit_compat_size "$commit_compat_content" "$commit_compat_content" + run_tests 'commit' $commit_compat_oid "" $commit_compat_size "$commit_compat_content" "$commit_compat_content" +fi tag_header_without_oid="type blob tag hellotag tagger $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL>" tag_header_without_timestamp="object $hello_oid $tag_header_without_oid" -tag_compat_header_without_timestamp="object $hello_compat_oid -$tag_header_without_oid" tag_description="This is a tag" tag_content="$tag_header_without_timestamp 0 +0000 $tag_description" -tag_compat_content="$tag_compat_header_without_timestamp 0 +0000 - -$tag_description" tag_oid=$(echo_without_newline "$tag_content" | git hash-object -t tag --stdin -w) tag_size=$(strlen "$tag_content") -tag_compat_oid=$(git rev-parse --output-object-format=$test_compat_hash_algo $tag_oid) -tag_compat_size=$(strlen "$tag_compat_content") - run_tests 'tag' $tag_oid "" $tag_size "$tag_content" "$tag_content" -run_tests 'tag' $tag_compat_oid "" $tag_compat_size "$tag_compat_content" "$tag_compat_content" + +if test_have_prereq RUST +then + tag_compat_header_without_timestamp="object $hello_compat_oid +$tag_header_without_oid" + tag_compat_content="$tag_compat_header_without_timestamp 0 +0000 + +$tag_description" + + tag_compat_oid=$(git rev-parse --output-object-format=$test_compat_hash_algo $tag_oid) + tag_compat_size=$(strlen "$tag_compat_content") + + run_tests 'tag' $tag_compat_oid "" $tag_compat_size "$tag_compat_content" "$tag_compat_content" +fi test_expect_success "Reach a blob from a tag pointing to it" ' echo_without_newline "$hello_content" >expect && @@ -590,7 +614,8 @@ flush" } batch_tests $hello_oid $tree_oid $tree_size $commit_oid $commit_size "$commit_content" $tag_oid $tag_size "$tag_content" -batch_tests $hello_compat_oid $tree_compat_oid $tree_compat_size $commit_compat_oid $commit_compat_size "$commit_compat_content" $tag_compat_oid $tag_compat_size "$tag_compat_content" + +test_have_prereq RUST && batch_tests $hello_compat_oid $tree_compat_oid $tree_compat_size $commit_compat_oid $commit_compat_size "$commit_compat_content" $tag_compat_oid $tag_compat_size "$tag_compat_content" test_expect_success FUNNYNAMES 'setup with newline in input' ' @@ -1226,7 +1251,10 @@ test_expect_success 'batch-check with a submodule' ' test_unconfig extensions.compatobjectformat && printf "160000 commit $(test_oid deadbeef)\tsub\n" >tree-with-sub && tree=$(git mktree <tree-with-sub) && - test_config extensions.compatobjectformat $test_compat_hash_algo && + if test_have_prereq RUST + then + test_config extensions.compatobjectformat $test_compat_hash_algo + fi && git cat-file --batch-check >actual <<-EOF && $tree:sub diff --git a/t/t1010-mktree.sh b/t/t1010-mktree.sh index e9973f7494..312fe6717a 100755 --- a/t/t1010-mktree.sh +++ b/t/t1010-mktree.sh @@ -11,10 +11,13 @@ test_expect_success setup ' git add "$d" || return 1 done && echo zero >one && - git update-index --add --info-only one && - git write-tree --missing-ok >tree.missing && - git ls-tree $(cat tree.missing) >top.missing && - git ls-tree -r $(cat tree.missing) >all.missing && + if test_have_prereq BROKEN_OBJECTS + then + git update-index --add --info-only one && + git write-tree --missing-ok >tree.missing && + git ls-tree $(cat tree.missing) >top.missing && + git ls-tree -r $(cat tree.missing) >all.missing + fi && echo one >one && git add one && git write-tree >tree && @@ -53,7 +56,7 @@ test_expect_success 'ls-tree output in wrong order given to mktree (2)' ' test_cmp tree.withsub actual ' -test_expect_success 'allow missing object with --missing' ' +test_expect_success BROKEN_OBJECTS 'allow missing object with --missing' ' git mktree --missing <top.missing >actual && test_cmp tree.missing actual ' diff --git a/t/t1016-compatObjectFormat.sh b/t/t1016-compatObjectFormat.sh index e88362fbe4..92d48b96a1 100755 --- a/t/t1016-compatObjectFormat.sh +++ b/t/t1016-compatObjectFormat.sh @@ -8,6 +8,12 @@ test_description='Test how well compatObjectFormat works' . ./test-lib.sh . "$TEST_DIRECTORY"/lib-gpg.sh +if ! test_have_prereq RUST +then + skip_all='interoperability requires a Git built with Rust' + test_done +fi + # All of the follow variables must be defined in the environment: # GIT_AUTHOR_NAME # GIT_AUTHOR_EMAIL @@ -21,6 +27,12 @@ test_description='Test how well compatObjectFormat works' # different hash functions result in the same content in the commits. # This means that when the commit is translated between hash functions # the commit is identical to the commit in the other repository. +# +# Similarly this test relies on: +# gpg --faked-system-time '20230918T154812! +# freezing the system time from gpg perspective so that two different +# runs of gpg applied to the same data result in identical signatures. +# compat_hash () { case "$1" in @@ -114,7 +126,7 @@ do git config core.repositoryformatversion 1 && git config extensions.objectformat $hash && git config extensions.compatobjectformat $(compat_hash $hash) && - test_config gpg.program $TEST_DIRECTORY/t1016/gpg && + git config gpg.program $TEST_DIRECTORY/t1016/gpg && echo "Hello World!" >hello && eval hello_${hash}_oid=$(git hash-object hello) && git update-index --add hello && diff --git a/t/t1016/gpg b/t/t1016/gpg index 2601cb18a5..34d6e055fc 100755 --- a/t/t1016/gpg +++ b/t/t1016/gpg @@ -1,2 +1,2 @@ #!/bin/sh -exec gpg --faked-system-time "20230918T154812" "$@" +exec gpg --faked-system-time '20230918T154812!' "$@" diff --git a/t/t1091-sparse-checkout-builtin.sh b/t/t1091-sparse-checkout-builtin.sh index ab3a105fff..b2da4feaef 100755 --- a/t/t1091-sparse-checkout-builtin.sh +++ b/t/t1091-sparse-checkout-builtin.sh @@ -1050,5 +1050,180 @@ test_expect_success 'check-rules null termination' ' test_cmp expect actual ' +test_expect_success 'clean' ' + git -C repo sparse-checkout set --cone deep/deeper1 && + git -C repo sparse-checkout reapply && + mkdir -p repo/deep/deeper2 repo/folder1/extra/inside && + + # Add untracked files + touch repo/deep/deeper2/file && + touch repo/folder1/extra/inside/file && + + test_must_fail git -C repo sparse-checkout clean 2>err && + grep "refusing to clean" err && + + git -C repo config clean.requireForce true && + test_must_fail git -C repo sparse-checkout clean 2>err && + grep "refusing to clean" err && + + cat >expect <<-\EOF && + Would remove deep/deeper2/ + Would remove folder1/ + EOF + + git -C repo sparse-checkout clean --dry-run >out && + test_cmp expect out && + test_path_exists repo/deep/deeper2 && + test_path_exists repo/folder1/extra/inside/file && + + cat >expect <<-\EOF && + Would remove deep/deeper2/file + Would remove folder1/extra/inside/file + EOF + + git -C repo sparse-checkout clean --dry-run --verbose >out && + test_cmp expect out && + + cat >expect <<-\EOF && + Removing deep/deeper2/ + Removing folder1/ + EOF + + git -C repo sparse-checkout clean -f >out && + test_cmp expect out && + + test_path_is_missing repo/deep/deeper2 && + test_path_is_missing repo/folder1 +' + +test_expect_success 'clean with sparse file states' ' + test_when_finished git reset --hard && + git -C repo sparse-checkout set --cone deep/deeper1 && + mkdir repo/folder2 && + + # The previous test case checked the -f option, so + # test the config option in this one. + git -C repo config clean.requireForce false && + + # create an untracked file and a modified file + touch repo/folder2/file && + echo dirty >repo/folder2/a && + + # First clean/reapply pass will do nothing. + git -C repo sparse-checkout clean >out && + test_must_be_empty out && + test_path_exists repo/folder2/a && + test_path_exists repo/folder2/file && + + git -C repo sparse-checkout reapply 2>err && + test_grep folder2 err && + test_path_exists repo/folder2/a && + test_path_exists repo/folder2/file && + + # Now, stage the change to the tracked file. + git -C repo add --sparse folder2/a && + + # Clean will continue not doing anything. + git -C repo sparse-checkout clean >out && + test_line_count = 0 out && + test_path_exists repo/folder2/a && + test_path_exists repo/folder2/file && + + # But we can reapply to remove the staged change. + git -C repo sparse-checkout reapply 2>err && + test_grep folder2 err && + test_path_is_missing repo/folder2/a && + test_path_exists repo/folder2/file && + + # We can clean now. + cat >expect <<-\EOF && + Removing folder2/ + EOF + git -C repo sparse-checkout clean >out && + test_cmp expect out && + test_path_is_missing repo/folder2 && + + # At the moment, the file is staged. + cat >expect <<-\EOF && + M folder2/a + EOF + + git -C repo status -s >out && + test_cmp expect out && + + # Reapply persists the modified state. + git -C repo sparse-checkout reapply && + cat >expect <<-\EOF && + M folder2/a + EOF + git -C repo status -s >out && + test_cmp expect out && + + # Committing the change leads to resolved status. + git -C repo commit -m "modified" && + git -C repo status -s >out && + test_must_be_empty out && + + # Repeat, but this time commit before reapplying. + mkdir repo/folder2/ && + echo dirtier >repo/folder2/a && + git -C repo add --sparse folder2/a && + git -C repo sparse-checkout clean >out && + test_must_be_empty out && + test_path_exists repo/folder2/a && + + # Committing without reapplying makes it look like a deletion + # due to no skip-worktree bit. + git -C repo commit -m "dirtier" && + git -C repo status -s >out && + test_must_be_empty out && + + git -C repo sparse-checkout reapply && + git -C repo status -s >out && + test_must_be_empty out +' + +test_expect_success 'sparse-checkout operations with merge conflicts' ' + git clone repo merge && + + ( + cd merge && + mkdir -p folder1/even/more/dirs && + echo base >folder1/even/more/dirs/file && + git add folder1 && + git commit -m "base" && + + git checkout -b right&& + echo right >folder1/even/more/dirs/file && + git commit -a -m "right" && + + git checkout -b left HEAD~1 && + echo left >folder1/even/more/dirs/file && + git commit -a -m "left" && + + git checkout -b merge && + git sparse-checkout set deep/deeper1 && + + test_must_fail git merge -m "will-conflict" right && + + test_must_fail git sparse-checkout clean -f 2>err && + grep "failed to convert index to a sparse index" err && + + echo merged >folder1/even/more/dirs/file && + git add --sparse folder1 && + git merge --continue && + + test_path_exists folder1/even/more/dirs/file && + + # clean does not remove the file, because the + # SKIP_WORKTREE bit was not cleared by the merge command. + git sparse-checkout clean -f >out && + test_line_count = 0 out && + test_path_exists folder1/even/more/dirs/file && + + git sparse-checkout reapply && + test_path_is_missing folder1 + ) +' test_done diff --git a/t/t1450-fsck.sh b/t/t1450-fsck.sh index 5ae86c42be..c4b651c2dc 100755 --- a/t/t1450-fsck.sh +++ b/t/t1450-fsck.sh @@ -454,6 +454,60 @@ test_expect_success 'tag with NUL in header' ' test_grep "error in tag $tag.*unterminated header: NUL at offset" out ' +test_expect_success 'tag accepts gpgsig header even if not validly signed' ' + test_oid_cache <<-\EOF && + header sha1:gpgsig-sha256 + header sha256:gpgsig + EOF + header=$(test_oid header) && + sha=$(git rev-parse HEAD) && + cat >good-tag <<-EOF && + object $sha + type commit + tag good + tagger T A Gger <tagger@example.com> 1234567890 -0000 + $header -----BEGIN PGP SIGNATURE----- + Not a valid signature + -----END PGP SIGNATURE----- + + This is a good tag. + EOF + + tag=$(git hash-object --literally -t tag -w --stdin <good-tag) && + test_when_finished "remove_object $tag" && + git update-ref refs/tags/good $tag && + test_when_finished "git update-ref -d refs/tags/good" && + git -c fsck.extraHeaderEntry=error fsck --tags +' + +test_expect_success 'tag rejects invalid headers' ' + test_oid_cache <<-\EOF && + header sha1:gpgsig-sha256 + header sha256:gpgsig + EOF + header=$(test_oid header) && + sha=$(git rev-parse HEAD) && + cat >bad-tag <<-EOF && + object $sha + type commit + tag good + tagger T A Gger <tagger@example.com> 1234567890 -0000 + $header -----BEGIN PGP SIGNATURE----- + Not a valid signature + -----END PGP SIGNATURE----- + junk + + This is a bad tag with junk at the end of the headers. + EOF + + tag=$(git hash-object --literally -t tag -w --stdin <bad-tag) && + test_when_finished "remove_object $tag" && + git update-ref refs/tags/bad $tag && + test_when_finished "git update-ref -d refs/tags/bad" && + test_must_fail git -c fsck.extraHeaderEntry=error fsck --tags 2>out && + test_grep "error in tag $tag.*invalid format - extra header" out +' + test_expect_success 'cleaned up' ' git fsck >actual 2>&1 && test_must_be_empty actual diff --git a/t/t1463-refs-optimize.sh b/t/t1463-refs-optimize.sh index c11c905d79..9afe3c1ed7 100755 --- a/t/t1463-refs-optimize.sh +++ b/t/t1463-refs-optimize.sh @@ -15,3 +15,5 @@ export GIT_TEST_DEFAULT_REF_FORMAT pack_refs='refs optimize' . "$TEST_DIRECTORY"/pack-refs-tests.sh + +test_done diff --git a/t/t1500-rev-parse.sh b/t/t1500-rev-parse.sh index 58a4583088..98c5a772bd 100755 --- a/t/t1500-rev-parse.sh +++ b/t/t1500-rev-parse.sh @@ -207,6 +207,40 @@ test_expect_success 'rev-parse --show-object-format in repo' ' grep "unknown mode for --show-object-format: squeamish-ossifrage" err ' + +test_expect_success RUST 'rev-parse --show-object-format in repo with compat mode' ' + mkdir repo && + ( + sane_unset GIT_DEFAULT_HASH && + cd repo && + git init --object-format=sha256 && + git config extensions.compatobjectformat sha1 && + echo sha256 >expect && + git rev-parse --show-object-format >actual && + test_cmp expect actual && + git rev-parse --show-object-format=storage >actual && + test_cmp expect actual && + git rev-parse --show-object-format=input >actual && + test_cmp expect actual && + git rev-parse --show-object-format=output >actual && + test_cmp expect actual && + echo sha1 >expect && + git rev-parse --show-object-format=compat >actual && + test_cmp expect actual && + test_must_fail git rev-parse --show-object-format=squeamish-ossifrage 2>err && + grep "unknown mode for --show-object-format: squeamish-ossifrage" err + ) && + mkdir repo2 && + ( + sane_unset GIT_DEFAULT_HASH && + cd repo2 && + git init --object-format=sha256 && + echo >expect && + git rev-parse --show-object-format=compat >actual && + test_cmp expect actual + ) +' + test_expect_success 'rev-parse --show-ref-format' ' test_detect_ref_format >expect && git rev-parse --show-ref-format >actual && diff --git a/t/t1900-repo.sh b/t/t1900-repo.sh index 2beba67889..51d55f11a5 100755 --- a/t/t1900-repo.sh +++ b/t/t1900-repo.sh @@ -4,6 +4,15 @@ test_description='test git repo-info' . ./test-lib.sh +# git-repo-info keys. It must contain the same keys listed in the const +# repo_info_fields, in lexicographical order. +REPO_INFO_KEYS=' + layout.bare + layout.shallow + object.format + references.format +' + # Test whether a key-value pair is correctly returned # # Usage: test_repo_info <label> <init command> <repo_name> <key> <expected value> @@ -110,4 +119,16 @@ test_expect_success 'git repo info uses the last requested format' ' test_cmp expected actual ' +test_expect_success 'git repo info --all returns all key-value pairs' ' + git repo info $REPO_INFO_KEYS >expect && + git repo info --all >actual && + test_cmp expect actual +' + +test_expect_success 'git repo info --all <key> aborts' ' + echo "fatal: --all and <key> cannot be used together" >expect && + test_must_fail git repo info --all object.format 2>actual && + test_cmp expect actual +' + test_done diff --git a/t/t1901-repo-structure.sh b/t/t1901-repo-structure.sh new file mode 100755 index 0000000000..36a71a144e --- /dev/null +++ b/t/t1901-repo-structure.sh @@ -0,0 +1,129 @@ +#!/bin/sh + +test_description='test git repo structure' + +. ./test-lib.sh + +test_expect_success 'empty repository' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + cat >expect <<-\EOF && + | Repository structure | Value | + | -------------------- | ----- | + | * References | | + | * Count | 0 | + | * Branches | 0 | + | * Tags | 0 | + | * Remotes | 0 | + | * Others | 0 | + | | | + | * Reachable objects | | + | * Count | 0 | + | * Commits | 0 | + | * Trees | 0 | + | * Blobs | 0 | + | * Tags | 0 | + EOF + + git repo structure >out 2>err && + + test_cmp expect out && + test_line_count = 0 err + ) +' + +test_expect_success 'repository with references and objects' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit_bulk 42 && + git tag -a foo -m bar && + + oid="$(git rev-parse HEAD)" && + git update-ref refs/remotes/origin/foo "$oid" && + + # Also creates a commit, tree, and blob. + git notes add -m foo && + + cat >expect <<-\EOF && + | Repository structure | Value | + | -------------------- | ----- | + | * References | | + | * Count | 4 | + | * Branches | 1 | + | * Tags | 1 | + | * Remotes | 1 | + | * Others | 1 | + | | | + | * Reachable objects | | + | * Count | 130 | + | * Commits | 43 | + | * Trees | 43 | + | * Blobs | 43 | + | * Tags | 1 | + EOF + + git repo structure >out 2>err && + + test_cmp expect out && + test_line_count = 0 err + ) +' + +test_expect_success 'keyvalue and nul format' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit_bulk 42 && + git tag -a foo -m bar && + + cat >expect <<-\EOF && + references.branches.count=1 + references.tags.count=1 + references.remotes.count=0 + references.others.count=0 + objects.commits.count=42 + objects.trees.count=42 + objects.blobs.count=42 + objects.tags.count=1 + EOF + + git repo structure --format=keyvalue >out 2>err && + + test_cmp expect out && + test_line_count = 0 err && + + # Replace key and value delimiters for nul format. + tr "\n=" "\0\n" <expect >expect_nul && + git repo structure --format=nul >out 2>err && + + test_cmp expect_nul out && + test_line_count = 0 err + ) +' + +test_expect_success 'progress meter option' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit foo && + + GIT_PROGRESS_DELAY=0 git repo structure --progress >out 2>err && + + test_file_not_empty out && + test_grep "Counting references: 2, done." err && + test_grep "Counting objects: 3, done." err && + + GIT_PROGRESS_DELAY=0 git repo structure --no-progress >out 2>err && + + test_file_not_empty out && + test_line_count = 0 err + ) +' + +test_done diff --git a/t/t2204-add-ignored.sh b/t/t2204-add-ignored.sh index 31eb233df5..aa55b219ab 100755 --- a/t/t2204-add-ignored.sh +++ b/t/t2204-add-ignored.sh @@ -89,4 +89,21 @@ do ' done +test_expect_success "exclude magic would not interfere with .gitignore" ' + test_write_lines dir file sub ign err out "*.o" >.gitignore && + >foo.o && + >foo.c && + test_must_fail git add foo.o 2>err && + test_grep "are ignored by one" err && + test_grep "hint: Use -f" err && + + git add ":(exclude)foo.o" && + git ls-files >actual && + cat >expect <<-\EOF && + .gitignore + foo.c + EOF + test_cmp expect actual +' + test_done diff --git a/t/t2401-worktree-prune.sh b/t/t2401-worktree-prune.sh index fe671d4197..f8f28c76ee 100755 --- a/t/t2401-worktree-prune.sh +++ b/t/t2401-worktree-prune.sh @@ -24,8 +24,8 @@ test_expect_success 'prune files inside $GIT_DIR/worktrees' ' Removing worktrees/abc: not a valid directory EOF test_cmp expect actual && - ! test -f .git/worktrees/abc && - ! test -d .git/worktrees + test_path_is_missing .git/worktrees/abc && + test_path_is_missing .git/worktrees ' test_expect_success 'prune directories without gitdir' ' @@ -36,8 +36,8 @@ Removing worktrees/def: gitdir file does not exist EOF git worktree prune --verbose 2>actual && test_cmp expect actual && - ! test -d .git/worktrees/def && - ! test -d .git/worktrees + test_path_is_missing .git/worktrees/def && + test_path_is_missing .git/worktrees ' test_expect_success SANITY 'prune directories with unreadable gitdir' ' @@ -47,8 +47,8 @@ test_expect_success SANITY 'prune directories with unreadable gitdir' ' chmod u-r .git/worktrees/def/gitdir && git worktree prune --verbose 2>actual && test_grep "Removing worktrees/def: unable to read gitdir file" actual && - ! test -d .git/worktrees/def && - ! test -d .git/worktrees + test_path_is_missing .git/worktrees/def && + test_path_is_missing .git/worktrees ' test_expect_success 'prune directories with invalid gitdir' ' @@ -57,8 +57,8 @@ test_expect_success 'prune directories with invalid gitdir' ' : >.git/worktrees/def/gitdir && git worktree prune --verbose 2>actual && test_grep "Removing worktrees/def: invalid gitdir file" actual && - ! test -d .git/worktrees/def && - ! test -d .git/worktrees + test_path_is_missing .git/worktrees/def && + test_path_is_missing .git/worktrees ' test_expect_success 'prune directories with gitdir pointing to nowhere' ' @@ -67,8 +67,8 @@ test_expect_success 'prune directories with gitdir pointing to nowhere' ' echo "$(pwd)"/nowhere >.git/worktrees/def/gitdir && git worktree prune --verbose 2>actual && test_grep "Removing worktrees/def: gitdir file points to non-existent location" actual && - ! test -d .git/worktrees/def && - ! test -d .git/worktrees + test_path_is_missing .git/worktrees/def && + test_path_is_missing .git/worktrees ' test_expect_success 'not prune locked checkout' ' @@ -76,23 +76,23 @@ test_expect_success 'not prune locked checkout' ' mkdir -p .git/worktrees/ghi && : >.git/worktrees/ghi/locked && git worktree prune && - test -d .git/worktrees/ghi + test_path_is_dir .git/worktrees/ghi ' test_expect_success 'not prune recent checkouts' ' test_when_finished rm -r .git/worktrees && git worktree add jlm HEAD && - test -d .git/worktrees/jlm && + test_path_is_dir .git/worktrees/jlm && rm -rf jlm && git worktree prune --verbose --expire=2.days.ago && - test -d .git/worktrees/jlm + test_path_is_dir .git/worktrees/jlm ' test_expect_success 'not prune proper checkouts' ' test_when_finished rm -r .git/worktrees && git worktree add --detach "$PWD/nop" main && git worktree prune && - test -d .git/worktrees/nop + test_path_is_dir .git/worktrees/nop ' test_expect_success 'prune duplicate (linked/linked)' ' @@ -103,8 +103,8 @@ test_expect_success 'prune duplicate (linked/linked)' ' mv .git/worktrees/w2/gitdir.new .git/worktrees/w2/gitdir && git worktree prune --verbose 2>actual && test_grep "duplicate entry" actual && - test -d .git/worktrees/w1 && - ! test -d .git/worktrees/w2 + test_path_is_dir .git/worktrees/w1 && + test_path_is_missing .git/worktrees/w2 ' test_expect_success 'prune duplicate (main/linked)' ' @@ -116,7 +116,7 @@ test_expect_success 'prune duplicate (main/linked)' ' mv repo wt && git -C wt worktree prune --verbose 2>actual && test_grep "duplicate entry" actual && - ! test -d .git/worktrees/wt + test_path_is_missing .git/worktrees/wt ' test_expect_success 'not prune proper worktrees inside linked worktree with relative paths' ' diff --git a/t/t3070-wildmatch.sh b/t/t3070-wildmatch.sh index 3da824117c..655bb1a0f2 100755 --- a/t/t3070-wildmatch.sh +++ b/t/t3070-wildmatch.sh @@ -235,6 +235,8 @@ match 1 1 1 1 aaaaaaabababab '*ab' match 1 1 1 1 'foo*' 'foo\*' match 0 0 0 0 foobar 'foo\*bar' match 1 1 1 1 'f\oo' 'f\\oo' +match 0 0 0 0 \ + 1 1 1 1 'foo\' 'foo\' match 1 1 1 1 ball '*[al]?' match 0 0 0 0 ten '[ten]' match 1 1 1 1 ten '**[!te]' diff --git a/t/t3440-rebase-trailer.sh b/t/t3440-rebase-trailer.sh new file mode 100755 index 0000000000..d0e0434664 --- /dev/null +++ b/t/t3440-rebase-trailer.sh @@ -0,0 +1,134 @@ +#!/bin/sh +# + +test_description='git rebase --trailer integration tests +We verify that --trailer works with the merge backend, +and that it is rejected early when the apply backend is requested.' + +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME + +. ./test-lib.sh +. "$TEST_DIRECTORY"/lib-rebase.sh # test_commit_message, helpers + +REVIEWED_BY_TRAILER="Reviewed-by: Dev <dev@example.com>" + +expect_trailer_msg() { + test_commit_message "$1" <<-EOF + $2 + + ${3:-$REVIEWED_BY_TRAILER} + EOF +} + +test_expect_success 'setup repo with a small history' ' + git commit --allow-empty -m "Initial empty commit" && + test_commit first file a && + test_commit second file && + git checkout -b conflict-branch first && + test_commit file-2 file-2 && + test_commit conflict file && + test_commit third file +' + +test_expect_success 'apply backend is rejected with --trailer' ' + head_before=$(git rev-parse HEAD) && + test_expect_code 128 \ + git rebase --apply --trailer "$REVIEWED_BY_TRAILER" \ + HEAD^ 2>err && + test_grep "fatal: --trailer requires the merge backend" err && + test_cmp_rev HEAD $head_before +' + +test_expect_success 'reject empty --trailer argument' ' + test_expect_code 128 git rebase -m --trailer "" HEAD^ 2>err && + test_grep "empty --trailer" err +' + +test_expect_success 'reject trailer with missing key before separator' ' + test_expect_code 128 git rebase -m --trailer ": no-key" HEAD^ 2>err && + test_grep "missing key before separator" err +' + +test_expect_success 'allow trailer with missing value after separator' ' + git rebase -m --trailer "Acked-by:" HEAD~1 third && + sed -e "s/_/ /g" <<-\EOF >expect && + third + + Acked-by:_ + EOF + test_commit_message HEAD expect +' + +test_expect_success 'CLI trailer duplicates allowed; replace policy keeps last' ' + git -c trailer.Bug.ifexists=replace -c trailer.Bug.ifmissing=add \ + rebase -m --trailer "Bug: 123" --trailer "Bug: 456" HEAD~1 third && + cat >expect <<-\EOF && + third + + Bug: 456 + EOF + test_commit_message HEAD expect +' + +test_expect_success 'multiple Signed-off-by trailers all preserved' ' + git rebase -m \ + --trailer "Signed-off-by: Dev A <a@example.com>" \ + --trailer "Signed-off-by: Dev B <b@example.com>" HEAD~1 third && + cat >expect <<-\EOF && + third + + Signed-off-by: Dev A <a@example.com> + Signed-off-by: Dev B <b@example.com> + EOF + test_commit_message HEAD expect +' + +test_expect_success 'rebase -m --trailer adds trailer after conflicts' ' + git checkout -B conflict-branch third && + test_commit fourth file && + test_must_fail git rebase -m \ + --trailer "$REVIEWED_BY_TRAILER" \ + second && + git checkout --theirs file && + git add file && + git rebase --continue && + expect_trailer_msg HEAD "fourth" && + expect_trailer_msg HEAD^ "third" +' + +test_expect_success '--trailer handles fixup commands in todo list' ' + git checkout -B fixup-trailer HEAD && + test_commit fixup-base base && + test_commit fixup-second second && + first_short=$(git rev-parse --short fixup-base) && + second_short=$(git rev-parse --short fixup-second) && + cat >todo <<EOF && +pick $first_short fixup-base +fixup $second_short fixup-second +EOF + ( + set_replace_editor todo && + git rebase -i --trailer "$REVIEWED_BY_TRAILER" HEAD~2 + ) && + expect_trailer_msg HEAD "fixup-base" && + git reset --hard fixup-second && + cat >todo <<EOF && +pick $first_short fixup-base +fixup -C $second_short fixup-second +EOF + ( + set_replace_editor todo && + git rebase -i --trailer "$REVIEWED_BY_TRAILER" HEAD~2 + ) && + expect_trailer_msg HEAD "fixup-second" +' + +test_expect_success 'rebase --root --trailer updates every commit' ' + git checkout first && + git -c trailer.review.key=Reviewed-by rebase --root \ + --trailer=review="Dev <dev@example.com>" && + expect_trailer_msg HEAD "first" && + expect_trailer_msg HEAD^ "Initial empty commit" +' +test_done diff --git a/t/t3450-history.sh b/t/t3450-history.sh new file mode 100755 index 0000000000..f513463b92 --- /dev/null +++ b/t/t3450-history.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +test_description='tests for git-history command' + +. ./test-lib.sh + +test_expect_success 'does nothing without any arguments' ' + test_must_fail git history 2>err && + test_grep "need a subcommand" err +' + +test_expect_success 'raises an error with unknown argument' ' + test_must_fail git history garbage 2>err && + test_grep "unknown subcommand: .garbage." err +' + +test_done diff --git a/t/t3451-history-reword.sh b/t/t3451-history-reword.sh new file mode 100755 index 0000000000..09dbc463c5 --- /dev/null +++ b/t/t3451-history-reword.sh @@ -0,0 +1,237 @@ +#!/bin/sh + +test_description='tests for git-history reword subcommand' + +. ./test-lib.sh + +reword_with_message () { + cat >message && + write_script fake-editor.sh <<-EOF && + cp "$(pwd)/message" "\$1" + EOF + test_set_editor "$(pwd)"/fake-editor.sh && + git history reword "$@" && + rm fake-editor.sh message +} + +test_expect_success 'refuses to work with merge commits' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit base && + git branch branch && + test_commit ours && + git switch branch && + test_commit theirs && + git switch - && + git merge theirs && + test_must_fail git history reword HEAD~ 2>err && + test_grep "cannot rearrange commit history with merges" err && + test_must_fail git history reword HEAD 2>err && + test_grep "cannot rearrange commit history with merges" err + ) +' + +test_expect_success 'refuses to work with unrelated commits' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit base && + git branch branch && + test_commit ours && + git switch branch && + test_commit theirs && + test_must_fail git history reword ours 2>err && + test_grep "split commit must be reachable from current HEAD commit" err + ) +' + +test_expect_success 'can reword tip of a branch' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit first && + test_commit second && + test_commit third && + + git symbolic-ref HEAD >expect && + reword_with_message HEAD <<-EOF && + third reworded + EOF + git symbolic-ref HEAD >actual && + test_cmp expect actual && + + cat >expect <<-EOF && + third reworded + second + first + EOF + git log --format=%s >actual && + test_cmp expect actual + ) +' + +test_expect_success 'can reword commit in the middle' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit first && + test_commit second && + test_commit third && + + git symbolic-ref HEAD >expect && + reword_with_message HEAD~ <<-EOF && + second reworded + EOF + git symbolic-ref HEAD >actual && + test_cmp expect actual && + + cat >expect <<-EOF && + third + second reworded + first + EOF + git log --format=%s >actual && + test_cmp expect actual + ) +' + +test_expect_success 'can reword root commit' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit first && + test_commit second && + test_commit third && + reword_with_message HEAD~2 <<-EOF && + first reworded + EOF + + cat >expect <<-EOF && + third + second + first reworded + EOF + git log --format=%s >actual && + test_cmp expect actual + ) +' + +test_expect_success 'can use editor to rewrite commit message' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit first && + + write_script fake-editor.sh <<-\EOF && + cp "$1" . && + printf "\namend a comment\n" >>"$1" + EOF + test_set_editor "$(pwd)"/fake-editor.sh && + git history reword HEAD && + + cat >expect <<-EOF && + first + + # Please enter the commit message for the reworded changes. Lines starting + # with ${SQ}#${SQ} will be ignored. + # Changes to be committed: + # new file: first.t + # + EOF + test_cmp expect COMMIT_EDITMSG && + + cat >expect <<-EOF && + first + + amend a comment + + EOF + git log --format=%B >actual && + test_cmp expect actual + ) +' + +# For now, git-history(1) does not yet execute any hooks. This is subject to +# change in the future, and if it does this test here is expected to start +# failing. In other words, this test is not an endorsement of the current +# status quo. +test_expect_success 'hooks are not executed for rewritten commits' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit first && + test_commit second && + test_commit third && + + write_script .git/hooks/prepare-commit-msg <<-EOF && + touch "$(pwd)/hooks.log + EOF + write_script .git/hooks/post-commit <<-EOF && + touch "$(pwd)/hooks.log + EOF + write_script .git/hooks/post-rewrite <<-EOF && + touch "$(pwd)/hooks.log + EOF + + reword_with_message HEAD~ <<-EOF && + second reworded + EOF + + cat >expect <<-EOF && + third + second reworded + first + EOF + git log --format=%s >actual && + test_cmp expect actual && + + test_path_is_missing hooks.log + ) +' + +test_expect_success 'aborts with empty commit message' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit first && + + ! reword_with_message HEAD 2>err </dev/null && + test_grep "Aborting commit due to empty commit message." err + ) +' + +test_expect_success 'retains changes in the worktree and index' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + touch a b && + git add . && + git commit -m "initial commit" && + echo foo >a && + echo bar >b && + git add b && + reword_with_message HEAD <<-EOF && + message + EOF + cat >expect <<-\EOF && + M a + M b + ?? actual + ?? expect + EOF + git status --porcelain >actual && + test_cmp expect actual + ) +' + +test_done diff --git a/t/t3452-history-split.sh b/t/t3452-history-split.sh new file mode 100755 index 0000000000..2aac28afdf --- /dev/null +++ b/t/t3452-history-split.sh @@ -0,0 +1,432 @@ +#!/bin/sh + +test_description='tests for git-history split subcommand' + +. ./test-lib.sh + +set_fake_editor () { + write_script fake-editor.sh <<-EOF && + echo "$@" >"\$1" + EOF + test_set_editor "$(pwd)"/fake-editor.sh +} + +expect_log () { + git log --format="%s" >actual && + cat >expect && + test_cmp expect actual +} + +expect_tree_entries () { + git ls-tree --name-only "$1" >actual && + cat >expect && + test_cmp expect actual +} + +test_expect_success 'refuses to work with merge commits' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit base && + git branch branch && + test_commit ours && + git switch branch && + test_commit theirs && + git switch - && + git merge theirs && + test_must_fail git history split HEAD 2>err && + test_grep "cannot rearrange commit history with merges" err && + test_must_fail git history split HEAD~ 2>err && + test_grep "cannot rearrange commit history with merges" err + ) +' + +test_expect_success 'refuses to work with unrelated commits' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit base && + git branch branch && + test_commit ours && + git switch branch && + test_commit theirs && + test_must_fail git history split ours 2>err && + test_grep "split commit must be reachable from current HEAD commit" err + ) +' + +test_expect_success 'can split up tip commit' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit initial && + touch bar foo && + git add . && + git commit -m split-me && + + git symbolic-ref HEAD >expect && + set_fake_editor "split-out commit" && + git history split HEAD <<-EOF && + y + n + EOF + git symbolic-ref HEAD >actual && + test_cmp expect actual && + + expect_log <<-EOF && + split-me + split-out commit + initial + EOF + + expect_tree_entries HEAD~ <<-EOF && + bar + initial.t + EOF + + expect_tree_entries HEAD <<-EOF + bar + foo + initial.t + EOF + ) +' + +test_expect_success 'can split up root commit' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + touch bar foo && + git add . && + git commit -m root && + test_commit tip && + + set_fake_editor "split-out commit" && + git history split HEAD~ <<-EOF && + y + n + EOF + + expect_log <<-EOF && + tip + root + split-out commit + EOF + + expect_tree_entries HEAD~2 <<-EOF && + bar + EOF + + expect_tree_entries HEAD~ <<-EOF && + bar + foo + EOF + + expect_tree_entries HEAD <<-EOF + bar + foo + tip.t + EOF + ) +' + +test_expect_success 'can split up in-between commit' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit initial && + touch bar foo && + git add . && + git commit -m split-me && + test_commit tip && + + set_fake_editor "split-out commit" && + git history split HEAD~ <<-EOF && + y + n + EOF + + expect_log <<-EOF && + tip + split-me + split-out commit + initial + EOF + + expect_tree_entries HEAD~2 <<-EOF && + bar + initial.t + EOF + + expect_tree_entries HEAD~ <<-EOF && + bar + foo + initial.t + EOF + + expect_tree_entries HEAD <<-EOF + bar + foo + initial.t + tip.t + EOF + ) +' + +test_expect_success 'can pick multiple hunks' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + touch bar baz foo qux && + git add . && + git commit -m split-me && + + set_fake_editor "split-out-commit" && + git history split HEAD <<-EOF && + y + n + y + n + EOF + + expect_tree_entries HEAD~ <<-EOF && + bar + foo + EOF + + expect_tree_entries HEAD <<-EOF + bar + baz + foo + qux + EOF + ) +' + + +test_expect_success 'can use only last hunk' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + touch bar foo && + git add . && + git commit -m split-me && + + set_fake_editor "split-out commit" && + git history split HEAD <<-EOF && + n + y + EOF + + expect_log <<-EOF && + split-me + split-out commit + EOF + + expect_tree_entries HEAD~ <<-EOF && + foo + EOF + + expect_tree_entries HEAD <<-EOF + bar + foo + EOF + ) +' + +test_expect_success 'aborts with empty commit message' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + touch bar foo && + git add . && + git commit -m split-me && + + set_fake_editor "" && + test_must_fail git history split HEAD <<-EOF 2>err && + y + n + EOF + test_grep "Aborting commit due to empty commit message." err + ) +' + +test_expect_success 'commit message editor sees split-out changes' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + touch bar foo && + git add . && + git commit -m split-me && + + write_script fake-editor.sh <<-\EOF && + cp "$1" . && + echo "some commit message" >>"$1" + EOF + test_set_editor "$(pwd)"/fake-editor.sh && + + git history split HEAD <<-EOF && + y + n + EOF + + cat >expect <<-EOF && + + # Please enter the commit message for the split-out changes. Lines starting + # with ${SQ}#${SQ} will be ignored. + # Changes to be committed: + # new file: bar + # + EOF + test_cmp expect COMMIT_EDITMSG && + + expect_log <<-EOF + split-me + some commit message + EOF + ) +' + +test_expect_success 'can use pathspec to limit what gets split' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + touch bar foo && + git add . && + git commit -m split-me && + + set_fake_editor "split-out commit" && + git history split HEAD -- foo <<-EOF && + y + EOF + + expect_tree_entries HEAD~ <<-EOF && + foo + EOF + + expect_tree_entries HEAD <<-EOF + bar + foo + EOF + ) +' + +test_expect_success 'refuses to create empty split-out commit' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit base && + touch bar foo && + git add . && + git commit -m split-me && + + test_must_fail git history split HEAD 2>err <<-EOF && + n + n + EOF + test_grep "split commit is empty" err + ) +' + +test_expect_success 'hooks are executed for rewritten commits' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + touch bar foo && + git add . && + git commit -m split-me && + old_head=$(git rev-parse HEAD) && + + write_script .git/hooks/prepare-commit-msg <<-EOF && + touch "$(pwd)/hooks.log" + EOF + write_script .git/hooks/post-commit <<-EOF && + touch "$(pwd)/hooks.log" + EOF + write_script .git/hooks/post-rewrite <<-EOF && + touch "$(pwd)/hooks.log" + EOF + + set_fake_editor "split-out commit" && + git history split HEAD <<-EOF && + y + n + EOF + + expect_log <<-EOF && + split-me + split-out commit + EOF + + test_path_is_missing hooks.log + ) +' + +test_expect_success 'refuses to create empty original commit' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + touch bar foo && + git add . && + git commit -m split-me && + + test_must_fail git history split HEAD 2>err <<-EOF && + y + y + EOF + test_grep "split commit tree matches original commit" err + ) +' + +test_expect_success 'retains changes in the worktree and index' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + echo a >a && + echo b >b && + git add . && + git commit -m "initial commit" && + echo a-modified >a && + echo b-modified >b && + git add b && + set_fake_editor "a-only" && + git history split HEAD <<-EOF && + y + n + EOF + + expect_tree_entries HEAD~ <<-EOF && + a + EOF + expect_tree_entries HEAD <<-EOF && + a + b + EOF + + cat >expect <<-\EOF && + M a + M b + ?? actual + ?? expect + ?? fake-editor.sh + EOF + git status --porcelain >actual && + test_cmp expect actual + ) +' + +test_done diff --git a/t/t3650-replay-basics.sh b/t/t3650-replay-basics.sh index 58b3759935..cf3aacf355 100755 --- a/t/t3650-replay-basics.sh +++ b/t/t3650-replay-basics.sh @@ -52,7 +52,7 @@ test_expect_success 'setup bare' ' ' test_expect_success 'using replay to rebase two branches, one on top of other' ' - git replay --onto main topic1..topic2 >result && + git replay --ref-action=print --onto main topic1..topic2 >result && test_line_count = 1 result && @@ -68,7 +68,7 @@ test_expect_success 'using replay to rebase two branches, one on top of other' ' ' test_expect_success 'using replay on bare repo to rebase two branches, one on top of other' ' - git -C bare replay --onto main topic1..topic2 >result-bare && + git -C bare replay --ref-action=print --onto main topic1..topic2 >result-bare && test_cmp expect result-bare ' @@ -86,7 +86,7 @@ test_expect_success 'using replay to perform basic cherry-pick' ' # 2nd field of result is refs/heads/main vs. refs/heads/topic2 # 4th field of result is hash for main instead of hash for topic2 - git replay --advance main topic1..topic2 >result && + git replay --ref-action=print --advance main topic1..topic2 >result && test_line_count = 1 result && @@ -102,7 +102,7 @@ test_expect_success 'using replay to perform basic cherry-pick' ' ' test_expect_success 'using replay on bare repo to perform basic cherry-pick' ' - git -C bare replay --advance main topic1..topic2 >result-bare && + git -C bare replay --ref-action=print --advance main topic1..topic2 >result-bare && test_cmp expect result-bare ' @@ -115,7 +115,7 @@ test_expect_success 'replay fails when both --advance and --onto are omitted' ' ' test_expect_success 'using replay to also rebase a contained branch' ' - git replay --contained --onto main main..topic3 >result && + git replay --ref-action=print --contained --onto main main..topic3 >result && test_line_count = 2 result && cut -f 3 -d " " result >new-branch-tips && @@ -139,12 +139,12 @@ test_expect_success 'using replay to also rebase a contained branch' ' ' test_expect_success 'using replay on bare repo to also rebase a contained branch' ' - git -C bare replay --contained --onto main main..topic3 >result-bare && + git -C bare replay --ref-action=print --contained --onto main main..topic3 >result-bare && test_cmp expect result-bare ' test_expect_success 'using replay to rebase multiple divergent branches' ' - git replay --onto main ^topic1 topic2 topic4 >result && + git replay --ref-action=print --onto main ^topic1 topic2 topic4 >result && test_line_count = 2 result && cut -f 3 -d " " result >new-branch-tips && @@ -168,7 +168,7 @@ test_expect_success 'using replay to rebase multiple divergent branches' ' ' test_expect_success 'using replay on bare repo to rebase multiple divergent branches, including contained ones' ' - git -C bare replay --contained --onto main ^main topic2 topic3 topic4 >result && + git -C bare replay --ref-action=print --contained --onto main ^main topic2 topic3 topic4 >result && test_line_count = 4 result && cut -f 3 -d " " result >new-branch-tips && @@ -217,4 +217,101 @@ test_expect_success 'merge.directoryRenames=false' ' --onto rename-onto rename-onto..rename-from ' +test_expect_success 'default atomic behavior updates refs directly' ' + # Use a separate branch to avoid contaminating topic2 for later tests + git branch test-atomic topic2 && + test_when_finished "git branch -D test-atomic" && + + # Test default atomic behavior (no output, refs updated) + git replay --onto main topic1..test-atomic >output && + test_must_be_empty output && + + # Verify ref was updated + git log --format=%s test-atomic >actual && + test_write_lines E D M L B A >expect && + test_cmp expect actual && + + # Verify reflog message includes SHA of onto commit + git reflog test-atomic -1 --format=%gs >reflog-msg && + ONTO_SHA=$(git rev-parse main) && + echo "replay --onto $ONTO_SHA" >expect-reflog && + test_cmp expect-reflog reflog-msg +' + +test_expect_success 'atomic behavior in bare repository' ' + # Store original state for cleanup + START=$(git -C bare rev-parse topic2) && + test_when_finished "git -C bare update-ref refs/heads/topic2 $START" && + + # Test atomic updates work in bare repo + git -C bare replay --onto main topic1..topic2 >output && + test_must_be_empty output && + + # Verify ref was updated in bare repo + git -C bare log --format=%s topic2 >actual && + test_write_lines E D M L B A >expect && + test_cmp expect actual +' + +test_expect_success 'reflog message for --advance mode' ' + # Store original state + START=$(git rev-parse main) && + test_when_finished "git update-ref refs/heads/main $START" && + + # Test --advance mode reflog message + git replay --advance main topic1..topic2 >output && + test_must_be_empty output && + + # Verify reflog message includes --advance and branch name + git reflog main -1 --format=%gs >reflog-msg && + echo "replay --advance main" >expect-reflog && + test_cmp expect-reflog reflog-msg +' + +test_expect_success 'replay.refAction=print config option' ' + # Store original state + START=$(git rev-parse topic2) && + test_when_finished "git branch -f topic2 $START" && + + # Test with config set to print + test_config replay.refAction print && + git replay --onto main topic1..topic2 >output && + test_line_count = 1 output && + test_grep "^update refs/heads/topic2 " output +' + +test_expect_success 'replay.refAction=update config option' ' + # Store original state + START=$(git rev-parse topic2) && + test_when_finished "git branch -f topic2 $START" && + + # Test with config set to update + test_config replay.refAction update && + git replay --onto main topic1..topic2 >output && + test_must_be_empty output && + + # Verify ref was updated + git log --format=%s topic2 >actual && + test_write_lines E D M L B A >expect && + test_cmp expect actual +' + +test_expect_success 'command-line --ref-action overrides config' ' + # Store original state + START=$(git rev-parse topic2) && + test_when_finished "git branch -f topic2 $START" && + + # Set config to update but use --ref-action=print + test_config replay.refAction update && + git replay --ref-action=print --onto main topic1..topic2 >output && + test_line_count = 1 output && + test_grep "^update refs/heads/topic2 " output +' + +test_expect_success 'invalid replay.refAction value' ' + test_config replay.refAction invalid && + test_must_fail git replay --onto main topic1..topic2 2>error && + test_grep "invalid.*replay.refAction.*value" error +' + test_done diff --git a/t/t3701-add-interactive.sh b/t/t3701-add-interactive.sh index b5e6edcb9e..4285314f35 100755 --- a/t/t3701-add-interactive.sh +++ b/t/t3701-add-interactive.sh @@ -48,8 +48,8 @@ test_expect_success 'unknown command' ' git add -N command && git diff command >expect && cat >>expect <<-EOF && - (1/1) Stage addition [y,n,q,a,d,e,p,?]? Unknown command ${SQ}W${SQ} (use ${SQ}?${SQ} for help) - (1/1) Stage addition [y,n,q,a,d,e,p,?]?$SP + (1/1) Stage addition [y,n,q,a,d,e,p,P,?]? Unknown command ${SQ}W${SQ} (use ${SQ}?${SQ} for help) + (1/1) Stage addition [y,n,q,a,d,e,p,P,?]?$SP EOF git add -p -- command <command >actual 2>&1 && test_cmp expect actual @@ -332,9 +332,9 @@ test_expect_success 'different prompts for mode change/deleted' ' git -c core.filemode=true add -p >actual && sed -n "s/^\(([0-9/]*) Stage .*?\).*/\1/p" actual >actual.filtered && cat >expect <<-\EOF && - (1/1) Stage deletion [y,n,q,a,d,p,?]? - (1/2) Stage mode change [y,n,q,a,d,k,K,j,J,g,/,p,?]? - (2/2) Stage this hunk [y,n,q,a,d,K,J,g,/,e,p,?]? + (1/1) Stage deletion [y,n,q,a,d,p,P,?]? + (1/2) Stage mode change [y,n,q,a,d,k,K,j,J,g,/,p,P,?]? + (2/2) Stage this hunk [y,n,q,a,d,K,J,g,/,e,p,P,?]? EOF test_cmp expect actual.filtered ' @@ -521,13 +521,13 @@ test_expect_success 'split hunk setup' ' test_expect_success 'goto hunk 1 with "g 1"' ' test_when_finished "git reset" && tr _ " " >expect <<-EOF && - (2/2) Stage this hunk [y,n,q,a,d,K,J,g,/,e,p,?]? + 1: -1,2 +1,3 +15 + (2/2) Stage this hunk [y,n,q,a,d,K,J,g,/,e,p,P,?]? + 1: -1,2 +1,3 +15 _ 2: -2,4 +3,8 +21 go to which hunk? @@ -1,2 +1,3 @@ _10 +15 _20 - (1/2) Stage this hunk [y,n,q,a,d,k,K,j,J,g,/,e,p,?]?_ + (1/2) Stage this hunk [y,n,q,a,d,k,K,j,J,g,/,e,p,P,?]?_ EOF test_write_lines s y g 1 | git add -p >actual && tail -n 7 <actual >actual.trimmed && @@ -540,7 +540,7 @@ test_expect_success 'goto hunk 1 with "g1"' ' _10 +15 _20 - (1/2) Stage this hunk [y,n,q,a,d,k,K,j,J,g,/,e,p,?]?_ + (1/2) Stage this hunk [y,n,q,a,d,k,K,j,J,g,/,e,p,P,?]?_ EOF test_write_lines s y g1 | git add -p >actual && tail -n 4 <actual >actual.trimmed && @@ -550,11 +550,11 @@ test_expect_success 'goto hunk 1 with "g1"' ' test_expect_success 'navigate to hunk via regex /pattern' ' test_when_finished "git reset" && tr _ " " >expect <<-EOF && - (2/2) Stage this hunk [y,n,q,a,d,K,J,g,/,e,p,?]? @@ -1,2 +1,3 @@ + (2/2) Stage this hunk [y,n,q,a,d,K,J,g,/,e,p,P,?]? @@ -1,2 +1,3 @@ _10 +15 _20 - (1/2) Stage this hunk [y,n,q,a,d,k,K,j,J,g,/,e,p,?]?_ + (1/2) Stage this hunk [y,n,q,a,d,k,K,j,J,g,/,e,p,P,?]?_ EOF test_write_lines s y /1,2 | git add -p >actual && tail -n 5 <actual >actual.trimmed && @@ -567,7 +567,7 @@ test_expect_success 'navigate to hunk via regex / pattern' ' _10 +15 _20 - (1/2) Stage this hunk [y,n,q,a,d,k,K,j,J,g,/,e,p,?]?_ + (1/2) Stage this hunk [y,n,q,a,d,k,K,j,J,g,/,e,p,P,?]?_ EOF test_write_lines s y / 1,2 | git add -p >actual && tail -n 4 <actual >actual.trimmed && @@ -579,11 +579,11 @@ test_expect_success 'print again the hunk' ' tr _ " " >expect <<-EOF && +15 20 - (1/2) Stage this hunk [y,n,q,a,d,k,K,j,J,g,/,e,p,?]? @@ -1,2 +1,3 @@ + (1/2) Stage this hunk [y,n,q,a,d,k,K,j,J,g,/,e,p,P,?]? @@ -1,2 +1,3 @@ 10 +15 20 - (1/2) Stage this hunk [y,n,q,a,d,k,K,j,J,g,/,e,p,?]?_ + (1/2) Stage this hunk [y,n,q,a,d,k,K,j,J,g,/,e,p,P,?]?_ EOF test_write_lines s y g 1 p | git add -p >actual && tail -n 7 <actual >actual.trimmed && @@ -595,11 +595,11 @@ test_expect_success TTY 'print again the hunk (PAGER)' ' cat >expect <<-EOF && <GREEN>+<RESET><GREEN>15<RESET> 20<RESET> - <BOLD;BLUE>(1/2) Stage this hunk [y,n,q,a,d,k,K,j,J,g,/,e,p,?]? <RESET>PAGER <CYAN>@@ -1,2 +1,3 @@<RESET> + <BOLD;BLUE>(1/2) Stage this hunk [y,n,q,a,d,k,K,j,J,g,/,e,p,P,?]? <RESET>PAGER <CYAN>@@ -1,2 +1,3 @@<RESET> PAGER 10<RESET> PAGER <GREEN>+<RESET><GREEN>15<RESET> PAGER 20<RESET> - <BOLD;BLUE>(1/2) Stage this hunk [y,n,q,a,d,k,K,j,J,g,/,e,p,?]? <RESET> + <BOLD;BLUE>(1/2) Stage this hunk [y,n,q,a,d,k,K,j,J,g,/,e,p,P,?]? <RESET> EOF test_write_lines s y g 1 P | ( @@ -796,21 +796,21 @@ test_expect_success 'colors can be overridden' ' <BLUE>+<RESET><BLUE>new<RESET> <CYAN> more-context<RESET> <BLUE>+<RESET><BLUE>another-one<RESET> - <YELLOW>(1/1) Stage this hunk [y,n,q,a,d,s,e,p,?]? <RESET><BOLD>Split into 2 hunks.<RESET> + <YELLOW>(1/1) Stage this hunk [y,n,q,a,d,s,e,p,P,?]? <RESET><BOLD>Split into 2 hunks.<RESET> <MAGENTA>@@ -1,3 +1,3 @@<RESET> <CYAN> context<RESET> <BOLD>-old<RESET> <BLUE>+<RESET><BLUE>new<RESET> <CYAN> more-context<RESET> - <YELLOW>(1/2) Stage this hunk [y,n,q,a,d,k,K,j,J,g,/,e,p,?]? <RESET><MAGENTA>@@ -3 +3,2 @@<RESET> + <YELLOW>(1/2) Stage this hunk [y,n,q,a,d,k,K,j,J,g,/,e,p,P,?]? <RESET><MAGENTA>@@ -3 +3,2 @@<RESET> <CYAN> more-context<RESET> <BLUE>+<RESET><BLUE>another-one<RESET> - <YELLOW>(2/2) Stage this hunk [y,n,q,a,d,K,J,g,/,e,p,?]? <RESET><MAGENTA>@@ -1,3 +1,3 @@<RESET> + <YELLOW>(2/2) Stage this hunk [y,n,q,a,d,K,J,g,/,e,p,P,?]? <RESET><MAGENTA>@@ -1,3 +1,3 @@<RESET> <CYAN> context<RESET> <BOLD>-old<RESET> <BLUE>+new<RESET> <CYAN> more-context<RESET> - <YELLOW>(1/2) Stage this hunk [y,n,q,a,d,k,K,j,J,g,/,e,p,?]? <RESET> + <YELLOW>(1/2) Stage this hunk [y,n,q,a,d,k,K,j,J,g,/,e,p,P,?]? <RESET> EOF test_cmp expect actual ' @@ -1424,11 +1424,22 @@ test_expect_success 'invalid option s is rejected' ' test_write_lines j s q | git add -p >out && sed -ne "s/ @@.*//" -e "s/ \$//" -e "/^(/p" <out >actual && cat >expect <<-EOF && - (1/2) Stage this hunk [y,n,q,a,d,k,K,j,J,g,/,s,e,p,?]? - (2/2) Stage this hunk [y,n,q,a,d,k,K,j,J,g,/,e,p,?]? Sorry, cannot split this hunk - (2/2) Stage this hunk [y,n,q,a,d,k,K,j,J,g,/,e,p,?]? + (1/2) Stage this hunk [y,n,q,a,d,k,K,j,J,g,/,s,e,p,P,?]? + (2/2) Stage this hunk [y,n,q,a,d,k,K,j,J,g,/,e,p,P,?]? Sorry, cannot split this hunk + (2/2) Stage this hunk [y,n,q,a,d,k,K,j,J,g,/,e,p,P,?]? EOF test_cmp expect actual ' +test_expect_success 'EOF quits' ' + echo a >file && + echo a >file2 && + git add file file2 && + echo X >file && + echo X >file2 && + git add -p </dev/null >out && + test_grep file out && + test_grep ! file2 out +' + test_done diff --git a/t/t4013-diff-various.sh b/t/t4013-diff-various.sh index 55a06eadb3..d35695f5b0 100755 --- a/t/t4013-diff-various.sh +++ b/t/t4013-diff-various.sh @@ -661,6 +661,43 @@ test_expect_success 'diff -I<regex>: ignore matching file' ' test_grep ! "file1" actual ' +test_expect_success 'diff -I<regex>: ignore all content changes' ' + test_when_finished "git rm -f file1 file2 file3" && + : >file1 && + git add file1 && + : >file2 && + git add file2 && + : >file3 && + git add file3 && + + rm -f file1 file2 && + mkdir file2 && + echo "A" >file3 && + A_hash=$(git hash-object -w file3) && + echo "B" >file3 && + B_hash=$(git hash-object -w file3) && + cat <<-EOF | git update-index --index-info && + 100644 $A_hash 1 file3 + 100644 $B_hash 2 file3 + EOF + + test_diff_no_content_changes () { + git diff $1 --ignore-blank-lines -I".*" >actual && + test_line_count = 3 actual && + test_grep "file1" actual && + test_grep "file2" actual && + test_grep "file3" actual && + test_grep ! "diff --git" actual + } && + test_diff_no_content_changes "--raw" && + test_diff_no_content_changes "--name-only" && + test_diff_no_content_changes "--name-status" && + + : >actual && + test_must_fail git diff --quiet -I".*" >actual && + test_must_be_empty actual +' + # check_prefix <patch> <src> <dst> # check only lines with paths to avoid dependency on exact oid/contents check_prefix () { diff --git a/t/t4015-diff-whitespace.sh b/t/t4015-diff-whitespace.sh index 9de7f73f42..138730cbce 100755 --- a/t/t4015-diff-whitespace.sh +++ b/t/t4015-diff-whitespace.sh @@ -43,6 +43,49 @@ do ' done +test_expect_success "incomplete line in both pre- and post-image context" ' + (echo foo && echo baz | tr -d "\012") >x && + git add x && + (echo bar && echo baz | tr -d "\012") >x && + git diff x && + git -c core.whitespace=incomplete diff --check x && + git diff -R x && + git -c core.whitespace=incomplete diff -R --check x +' + +test_expect_success "incomplete lines on both pre- and post-image" ' + # The interpretation taken here is "since you are toucing + # the line anyway, you would better fix the incomplete line + # while you are at it." but this is debatable. + echo foo | tr -d "\012" >x && + git add x && + echo bar | tr -d "\012" >x && + git diff x && + test_must_fail git -c core.whitespace=incomplete diff --check x && + git diff -R x && + test_must_fail git -c core.whitespace=incomplete diff -R --check x +' + +test_expect_success "fix incomplete line in pre-image" ' + echo foo | tr -d "\012" >x && + git add x && + echo bar >x && + git diff x && + git -c core.whitespace=incomplete diff --check x && + git diff -R x && + test_must_fail git -c core.whitespace=incomplete diff -R --check x +' + +test_expect_success "new incomplete line in post-image" ' + echo foo >x && + git add x && + echo bar | tr -d "\012" >x && + git diff x && + test_must_fail git -c core.whitespace=incomplete diff --check x && + git diff -R x && + git -c core.whitespace=incomplete diff -R --check x +' + test_expect_success "Ray Lehtiniemi's example" ' cat <<-\EOF >x && do { @@ -1040,7 +1083,8 @@ test_expect_success 'ws-error-highlight test setup' ' { echo "0. blank-at-eol " && echo "1. still-blank-at-eol " && - echo "2. and a new line " + echo "2. and a new line " && + printf "3. and more" } >x && new_hash_x=$(git hash-object x) && after=$(git rev-parse --short "$new_hash_x") && @@ -1050,11 +1094,13 @@ test_expect_success 'ws-error-highlight test setup' ' <BOLD>index $before..$after 100644<RESET> <BOLD>--- a/x<RESET> <BOLD>+++ b/x<RESET> - <CYAN>@@ -1,2 +1,3 @@<RESET> + <CYAN>@@ -1,2 +1,4 @@<RESET> 0. blank-at-eol <RESET> <RED>-<RESET><RED>1. blank-at-eol<RESET><BLUE> <RESET> <GREEN>+<RESET><GREEN>1. still-blank-at-eol<RESET><BLUE> <RESET> <GREEN>+<RESET><GREEN>2. and a new line<RESET><BLUE> <RESET> + <GREEN>+<RESET><GREEN>3. and more<RESET> + <BLUE>\ No newline at end of file<RESET> EOF cat >expect.all <<-EOF && @@ -1062,11 +1108,13 @@ test_expect_success 'ws-error-highlight test setup' ' <BOLD>index $before..$after 100644<RESET> <BOLD>--- a/x<RESET> <BOLD>+++ b/x<RESET> - <CYAN>@@ -1,2 +1,3 @@<RESET> + <CYAN>@@ -1,2 +1,4 @@<RESET> <RESET>0. blank-at-eol<RESET><BLUE> <RESET> <RED>-<RESET><RED>1. blank-at-eol<RESET><BLUE> <RESET> <GREEN>+<RESET><GREEN>1. still-blank-at-eol<RESET><BLUE> <RESET> <GREEN>+<RESET><GREEN>2. and a new line<RESET><BLUE> <RESET> + <GREEN>+<RESET><GREEN>3. and more<RESET> + <BLUE>\ No newline at end of file<RESET> EOF cat >expect.none <<-EOF @@ -1074,16 +1122,19 @@ test_expect_success 'ws-error-highlight test setup' ' <BOLD>index $before..$after 100644<RESET> <BOLD>--- a/x<RESET> <BOLD>+++ b/x<RESET> - <CYAN>@@ -1,2 +1,3 @@<RESET> + <CYAN>@@ -1,2 +1,4 @@<RESET> 0. blank-at-eol <RESET> <RED>-1. blank-at-eol <RESET> <GREEN>+1. still-blank-at-eol <RESET> <GREEN>+2. and a new line <RESET> + <GREEN>+3. and more<RESET> + \ No newline at end of file<RESET> EOF ' test_expect_success 'test --ws-error-highlight option' ' + git config core.whitespace blank-at-eol,incomplete-line && git diff --color --ws-error-highlight=default,old >current.raw && test_decode_color <current.raw >current && @@ -1100,6 +1151,7 @@ test_expect_success 'test --ws-error-highlight option' ' ' test_expect_success 'test diff.wsErrorHighlight config' ' + git config core.whitespace blank-at-eol,incomplete-line && git -c diff.wsErrorHighlight=default,old diff --color >current.raw && test_decode_color <current.raw >current && @@ -1116,6 +1168,7 @@ test_expect_success 'test diff.wsErrorHighlight config' ' ' test_expect_success 'option overrides diff.wsErrorHighlight' ' + git config core.whitespace blank-at-eol,incomplete-line && git -c diff.wsErrorHighlight=none \ diff --color --ws-error-highlight=default,old >current.raw && @@ -1135,6 +1188,8 @@ test_expect_success 'option overrides diff.wsErrorHighlight' ' ' test_expect_success 'detect moved code, complete file' ' + git config core.whitespace blank-at-eol && + git reset --hard && cat <<-\EOF >test.c && #include<stdio.h> diff --git a/t/t4020-diff-external.sh b/t/t4020-diff-external.sh index c8a23d5148..7ec5854f74 100755 --- a/t/t4020-diff-external.sh +++ b/t/t4020-diff-external.sh @@ -44,6 +44,16 @@ test_expect_success 'GIT_EXTERNAL_DIFF environment and --no-ext-diff' ' ' +test_expect_success 'GIT_EXTERNAL_DIFF and --output' ' + cat >expect <<-EOF && + file $(git rev-parse --verify HEAD:file) 100644 file $(test_oid zero) 100644 + EOF + GIT_EXTERNAL_DIFF=echo git diff --output=out >stdout && + cut -d" " -f1,3- <out >actual && + test_must_be_empty stdout && + test_cmp expect actual +' + test_expect_success SYMLINKS 'typechange diff' ' rm -f file && ln -s elif file && diff --git a/t/t4035-diff-quiet.sh b/t/t4035-diff-quiet.sh index 0352bf81a9..35eaf0855f 100755 --- a/t/t4035-diff-quiet.sh +++ b/t/t4035-diff-quiet.sh @@ -50,6 +50,10 @@ test_expect_success 'git diff-tree HEAD HEAD' ' test_expect_code 0 git diff-tree --quiet HEAD HEAD >cnt && test_line_count = 0 cnt ' +test_expect_success 'git diff-tree -w HEAD^ HEAD' ' + test_expect_code 1 git diff-tree --quiet -w HEAD^ HEAD >cnt && + test_line_count = 0 cnt +' test_expect_success 'git diff-files' ' test_expect_code 0 git diff-files --quiet >cnt && test_line_count = 0 cnt diff --git a/t/t4124-apply-ws-rule.sh b/t/t4124-apply-ws-rule.sh index 485c7d2d12..115a0f8579 100755 --- a/t/t4124-apply-ws-rule.sh +++ b/t/t4124-apply-ws-rule.sh @@ -556,4 +556,191 @@ test_expect_success 'whitespace check skipped for excluded paths' ' git apply --include=used --stat --whitespace=error <patch ' +test_expect_success 'check incomplete lines (setup)' ' + rm -f .gitattributes && + git config core.whitespace incomplete-line +' + +test_expect_success 'incomplete context line (not an error)' ' + (test_write_lines 1 2 3 4 5 && printf 6) >sample-i && + (test_write_lines 1 2 3 0 5 && printf 6) >sample2-i && + cat sample-i >target && + git add target && + cat sample2-i >target && + git diff-files -p target >patch && + + cat sample-i >target && + git apply --whitespace=error <patch && + test_cmp sample2-i target && + + cat sample-i >target && + git apply --whitespace=error --check <patch 2>error && + test_cmp sample-i target && + test_must_be_empty error && + + cat sample2-i >target && + git apply --whitespace=error -R <patch && + test_cmp sample-i target && + + cat sample2-i >target && + git apply -R --whitespace=error --check <patch 2>error && + test_cmp sample2-i target && + test_must_be_empty error +' + +test_expect_success 'last line made incomplete (error)' ' + test_write_lines 1 2 3 4 5 6 >sample && + (test_write_lines 1 2 3 4 5 && printf 6) >sample-i && + cat sample >target && + git add target && + cat sample-i >target && + git diff-files -p target >patch && + + cat sample >target && + test_must_fail git apply --whitespace=error <patch 2>error && + test_grep "no newline" error && + + cat sample >target && + test_must_fail git apply --whitespace=error --check <patch 2>actual && + test_cmp sample target && + cat >expect <<-\EOF && + <stdin>:10: no newline at the end of file. + 6 + error: 1 line adds whitespace errors. + EOF + test_cmp expect actual && + + cat sample-i >target && + git apply --whitespace=error -R <patch && + test_cmp sample target && + + cat sample-i >target && + git apply --whitespace=error --check -R <patch 2>error && + test_cmp sample-i target && + test_must_be_empty error && + + cat sample >target && + git apply --whitespace=fix <patch && + test_cmp sample target +' + +test_expect_success 'incomplete line removed at the end (not an error)' ' + (test_write_lines 1 2 3 4 5 && printf 6) >sample-i && + test_write_lines 1 2 3 4 5 6 >sample && + cat sample-i >target && + git add target && + cat sample >target && + git diff-files -p target >patch && + + cat sample-i >target && + git apply --whitespace=error <patch && + test_cmp sample target && + + cat sample-i >target && + git apply --whitespace=error --check <patch 2>error && + test_cmp sample-i target && + test_must_be_empty error && + + cat sample >target && + test_must_fail git apply --whitespace=error -R <patch 2>error && + test_grep "no newline" error && + + cat sample >target && + test_must_fail git apply --whitespace=error --check -R <patch 2>actual && + test_cmp sample target && + cat >expect <<-\EOF && + <stdin>:9: no newline at the end of file. + 6 + error: 1 line adds whitespace errors. + EOF + test_cmp expect actual && + + cat sample >target && + git apply --whitespace=fix -R <patch && + test_cmp sample target +' + +test_expect_success 'incomplete line corrected at the end (not an error)' ' + (test_write_lines 1 2 3 4 5 && printf 6) >sample-i && + test_write_lines 1 2 3 4 5 7 >sample3 && + cat sample-i >target && + git add target && + cat sample3 >target && + git diff-files -p target >patch && + + cat sample-i >target && + git apply --whitespace=error <patch && + test_cmp sample3 target && + + cat sample-i >target && + git apply --whitespace=error --check <patch 2>error && + test_cmp sample-i target && + test_must_be_empty error && + + cat sample3 >target && + test_must_fail git apply --whitespace=error -R <patch 2>error && + test_grep "no newline" error && + + cat sample3 >target && + test_must_fail git apply --whitespace=error -R --check <patch 2>actual && + test_cmp sample3 target && + cat >expect <<-\EOF && + <stdin>:9: no newline at the end of file. + 6 + error: 1 line adds whitespace errors. + EOF + test_cmp expect actual && + + cat sample3 >target && + git apply --whitespace=fix -R <patch && + test_cmp sample target +' + +test_expect_success 'incomplete line modified at the end (error)' ' + (test_write_lines 1 2 3 4 5 && printf 6) >sample-i && + (test_write_lines 1 2 3 4 5 && printf 7) >sample3-i && + test_write_lines 1 2 3 4 5 6 >sample && + test_write_lines 1 2 3 4 5 7 >sample3 && + cat sample-i >target && + git add target && + cat sample3-i >target && + git diff-files -p target >patch && + + cat sample-i >target && + test_must_fail git apply --whitespace=error <patch 2>error && + test_grep "no newline" error && + + cat sample-i >target && + test_must_fail git apply --whitespace=error --check <patch 2>actual && + test_cmp sample-i target && + cat >expect <<-\EOF && + <stdin>:11: no newline at the end of file. + 7 + error: 1 line adds whitespace errors. + EOF + test_cmp expect actual && + + cat sample3-i >target && + test_must_fail git apply --whitespace=error -R <patch 2>error && + test_grep "no newline" error && + + cat sample3-i >target && + test_must_fail git apply --whitespace=error --check -R <patch 2>actual && + test_cmp sample3-i target && + cat >expect <<-\EOF && + <stdin>:9: no newline at the end of file. + 6 + error: 1 line adds whitespace errors. + EOF + test_cmp expect actual && + + cat sample-i >target && + git apply --whitespace=fix <patch && + test_cmp sample3 target && + + cat sample3-i >target && + git apply --whitespace=fix -R <patch && + test_cmp sample target +' + test_done diff --git a/t/t5318-commit-graph.sh b/t/t5318-commit-graph.sh index 0b3404f58f..98c6910963 100755 --- a/t/t5318-commit-graph.sh +++ b/t/t5318-commit-graph.sh @@ -946,4 +946,48 @@ test_expect_success 'stale commit cannot be parsed when traversing graph' ' ) ' +test_expect_success 'config commitGraph.changedPaths acts like --changed-paths' ' + git init config-changed-paths && + ( + cd config-changed-paths && + + # commitGraph.changedPaths is not set and it should not write Bloom filters + test_commit first && + GIT_PROGRESS_DELAY=0 git commit-graph write --reachable --progress 2>error && + test_grep ! "Bloom filters" error && + + # Set commitGraph.changedPaths to true and it should write Bloom filters + test_commit second && + git config commitGraph.changedPaths true && + GIT_PROGRESS_DELAY=0 git commit-graph write --reachable --progress 2>error && + test_grep "Bloom filters" error && + + # Add one more config commitGraph.changedPaths as false to disable the previous true config value + # It should still write Bloom filters due to existing filters + test_commit third && + git config --add commitGraph.changedPaths false && + GIT_PROGRESS_DELAY=0 git commit-graph write --reachable --progress 2>error && + test_grep "Bloom filters" error && + + # commitGraph.changedPaths is still false and command line options should take precedence + test_commit fourth && + GIT_PROGRESS_DELAY=0 git commit-graph write --no-changed-paths --reachable --progress 2>error && + test_grep ! "Bloom filters" error && + GIT_PROGRESS_DELAY=0 git commit-graph write --reachable --progress 2>error && + test_grep ! "Bloom filters" error && + + # commitGraph.changedPaths is all cleared and then set to false again, command line options should take precedence + test_commit fifth && + git config --unset-all commitGraph.changedPaths && + git config commitGraph.changedPaths false && + GIT_PROGRESS_DELAY=0 git commit-graph write --changed-paths --reachable --progress 2>error && + test_grep "Bloom filters" error && + + # commitGraph.changedPaths is still false and it should write Bloom filters due to existing filters + test_commit sixth && + GIT_PROGRESS_DELAY=0 git commit-graph write --reachable --progress 2>error && + test_grep "Bloom filters" error + ) +' + test_done diff --git a/t/t6429-merge-sequence-rename-caching.sh b/t/t6429-merge-sequence-rename-caching.sh index 0f39ed0d08..15dd2d94b7 100755 --- a/t/t6429-merge-sequence-rename-caching.sh +++ b/t/t6429-merge-sequence-rename-caching.sh @@ -11,14 +11,13 @@ test_description="remember regular & dir renames in sequence of merges" # sure that we are triggering rename caching rather than rename # bypassing. # -# NOTE 2: this testfile uses 'test-tool fast-rebase' instead of either -# cherry-pick or rebase. sequencer.c is only superficially -# integrated with merge-ort; it calls merge_switch_to_result() -# after EACH merge, which updates the index and working copy AND -# throws away the cached results (because merge_switch_to_result() -# is only supposed to be called at the end of the sequence). -# Integrating them more deeply is a big task, so for now the tests -# use 'test-tool fast-rebase'. +# NOTE 2: this testfile uses replay instead of either cherry-pick or rebase. +# sequencer.c is only superficially integrated with merge-ort; it +# calls merge_switch_to_result() after EACH merge, which updates the +# index and working copy AND throws away the cached results (because +# merge_switch_to_result() is only supposed to be called at the end +# of the sequence). Integrating them more deeply is a big task, so +# for now the tests use 'git replay'. # @@ -769,4 +768,82 @@ test_expect_success 'avoid assuming we detected renames' ' ) ' +# +# In the following testcase: +# Base: olddir/{valuesX_1, valuesY_1, valuesZ_1} +# other/content +# Upstream: rename olddir/valuesX_1 -> newdir/valuesX_2 +# Topic_1: modify olddir/valuesX_1 -> olddir/valuesX_3 +# Topic_2: modify olddir/valuesY, +# modify other/content +# Expected Pick1: olddir/{valuesY, valuesZ}, newdir/valuesX, other/content +# Expected Pick2: olddir/{valuesY, valuesZ}, newdir/valuesX, other/content +# +# This testcase presents no problems for git traditionally, but the fact that +# olddir/valuesX -> newdir/valuesX +# gets cached after the first pick presents a problem for the second commit to +# be replayed, because it appears to be an irrelevant rename, so the trivial +# directory resolution will resolve newdir/ without recursing into it, giving +# us no way to apply the cached rename to anything. +# +test_expect_success 'rename a file, use it on first pick, but irrelevant on second' ' + git init rename_a_file_use_it_once_irrelevant_on_second && + ( + cd rename_a_file_use_it_once_irrelevant_on_second && + + mkdir olddir/ other/ && + test_seq 3 8 >olddir/valuesX && + test_seq 3 8 >olddir/valuesY && + test_seq 3 8 >olddir/valuesZ && + printf "%s\n" A B C D E F G >other/content && + git add olddir other && + git commit -m orig && + + git branch upstream && + git branch topic && + + git switch upstream && + test_seq 1 8 >olddir/valuesX && + git add olddir && + mkdir newdir && + git mv olddir/valuesX newdir && + git commit -m "Renamed (and modified) olddir/valuesX into newdir/" && + + git switch topic && + + test_seq 3 10 >olddir/valuesX && + git add olddir && + git commit -m A && + + test_seq 1 8 >olddir/valuesY && + printf "%s\n" A B C D E F G H I >other/content && + git add olddir/valuesY other && + git commit -m B && + + # + # Actual testing; mostly we want to verify that we do not hit + # git: merge-ort.c:3032: process_renames: Assertion `newinfo && !newinfo->merged.clean` failed. + # + + git switch upstream && + git config merge.directoryRenames true && + + git replay --onto HEAD upstream~1..topic >out && + + # + # ...but we may as well check that the replay gave us a reasonable result + # + + git update-ref --stdin <out && + git checkout topic && + + git ls-files >tracked && + test_line_count = 4 tracked && + test_path_is_file newdir/valuesX && + test_path_is_file olddir/valuesY && + test_path_is_file olddir/valuesZ && + test_path_is_file other/content + ) +' + test_done diff --git a/t/t7004-tag.sh b/t/t7004-tag.sh index 10835631ca..ce2ff2a28a 100755 --- a/t/t7004-tag.sh +++ b/t/t7004-tag.sh @@ -2293,24 +2293,26 @@ test_expect_success '--contains combined with --no-contains' ' # don't recurse down to tags for trees or blobs pointed to by *those* # commits. test_expect_success 'Does --[no-]contains stop at commits? Yes!' ' - cd no-contains && - blob=$(git rev-parse v0.3:v0.3.t) && - tree=$(git rev-parse v0.3^{tree}) && - git tag tag-blob $blob && - git tag tag-tree $tree && - git tag --contains v0.3 >actual && - cat >expected <<-\EOF && - v0.3 - v0.4 - v0.5 - EOF - test_cmp expected actual && - git tag --no-contains v0.3 >actual && - cat >expected <<-\EOF && - v0.1 - v0.2 - EOF - test_cmp expected actual + ( + cd no-contains && + blob=$(git rev-parse v0.3:v0.3.t) && + tree=$(git rev-parse v0.3^{tree}) && + git tag tag-blob $blob && + git tag tag-tree $tree && + git tag --contains v0.3 >actual && + cat >expected <<-\EOF && + v0.3 + v0.4 + v0.5 + EOF + test_cmp expected actual && + git tag --no-contains v0.3 >actual && + cat >expected <<-\EOF && + v0.1 + v0.2 + EOF + test_cmp expected actual + ) ' test_expect_success 'If tag is created then tag message file is unlinked' ' @@ -2332,4 +2334,24 @@ test_expect_success 'If tag cannot be created then tag message file is not unlin test_path_exists .git/TAG_EDITMSG ' +test_expect_success 'annotated tag version sort' ' + git tag -a -m "sample 1.0" vsample-1.0 && + git tag -a -m "sample 2.0" vsample-2.0 && + git tag -a -m "sample 10.0" vsample-10.0 && + cat >expect <<-EOF && + vsample-1.0 + vsample-2.0 + vsample-10.0 + EOF + + git tag --list --sort=version:tag vsample-\* >actual && + test_cmp expect actual && + + # Ensure that we also handle this case alright in the case we have the + # peeled values cached e.g. via the packed-refs file. + git pack-refs --all && + git tag --list --sort=version:tag vsample-\* && + test_cmp expect actual +' + test_done diff --git a/t/t7425-submodule-encoding.sh b/t/t7425-submodule-encoding.sh new file mode 100755 index 0000000000..f92b3e6338 --- /dev/null +++ b/t/t7425-submodule-encoding.sh @@ -0,0 +1,161 @@ +#!/bin/sh + +test_description='submodules handle mixed legacy and new (encoded) style gitdir paths' + +. ./test-lib.sh +. "$TEST_DIRECTORY"/lib-verify-submodule-gitdir-path.sh + +test_expect_success 'setup: allow file protocol' ' + git config --global protocol.file.allow always +' + +test_expect_success 'create repo with mixed encoded and non-encoded submodules' ' + git init -b main legacy-sub && + test_commit -C legacy-sub legacy-initial && + legacy_rev=$(git -C legacy-sub rev-parse HEAD) && + + git init -b main new-sub && + test_commit -C new-sub new-initial && + new_rev=$(git -C new-sub rev-parse HEAD) && + + git init -b main main && + ( + cd main && + git submodule add ../legacy-sub legacy && + test_commit legacy-sub && + + git config core.repositoryformatversion 1 && + git config extensions.submoduleEncoding true && + + git submodule add ../new-sub "New Sub" && + test_commit new + ) +' + +test_expect_success 'verify submodule name is properly encoded' ' + verify_submodule_gitdir_path main legacy modules/legacy && + verify_submodule_gitdir_path main "New Sub" "modules/New Sub" +' + +test_expect_success 'clone from repo with both legacy and new-style submodules' ' + git clone --recurse-submodules main cloned-non-encoding && + ( + cd cloned-non-encoding && + + test_path_is_dir .git/modules/legacy && + test_path_is_dir .git/modules/"New Sub" && + + git submodule status >list && + test_grep "$legacy_rev legacy" list && + test_grep "$new_rev New Sub" list + ) && + + git clone -c extensions.submoduleEncoding=true --recurse-submodules main cloned-encoding && + ( + cd cloned-encoding && + + test_path_is_dir .git/modules/legacy && + test_path_is_dir ".git/modules/New Sub" && + + git submodule status >list && + test_grep "$legacy_rev legacy" list && + test_grep "$new_rev New Sub" list + ) +' + +test_expect_success 'commit and push changes to encoded submodules' ' + git -C legacy-sub config receive.denyCurrentBranch updateInstead && + git -C new-sub config receive.denyCurrentBranch updateInstead && + git -C main config receive.denyCurrentBranch updateInstead && + ( + cd cloned-encoding && + + git -C legacy switch --track -C main origin/main && + test_commit -C legacy second-commit && + git -C legacy push && + + git -C "New Sub" switch --track -C main origin/main && + test_commit -C "New Sub" second-commit && + git -C "New Sub" push && + + # Stage and commit submodule changes in superproject + git switch --track -C main origin/main && + git add legacy "New Sub" && + git commit -m "update submodules" && + + # push superproject commit to main repo + git push + ) && + + # update expected legacy & new submodule checksums + legacy_rev=$(git -C legacy-sub rev-parse HEAD) && + new_rev=$(git -C new-sub rev-parse HEAD) +' + +test_expect_success 'fetch mixed submodule changes and verify updates' ' + ( + cd main && + + # only update submodules because superproject was + # pushed into at the end of last test + git submodule update --init --recursive && + + test_path_is_dir .git/modules/legacy && + test_path_is_dir ".git/modules/New Sub" && + + # Verify both submodules are at the expected commits + git submodule status >list && + test_grep "$legacy_rev legacy" list && + test_grep "$new_rev New Sub" list + ) +' + +test_expect_success 'setup submodules with nested git dirs' ' + git init nested && + test_commit -C nested nested && + ( + cd nested && + cat >.gitmodules <<-EOF && + [submodule "hippo"] + url = . + path = thing1 + [submodule "hippo/hooks"] + url = . + path = thing2 + EOF + git clone . thing1 && + git clone . thing2 && + git add .gitmodules thing1 thing2 && + test_tick && + git commit -m nested + ) +' + +test_expect_success 'git dirs of encoded sibling submodules must not be nested' ' + git clone -c extensions.submoduleEncoding=true --recurse-submodules nested clone_nested && + verify_submodule_gitdir_path clone_nested hippo modules/hippo && + verify_submodule_gitdir_path clone_nested hippo/hooks modules/hippo%2fhooks +' + +test_expect_success 'submodule git dir nesting detection must work with parallel cloning' ' + git clone -c extensions.submoduleEncoding=true --recurse-submodules --jobs=2 nested clone_parallel && + verify_submodule_gitdir_path clone_parallel hippo modules/hippo && + verify_submodule_gitdir_path clone_parallel hippo/hooks modules/hippo%2fhooks +' + +test_expect_success 'verify case-folding conflict is correctly encoded' ' + git clone -c extensions.submoduleEncoding=true -c core.ignoreCase=true main cloned-folding && + ( + cd cloned-folding && + + git submodule add ../new-sub "folding" && + test_commit lowercase && + + git submodule add ../new-sub "FoldinG" && + test_commit uppercase + ) && + verify_submodule_gitdir_path cloned-folding "folding" "modules/folding" && + verify_submodule_gitdir_path cloned-folding "FoldinG" "modules/%46oldin%47" +' + +test_done diff --git a/t/t7500-commit-template-squash-signoff.sh b/t/t7500-commit-template-squash-signoff.sh index 1935171d68..66aff8e097 100755 --- a/t/t7500-commit-template-squash-signoff.sh +++ b/t/t7500-commit-template-squash-signoff.sh @@ -33,7 +33,7 @@ test_expect_success 'nonexistent template file should return error' ' ( GIT_EDITOR="echo hello >" && export GIT_EDITOR && - test_must_fail git commit --template "$PWD"/notexist + test_must_fail git commit --template "$(pwd)"/notexist ) ' @@ -43,12 +43,12 @@ test_expect_success 'nonexistent optional template file on command line' ' ( GIT_EDITOR="echo hello >\"\$1\"" && export GIT_EDITOR && - git commit --template ":(optional)$PWD/notexist" + git commit --template ":(optional)$(pwd)/notexist" ) ' test_expect_success 'nonexistent template file in config should return error' ' - test_config commit.template "$PWD"/notexist && + test_config commit.template "$(pwd)"/notexist && ( GIT_EDITOR="echo hello >" && export GIT_EDITOR && @@ -57,7 +57,7 @@ test_expect_success 'nonexistent template file in config should return error' ' ' test_expect_success 'nonexistent optional template file in config' ' - test_config commit.template ":(optional)$PWD"/notexist && + test_config commit.template ":(optional)$(pwd)"/notexist && GIT_EDITOR="echo hello >" git commit --allow-empty && git cat-file commit HEAD | sed -e "1,/^$/d" >actual && echo hello >expect && @@ -65,7 +65,7 @@ test_expect_success 'nonexistent optional template file in config' ' ' # From now on we'll use a template file that exists. -TEMPLATE="$PWD"/template +TEMPLATE="$(pwd)"/template test_expect_success 'unedited template should not commit' ' echo "template line" >"$TEMPLATE" && @@ -99,6 +99,15 @@ test_expect_success 'adding real content to a template should commit' ' commit_msg_is "template linecommit message" ' +test_expect_success 'existent template marked optional should commit' ' + echo "existent template" >"$TEMPLATE" && + ( + test_set_editor "$TEST_DIRECTORY"/t7500/add-content && + git commit --allow-empty --template ":(optional)$TEMPLATE" + ) && + commit_msg_is "existent templatecommit message" +' + test_expect_success '-t option should be short for --template' ' echo "short template" > "$TEMPLATE" && echo "new content" >> foo && diff --git a/t/t7508-status.sh b/t/t7508-status.sh index cdc1d6fcc7..abad229e9d 100755 --- a/t/t7508-status.sh +++ b/t/t7508-status.sh @@ -717,6 +717,17 @@ test_expect_success TTY 'status -s with color.status' ' ' +test_expect_success TTY 'status -s keeps colors with -z' ' + test_when_finished "rm -f output.*" && + test_terminal git status -s -z >output.raw && + # convert back to newlines to avoid portability issues with + # test_decode_color and test_cmp, and to let us use the same expected + # output as earlier tests + tr "\0" "\n" <output.raw >output.nl && + test_decode_color <output.nl >output && + test_cmp expect output +' + cat >expect <<\EOF ## <YELLOW>main<RESET>...<CYAN>upstream<RESET> [ahead <YELLOW>1<RESET>, behind <CYAN>2<RESET>] <RED>M<RESET> dir1/modified diff --git a/t/t7528-signed-commit-ssh.sh b/t/t7528-signed-commit-ssh.sh index 0f887a3ebe..b50306b9b3 100755 --- a/t/t7528-signed-commit-ssh.sh +++ b/t/t7528-signed-commit-ssh.sh @@ -82,7 +82,7 @@ test_expect_success GPGSSH 'create signed commits' ' test_expect_success GPGSSH 'sign commits using literal public keys with ssh-agent' ' test_when_finished "test_unconfig commit.gpgsign" && test_config gpg.format ssh && - eval $(ssh-agent) && + eval $(ssh-agent -T || ssh-agent) && test_when_finished "kill ${SSH_AGENT_PID}" && test_when_finished "test_unconfig user.signingkey" && mkdir tmpdir && diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh index ddd273d8dc..6b36f52df7 100755 --- a/t/t7900-maintenance.sh +++ b/t/t7900-maintenance.sh @@ -49,7 +49,9 @@ test_expect_success 'run [--auto|--quiet]' ' git maintenance run --auto 2>/dev/null && GIT_TRACE2_EVENT="$(pwd)/run-no-quiet.txt" \ git maintenance run --no-quiet 2>/dev/null && + git maintenance is-needed && test_subcommand git gc --quiet --no-detach --skip-foreground-tasks <run-no-auto.txt && + ! git maintenance is-needed --auto && test_subcommand ! git gc --auto --quiet --no-detach --skip-foreground-tasks <run-auto.txt && test_subcommand git gc --no-quiet --no-detach --skip-foreground-tasks <run-no-quiet.txt ' @@ -180,6 +182,11 @@ test_expect_success 'commit-graph auto condition' ' test_commit first && + ! git -c maintenance.commit-graph.auto=0 \ + maintenance is-needed --auto --task=commit-graph && + git -c maintenance.commit-graph.auto=1 \ + maintenance is-needed --auto --task=commit-graph && + GIT_TRACE2_EVENT="$(pwd)/cg-zero-means-no.txt" \ git -c maintenance.commit-graph.auto=0 $COMMAND && GIT_TRACE2_EVENT="$(pwd)/cg-one-satisfied.txt" \ @@ -290,16 +297,23 @@ test_expect_success 'maintenance.loose-objects.auto' ' git -c maintenance.loose-objects.auto=1 maintenance \ run --auto --task=loose-objects 2>/dev/null && test_subcommand ! git prune-packed --quiet <trace-lo1.txt && + printf data-A | git hash-object -t blob --stdin -w && + ! git -c maintenance.loose-objects.auto=2 \ + maintenance is-needed --auto --task=loose-objects && GIT_TRACE2_EVENT="$(pwd)/trace-loA" \ git -c maintenance.loose-objects.auto=2 \ maintenance run --auto --task=loose-objects 2>/dev/null && test_subcommand ! git prune-packed --quiet <trace-loA && + printf data-B | git hash-object -t blob --stdin -w && + git -c maintenance.loose-objects.auto=2 \ + maintenance is-needed --auto --task=loose-objects && GIT_TRACE2_EVENT="$(pwd)/trace-loB" \ git -c maintenance.loose-objects.auto=2 \ maintenance run --auto --task=loose-objects 2>/dev/null && test_subcommand git prune-packed --quiet <trace-loB && + GIT_TRACE2_EVENT="$(pwd)/trace-loC" \ git -c maintenance.loose-objects.auto=2 \ maintenance run --auto --task=loose-objects 2>/dev/null && @@ -421,10 +435,13 @@ run_incremental_repack_and_verify () { test_commit A && git repack -adk && git multi-pack-index write && + ! git -c maintenance.incremental-repack.auto=1 \ + maintenance is-needed --auto --task=incremental-repack && GIT_TRACE2_EVENT="$(pwd)/midx-init.txt" git \ -c maintenance.incremental-repack.auto=1 \ maintenance run --auto --task=incremental-repack 2>/dev/null && test_subcommand ! git multi-pack-index write --no-progress <midx-init.txt && + test_commit B && git pack-objects --revs .git/objects/pack/pack <<-\EOF && HEAD @@ -434,11 +451,14 @@ run_incremental_repack_and_verify () { -c maintenance.incremental-repack.auto=2 \ maintenance run --auto --task=incremental-repack 2>/dev/null && test_subcommand ! git multi-pack-index write --no-progress <trace-A && + test_commit C && git pack-objects --revs .git/objects/pack/pack <<-\EOF && HEAD ^HEAD~1 EOF + git -c maintenance.incremental-repack.auto=2 \ + maintenance is-needed --auto --task=incremental-repack && GIT_TRACE2_EVENT=$(pwd)/trace-B git \ -c maintenance.incremental-repack.auto=2 \ maintenance run --auto --task=incremental-repack 2>/dev/null && @@ -465,6 +485,176 @@ test_expect_success 'maintenance.incremental-repack.auto (when config is unset)' ) ' +run_and_verify_geometric_pack () { + EXPECTED_PACKS="$1" && + + # Verify that we perform a geometric repack. + rm -f "trace2.txt" && + GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \ + git maintenance run --task=geometric-repack 2>/dev/null && + test_subcommand git repack -d -l --geometric=2 \ + --quiet --write-midx <trace2.txt && + + # Verify that the number of packfiles matches our expectation. + ls -l .git/objects/pack/*.pack >packfiles && + test_line_count = "$EXPECTED_PACKS" packfiles && + + # And verify that there are no loose objects anymore. + git count-objects -v >count && + test_grep '^count: 0$' count +} + +test_expect_success 'geometric repacking task' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + git config set maintenance.auto false && + test_commit initial && + + # The initial repack causes an all-into-one repack. + GIT_TRACE2_EVENT="$(pwd)/initial-repack.txt" \ + git maintenance run --task=geometric-repack 2>/dev/null && + test_subcommand git repack -d -l --cruft --cruft-expiration=2.weeks.ago \ + --quiet --write-midx <initial-repack.txt && + + # Repacking should now cause a no-op geometric repack because + # no packfiles need to be combined. + ls -l .git/objects/pack/*.pack >before && + run_and_verify_geometric_pack 1 && + ls -l .git/objects/pack/*.pack >after && + test_cmp before after && + + # This incremental change creates a new packfile that only + # soaks up loose objects. The packfiles are not getting merged + # at this point. + test_commit loose && + run_and_verify_geometric_pack 2 && + + # Both packfiles have 3 objects, so the next run would cause us + # to merge all packfiles together. This should be turned into + # an all-into-one-repack. + GIT_TRACE2_EVENT="$(pwd)/all-into-one-repack.txt" \ + git maintenance run --task=geometric-repack 2>/dev/null && + test_subcommand git repack -d -l --cruft --cruft-expiration=2.weeks.ago \ + --quiet --write-midx <all-into-one-repack.txt && + + # The geometric repack soaks up unreachable objects. + echo blob-1 | git hash-object -w --stdin -t blob && + run_and_verify_geometric_pack 2 && + + # A second unreachable object should be written into another packfile. + echo blob-2 | git hash-object -w --stdin -t blob && + run_and_verify_geometric_pack 3 && + + # And these two small packs should now be merged via the + # geometric repack. The large packfile should remain intact. + run_and_verify_geometric_pack 2 && + + # If we now add two more objects and repack twice we should + # then see another all-into-one repack. This time around + # though, as we have unreachable objects, we should also see a + # cruft pack. + echo blob-3 | git hash-object -w --stdin -t blob && + echo blob-4 | git hash-object -w --stdin -t blob && + run_and_verify_geometric_pack 3 && + GIT_TRACE2_EVENT="$(pwd)/cruft-repack.txt" \ + git maintenance run --task=geometric-repack 2>/dev/null && + test_subcommand git repack -d -l --cruft --cruft-expiration=2.weeks.ago \ + --quiet --write-midx <cruft-repack.txt && + ls .git/objects/pack/*.pack >packs && + test_line_count = 2 packs && + ls .git/objects/pack/*.mtimes >cruft && + test_line_count = 1 cruft + ) +' + +test_geometric_repack_needed () { + NEEDED="$1" + GEOMETRIC_CONFIG="$2" && + rm -f trace2.txt && + GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \ + git ${GEOMETRIC_CONFIG:+-c maintenance.geometric-repack.$GEOMETRIC_CONFIG} \ + maintenance run --auto --task=geometric-repack 2>/dev/null && + case "$NEEDED" in + true) + test_grep "\[\"git\",\"repack\"," trace2.txt;; + false) + ! test_grep "\[\"git\",\"repack\"," trace2.txt;; + *) + BUG "invalid parameter: $NEEDED";; + esac +} + +test_expect_success 'geometric repacking with --auto' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + + # An empty repository does not need repacking, except when + # explicitly told to do it. + test_geometric_repack_needed false && + test_geometric_repack_needed false auto=0 && + test_geometric_repack_needed false auto=1 && + test_geometric_repack_needed true auto=-1 && + + test_oid_init && + + # Loose objects cause a repack when crossing the limit. Note + # that the number of objects gets extrapolated by having a look + # at the "objects/17/" shard. + test_commit "$(test_oid blob17_1)" && + test_geometric_repack_needed false && + test_commit "$(test_oid blob17_2)" && + test_geometric_repack_needed false auto=257 && + test_geometric_repack_needed true auto=256 && + + # Force another repack. + test_commit first && + test_commit second && + test_geometric_repack_needed true auto=-1 && + + # We now have two packfiles that would be merged together. As + # such, the repack should always happen unless the user has + # disabled the auto task. + test_geometric_repack_needed false auto=0 && + test_geometric_repack_needed true auto=9000 + ) +' + +test_expect_success 'geometric repacking honors configured split factor' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + git config set maintenance.auto false && + + # Create three different packs with 9, 2 and 1 object, respectively. + # This is done so that only a subset of packs would be merged + # together so that we can verify that `git repack` receives the + # correct geometric factor. + for i in $(test_seq 9) + do + echo first-$i | git hash-object -w --stdin -t blob || return 1 + done && + git repack --geometric=2 -d && + + for i in $(test_seq 2) + do + echo second-$i | git hash-object -w --stdin -t blob || return 1 + done && + git repack --geometric=2 -d && + + echo third | git hash-object -w --stdin -t blob && + git repack --geometric=2 -d && + + test_geometric_repack_needed false splitFactor=2 && + test_geometric_repack_needed true splitFactor=3 && + test_subcommand git repack -d -l --geometric=3 --quiet --write-midx <trace2.txt + ) +' + test_expect_success 'pack-refs task' ' for n in $(test_seq 1 5) do @@ -485,9 +675,15 @@ test_expect_success 'reflog-expire task --auto only packs when exceeding limits' git reflog expire --all --expire=now && test_commit reflog-one && test_commit reflog-two && + + ! git -c maintenance.reflog-expire.auto=3 \ + maintenance is-needed --auto --task=reflog-expire && GIT_TRACE2_EVENT="$(pwd)/reflog-expire-auto.txt" \ git -c maintenance.reflog-expire.auto=3 maintenance run --auto --task=reflog-expire && test_subcommand ! git reflog expire --all <reflog-expire-auto.txt && + + git -c maintenance.reflog-expire.auto=2 \ + maintenance is-needed --auto --task=reflog-expire && GIT_TRACE2_EVENT="$(pwd)/reflog-expire-auto.txt" \ git -c maintenance.reflog-expire.auto=2 maintenance run --auto --task=reflog-expire && test_subcommand git reflog expire --all <reflog-expire-auto.txt @@ -514,6 +710,7 @@ test_expect_success 'worktree-prune task --auto only prunes with prunable worktr test_expect_worktree_prune ! git maintenance run --auto --task=worktree-prune && mkdir .git/worktrees && : >.git/worktrees/abc && + git maintenance is-needed --auto --task=worktree-prune && test_expect_worktree_prune git maintenance run --auto --task=worktree-prune ' @@ -530,22 +727,7 @@ test_expect_success 'worktree-prune task with --auto honors maintenance.worktree test_expect_worktree_prune ! git -c maintenance.worktree-prune.auto=0 maintenance run --auto --task=worktree-prune && # A positive value should require at least this many prunable worktrees. test_expect_worktree_prune ! git -c maintenance.worktree-prune.auto=4 maintenance run --auto --task=worktree-prune && - test_expect_worktree_prune git -c maintenance.worktree-prune.auto=3 maintenance run --auto --task=worktree-prune -' - -test_expect_success 'worktree-prune task with --auto honors maintenance.worktree-prune.auto' ' - # A negative value should always prune. - test_expect_worktree_prune git -c maintenance.worktree-prune.auto=-1 maintenance run --auto --task=worktree-prune && - - mkdir .git/worktrees && - : >.git/worktrees/first && - : >.git/worktrees/second && - : >.git/worktrees/third && - - # Zero should never prune. - test_expect_worktree_prune ! git -c maintenance.worktree-prune.auto=0 maintenance run --auto --task=worktree-prune && - # A positive value should require at least this many prunable worktrees. - test_expect_worktree_prune ! git -c maintenance.worktree-prune.auto=4 maintenance run --auto --task=worktree-prune && + git -c maintenance.worktree-prune.auto=3 maintenance is-needed --auto --task=worktree-prune && test_expect_worktree_prune git -c maintenance.worktree-prune.auto=3 maintenance run --auto --task=worktree-prune ' @@ -554,11 +736,13 @@ test_expect_success 'worktree-prune task honors gc.worktreePruneExpire' ' rm -rf worktree && rm -f worktree-prune.txt && + ! git -c gc.worktreePruneExpire=1.week.ago maintenance is-needed --auto --task=worktree-prune && GIT_TRACE2_EVENT="$(pwd)/worktree-prune.txt" git -c gc.worktreePruneExpire=1.week.ago maintenance run --auto --task=worktree-prune && test_subcommand ! git worktree prune --expire 1.week.ago <worktree-prune.txt && test_path_is_dir .git/worktrees/worktree && rm -f worktree-prune.txt && + git -c gc.worktreePruneExpire=now maintenance is-needed --auto --task=worktree-prune && GIT_TRACE2_EVENT="$(pwd)/worktree-prune.txt" git -c gc.worktreePruneExpire=now maintenance run --auto --task=worktree-prune && test_subcommand git worktree prune --expire now <worktree-prune.txt && test_path_is_missing .git/worktrees/worktree @@ -583,10 +767,13 @@ test_expect_success 'rerere-gc task without --auto always collects garbage' ' test_expect_success 'rerere-gc task with --auto only prunes with prunable entries' ' test_when_finished "rm -rf .git/rr-cache" && + ! git maintenance is-needed --auto --task=rerere-gc && test_expect_rerere_gc ! git maintenance run --auto --task=rerere-gc && mkdir .git/rr-cache && + ! git maintenance is-needed --auto --task=rerere-gc && test_expect_rerere_gc ! git maintenance run --auto --task=rerere-gc && : >.git/rr-cache/entry && + git maintenance is-needed --auto --task=rerere-gc && test_expect_rerere_gc git maintenance run --auto --task=rerere-gc ' @@ -594,17 +781,22 @@ test_expect_success 'rerere-gc task with --auto honors maintenance.rerere-gc.aut test_when_finished "rm -rf .git/rr-cache" && # A negative value should always prune. + git -c maintenance.rerere-gc.auto=-1 maintenance is-needed --auto --task=rerere-gc && test_expect_rerere_gc git -c maintenance.rerere-gc.auto=-1 maintenance run --auto --task=rerere-gc && # A positive value prunes when there is at least one entry. + ! git -c maintenance.rerere-gc.auto=9000 maintenance is-needed --auto --task=rerere-gc && test_expect_rerere_gc ! git -c maintenance.rerere-gc.auto=9000 maintenance run --auto --task=rerere-gc && mkdir .git/rr-cache && + ! git -c maintenance.rerere-gc.auto=9000 maintenance is-needed --auto --task=rerere-gc && test_expect_rerere_gc ! git -c maintenance.rerere-gc.auto=9000 maintenance run --auto --task=rerere-gc && : >.git/rr-cache/entry-1 && + git -c maintenance.rerere-gc.auto=9000 maintenance is-needed --auto --task=rerere-gc && test_expect_rerere_gc git -c maintenance.rerere-gc.auto=9000 maintenance run --auto --task=rerere-gc && # Zero should never prune. : >.git/rr-cache/entry-1 && + ! git -c maintenance.rerere-gc.auto=0 maintenance is-needed --auto --task=rerere-gc && test_expect_rerere_gc ! git -c maintenance.rerere-gc.auto=0 maintenance run --auto --task=rerere-gc ' @@ -716,6 +908,76 @@ test_expect_success 'maintenance.strategy inheritance' ' <modified-daily.txt ' +test_strategy () { + STRATEGY="$1" + shift + + cat >expect && + rm -f trace2.txt && + GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \ + git -c maintenance.strategy=$STRATEGY maintenance run --quiet "$@" && + sed -n 's/{"event":"child_start","sid":"[^/"]*",.*,"argv":\["\(.*\)\"]}/\1/p' <trace2.txt | + sed 's/","/ /g' >actual + test_cmp expect actual +} + +test_expect_success 'maintenance.strategy is respected' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit initial && + + test_must_fail git -c maintenance.strategy=unknown maintenance run 2>err && + test_grep "unknown maintenance strategy: .unknown." err && + + test_strategy incremental <<-\EOF && + git pack-refs --all --prune + git reflog expire --all + git gc --quiet --no-detach --skip-foreground-tasks + EOF + + test_strategy incremental --schedule=weekly <<-\EOF && + git pack-refs --all --prune + git prune-packed --quiet + git multi-pack-index write --no-progress + git multi-pack-index expire --no-progress + git multi-pack-index repack --no-progress --batch-size=1 + git commit-graph write --split --reachable --no-progress + EOF + + test_strategy gc <<-\EOF && + git pack-refs --all --prune + git reflog expire --all + git gc --quiet --no-detach --skip-foreground-tasks + EOF + + test_strategy gc --schedule=weekly <<-\EOF && + git pack-refs --all --prune + git reflog expire --all + git gc --quiet --no-detach --skip-foreground-tasks + EOF + + test_strategy geometric <<-\EOF && + git pack-refs --all --prune + git reflog expire --all + git repack -d -l --geometric=2 --quiet --write-midx + git commit-graph write --split --reachable --no-progress + git worktree prune --expire 3.months.ago + git rerere gc + EOF + + test_strategy geometric --schedule=weekly <<-\EOF + git pack-refs --all --prune + git reflog expire --all + git repack -d -l --geometric=2 --quiet --write-midx + git commit-graph write --split --reachable --no-progress + git worktree prune --expire 3.months.ago + git rerere gc + EOF + ) +' + test_expect_success 'register and unregister' ' test_when_finished git config --global --unset-all maintenance.repo && @@ -1093,6 +1355,11 @@ test_expect_success 'fails when running outside of a repository' ' nongit test_must_fail git maintenance unregister ' +test_expect_success 'fails when configured to use an invalid strategy' ' + test_must_fail git -c maintenance.strategy=invalid maintenance run --schedule=hourly 2>err && + test_grep "unknown maintenance strategy: .invalid." err +' + test_expect_success 'register and unregister bare repo' ' test_when_finished "git config --global --unset-all maintenance.repo || :" && test_might_fail git config --global --unset-all maintenance.repo && diff --git a/t/t8015-blame-diff-algorithm.sh b/t/t8015-blame-diff-algorithm.sh new file mode 100755 index 0000000000..55e1d540dc --- /dev/null +++ b/t/t8015-blame-diff-algorithm.sh @@ -0,0 +1,203 @@ +#!/bin/sh + +test_description='git blame with specific diff algorithm' + +. ./test-lib.sh + +test_expect_success setup ' + cat >file.c <<-\EOF && + int f(int x, int y) + { + if (x == 0) + { + return y; + } + return x; + } + + int g(size_t u) + { + while (u < 30) + { + u++; + } + return u; + } + EOF + test_write_lines x x x x >file.txt && + git add file.c file.txt && + GIT_AUTHOR_NAME=Commit_1 git commit -m Commit_1 && + + cat >file.c <<-\EOF && + int g(size_t u) + { + while (u < 30) + { + u++; + } + return u; + } + + int h(int x, int y, int z) + { + if (z == 0) + { + return x; + } + return y; + } + EOF + test_write_lines x x x A B C D x E F G >file.txt && + git add file.c file.txt && + GIT_AUTHOR_NAME=Commit_2 git commit -m Commit_2 +' + +test_expect_success 'blame uses Myers diff algorithm by default' ' + cat >expected <<-\EOF && + Commit_2 int g(size_t u) + Commit_1 { + Commit_2 while (u < 30) + Commit_1 { + Commit_2 u++; + Commit_1 } + Commit_2 return u; + Commit_1 } + Commit_1 + Commit_2 int h(int x, int y, int z) + Commit_1 { + Commit_2 if (z == 0) + Commit_1 { + Commit_2 return x; + Commit_1 } + Commit_2 return y; + Commit_1 } + EOF + + git blame file.c > output && + sed -e "s/^[^ ]* (\([^ ]*\) [^)]*)/\1/g" output > without_varying_parts && + sed -e "s/ *$//g" without_varying_parts > actual && + test_cmp expected actual +' + +test_expect_success 'blame honors --diff-algorithm option' ' + cat >expected <<-\EOF && + Commit_1 int g(size_t u) + Commit_1 { + Commit_1 while (u < 30) + Commit_1 { + Commit_1 u++; + Commit_1 } + Commit_1 return u; + Commit_1 } + Commit_2 + Commit_2 int h(int x, int y, int z) + Commit_2 { + Commit_2 if (z == 0) + Commit_2 { + Commit_2 return x; + Commit_2 } + Commit_2 return y; + Commit_2 } + EOF + + git blame file.c --diff-algorithm histogram > output && + sed -e "s/^[^ ]* (\([^ ]*\) [^)]*)/\1/g" output > without_varying_parts && + sed -e "s/ *$//g" without_varying_parts > actual && + test_cmp expected actual +' + +test_expect_success 'blame honors diff.algorithm config variable' ' + cat >expected <<-\EOF && + Commit_1 int g(size_t u) + Commit_1 { + Commit_1 while (u < 30) + Commit_1 { + Commit_1 u++; + Commit_1 } + Commit_1 return u; + Commit_1 } + Commit_2 + Commit_2 int h(int x, int y, int z) + Commit_2 { + Commit_2 if (z == 0) + Commit_2 { + Commit_2 return x; + Commit_2 } + Commit_2 return y; + Commit_2 } + EOF + + git -c diff.algorithm=histogram blame file.c > output && + sed -e "s/^[^ ]* (\([^ ]*\) [^)]*)/\1/g" \ + -e "s/ *$//g" output > actual && + test_cmp expected actual +' + +test_expect_success 'blame gives priority to --diff-algorithm over diff.algorithm' ' + cat >expected <<-\EOF && + Commit_1 int g(size_t u) + Commit_1 { + Commit_1 while (u < 30) + Commit_1 { + Commit_1 u++; + Commit_1 } + Commit_1 return u; + Commit_1 } + Commit_2 + Commit_2 int h(int x, int y, int z) + Commit_2 { + Commit_2 if (z == 0) + Commit_2 { + Commit_2 return x; + Commit_2 } + Commit_2 return y; + Commit_2 } + EOF + + git -c diff.algorithm=myers blame file.c --diff-algorithm histogram > output && + sed -e "s/^[^ ]* (\([^ ]*\) [^)]*)/\1/g" \ + -e "s/ *$//g" output > actual && + test_cmp expected actual +' + +test_expect_success 'blame honors --minimal option' ' + cat >expected <<-\EOF && + Commit_1 x + Commit_1 x + Commit_1 x + Commit_2 A + Commit_2 B + Commit_2 C + Commit_2 D + Commit_1 x + Commit_2 E + Commit_2 F + Commit_2 G + EOF + + git blame file.txt --minimal > output && + sed -e "s/^[^ ]* (\([^ ]*\) [^)]*)/\1/g" output > actual && + test_cmp expected actual +' + +test_expect_success 'blame respects the order of diff options' ' + cat >expected <<-\EOF && + Commit_1 x + Commit_1 x + Commit_1 x + Commit_2 A + Commit_2 B + Commit_2 C + Commit_2 D + Commit_2 x + Commit_2 E + Commit_2 F + Commit_2 G + EOF + + git blame file.txt --minimal --diff-algorithm myers > output && + sed -e "s/^[^ ]* (\([^ ]*\) [^)]*)/\1/g" output > actual && + test_cmp expected actual +' + +test_done diff --git a/t/t8020-last-modified.sh b/t/t8020-last-modified.sh index 61f00bc15c..a4c1114ee2 100755 --- a/t/t8020-last-modified.sh +++ b/t/t8020-last-modified.sh @@ -57,9 +57,9 @@ test_expect_success 'last-modified recursive' ' test_expect_success 'last-modified recursive with show-trees' ' check_last_modified -r -t <<-\EOF - 3 a 3 a/b 3 a/b/file + 3 a 2 a/file 1 file EOF diff --git a/t/t9300-fast-import.sh b/t/t9300-fast-import.sh index 4dc3d645bf..5685cce6fe 100755 --- a/t/t9300-fast-import.sh +++ b/t/t9300-fast-import.sh @@ -2927,16 +2927,16 @@ test_expect_success 'R: blob appears only once' ' # The error message when a space is missing not at the # end of the line is: # -# Missing space after .. +# missing space after .. # # or when extra characters come after the mark at the end # of the line: # -# Garbage after .. +# garbage after .. # # or when the dataref is neither "inline " or a known SHA1, # -# Invalid dataref .. +# invalid dataref .. # test_expect_success 'S: initialize for S tests' ' test_tick && @@ -3405,15 +3405,15 @@ test_path_fail () { test_path_base_fail () { local change="$1" prefix="$2" field="$3" suffix="$4" - test_path_fail "$change" 'unclosed " in '"$field" "$prefix" '"hello.c' "$suffix" "Invalid $field" - test_path_fail "$change" "invalid escape in quoted $field" "$prefix" '"hello\xff"' "$suffix" "Invalid $field" + test_path_fail "$change" 'unclosed " in '"$field" "$prefix" '"hello.c' "$suffix" "invalid $field" + test_path_fail "$change" "invalid escape in quoted $field" "$prefix" '"hello\xff"' "$suffix" "invalid $field" test_path_fail "$change" "escaped NUL in quoted $field" "$prefix" '"hello\000"' "$suffix" "NUL in $field" } test_path_eol_quoted_fail () { local change="$1" prefix="$2" field="$3" test_path_base_fail "$change" "$prefix" "$field" '' - test_path_fail "$change" "garbage after quoted $field" "$prefix" '"hello.c"' 'x' "Garbage after $field" - test_path_fail "$change" "space after quoted $field" "$prefix" '"hello.c"' ' ' "Garbage after $field" + test_path_fail "$change" "garbage after quoted $field" "$prefix" '"hello.c"' 'x' "garbage after $field" + test_path_fail "$change" "space after quoted $field" "$prefix" '"hello.c"' ' ' "garbage after $field" } test_path_eol_fail () { local change="$1" prefix="$2" field="$3" @@ -3422,8 +3422,8 @@ test_path_eol_fail () { test_path_space_fail () { local change="$1" prefix="$2" field="$3" test_path_base_fail "$change" "$prefix" "$field" ' world.c' - test_path_fail "$change" "missing space after quoted $field" "$prefix" '"hello.c"' 'x world.c' "Missing space after $field" - test_path_fail "$change" "missing space after unquoted $field" "$prefix" 'hello.c' '' "Missing space after $field" + test_path_fail "$change" "missing space after quoted $field" "$prefix" '"hello.c"' 'x world.c' "missing space after $field" + test_path_fail "$change" "missing space after unquoted $field" "$prefix" 'hello.c' '' "missing space after $field" } test_path_eol_fail filemodify 'M 100644 :1 ' path @@ -3820,7 +3820,7 @@ test_expect_success 'X: replace ref that becomes useless is removed' ' sed -e s/othername/somename/ tmp >tmp2 && git fast-import --force <tmp2 2>msgs && - grep "Dropping.*since it would point to itself" msgs && + grep "dropping.*since it would point to itself" msgs && git show-ref >refs && ! grep refs/replace refs ) diff --git a/t/t9305-fast-import-signatures.sh b/t/t9305-fast-import-signatures.sh index c2b4271658..63c0a2b5c4 100755 --- a/t/t9305-fast-import-signatures.sh +++ b/t/t9305-fast-import-signatures.sh @@ -70,7 +70,7 @@ test_expect_success GPGSSH 'strip SSH signature with --signed-commits=strip' ' test_must_be_empty log ' -test_expect_success GPG 'setup a commit with dual OpenPGP signatures on its SHA-1 and SHA-256 formats' ' +test_expect_success RUST,GPG 'setup a commit with dual OpenPGP signatures on its SHA-1 and SHA-256 formats' ' # Create a signed SHA-256 commit git init --object-format=sha256 explicit-sha256 && git -C explicit-sha256 config extensions.compatObjectFormat sha1 && @@ -91,7 +91,7 @@ test_expect_success GPG 'setup a commit with dual OpenPGP signatures on its SHA- test_grep -E "^gpgsig-sha256 " out ' -test_expect_success GPG 'strip both OpenPGP signatures with --signed-commits=warn-strip' ' +test_expect_success RUST,GPG 'strip both OpenPGP signatures with --signed-commits=warn-strip' ' git -C explicit-sha256 fast-export --signed-commits=verbatim dual-signed >output && test_grep -E "^gpgsig sha1 openpgp" output && test_grep -E "^gpgsig sha256 openpgp" output && diff --git a/t/t9306-fast-import-signed-tags.sh b/t/t9306-fast-import-signed-tags.sh new file mode 100755 index 0000000000..363619e7d1 --- /dev/null +++ b/t/t9306-fast-import-signed-tags.sh @@ -0,0 +1,80 @@ +#!/bin/sh + +test_description='git fast-import --signed-tags=<mode>' + +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main + +. ./test-lib.sh +. "$TEST_DIRECTORY/lib-gpg.sh" + +test_expect_success 'set up unsigned initial commit and import repo' ' + test_commit first && + git init new +' + +test_expect_success 'import no signed tag with --signed-tags=abort' ' + git fast-export --signed-tags=verbatim >output && + git -C new fast-import --quiet --signed-tags=abort <output +' + +test_expect_success GPG 'set up OpenPGP signed tag' ' + git tag -s -m "OpenPGP signed tag" openpgp-signed first && + OPENPGP_SIGNED=$(git rev-parse --verify refs/tags/openpgp-signed) && + git fast-export --signed-tags=verbatim openpgp-signed >output +' + +test_expect_success GPG 'import OpenPGP signed tag with --signed-tags=abort' ' + test_must_fail git -C new fast-import --quiet --signed-tags=abort <output +' + +test_expect_success GPG 'import OpenPGP signed tag with --signed-tags=verbatim' ' + git -C new fast-import --quiet --signed-tags=verbatim <output >log 2>&1 && + IMPORTED=$(git -C new rev-parse --verify refs/tags/openpgp-signed) && + test $OPENPGP_SIGNED = $IMPORTED && + test_must_be_empty log +' + +test_expect_success GPGSM 'setup X.509 signed tag' ' + test_config gpg.format x509 && + test_config user.signingkey $GIT_COMMITTER_EMAIL && + + git tag -s -m "X.509 signed tag" x509-signed first && + X509_SIGNED=$(git rev-parse --verify refs/tags/x509-signed) && + git fast-export --signed-tags=verbatim x509-signed >output +' + +test_expect_success GPGSM 'import X.509 signed tag with --signed-tags=warn-strip' ' + git -C new fast-import --quiet --signed-tags=warn-strip <output >log 2>&1 && + test_grep "stripping a tag signature for tag '\''x509-signed'\''" log && + IMPORTED=$(git -C new rev-parse --verify refs/tags/x509-signed) && + test $X509_SIGNED != $IMPORTED && + git -C new cat-file -p x509-signed >out && + test_grep ! "SIGNED MESSAGE" out +' + +test_expect_success GPGSSH 'setup SSH signed tag' ' + test_config gpg.format ssh && + test_config user.signingkey "${GPGSSH_KEY_PRIMARY}" && + + git tag -s -m "SSH signed tag" ssh-signed first && + SSH_SIGNED=$(git rev-parse --verify refs/tags/ssh-signed) && + git fast-export --signed-tags=verbatim ssh-signed >output +' + +test_expect_success GPGSSH 'import SSH signed tag with --signed-tags=warn-verbatim' ' + git -C new fast-import --quiet --signed-tags=warn-verbatim <output >log 2>&1 && + test_grep "importing a tag signature verbatim for tag '\''ssh-signed'\''" log && + IMPORTED=$(git -C new rev-parse --verify refs/tags/ssh-signed) && + test $SSH_SIGNED = $IMPORTED +' + +test_expect_success GPGSSH 'import SSH signed tag with --signed-tags=strip' ' + git -C new fast-import --quiet --signed-tags=strip <output >log 2>&1 && + test_must_be_empty log && + IMPORTED=$(git -C new rev-parse --verify refs/tags/ssh-signed) && + test $SSH_SIGNED != $IMPORTED && + git -C new cat-file -p ssh-signed >out && + test_grep ! "SSH SIGNATURE" out +' + +test_done diff --git a/t/t9350-fast-export.sh b/t/t9350-fast-export.sh index 8f85c69d62..784d68b6e5 100755 --- a/t/t9350-fast-export.sh +++ b/t/t9350-fast-export.sh @@ -35,6 +35,7 @@ test_expect_success 'setup' ' git commit -m sitzt file2 && test_tick && git tag -a -m valentin muss && + ANNOTATED_TAG_COUNT=1 && git merge -s ours main ' @@ -229,7 +230,8 @@ EOF test_expect_success 'set up faked signed tag' ' - git fast-import <signed-tag-import + git fast-import <signed-tag-import && + ANNOTATED_TAG_COUNT=$((ANNOTATED_TAG_COUNT + 1)) ' @@ -277,6 +279,42 @@ test_expect_success 'signed-tags=warn-strip' ' test -s err ' +test_expect_success GPGSM 'setup X.509 signed tag' ' + test_config gpg.format x509 && + test_config user.signingkey $GIT_COMMITTER_EMAIL && + + git tag -s -m "X.509 signed tag" x509-signed $(git rev-parse HEAD) && + ANNOTATED_TAG_COUNT=$((ANNOTATED_TAG_COUNT + 1)) +' + +test_expect_success GPGSM 'signed-tags=verbatim with X.509' ' + git fast-export --signed-tags=verbatim x509-signed > output && + test_grep "SIGNED MESSAGE" output +' + +test_expect_success GPGSM 'signed-tags=strip with X.509' ' + git fast-export --signed-tags=strip x509-signed > output && + test_grep ! "SIGNED MESSAGE" output +' + +test_expect_success GPGSSH 'setup SSH signed tag' ' + test_config gpg.format ssh && + test_config user.signingkey "${GPGSSH_KEY_PRIMARY}" && + + git tag -s -m "SSH signed tag" ssh-signed $(git rev-parse HEAD) && + ANNOTATED_TAG_COUNT=$((ANNOTATED_TAG_COUNT + 1)) +' + +test_expect_success GPGSSH 'signed-tags=verbatim with SSH' ' + git fast-export --signed-tags=verbatim ssh-signed > output && + test_grep "SSH SIGNATURE" output +' + +test_expect_success GPGSSH 'signed-tags=strip with SSH' ' + git fast-export --signed-tags=strip ssh-signed > output && + test_grep ! "SSH SIGNATURE" output +' + test_expect_success GPG 'set up signed commit' ' # Generate a commit with both "gpgsig" and "encoding" set, so @@ -491,8 +529,9 @@ test_expect_success 'fast-export -C -C | fast-import' ' test_expect_success 'fast-export | fast-import when main is tagged' ' git tag -m msg last && + ANNOTATED_TAG_COUNT=$((ANNOTATED_TAG_COUNT + 1)) && git fast-export -C -C --signed-tags=strip --all > output && - test $(grep -c "^tag " output) = 3 + test $(grep -c "^tag " output) = $ANNOTATED_TAG_COUNT ' @@ -506,12 +545,13 @@ test_expect_success 'cope with tagger-less tags' ' TAG=$(git hash-object --literally -t tag -w tag-content) && git update-ref refs/tags/sonnenschein $TAG && + ANNOTATED_TAG_COUNT=$((ANNOTATED_TAG_COUNT + 1)) && git fast-export -C -C --signed-tags=strip --all > output && - test $(grep -c "^tag " output) = 4 && + test $(grep -c "^tag " output) = $ANNOTATED_TAG_COUNT && ! grep "Unspecified Tagger" output && git fast-export -C -C --signed-tags=strip --all \ --fake-missing-tagger > output && - test $(grep -c "^tag " output) = 4 && + test $(grep -c "^tag " output) = $ANNOTATED_TAG_COUNT && grep "Unspecified Tagger" output ' @@ -932,7 +972,7 @@ test_expect_success 'fast-export handles --end-of-options' ' test_cmp expect actual ' -test_expect_success GPG 'setup a commit with dual signatures on its SHA-1 and SHA-256 formats' ' +test_expect_success GPG,RUST 'setup a commit with dual signatures on its SHA-1 and SHA-256 formats' ' # Create a signed SHA-256 commit git init --object-format=sha256 explicit-sha256 && git -C explicit-sha256 config extensions.compatObjectFormat sha1 && @@ -953,7 +993,7 @@ test_expect_success GPG 'setup a commit with dual signatures on its SHA-1 and SH test_grep -E "^gpgsig-sha256 " out ' -test_expect_success GPG 'export and import of doubly signed commit' ' +test_expect_success GPG,RUST 'export and import of doubly signed commit' ' git -C explicit-sha256 fast-export --signed-commits=verbatim dual-signed >output && test_grep -E "^gpgsig sha1 openpgp" output && test_grep -E "^gpgsig sha256 openpgp" output && diff --git a/t/t9902-completion.sh b/t/t9902-completion.sh index 964e1f1569..ffb9c8b522 100755 --- a/t/t9902-completion.sh +++ b/t/t9902-completion.sh @@ -3053,6 +3053,7 @@ test_expect_success 'git config set - variable name - __git_compute_second_level submodule.sub.fetchRecurseSubmodules Z submodule.sub.ignore Z submodule.sub.active Z + submodule.sub.gitdir Z EOF ' diff --git a/t/test-lib-functions.sh b/t/test-lib-functions.sh index a28de7b19b..52d7759bf5 100644 --- a/t/test-lib-functions.sh +++ b/t/test-lib-functions.sh @@ -1708,11 +1708,16 @@ test_set_hash () { # Detect the hash algorithm in use. test_detect_hash () { case "${GIT_TEST_DEFAULT_HASH:-$GIT_TEST_BUILTIN_HASH}" in - "sha256") + *:*) + test_hash_algo="${GIT_TEST_DEFAULT_HASH%%:*}" + test_compat_hash_algo="${GIT_TEST_DEFAULT_HASH##*:}" + test_repo_compat_hash_algo="$test_compat_hash_algo" + ;; + sha256) test_hash_algo=sha256 test_compat_hash_algo=sha1 ;; - *) + sha1) test_hash_algo=sha1 test_compat_hash_algo=sha256 ;; diff --git a/t/test-lib.sh b/t/test-lib.sh index 562f950fb0..3499a83806 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -1890,6 +1890,10 @@ test_lazy_prereq LONG_IS_64BIT ' test 8 -le "$(build_option sizeof-long)" ' +test_lazy_prereq RUST ' + test "$(build_option rust)" = enabled +' + test_lazy_prereq TIME_IS_64BIT 'test-tool date is64bit' test_lazy_prereq TIME_T_IS_64BIT 'test-tool date time_t-is64bit' @@ -1924,6 +1928,19 @@ test_lazy_prereq DEFAULT_HASH_ALGORITHM ' test_lazy_prereq DEFAULT_REPO_FORMAT ' test_have_prereq SHA1,REFFILES ' +# BROKEN_OBJECTS is a test whether we can write deliberately broken objects and +# expect them to work. When running using SHA-256 mode with SHA-1 +# compatibility, we cannot write such objects because there's no SHA-1 +# compatibility value for a nonexistent object. +test_lazy_prereq BROKEN_OBJECTS ' + ! test_have_prereq COMPAT_HASH +' + +# COMPAT_HASH is a test if we're operating in a repository with SHA-256 with +# SHA-1 compatibility. +test_lazy_prereq COMPAT_HASH ' + test -n "$test_repo_compat_hash_algo" +' # Ensure that no test accidentally triggers a Git command # that runs the actual maintenance scheduler, affecting a user's diff --git a/t/unit-tests/u-reftable-stack.c b/t/unit-tests/u-reftable-stack.c index a8b91812e8..b8110cdeee 100644 --- a/t/unit-tests/u-reftable-stack.c +++ b/t/unit-tests/u-reftable-stack.c @@ -1067,6 +1067,7 @@ void test_reftable_stack__add_performs_auto_compaction(void) .value_type = REFTABLE_REF_SYMREF, .value.symref = (char *) "master", }; + bool required = false; char buf[128]; /* @@ -1087,10 +1088,17 @@ void test_reftable_stack__add_performs_auto_compaction(void) * auto compaction is disabled. When enabled, we should merge * all tables in the stack. */ - if (i != n) + cl_assert_equal_i(reftable_stack_compaction_required(st, true, &required), 0); + if (i != n) { cl_assert_equal_i(st->merged->tables_len, i + 1); - else + if (i < 1) + cl_assert_equal_b(required, false); + else + cl_assert_equal_b(required, true); + } else { cl_assert_equal_i(st->merged->tables_len, 1); + cl_assert_equal_b(required, false); + } } reftable_stack_destroy(st); @@ -94,18 +94,6 @@ struct object *deref_tag(struct repository *r, struct object *o, const char *war return o; } -struct object *deref_tag_noverify(struct repository *r, struct object *o) -{ - while (o && o->type == OBJ_TAG) { - o = parse_object(r, &o->oid); - if (o && o->type == OBJ_TAG && ((struct tag *)o)->tagged) - o = ((struct tag *)o)->tagged; - else - o = NULL; - } - return o; -} - struct tag *lookup_tag(struct repository *r, const struct object_id *oid) { struct object *obj = lookup_object(r, oid); @@ -16,7 +16,6 @@ int parse_tag_buffer(struct repository *r, struct tag *item, const void *data, u int parse_tag(struct tag *item); void release_tag_memory(struct tag *t); struct object *deref_tag(struct repository *r, struct object *, const char *, int); -struct object *deref_tag_noverify(struct repository *r, struct object *); int gpg_verify_tag(const struct object_id *oid, const char *name_to_report, unsigned flags); struct object_id *get_tagged_oid(struct tag *tag); @@ -7,8 +7,11 @@ #include "string-list.h" #include "run-command.h" #include "commit.h" +#include "strvec.h" #include "trailer.h" #include "list.h" +#include "wrapper.h" + /* * Copyright (c) 2013, 2014 Christian Couder <chriscool@tuxfamily.org> */ @@ -772,6 +775,30 @@ void parse_trailers_from_command_line_args(struct list_head *arg_head, free(cl_separators); } +void validate_trailer_args_after_config(const struct strvec *cli_args) +{ + char *cl_separators; + + trailer_config_init(); + + cl_separators = xstrfmt("=%s", separators); + + for (size_t i = 0; i < cli_args->nr; i++) { + const char *txt = cli_args->v[i]; + ssize_t separator_pos; + + if (!*txt) + die(_("empty --trailer argument")); + + separator_pos = find_separator(txt, cl_separators); + if (separator_pos == 0) + die(_("invalid trailer '%s': missing key before separator"), + txt); + } + + free(cl_separators); +} + static const char *next_line(const char *str) { const char *nl = strchrnul(str, '\n'); @@ -1224,14 +1251,98 @@ void trailer_iterator_release(struct trailer_iterator *iter) strbuf_release(&iter->key); } -int amend_file_with_trailers(const char *path, const struct strvec *trailer_args) +int amend_strbuf_with_trailers(struct strbuf *buf, + const struct strvec *trailer_args) { - struct child_process run_trailer = CHILD_PROCESS_INIT; - - run_trailer.git_cmd = 1; - strvec_pushl(&run_trailer.args, "interpret-trailers", - "--in-place", "--no-divider", - path, NULL); - strvec_pushv(&run_trailer.args, trailer_args->v); - return run_command(&run_trailer); + struct process_trailer_options opts = PROCESS_TRAILER_OPTIONS_INIT; + LIST_HEAD(new_trailer_head); + struct strbuf out = STRBUF_INIT; + size_t i; + + opts.no_divider = 1; + + for (i = 0; i < trailer_args->nr; i++) { + const char *text = trailer_args->v[i]; + struct new_trailer_item *item; + + if (!*text) + continue; + item = xcalloc(1, sizeof(*item)); + INIT_LIST_HEAD(&item->list); + item->text = text; + list_add_tail(&item->list, &new_trailer_head); + } + + process_trailers(&opts, &new_trailer_head, buf, &out); + + strbuf_swap(buf, &out); + strbuf_release(&out); + while (!list_empty(&new_trailer_head)) { + struct new_trailer_item *item = + list_first_entry(&new_trailer_head, struct new_trailer_item, list); + list_del(&item->list); + free(item); + } + return 0; +} + +int amend_file_with_trailers(const char *path, + const struct strvec *trailer_args) +{ + struct strbuf buf = STRBUF_INIT; + + if (!trailer_args || !trailer_args->nr) + return 0; + + if (strbuf_read_file(&buf, path, 0) < 0) + return error_errno("could not read '%s'", path); + + if (amend_strbuf_with_trailers(&buf, trailer_args)) { + strbuf_release(&buf); + return error("failed to append trailers"); + } + + if (write_file_buf_gently(path, buf.buf, buf.len)) { + strbuf_release(&buf); + return -1; + } + + strbuf_release(&buf); + return 0; + } + +void process_trailers(const struct process_trailer_options *opts, + struct list_head *new_trailer_head, + struct strbuf *sb, struct strbuf *out) +{ + LIST_HEAD(head); + struct trailer_block *trailer_block; + + trailer_block = parse_trailers(opts, sb->buf, &head); + + /* Print the lines before the trailer block */ + if (!opts->only_trailers) + strbuf_add(out, sb->buf, trailer_block_start(trailer_block)); + + if (!opts->only_trailers && !blank_line_before_trailer_block(trailer_block)) + strbuf_addch(out, '\n'); + + if (!opts->only_input) { + LIST_HEAD(config_head); + LIST_HEAD(arg_head); + parse_trailers_from_config(&config_head); + parse_trailers_from_command_line_args(&arg_head, new_trailer_head); + list_splice(&config_head, &arg_head); + process_trailers_lists(&head, &arg_head); + } + + /* Print trailer block. */ + format_trailers(opts, &head, out); + free_trailers(&head); + + /* Print the lines after the trailer block as is. */ + if (!opts->only_trailers) + strbuf_add(out, sb->buf + trailer_block_end(trailer_block), + sb->len - trailer_block_end(trailer_block)); + trailer_block_release(trailer_block); } @@ -68,6 +68,8 @@ void parse_trailers_from_config(struct list_head *config_head); void parse_trailers_from_command_line_args(struct list_head *arg_head, struct list_head *new_trailer_head); +void validate_trailer_args_after_config(const struct strvec *cli_args); + void process_trailers_lists(struct list_head *head, struct list_head *arg_head); @@ -195,11 +197,16 @@ int trailer_iterator_advance(struct trailer_iterator *iter); */ void trailer_iterator_release(struct trailer_iterator *iter); +int amend_strbuf_with_trailers(struct strbuf *buf, + const struct strvec *trailer_args); + /* - * Augment a file to add trailers to it by running git-interpret-trailers. - * This calls run_command() and its return value is the same (i.e. 0 for - * success, various non-zero for other errors). See run-command.h. + * Augment a file to add trailers to it (similar to 'git interpret-trailers'). + * Returns 0 on success or a non-zero error code on failure. */ int amend_file_with_trailers(const char *path, const struct strvec *trailer_args); +void process_trailers(const struct process_trailer_options *opts, + struct list_head *new_trailer_head, + struct strbuf *sb, struct strbuf *out); #endif /* TRAILER_H */ diff --git a/transport.c b/transport.c index c7f06a7382..67368754bf 100644 --- a/transport.c +++ b/transport.c @@ -1316,65 +1316,56 @@ static void die_with_unpushed_submodules(struct string_list *needs_pushing) die(_("Aborting.")); } -static int run_pre_push_hook(struct transport *transport, - struct ref *remote_refs) +static int pre_push_hook_feed_stdin(int hook_stdin_fd, void *pp_cb, void *pp_task_cb UNUSED) { - int ret = 0, x; - struct ref *r; - struct child_process proc = CHILD_PROCESS_INIT; - struct strbuf buf; - const char *hook_path = find_hook(the_repository, "pre-push"); + struct hook_cb_data *hook_cb = pp_cb; + struct ref *r = hook_cb->options->feed_pipe_ctx; + struct strbuf *buf = hook_cb->options->feed_pipe_cb_data; + int ret = 0; - if (!hook_path) - return 0; + if (!r) + return 1; /* no more refs */ - strvec_push(&proc.args, hook_path); - strvec_push(&proc.args, transport->remote->name); - strvec_push(&proc.args, transport->url); + if (!buf) + BUG("pipe_task_cb must contain a valid strbuf"); - proc.in = -1; - proc.trace2_hook_name = "pre-push"; + hook_cb->options->feed_pipe_ctx = r->next; + strbuf_reset(buf); - if (start_command(&proc)) { - finish_command(&proc); - return -1; - } - - sigchain_push(SIGPIPE, SIG_IGN); + if (!r->peer_ref) return 0; + if (r->status == REF_STATUS_REJECT_NONFASTFORWARD) return 0; + if (r->status == REF_STATUS_REJECT_STALE) return 0; + if (r->status == REF_STATUS_REJECT_REMOTE_UPDATED) return 0; + if (r->status == REF_STATUS_UPTODATE) return 0; - strbuf_init(&buf, 256); + strbuf_addf(buf, "%s %s %s %s\n", + r->peer_ref->name, oid_to_hex(&r->new_oid), + r->name, oid_to_hex(&r->old_oid)); - for (r = remote_refs; r; r = r->next) { - if (!r->peer_ref) continue; - if (r->status == REF_STATUS_REJECT_NONFASTFORWARD) continue; - if (r->status == REF_STATUS_REJECT_STALE) continue; - if (r->status == REF_STATUS_REJECT_REMOTE_UPDATED) continue; - if (r->status == REF_STATUS_UPTODATE) continue; + ret = write_in_full(hook_stdin_fd, buf->buf, buf->len); + if (ret < 0 && errno != EPIPE) + return ret; /* We do not mind if a hook does not read all refs. */ - strbuf_reset(&buf); - strbuf_addf( &buf, "%s %s %s %s\n", - r->peer_ref->name, oid_to_hex(&r->new_oid), - r->name, oid_to_hex(&r->old_oid)); + return 0; +} - if (write_in_full(proc.in, buf.buf, buf.len) < 0) { - /* We do not mind if a hook does not read all refs. */ - if (errno != EPIPE) - ret = -1; - break; - } - } +static int run_pre_push_hook(struct transport *transport, + struct ref *remote_refs) +{ + struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; + struct strbuf buf = STRBUF_INIT; + int ret = 0; - strbuf_release(&buf); + strvec_push(&opt.args, transport->remote->name); + strvec_push(&opt.args, transport->url); - x = close(proc.in); - if (!ret) - ret = x; + opt.feed_pipe = pre_push_hook_feed_stdin; + opt.feed_pipe_ctx = remote_refs; + opt.feed_pipe_cb_data = &buf; - sigchain_pop(SIGPIPE); + ret = run_hooks_opt(the_repository, "pre-push", &opt); - x = finish_command(&proc); - if (!ret) - ret = x; + strbuf_release(&buf); return ret; } diff --git a/unicode-width.h b/unicode-width.h index 3ffee123a0..b701129515 100644 --- a/unicode-width.h +++ b/unicode-width.h @@ -143,7 +143,8 @@ static const struct interval zero_width[] = { { 0x1A65, 0x1A6C }, { 0x1A73, 0x1A7C }, { 0x1A7F, 0x1A7F }, -{ 0x1AB0, 0x1ACE }, +{ 0x1AB0, 0x1ADD }, +{ 0x1AE0, 0x1AEB }, { 0x1B00, 0x1B03 }, { 0x1B34, 0x1B34 }, { 0x1B36, 0x1B3A }, @@ -229,7 +230,7 @@ static const struct interval zero_width[] = { { 0x10D24, 0x10D27 }, { 0x10D69, 0x10D6D }, { 0x10EAB, 0x10EAC }, -{ 0x10EFC, 0x10EFF }, +{ 0x10EFA, 0x10EFF }, { 0x10F46, 0x10F50 }, { 0x10F82, 0x10F85 }, { 0x11001, 0x11001 }, @@ -306,6 +307,9 @@ static const struct interval zero_width[] = { { 0x11A59, 0x11A5B }, { 0x11A8A, 0x11A96 }, { 0x11A98, 0x11A99 }, +{ 0x11B60, 0x11B60 }, +{ 0x11B62, 0x11B64 }, +{ 0x11B66, 0x11B66 }, { 0x11C30, 0x11C36 }, { 0x11C38, 0x11C3D }, { 0x11C3F, 0x11C3F }, @@ -362,6 +366,10 @@ static const struct interval zero_width[] = { { 0x1E2EC, 0x1E2EF }, { 0x1E4EC, 0x1E4EF }, { 0x1E5EE, 0x1E5EF }, +{ 0x1E6E3, 0x1E6E3 }, +{ 0x1E6E6, 0x1E6E6 }, +{ 0x1E6EE, 0x1E6EF }, +{ 0x1E6F5, 0x1E6F5 }, { 0x1E8D0, 0x1E8D6 }, { 0x1E944, 0x1E94A }, { 0xE0001, 0xE0001 }, @@ -429,10 +437,10 @@ static const struct interval double_width[] = { { 0xFF01, 0xFF60 }, { 0xFFE0, 0xFFE6 }, { 0x16FE0, 0x16FE4 }, -{ 0x16FF0, 0x16FF1 }, -{ 0x17000, 0x187F7 }, -{ 0x18800, 0x18CD5 }, -{ 0x18CFF, 0x18D08 }, +{ 0x16FF0, 0x16FF6 }, +{ 0x17000, 0x18CD5 }, +{ 0x18CFF, 0x18D1E }, +{ 0x18D80, 0x18DF2 }, { 0x1AFF0, 0x1AFF3 }, { 0x1AFF5, 0x1AFFB }, { 0x1AFFD, 0x1AFFE }, @@ -474,7 +482,7 @@ static const struct interval double_width[] = { { 0x1F680, 0x1F6C5 }, { 0x1F6CC, 0x1F6CC }, { 0x1F6D0, 0x1F6D2 }, -{ 0x1F6D5, 0x1F6D7 }, +{ 0x1F6D5, 0x1F6D8 }, { 0x1F6DC, 0x1F6DF }, { 0x1F6EB, 0x1F6EC }, { 0x1F6F4, 0x1F6FC }, @@ -484,11 +492,12 @@ static const struct interval double_width[] = { { 0x1F93C, 0x1F945 }, { 0x1F947, 0x1F9FF }, { 0x1FA70, 0x1FA7C }, -{ 0x1FA80, 0x1FA89 }, -{ 0x1FA8F, 0x1FAC6 }, -{ 0x1FACE, 0x1FADC }, -{ 0x1FADF, 0x1FAE9 }, -{ 0x1FAF0, 0x1FAF8 }, +{ 0x1FA80, 0x1FA8A }, +{ 0x1FA8E, 0x1FAC6 }, +{ 0x1FAC8, 0x1FAC8 }, +{ 0x1FACD, 0x1FADC }, +{ 0x1FADF, 0x1FAEA }, +{ 0x1FAEF, 0x1FAF8 }, { 0x20000, 0x2FFFD }, { 0x30000, 0x3FFFD } }; diff --git a/upload-pack.c b/upload-pack.c index 1e87ae9559..2d2b70cbf2 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -870,8 +870,8 @@ static void send_unshallow(struct upload_pack_data *data) } } -static int check_ref(const char *refname_full, const char *referent UNUSED, const struct object_id *oid, - int flag, void *cb_data); +static int check_ref(const struct reference *ref, void *cb_data); + static void deepen(struct upload_pack_data *data, int depth) { if (depth == INFINITE_DEPTH && !is_repository_shallow(the_repository)) { @@ -1224,13 +1224,12 @@ static int mark_our_ref(const char *refname, const char *refname_full, return 0; } -static int check_ref(const char *refname_full, const char *referent UNUSED,const struct object_id *oid, - int flag UNUSED, void *cb_data) +static int check_ref(const struct reference *ref, void *cb_data) { - const char *refname = strip_namespace(refname_full); + const char *refname = strip_namespace(ref->name); struct upload_pack_data *data = cb_data; - mark_our_ref(refname, refname_full, oid, &data->hidden_refs); + mark_our_ref(refname, ref->name, ref->oid, &data->hidden_refs); return 0; } @@ -1250,15 +1249,15 @@ static void format_session_id(struct strbuf *buf, struct upload_pack_data *d) { } static void write_v0_ref(struct upload_pack_data *data, - const char *refname, const char *refname_nons, - const struct object_id *oid) + const struct reference *ref, + const char *refname_nons) { static const char *capabilities = "multi_ack thin-pack side-band" " side-band-64k ofs-delta shallow deepen-since deepen-not" " deepen-relative no-progress include-tag multi_ack_detailed"; struct object_id peeled; - if (mark_our_ref(refname_nons, refname, oid, &data->hidden_refs)) + if (mark_our_ref(refname_nons, ref->name, ref->oid, &data->hidden_refs)) return; if (capabilities) { @@ -1268,7 +1267,7 @@ static void write_v0_ref(struct upload_pack_data *data, format_symref_info(&symref_info, &data->symref); format_session_id(&session_id, data); packet_fwrite_fmt(stdout, "%s %s%c%s%s%s%s%s%s%s object-format=%s agent=%s\n", - oid_to_hex(oid), refname_nons, + oid_to_hex(ref->oid), refname_nons, 0, capabilities, (data->allow_uor & ALLOW_TIP_SHA1) ? " allow-tip-sha1-in-want" : "", @@ -1284,35 +1283,33 @@ static void write_v0_ref(struct upload_pack_data *data, strbuf_release(&session_id); data->sent_capabilities = 1; } else { - packet_fwrite_fmt(stdout, "%s %s\n", oid_to_hex(oid), refname_nons); + packet_fwrite_fmt(stdout, "%s %s\n", oid_to_hex(ref->oid), refname_nons); } capabilities = NULL; - if (!peel_iterated_oid(the_repository, oid, &peeled)) + if (!reference_get_peeled_oid(the_repository, ref, &peeled)) packet_fwrite_fmt(stdout, "%s %s^{}\n", oid_to_hex(&peeled), refname_nons); return; } -static int send_ref(const char *refname, const char *referent UNUSED, const struct object_id *oid, - int flag UNUSED, void *cb_data) +static int send_ref(const struct reference *ref, void *cb_data) { - write_v0_ref(cb_data, refname, strip_namespace(refname), oid); + write_v0_ref(cb_data, ref, strip_namespace(ref->name)); return 0; } -static int find_symref(const char *refname, const char *referent UNUSED, - const struct object_id *oid UNUSED, - int flag, void *cb_data) +static int find_symref(const struct reference *ref, void *cb_data) { const char *symref_target; struct string_list_item *item; + int flag; - if ((flag & REF_ISSYMREF) == 0) + if ((ref->flags & REF_ISSYMREF) == 0) return 0; symref_target = refs_resolve_ref_unsafe(get_main_ref_store(the_repository), - refname, 0, NULL, &flag); + ref->name, 0, NULL, &flag); if (!symref_target || (flag & REF_ISSYMREF) == 0) - die("'%s' is a symref but it is not?", refname); - item = string_list_append(cb_data, strip_namespace(refname)); + die("'%s' is a symref but it is not?", ref->name); + item = string_list_append(cb_data, strip_namespace(ref->name)); item->util = xstrdup(strip_namespace(symref_target)); return 0; } @@ -1445,8 +1442,12 @@ void upload_pack(const int advertise_refs, const int stateless_rpc, send_ref, &data); for_each_namespaced_ref_1(send_ref, &data); if (!data.sent_capabilities) { - const char *refname = "capabilities^{}"; - write_v0_ref(&data, refname, refname, null_oid(the_hash_algo)); + struct reference ref = { + .name = "capabilities^{}", + .oid = null_oid(the_hash_algo), + }; + + write_v0_ref(&data, &ref, ref.name); } /* * fflush stdout before calling advertise_shallow_grafts because send_ref @@ -3,6 +3,29 @@ #include "strbuf.h" #include "url.h" +/* + * The set of unreserved characters as per STD66 (RFC3986) is + * '[A-Za-z0-9-._~]'. These characters are safe to appear in URI + * components without percent-encoding. + */ +int is_rfc3986_unreserved(char ch) +{ + return isalnum(ch) || + ch == '-' || ch == '_' || ch == '.' || ch == '~'; +} + +/* + * This is a variant of is_rfc3986_unreserved() that treats uppercase + * letters as "reserved". This forces them to be percent-encoded, allowing + * 'Foo' (%46oo) and 'foo' (foo) to be distinct on case-folding filesystems. + */ +int is_casefolding_rfc3986_unreserved(char c) +{ + return (c >= 'a' && c <= 'z') || + (c >= '0' && c <= '9') || + c == '-' || c == '.' || c == '_' || c == '~'; +} + int is_urlschemechar(int first_flag, int ch) { /* @@ -21,4 +21,7 @@ char *url_decode_parameter_value(const char **query); void end_url_with_slash(struct strbuf *buf, const char *url); void str_end_url_with_slash(const char *url, char **dest); +int is_rfc3986_unreserved(char ch); +int is_casefolding_rfc3986_unreserved(char c); + #endif /* URL_H */ @@ -226,14 +226,10 @@ static int interpret_target(struct walker *walker, char *target, struct object_i return -1; } -static int mark_complete(const char *path UNUSED, - const char *referent UNUSED, - const struct object_id *oid, - int flag UNUSED, - void *cb_data UNUSED) +static int mark_complete(const struct reference *ref, void *cb_data UNUSED) { struct commit *commit = lookup_commit_reference_gently(the_repository, - oid, 1); + ref->oid, 1); if (commit) { commit->object.flags |= COMPLETE; diff --git a/worktree.c b/worktree.c index a2a5f51f29..9308389cb6 100644 --- a/worktree.c +++ b/worktree.c @@ -595,8 +595,15 @@ int other_head_refs(each_ref_fn fn, void *cb_data) if (refs_resolve_ref_unsafe(get_main_ref_store(the_repository), refname.buf, RESOLVE_REF_READING, - &oid, &flag)) - ret = fn(refname.buf, NULL, &oid, flag, cb_data); + &oid, &flag)) { + struct reference ref = { + .name = refname.buf, + .oid = &oid, + .flags = flag, + }; + + ret = fn(&ref, cb_data); + } if (ret) break; } @@ -688,6 +688,22 @@ void write_file_buf(const char *path, const char *buf, size_t len) die_errno(_("could not close '%s'"), path); } +int write_file_buf_gently(const char *path, const char *buf, size_t len) +{ + int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0666); + + if (fd < 0) + return error_errno(_("could not open '%s'"), path); + if (write_in_full(fd, buf, len) < 0) { + int ret = error_errno(_("could not write to '%s'"), path); + close(fd); + return ret; + } + if (close(fd)) + return error_errno(_("could not close '%s'"), path); + return 0; +} + void write_file(const char *path, const char *fmt, ...) { va_list params; @@ -57,6 +57,12 @@ static inline ssize_t write_str_in_full(int fd, const char *str) void write_file_buf(const char *path, const char *buf, size_t len); /** + * Like write_file_buf(), but report errors instead of exiting. Returns 0 on + * success or a negative value on error after emitting a message. + */ +int write_file_buf_gently(const char *path, const char *buf, size_t len); + +/** * Like write_file_buf(), but format the contents into a buffer first. * Additionally, write_file() will append a newline if one is not already * present, making it convenient to write text files: diff --git a/write-or-die.h b/write-or-die.h index 65a5c42a47..8d5ec23e1f 100644 --- a/write-or-die.h +++ b/write-or-die.h @@ -21,6 +21,7 @@ enum fsync_component { FSYNC_COMPONENT_COMMIT_GRAPH = 1 << 3, FSYNC_COMPONENT_INDEX = 1 << 4, FSYNC_COMPONENT_REFERENCE = 1 << 5, + FSYNC_COMPONENT_LOOSE_OBJECT_MAP = 1 << 6, }; #define FSYNC_COMPONENTS_OBJECTS (FSYNC_COMPONENT_LOOSE_OBJECT | \ @@ -44,7 +45,8 @@ enum fsync_component { FSYNC_COMPONENT_PACK_METADATA | \ FSYNC_COMPONENT_COMMIT_GRAPH | \ FSYNC_COMPONENT_INDEX | \ - FSYNC_COMPONENT_REFERENCE) + FSYNC_COMPONENT_REFERENCE | \ + FSYNC_COMPONENT_LOOSE_OBJECT_MAP) #ifndef FSYNC_COMPONENTS_PLATFORM_DEFAULT #define FSYNC_COMPONENTS_PLATFORM_DEFAULT FSYNC_COMPONENTS_DEFAULT @@ -26,6 +26,7 @@ static struct whitespace_rule { { "blank-at-eol", WS_BLANK_AT_EOL, 0 }, { "blank-at-eof", WS_BLANK_AT_EOF, 0 }, { "tab-in-indent", WS_TAB_IN_INDENT, 0, 1 }, + { "incomplete-line", WS_INCOMPLETE_LINE, 0, 0 }, }; unsigned parse_whitespace_rule(const char *string) @@ -139,6 +140,11 @@ char *whitespace_error_string(unsigned ws) strbuf_addstr(&err, ", "); strbuf_addstr(&err, "tab in indent"); } + if (ws & WS_INCOMPLETE_LINE) { + if (err.len) + strbuf_addstr(&err, ", "); + strbuf_addstr(&err, "no newline at the end of file"); + } return strbuf_detach(&err, NULL); } @@ -180,6 +186,9 @@ static unsigned ws_check_emit_1(const char *line, int len, unsigned ws_rule, if (trailing_whitespace == -1) trailing_whitespace = len; + if (!trailing_newline && (ws_rule & WS_INCOMPLETE_LINE)) + result |= WS_INCOMPLETE_LINE; + /* Check indentation */ for (i = 0; i < trailing_whitespace; i++) { if (line[i] == ' ') @@ -292,6 +301,17 @@ void ws_fix_copy(struct strbuf *dst, const char *src, int len, unsigned ws_rule, int need_fix_leading_space = 0; /* + * Remembering that we need to add '\n' at the end + * is sufficient to fix an incomplete line. + */ + if (ws_rule & WS_INCOMPLETE_LINE) { + if (0 < len && src[len - 1] != '\n') { + fixed = 1; + add_nl_to_tail = 1; + } + } + + /* * Strip trailing whitespace */ if (ws_rule & WS_BLANK_AT_EOL) { @@ -7,19 +7,23 @@ struct strbuf; /* * whitespace rules. * used by both diff and apply - * last two digits are tab width + * last two octal-digits are tab width (we support only up to 63). */ -#define WS_BLANK_AT_EOL 0100 -#define WS_SPACE_BEFORE_TAB 0200 -#define WS_INDENT_WITH_NON_TAB 0400 -#define WS_CR_AT_EOL 01000 -#define WS_BLANK_AT_EOF 02000 -#define WS_TAB_IN_INDENT 04000 -#define WS_TRAILING_SPACE (WS_BLANK_AT_EOL|WS_BLANK_AT_EOF) +#define WS_BLANK_AT_EOL (1<<6) +#define WS_SPACE_BEFORE_TAB (1<<7) +#define WS_INDENT_WITH_NON_TAB (1<<8) +#define WS_CR_AT_EOL (1<<9) +#define WS_BLANK_AT_EOF (1<<10) +#define WS_TAB_IN_INDENT (1<<11) +#define WS_INCOMPLETE_LINE (1<<12) + +#define WS_TRAILING_SPACE (WS_BLANK_AT_EOL|WS_BLANK_AT_EOF) #define WS_DEFAULT_RULE (WS_TRAILING_SPACE|WS_SPACE_BEFORE_TAB|8) -#define WS_TAB_WIDTH_MASK 077 -/* All WS_* -- when extended, adapt diff.c emit_symbol */ -#define WS_RULE_MASK 07777 +#define WS_TAB_WIDTH_MASK ((1<<6)-1) + +/* All WS_* -- when extended, adapt constants defined after diff.c:diff_symbol */ +#define WS_RULE_MASK ((1<<16)-1) + extern unsigned whitespace_rule_cfg; unsigned whitespace_rule(struct index_state *, const char *); unsigned parse_whitespace_rule(const char *); diff --git a/wt-status.c b/wt-status.c index 8ffe6d3988..95942399f8 100644 --- a/wt-status.c +++ b/wt-status.c @@ -612,6 +612,30 @@ static void wt_status_collect_updated_cb(struct diff_queue_struct *q, } } +void wt_status_collect_changes_trees(struct wt_status *s, + const struct object_id *old_treeish, + const struct object_id *new_treeish) +{ + struct diff_options opts = { 0 }; + + repo_diff_setup(s->repo, &opts); + opts.output_format = DIFF_FORMAT_CALLBACK; + opts.format_callback = wt_status_collect_updated_cb; + opts.format_callback_data = s; + opts.detect_rename = s->detect_rename >= 0 ? s->detect_rename : opts.detect_rename; + opts.rename_limit = s->rename_limit >= 0 ? s->rename_limit : opts.rename_limit; + opts.rename_score = s->rename_score >= 0 ? s->rename_score : opts.rename_score; + opts.flags.recursive = 1; + diff_setup_done(&opts); + + diff_tree_oid(old_treeish, new_treeish, "", &opts); + diffcore_std(&opts); + diff_flush(&opts); + wt_status_get_state(s->repo, &s->state, 0); + + diff_free(&opts); +} + static void wt_status_collect_changes_worktree(struct wt_status *s) { struct rev_info rev; @@ -2042,13 +2066,13 @@ static void wt_shortstatus_status(struct string_list_item *it, static void wt_shortstatus_other(struct string_list_item *it, struct wt_status *s, const char *sign) { + color_fprintf(s->fp, color(WT_STATUS_UNTRACKED, s), "%s", sign); if (s->null_termination) { - fprintf(s->fp, "%s %s%c", sign, it->string, 0); + fprintf(s->fp, " %s%c", it->string, 0); } else { struct strbuf onebuf = STRBUF_INIT; const char *one; one = quote_path(it->string, s->prefix, &onebuf, QUOTE_PATH_QUOTE_SP); - color_fprintf(s->fp, color(WT_STATUS_UNTRACKED, s), "%s", sign); fprintf(s->fp, " %s\n", one); strbuf_release(&onebuf); } diff --git a/wt-status.h b/wt-status.h index e40a27214a..e9fe32e98c 100644 --- a/wt-status.h +++ b/wt-status.h @@ -153,6 +153,15 @@ void wt_status_add_cut_line(struct wt_status *s); void wt_status_prepare(struct repository *r, struct wt_status *s); void wt_status_print(struct wt_status *s); void wt_status_collect(struct wt_status *s); + +/* + * Collect all changes between the two trees. Changes will be displayed as if + * they were staged into the index. + */ +void wt_status_collect_changes_trees(struct wt_status *s, + const struct object_id *old_treeish, + const struct object_id *new_treeish); + /* * Frees the buffers allocated by wt_status_collect. */ diff --git a/xdiff-interface.c b/xdiff-interface.c index 4971f722b3..1a35556380 100644 --- a/xdiff-interface.c +++ b/xdiff-interface.c @@ -300,7 +300,7 @@ void xdiff_clear_find_func(xdemitconf_t *xecfg) unsigned long xdiff_hash_string(const char *s, size_t len, long flags) { - return xdl_hash_record(&s, s + len, flags); + return xdl_hash_record((uint8_t const**)&s, (uint8_t const*)s + len, flags); } int xdiff_compare_lines(const char *l1, long s1, diff --git a/xdiff/xdiff.h b/xdiff/xdiff.h index 2cecde5afe..dc370712e9 100644 --- a/xdiff/xdiff.h +++ b/xdiff/xdiff.h @@ -43,7 +43,7 @@ extern "C" { #define XDF_PATIENCE_DIFF (1 << 14) #define XDF_HISTOGRAM_DIFF (1 << 15) -#define XDF_DIFF_ALGORITHM_MASK (XDF_PATIENCE_DIFF | XDF_HISTOGRAM_DIFF) +#define XDF_DIFF_ALGORITHM_MASK (XDF_PATIENCE_DIFF | XDF_HISTOGRAM_DIFF | XDF_NEED_MINIMAL) #define XDF_DIFF_ALG(x) ((x) & XDF_DIFF_ALGORITHM_MASK) #define XDF_INDENT_HEURISTIC (1 << 23) diff --git a/xdiff/xdiffi.c b/xdiff/xdiffi.c index 6f3998ee54..8eb664be3e 100644 --- a/xdiff/xdiffi.c +++ b/xdiff/xdiffi.c @@ -22,9 +22,9 @@ #include "xinclude.h" -static unsigned long get_hash(xdfile_t *xdf, long index) +static size_t get_hash(xdfile_t *xdf, long index) { - return xdf->recs[xdf->rindex[index]].ha; + return xdf->recs[xdf->reference_index[index]].minimal_perfect_hash; } #define XDL_MAX_COST_MIN 256 @@ -278,10 +278,10 @@ int xdl_recs_cmp(xdfile_t *xdf1, long off1, long lim1, */ if (off1 == lim1) { for (; off2 < lim2; off2++) - xdf2->changed[xdf2->rindex[off2]] = true; + xdf2->changed[xdf2->reference_index[off2]] = true; } else if (off2 == lim2) { for (; off1 < lim1; off1++) - xdf1->changed[xdf1->rindex[off1]] = true; + xdf1->changed[xdf1->reference_index[off1]] = true; } else { xdpsplit_t spl; spl.i1 = spl.i2 = 0; @@ -385,7 +385,7 @@ static xdchange_t *xdl_add_change(xdchange_t *xscr, long i1, long i2, long chg1, static int recs_match(xrecord_t *rec1, xrecord_t *rec2) { - return (rec1->ha == rec2->ha); + return rec1->minimal_perfect_hash == rec2->minimal_perfect_hash; } /* @@ -403,11 +403,10 @@ static int recs_match(xrecord_t *rec1, xrecord_t *rec2) */ static int get_indent(xrecord_t *rec) { - long i; int ret = 0; - for (i = 0; i < rec->size; i++) { - char c = rec->ptr[i]; + for (size_t i = 0; i < rec->size; i++) { + uint8_t c = rec->ptr[i]; if (!XDL_ISSPACE(c)) return ret; @@ -484,7 +483,7 @@ static void measure_split(const xdfile_t *xdf, long split, { long i; - if (split >= xdf->nrec) { + if (split >= (long)xdf->nrec) { m->end_of_file = 1; m->indent = -1; } else { @@ -507,7 +506,7 @@ static void measure_split(const xdfile_t *xdf, long split, m->post_blank = 0; m->post_indent = -1; - for (i = split + 1; i < xdf->nrec; i++) { + for (i = split + 1; i < (long)xdf->nrec; i++) { m->post_indent = get_indent(&xdf->recs[i]); if (m->post_indent != -1) break; @@ -718,7 +717,7 @@ static void group_init(xdfile_t *xdf, struct xdlgroup *g) */ static inline int group_next(xdfile_t *xdf, struct xdlgroup *g) { - if (g->end == xdf->nrec) + if (g->end == (long)xdf->nrec) return -1; g->start = g->end + 1; @@ -751,7 +750,7 @@ static inline int group_previous(xdfile_t *xdf, struct xdlgroup *g) */ static int group_slide_down(xdfile_t *xdf, struct xdlgroup *g) { - if (g->end < xdf->nrec && + if (g->end < (long)xdf->nrec && recs_match(&xdf->recs[g->start], &xdf->recs[g->end])) { xdf->changed[g->start++] = false; xdf->changed[g->end++] = true; @@ -993,11 +992,11 @@ static void xdl_mark_ignorable_lines(xdchange_t *xscr, xdfenv_t *xe, long flags) rec = &xe->xdf1.recs[xch->i1]; for (i = 0; i < xch->chg1 && ignore; i++) - ignore = xdl_blankline(rec[i].ptr, rec[i].size, flags); + ignore = xdl_blankline((const char *)rec[i].ptr, (long)rec[i].size, flags); rec = &xe->xdf2.recs[xch->i2]; for (i = 0; i < xch->chg2 && ignore; i++) - ignore = xdl_blankline(rec[i].ptr, rec[i].size, flags); + ignore = xdl_blankline((const char *)rec[i].ptr, (long)rec[i].size, flags); xch->ignore = ignore; } @@ -1008,7 +1007,7 @@ static int record_matches_regex(xrecord_t *rec, xpparam_t const *xpp) { size_t i; for (i = 0; i < xpp->ignore_regex_nr; i++) - if (!regexec_buf(xpp->ignore_regex[i], rec->ptr, rec->size, 1, + if (!regexec_buf(xpp->ignore_regex[i], (const char *)rec->ptr, rec->size, 1, ®match, 0)) return 1; diff --git a/xdiff/xemit.c b/xdiff/xemit.c index b2f1f30cd3..04f7e9193b 100644 --- a/xdiff/xemit.c +++ b/xdiff/xemit.c @@ -27,7 +27,7 @@ static int xdl_emit_record(xdfile_t *xdf, long ri, char const *pre, xdemitcb_t * { xrecord_t *rec = &xdf->recs[ri]; - if (xdl_emit_diffrec(rec->ptr, rec->size, pre, strlen(pre), ecb) < 0) + if (xdl_emit_diffrec((char const *)rec->ptr, (long)rec->size, pre, strlen(pre), ecb) < 0) return -1; return 0; @@ -113,8 +113,8 @@ static long match_func_rec(xdfile_t *xdf, xdemitconf_t const *xecfg, long ri, xrecord_t *rec = &xdf->recs[ri]; if (!xecfg->find_func) - return def_ff(rec->ptr, rec->size, buf, sz); - return xecfg->find_func(rec->ptr, rec->size, buf, sz, xecfg->find_func_priv); + return def_ff((const char *)rec->ptr, (long)rec->size, buf, sz); + return xecfg->find_func((const char *)rec->ptr, (long)rec->size, buf, sz, xecfg->find_func_priv); } static int is_func_rec(xdfile_t *xdf, xdemitconf_t const *xecfg, long ri) @@ -137,7 +137,7 @@ static long get_func_line(xdfenv_t *xe, xdemitconf_t const *xecfg, buf = func_line ? func_line->buf : dummy; size = func_line ? sizeof(func_line->buf) : sizeof(dummy); - for (l = start; l != limit && 0 <= l && l < xe->xdf1.nrec; l += step) { + for (l = start; l != limit && 0 <= l && l < (long)xe->xdf1.nrec; l += step) { long len = match_func_rec(&xe->xdf1, xecfg, l, buf, size); if (len >= 0) { if (func_line) @@ -151,7 +151,7 @@ static long get_func_line(xdfenv_t *xe, xdemitconf_t const *xecfg, static int is_empty_rec(xdfile_t *xdf, long ri) { xrecord_t *rec = &xdf->recs[ri]; - long i = 0; + size_t i = 0; for (; i < rec->size && XDL_ISSPACE(rec->ptr[i]); i++); @@ -179,14 +179,14 @@ pre_context_calculation: long fs1, i1 = xch->i1; /* Appended chunk? */ - if (i1 >= xe->xdf1.nrec) { + if (i1 >= (long)xe->xdf1.nrec) { long i2 = xch->i2; /* * We don't need additional context if * a whole function was added. */ - while (i2 < xe->xdf2.nrec) { + while (i2 < (long)xe->xdf2.nrec) { if (is_func_rec(&xe->xdf2, xecfg, i2)) goto post_context_calculation; i2++; @@ -196,7 +196,7 @@ pre_context_calculation: * Otherwise get more context from the * pre-image. */ - i1 = xe->xdf1.nrec - 1; + i1 = (long)xe->xdf1.nrec - 1; } fs1 = get_func_line(xe, xecfg, NULL, i1, -1); @@ -228,8 +228,8 @@ pre_context_calculation: post_context_calculation: lctx = xecfg->ctxlen; - lctx = XDL_MIN(lctx, xe->xdf1.nrec - (xche->i1 + xche->chg1)); - lctx = XDL_MIN(lctx, xe->xdf2.nrec - (xche->i2 + xche->chg2)); + lctx = XDL_MIN(lctx, (long)xe->xdf1.nrec - (xche->i1 + xche->chg1)); + lctx = XDL_MIN(lctx, (long)xe->xdf2.nrec - (xche->i2 + xche->chg2)); e1 = xche->i1 + xche->chg1 + lctx; e2 = xche->i2 + xche->chg2 + lctx; @@ -237,13 +237,13 @@ pre_context_calculation: if (xecfg->flags & XDL_EMIT_FUNCCONTEXT) { long fe1 = get_func_line(xe, xecfg, NULL, xche->i1 + xche->chg1, - xe->xdf1.nrec); + (long)xe->xdf1.nrec); while (fe1 > 0 && is_empty_rec(&xe->xdf1, fe1 - 1)) fe1--; if (fe1 < 0) - fe1 = xe->xdf1.nrec; + fe1 = (long)xe->xdf1.nrec; if (fe1 > e1) { - e2 = XDL_MIN(e2 + (fe1 - e1), xe->xdf2.nrec); + e2 = XDL_MIN(e2 + (fe1 - e1), (long)xe->xdf2.nrec); e1 = fe1; } @@ -254,7 +254,7 @@ pre_context_calculation: */ if (xche->next) { long l = XDL_MIN(xche->next->i1, - xe->xdf1.nrec - 1); + (long)xe->xdf1.nrec - 1); if (l - xecfg->ctxlen <= e1 || get_func_line(xe, xecfg, NULL, l, e1) < 0) { xche = xche->next; diff --git a/xdiff/xhistogram.c b/xdiff/xhistogram.c index 6dc450b1fe..5ae1282c27 100644 --- a/xdiff/xhistogram.c +++ b/xdiff/xhistogram.c @@ -90,7 +90,7 @@ struct region { static int cmp_recs(xrecord_t *r1, xrecord_t *r2) { - return r1->ha == r2->ha; + return r1->minimal_perfect_hash == r2->minimal_perfect_hash; } @@ -98,7 +98,7 @@ static int cmp_recs(xrecord_t *r1, xrecord_t *r2) (cmp_recs(REC(i->env, s1, l1), REC(i->env, s2, l2))) #define TABLE_HASH(index, side, line) \ - XDL_HASHLONG((REC(index->env, side, line))->ha, index->table_bits) + XDL_HASHLONG((REC(index->env, side, line))->minimal_perfect_hash, index->table_bits) static int scanA(struct histindex *index, int line1, int count1) { diff --git a/xdiff/xmerge.c b/xdiff/xmerge.c index fd600cbb5d..29dad98c49 100644 --- a/xdiff/xmerge.c +++ b/xdiff/xmerge.c @@ -101,8 +101,8 @@ static int xdl_merge_cmp_lines(xdfenv_t *xe1, int i1, xdfenv_t *xe2, int i2, xrecord_t *rec2 = xe2->xdf2.recs + i2; for (i = 0; i < line_count; i++) { - int result = xdl_recmatch(rec1[i].ptr, rec1[i].size, - rec2[i].ptr, rec2[i].size, flags); + int result = xdl_recmatch((const char *)rec1[i].ptr, (long)rec1[i].size, + (const char *)rec2[i].ptr, (long)rec2[i].size, flags); if (!result) return -1; } @@ -119,11 +119,11 @@ static int xdl_recs_copy_0(int use_orig, xdfenv_t *xe, int i, int count, int nee if (count < 1) return 0; - for (i = 0; i < count; size += recs[i++].size) + for (i = 0; i < count; size += (int)recs[i++].size) if (dest) memcpy(dest + size, recs[i].ptr, recs[i].size); if (add_nl) { - i = recs[count - 1].size; + i = (int)recs[count - 1].size; if (i == 0 || recs[count - 1].ptr[i - 1] != '\n') { if (needs_cr) { if (dest) @@ -156,9 +156,9 @@ static int xdl_orig_copy(xdfenv_t *xe, int i, int count, int needs_cr, int add_n */ static int is_eol_crlf(xdfile_t *file, int i) { - long size; + size_t size; - if (i < file->nrec - 1) + if (i < (long)file->nrec - 1) /* All lines before the last *must* end in LF */ return (size = file->recs[i].size) > 1 && file->recs[i].ptr[size - 2] == '\r'; @@ -317,15 +317,15 @@ static int xdl_fill_merge_buffer(xdfenv_t *xe1, const char *name1, continue; i = m->i1 + m->chg1; } - size += xdl_recs_copy(xe1, i, xe1->xdf2.nrec - i, 0, 0, + size += xdl_recs_copy(xe1, i, (int)xe1->xdf2.nrec - i, 0, 0, dest ? dest + size : NULL); return size; } static int recmatch(xrecord_t *rec1, xrecord_t *rec2, unsigned long flags) { - return xdl_recmatch(rec1->ptr, rec1->size, - rec2->ptr, rec2->size, flags); + return xdl_recmatch((const char *)rec1->ptr, (long)rec1->size, + (const char *)rec2->ptr, (long)rec2->size, flags); } /* @@ -382,10 +382,10 @@ static int xdl_refine_conflicts(xdfenv_t *xe1, xdfenv_t *xe2, xdmerge_t *m, * we have a very simple mmfile structure. */ t1.ptr = (char *)xe1->xdf2.recs[m->i1].ptr; - t1.size = xe1->xdf2.recs[m->i1 + m->chg1 - 1].ptr + t1.size = (char *)xe1->xdf2.recs[m->i1 + m->chg1 - 1].ptr + xe1->xdf2.recs[m->i1 + m->chg1 - 1].size - t1.ptr; t2.ptr = (char *)xe2->xdf2.recs[m->i2].ptr; - t2.size = xe2->xdf2.recs[m->i2 + m->chg2 - 1].ptr + t2.size = (char *)xe2->xdf2.recs[m->i2 + m->chg2 - 1].ptr + xe2->xdf2.recs[m->i2 + m->chg2 - 1].size - t2.ptr; if (xdl_do_diff(&t1, &t2, xpp, &xe) < 0) return -1; @@ -440,8 +440,8 @@ static int line_contains_alnum(const char *ptr, long size) static int lines_contain_alnum(xdfenv_t *xe, int i, int chg) { for (; chg; chg--, i++) - if (line_contains_alnum(xe->xdf2.recs[i].ptr, - xe->xdf2.recs[i].size)) + if (line_contains_alnum((const char *)xe->xdf2.recs[i].ptr, + (long)xe->xdf2.recs[i].size)) return 1; return 0; } @@ -622,7 +622,7 @@ static int xdl_do_merge(xdfenv_t *xe1, xdchange_t *xscr1, changes = c; i0 = xscr1->i1; i1 = xscr1->i2; - i2 = xscr1->i1 + xe2->xdf2.nrec - xe2->xdf1.nrec; + i2 = xscr1->i1 + (long)xe2->xdf2.nrec - (long)xe2->xdf1.nrec; chg0 = xscr1->chg1; chg1 = xscr1->chg2; chg2 = xscr1->chg1; @@ -637,7 +637,7 @@ static int xdl_do_merge(xdfenv_t *xe1, xdchange_t *xscr1, if (!changes) changes = c; i0 = xscr2->i1; - i1 = xscr2->i1 + xe1->xdf2.nrec - xe1->xdf1.nrec; + i1 = xscr2->i1 + (long)xe1->xdf2.nrec - (long)xe1->xdf1.nrec; i2 = xscr2->i2; chg0 = xscr2->chg1; chg1 = xscr2->chg1; diff --git a/xdiff/xpatience.c b/xdiff/xpatience.c index 669b653580..a0b31eb5d8 100644 --- a/xdiff/xpatience.c +++ b/xdiff/xpatience.c @@ -48,7 +48,7 @@ struct hashmap { int nr, alloc; struct entry { - unsigned long hash; + size_t minimal_perfect_hash; /* * 0 = unused entry, 1 = first line, 2 = second, etc. * line2 is NON_UNIQUE if the line is not unique @@ -101,10 +101,10 @@ static void insert_record(xpparam_t const *xpp, int line, struct hashmap *map, * So we multiply ha by 2 in the hope that the hashing was * "unique enough". */ - int index = (int)((record->ha << 1) % map->alloc); + int index = (int)((record->minimal_perfect_hash << 1) % map->alloc); while (map->entries[index].line1) { - if (map->entries[index].hash != record->ha) { + if (map->entries[index].minimal_perfect_hash != record->minimal_perfect_hash) { if (++index >= map->alloc) index = 0; continue; @@ -120,8 +120,8 @@ static void insert_record(xpparam_t const *xpp, int line, struct hashmap *map, if (pass == 2) return; map->entries[index].line1 = line; - map->entries[index].hash = record->ha; - map->entries[index].anchor = is_anchor(xpp, map->env->xdf1.recs[line - 1].ptr); + map->entries[index].minimal_perfect_hash = record->minimal_perfect_hash; + map->entries[index].anchor = is_anchor(xpp, (const char *)map->env->xdf1.recs[line - 1].ptr); if (!map->first) map->first = map->entries + index; if (map->last) { @@ -248,7 +248,7 @@ static int match(struct hashmap *map, int line1, int line2) { xrecord_t *record1 = &map->env->xdf1.recs[line1 - 1]; xrecord_t *record2 = &map->env->xdf2.recs[line2 - 1]; - return record1->ha == record2->ha; + return record1->minimal_perfect_hash == record2->minimal_perfect_hash; } static int patience_diff(xpparam_t const *xpp, xdfenv_t *env, @@ -370,5 +370,5 @@ static int patience_diff(xpparam_t const *xpp, xdfenv_t *env, int xdl_do_patience_diff(xpparam_t const *xpp, xdfenv_t *env) { - return patience_diff(xpp, env, 1, env->xdf1.nrec, 1, env->xdf2.nrec); + return patience_diff(xpp, env, 1, (int)env->xdf1.nrec, 1, (int)env->xdf2.nrec); } diff --git a/xdiff/xprepare.c b/xdiff/xprepare.c index 192334f1b7..1dd420a2ff 100644 --- a/xdiff/xprepare.c +++ b/xdiff/xprepare.c @@ -96,11 +96,11 @@ static int xdl_classify_record(unsigned int pass, xdlclassifier_t *cf, xrecord_t long hi; xdlclass_t *rcrec; - hi = (long) XDL_HASHLONG(rec->ha, cf->hbits); + hi = (long) XDL_HASHLONG(rec->line_hash, cf->hbits); for (rcrec = cf->rchash[hi]; rcrec; rcrec = rcrec->next) - if (rcrec->rec.ha == rec->ha && - xdl_recmatch(rcrec->rec.ptr, rcrec->rec.size, - rec->ptr, rec->size, cf->flags)) + if (rcrec->rec.line_hash == rec->line_hash && + xdl_recmatch((const char *)rcrec->rec.ptr, (long)rcrec->rec.size, + (const char *)rec->ptr, (long)rec->size, cf->flags)) break; if (!rcrec) { @@ -120,7 +120,7 @@ static int xdl_classify_record(unsigned int pass, xdlclassifier_t *cf, xrecord_t (pass == 1) ? rcrec->len1++ : rcrec->len2++; - rec->ha = (unsigned long) rcrec->idx; + rec->minimal_perfect_hash = (size_t)rcrec->idx; return 0; } @@ -128,7 +128,7 @@ static int xdl_classify_record(unsigned int pass, xdlclassifier_t *cf, xrecord_t static void xdl_free_ctx(xdfile_t *xdf) { - xdl_free(xdf->rindex); + xdl_free(xdf->reference_index); xdl_free(xdf->changed - 1); xdl_free(xdf->recs); } @@ -137,11 +137,11 @@ static void xdl_free_ctx(xdfile_t *xdf) static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_t const *xpp, xdlclassifier_t *cf, xdfile_t *xdf) { long bsize; - unsigned long hav; - char const *blk, *cur, *top, *prev; + uint64_t hav; + uint8_t const *blk, *cur, *top, *prev; xrecord_t *crec; - xdf->rindex = NULL; + xdf->reference_index = NULL; xdf->changed = NULL; xdf->recs = NULL; @@ -153,12 +153,12 @@ static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_ for (top = blk + bsize; cur < top; ) { prev = cur; hav = xdl_hash_record(&cur, top, xpp->flags); - if (XDL_ALLOC_GROW(xdf->recs, xdf->nrec + 1, narec)) + if (XDL_ALLOC_GROW(xdf->recs, (long)xdf->nrec + 1, narec)) goto abort; crec = &xdf->recs[xdf->nrec++]; crec->ptr = prev; - crec->size = (long) (cur - prev); - crec->ha = hav; + crec->size = cur - prev; + crec->line_hash = hav; if (xdl_classify_record(pass, cf, crec) < 0) goto abort; } @@ -169,7 +169,7 @@ static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_ if ((XDF_DIFF_ALG(xpp->flags) != XDF_PATIENCE_DIFF) && (XDF_DIFF_ALG(xpp->flags) != XDF_HISTOGRAM_DIFF)) { - if (!XDL_ALLOC_ARRAY(xdf->rindex, xdf->nrec + 1)) + if (!XDL_ALLOC_ARRAY(xdf->reference_index, xdf->nrec + 1)) goto abort; } @@ -264,7 +264,7 @@ static bool xdl_clean_mmatch(uint8_t const *action, long i, long s, long e) { * might be potentially discarded if they appear in a run of discardable. */ static int xdl_cleanup_records(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xdf2) { - long i, nm, nreff, mlim; + long i, nm, mlim; xrecord_t *recs; xdlclass_t *rcrec; uint8_t *action1 = NULL, *action2 = NULL; @@ -287,18 +287,18 @@ static int xdl_cleanup_records(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xd /* * Initialize temporary arrays with DISCARD, KEEP, or INVESTIGATE. */ - if ((mlim = xdl_bogosqrt(xdf1->nrec)) > XDL_MAX_EQLIMIT) + if ((mlim = xdl_bogosqrt((long)xdf1->nrec)) > XDL_MAX_EQLIMIT) mlim = XDL_MAX_EQLIMIT; for (i = xdf1->dstart, recs = &xdf1->recs[xdf1->dstart]; i <= xdf1->dend; i++, recs++) { - rcrec = cf->rcrecs[recs->ha]; + rcrec = cf->rcrecs[recs->minimal_perfect_hash]; nm = rcrec ? rcrec->len2 : 0; action1[i] = (nm == 0) ? DISCARD: (nm >= mlim && !need_min) ? INVESTIGATE: KEEP; } - if ((mlim = xdl_bogosqrt(xdf2->nrec)) > XDL_MAX_EQLIMIT) + if ((mlim = xdl_bogosqrt((long)xdf2->nrec)) > XDL_MAX_EQLIMIT) mlim = XDL_MAX_EQLIMIT; for (i = xdf2->dstart, recs = &xdf2->recs[xdf2->dstart]; i <= xdf2->dend; i++, recs++) { - rcrec = cf->rcrecs[recs->ha]; + rcrec = cf->rcrecs[recs->minimal_perfect_hash]; nm = rcrec ? rcrec->len1 : 0; action2[i] = (nm == 0) ? DISCARD: (nm >= mlim && !need_min) ? INVESTIGATE: KEEP; } @@ -307,29 +307,29 @@ static int xdl_cleanup_records(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xd * Use temporary arrays to decide if changed[i] should remain * false, or become true. */ - for (nreff = 0, i = xdf1->dstart, recs = &xdf1->recs[xdf1->dstart]; + xdf1->nreff = 0; + for (i = xdf1->dstart, recs = &xdf1->recs[xdf1->dstart]; i <= xdf1->dend; i++, recs++) { if (action1[i] == KEEP || (action1[i] == INVESTIGATE && !xdl_clean_mmatch(action1, i, xdf1->dstart, xdf1->dend))) { - xdf1->rindex[nreff++] = i; + xdf1->reference_index[xdf1->nreff++] = i; /* changed[i] remains false, i.e. keep */ } else xdf1->changed[i] = true; /* i.e. discard */ } - xdf1->nreff = nreff; - for (nreff = 0, i = xdf2->dstart, recs = &xdf2->recs[xdf2->dstart]; + xdf2->nreff = 0; + for (i = xdf2->dstart, recs = &xdf2->recs[xdf2->dstart]; i <= xdf2->dend; i++, recs++) { if (action2[i] == KEEP || (action2[i] == INVESTIGATE && !xdl_clean_mmatch(action2, i, xdf2->dstart, xdf2->dend))) { - xdf2->rindex[nreff++] = i; + xdf2->reference_index[xdf2->nreff++] = i; /* changed[i] remains false, i.e. keep */ } else xdf2->changed[i] = true; /* i.e. discard */ } - xdf2->nreff = nreff; cleanup: xdl_free(action1); @@ -348,9 +348,9 @@ static int xdl_trim_ends(xdfile_t *xdf1, xdfile_t *xdf2) { recs1 = xdf1->recs; recs2 = xdf2->recs; - for (i = 0, lim = XDL_MIN(xdf1->nrec, xdf2->nrec); i < lim; + for (i = 0, lim = (long)XDL_MIN(xdf1->nrec, xdf2->nrec); i < lim; i++, recs1++, recs2++) - if (recs1->ha != recs2->ha) + if (recs1->minimal_perfect_hash != recs2->minimal_perfect_hash) break; xdf1->dstart = xdf2->dstart = i; @@ -358,11 +358,11 @@ static int xdl_trim_ends(xdfile_t *xdf1, xdfile_t *xdf2) { recs1 = xdf1->recs + xdf1->nrec - 1; recs2 = xdf2->recs + xdf2->nrec - 1; for (lim -= i, i = 0; i < lim; i++, recs1--, recs2--) - if (recs1->ha != recs2->ha) + if (recs1->minimal_perfect_hash != recs2->minimal_perfect_hash) break; - xdf1->dend = xdf1->nrec - i - 1; - xdf2->dend = xdf2->nrec - i - 1; + xdf1->dend = (long)xdf1->nrec - i - 1; + xdf2->dend = (long)xdf2->nrec - i - 1; return 0; } diff --git a/xdiff/xtypes.h b/xdiff/xtypes.h index f145abba3e..5accbec284 100644 --- a/xdiff/xtypes.h +++ b/xdiff/xtypes.h @@ -39,18 +39,19 @@ typedef struct s_chastore { } chastore_t; typedef struct s_xrecord { - char const *ptr; - long size; - unsigned long ha; + uint8_t const *ptr; + size_t size; + uint64_t line_hash; + size_t minimal_perfect_hash; } xrecord_t; typedef struct s_xdfile { xrecord_t *recs; - long nrec; - long dstart, dend; + size_t nrec; bool *changed; - long *rindex; - long nreff; + size_t *reference_index; + size_t nreff; + ptrdiff_t dstart, dend; } xdfile_t; typedef struct s_xdfenv { diff --git a/xdiff/xutils.c b/xdiff/xutils.c index 447e66c719..77ee1ad9c8 100644 --- a/xdiff/xutils.c +++ b/xdiff/xutils.c @@ -249,11 +249,11 @@ int xdl_recmatch(const char *l1, long s1, const char *l2, long s2, long flags) return 1; } -unsigned long xdl_hash_record_with_whitespace(char const **data, - char const *top, long flags) { - unsigned long ha = 5381; - char const *ptr = *data; - int cr_at_eol_only = (flags & XDF_WHITESPACE_FLAGS) == XDF_IGNORE_CR_AT_EOL; +uint64_t xdl_hash_record_with_whitespace(uint8_t const **data, + uint8_t const *top, uint64_t flags) { + uint64_t ha = 5381; + uint8_t const *ptr = *data; + bool cr_at_eol_only = (flags & XDF_WHITESPACE_FLAGS) == XDF_IGNORE_CR_AT_EOL; for (; ptr < top && *ptr != '\n'; ptr++) { if (cr_at_eol_only) { @@ -263,8 +263,8 @@ unsigned long xdl_hash_record_with_whitespace(char const **data, continue; } else if (XDL_ISSPACE(*ptr)) { - const char *ptr2 = ptr; - int at_eol; + const uint8_t *ptr2 = ptr; + bool at_eol; while (ptr + 1 < top && XDL_ISSPACE(ptr[1]) && ptr[1] != '\n') ptr++; @@ -274,20 +274,20 @@ unsigned long xdl_hash_record_with_whitespace(char const **data, else if (flags & XDF_IGNORE_WHITESPACE_CHANGE && !at_eol) { ha += (ha << 5); - ha ^= (unsigned long) ' '; + ha ^= (uint64_t) ' '; } else if (flags & XDF_IGNORE_WHITESPACE_AT_EOL && !at_eol) { while (ptr2 != ptr + 1) { ha += (ha << 5); - ha ^= (unsigned long) *ptr2; + ha ^= (uint64_t) *ptr2; ptr2++; } } continue; } ha += (ha << 5); - ha ^= (unsigned long) *ptr; + ha ^= (uint64_t) *ptr; } *data = ptr < top ? ptr + 1: ptr; @@ -304,9 +304,9 @@ unsigned long xdl_hash_record_with_whitespace(char const **data, #define REASSOC_FENCE(x, y) #endif -unsigned long xdl_hash_record_verbatim(char const **data, char const *top) { - unsigned long ha = 5381, c0, c1; - char const *ptr = *data; +uint64_t xdl_hash_record_verbatim(uint8_t const **data, uint8_t const *top) { + uint64_t ha = 5381, c0, c1; + uint8_t const *ptr = *data; #if 0 /* * The baseline form of the optimized loop below. This is the djb2 @@ -314,7 +314,7 @@ unsigned long xdl_hash_record_verbatim(char const **data, char const *top) { */ for (; ptr < top && *ptr != '\n'; ptr++) { ha += (ha << 5); - ha += (unsigned long) *ptr; + ha += (uint64_t) *ptr; } *data = ptr < top ? ptr + 1: ptr; #else @@ -465,10 +465,10 @@ int xdl_fall_back_diff(xdfenv_t *diff_env, xpparam_t const *xpp, xdfenv_t env; subfile1.ptr = (char *)diff_env->xdf1.recs[line1 - 1].ptr; - subfile1.size = diff_env->xdf1.recs[line1 + count1 - 2].ptr + + subfile1.size = (char *)diff_env->xdf1.recs[line1 + count1 - 2].ptr + diff_env->xdf1.recs[line1 + count1 - 2].size - subfile1.ptr; subfile2.ptr = (char *)diff_env->xdf2.recs[line2 - 1].ptr; - subfile2.size = diff_env->xdf2.recs[line2 + count2 - 2].ptr + + subfile2.size = (char *)diff_env->xdf2.recs[line2 + count2 - 2].ptr + diff_env->xdf2.recs[line2 + count2 - 2].size - subfile2.ptr; if (xdl_do_diff(&subfile1, &subfile2, xpp, &env) < 0) return -1; diff --git a/xdiff/xutils.h b/xdiff/xutils.h index 13f6831047..615b4a9d35 100644 --- a/xdiff/xutils.h +++ b/xdiff/xutils.h @@ -34,9 +34,9 @@ void *xdl_cha_alloc(chastore_t *cha); long xdl_guess_lines(mmfile_t *mf, long sample); int xdl_blankline(const char *line, long size, long flags); int xdl_recmatch(const char *l1, long s1, const char *l2, long s2, long flags); -unsigned long xdl_hash_record_verbatim(char const **data, char const *top); -unsigned long xdl_hash_record_with_whitespace(char const **data, char const *top, long flags); -static inline unsigned long xdl_hash_record(char const **data, char const *top, long flags) +uint64_t xdl_hash_record_verbatim(uint8_t const **data, uint8_t const *top); +uint64_t xdl_hash_record_with_whitespace(uint8_t const **data, uint8_t const *top, uint64_t flags); +static inline uint64_t xdl_hash_record(uint8_t const **data, uint8_t const *top, uint64_t flags) { if (flags & XDF_WHITESPACE_FLAGS) return xdl_hash_record_with_whitespace(data, top, flags); |