From 61df89c8e5559c2b524968f492d445421913bfdb Mon Sep 17 00:00:00 2001 From: Ævar Arnfjörð Bjarmason Date: Mon, 25 Mar 2019 13:08:30 +0100 Subject: commit-graph: don't early exit(1) on e.g. "git status" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make the commit-graph loading code work as a library that returns an error code instead of calling exit(1) when the commit-graph is corrupt. This means that e.g. "status" will now report commit-graph corruption as an "error: [...]" at the top of its output, but then proceed to work normally. This required splitting up the load_commit_graph_one() function so that the code that deals with open()-ing and stat()-ing the graph can now be called independently as open_commit_graph(). This is needed because "commit-graph verify" where the graph doesn't exist isn't an error. See the third paragraph in 283e68c72f ("commit-graph: add 'verify' subcommand", 2018-06-27). There's a bug in that logic where we conflate the intended ENOENT with other errno values (e.g. EACCES), but this change doesn't address that. That'll be addressed in a follow-up change. I'm then splitting most of the logic out of load_commit_graph_one() into load_commit_graph_one_fd_st(), which allows for providing an existing file descriptor and stat information to the loading code. This isn't strictly needed, but it would be redundant and confusing to open() and stat() the file twice for some of the codepaths, this allows for calling open_commit_graph() followed by load_commit_graph_one_fd_st(). The "graph_file" still needs to be passed to that function for the the "graph file %s is too small" error message. This leaves load_commit_graph_one() unused by everything except the internal prepare_commit_graph_one() function, so let's mark it as "static". If someone needs it in the future we can remove the "static" attribute. I could also rewrite its sole remaining user ("prepare_commit_graph_one()") to use load_commit_graph_one_fd_st() instead, but let's leave it at this. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Ramsay Jones Signed-off-by: Junio C Hamano --- commit-graph.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'commit-graph.h') diff --git a/commit-graph.h b/commit-graph.h index 096d8bac34..77cc739bc0 100644 --- a/commit-graph.h +++ b/commit-graph.h @@ -11,6 +11,7 @@ struct commit; char *get_commit_graph_filename(const char *obj_dir); +int open_commit_graph(const char *graph_file, int *fd, struct stat *st); /* * Given a commit struct, try to fill the commit struct info, including: @@ -52,7 +53,8 @@ struct commit_graph { const unsigned char *chunk_extra_edges; }; -struct commit_graph *load_commit_graph_one(const char *graph_file); +struct commit_graph *load_commit_graph_one_fd_st(const char *graph_file, + int fd, struct stat *st); struct commit_graph *parse_commit_graph(void *graph_map, int fd, size_t graph_size); -- cgit v1.2.3 From 67a530fab3bbd1d4c3390d6f18e35ab10442c015 Mon Sep 17 00:00:00 2001 From: Ævar Arnfjörð Bjarmason Date: Mon, 25 Mar 2019 13:08:31 +0100 Subject: commit-graph: don't pass filename to load_commit_graph_one_fd_st() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An earlier change implemented load_commit_graph_one_fd_st() in a way that was bug-compatible with earlier code in terms of the "graph file %s is too small" error message printing out the path to the commit-graph (".git/objects/info/commit-graph"). But change that, because: * A function that takes an already-open file descriptor also needing the filename isn't very intuitive. * The vast majority of errors we might emit when loading the graph come from parse_commit_graph(), which doesn't report the filename. Let's not do that either in this case for consistency. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- builtin/commit-graph.c | 4 ++-- commit-graph.c | 7 +++---- commit-graph.h | 3 +-- 3 files changed, 6 insertions(+), 8 deletions(-) (limited to 'commit-graph.h') diff --git a/builtin/commit-graph.c b/builtin/commit-graph.c index 32bcc63427..8196fdbe9c 100644 --- a/builtin/commit-graph.c +++ b/builtin/commit-graph.c @@ -64,7 +64,7 @@ static int graph_verify(int argc, const char **argv) open_ok = open_commit_graph(graph_name, &fd, &st); if (!open_ok) return 0; - graph = load_commit_graph_one_fd_st(graph_name, fd, &st); + graph = load_commit_graph_one_fd_st(fd, &st); FREE_AND_NULL(graph_name); if (!graph) @@ -102,7 +102,7 @@ static int graph_read(int argc, const char **argv) if (!open_ok) die_errno(_("Could not open commit-graph '%s'"), graph_name); - graph = load_commit_graph_one_fd_st(graph_name, fd, &st); + graph = load_commit_graph_one_fd_st(fd, &st); if (!graph) return 1; diff --git a/commit-graph.c b/commit-graph.c index 3acc032c1b..a26d266663 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -92,8 +92,7 @@ int open_commit_graph(const char *graph_file, int *fd, struct stat *st) return 1; } -struct commit_graph *load_commit_graph_one_fd_st(const char *graph_file, - int fd, struct stat *st) +struct commit_graph *load_commit_graph_one_fd_st(int fd, struct stat *st) { void *graph_map; size_t graph_size; @@ -103,7 +102,7 @@ struct commit_graph *load_commit_graph_one_fd_st(const char *graph_file, if (graph_size < GRAPH_MIN_SIZE) { close(fd); - error(_("graph file %s is too small"), graph_file); + error(_("commit-graph file is too small")); return NULL; } graph_map = xmmap(NULL, graph_size, PROT_READ, MAP_PRIVATE, fd, 0); @@ -284,7 +283,7 @@ static struct commit_graph *load_commit_graph_one(const char *graph_file) if (!open_ok) return NULL; - return load_commit_graph_one_fd_st(graph_file, fd, &st); + return load_commit_graph_one_fd_st(fd, &st); } static void prepare_commit_graph_one(struct repository *r, const char *obj_dir) diff --git a/commit-graph.h b/commit-graph.h index 77cc739bc0..ada7aea9ed 100644 --- a/commit-graph.h +++ b/commit-graph.h @@ -53,8 +53,7 @@ struct commit_graph { const unsigned char *chunk_extra_edges; }; -struct commit_graph *load_commit_graph_one_fd_st(const char *graph_file, - int fd, struct stat *st); +struct commit_graph *load_commit_graph_one_fd_st(int fd, struct stat *st); struct commit_graph *parse_commit_graph(void *graph_map, int fd, size_t graph_size); -- cgit v1.2.3 From 43d356180556180b4ef6ac232a14498a5bb2b446 Mon Sep 17 00:00:00 2001 From: Ævar Arnfjörð Bjarmason Date: Mon, 25 Mar 2019 13:08:33 +0100 Subject: commit-graph write: don't die if the existing graph is corrupt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the commit-graph is written we end up calling parse_commit(). This will in turn invoke code that'll consult the existing commit-graph about the commit, if the graph is corrupted we die. We thus get into a state where a failing "commit-graph verify" can't be followed-up with a "commit-graph write" if core.commitGraph=true is set, the graph either needs to be manually removed to proceed, or core.commitGraph needs to be set to "false". Change the "commit-graph write" codepath to use a new parse_commit_no_graph() helper instead of parse_commit() to avoid this. The latter will call repo_parse_commit_internal() with use_commit_graph=1 as seen in 177722b344 ("commit: integrate commit graph with commit parsing", 2018-04-10). Not using the old graph at all slows down the writing of the new graph by some small amount, but is a sensible way to prevent an error in the existing commit-graph from spreading. Just fixing the current issue would be likely to result in code that's inadvertently broken in the future. New code might use the commit-graph at a distance. To detect such cases introduce a "GIT_TEST_COMMIT_GRAPH_DIE_ON_LOAD" setting used when we do our corruption tests, and test that a "write/verify" combo works after every one of our current test cases where we now detect commit-graph corruption. Some of the code changes here might be strictly unnecessary, e.g. I was unable to find cases where the parse_commit() called from write_graph_chunk_data() didn't exit early due to "item->object.parsed" being true in repo_parse_commit_internal() (before the use_commit_graph=1 has any effect). But let's also convert those cases for good measure, we do not have exhaustive tests for all possible types of commit-graph corruption. This might need to be re-visited if we learn to write the commit-graph incrementally, but probably not. Hopefully we'll just start by finding out what commits we have in total, then read the old graph(s) to see what they cover, and finally write a new graph file with everything that's missing. In that case the new graph writing code just needs to continue to use e.g. a parse_commit() that doesn't consult the existing commit-graphs. Signed-off-by: Ævar Arnfjörð Bjarmason Signed-off-by: Junio C Hamano --- commit-graph.c | 10 +++++++--- commit-graph.h | 1 + commit.h | 6 ++++++ t/t5318-commit-graph.sh | 11 +++++++++-- 4 files changed, 23 insertions(+), 5 deletions(-) (limited to 'commit-graph.h') diff --git a/commit-graph.c b/commit-graph.c index a26d266663..34ecaaf857 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -311,6 +311,10 @@ static int prepare_commit_graph(struct repository *r) struct object_directory *odb; int config_value; + if (git_env_bool(GIT_TEST_COMMIT_GRAPH_DIE_ON_LOAD, 0)) + die("dying as requested by the '%s' variable on commit-graph load!", + GIT_TEST_COMMIT_GRAPH_DIE_ON_LOAD); + if (r->objects->commit_graph_attempted) return !!r->objects->commit_graph; r->objects->commit_graph_attempted = 1; @@ -575,7 +579,7 @@ static void write_graph_chunk_data(struct hashfile *f, int hash_len, uint32_t packedDate[2]; display_progress(progress, ++*progress_cnt); - parse_commit(*list); + parse_commit_no_graph(*list); hashwrite(f, get_commit_tree_oid(*list)->hash, hash_len); parent = (*list)->parents; @@ -772,7 +776,7 @@ static void close_reachable(struct packed_oid_list *oids, int report_progress) display_progress(progress, i + 1); commit = lookup_commit(the_repository, &oids->list[i]); - if (commit && !parse_commit(commit)) + if (commit && !parse_commit_no_graph(commit)) add_missing_parents(oids, commit); } stop_progress(&progress); @@ -1021,7 +1025,7 @@ void write_commit_graph(const char *obj_dir, continue; commits.list[commits.nr] = lookup_commit(the_repository, &oids.list[i]); - parse_commit(commits.list[commits.nr]); + parse_commit_no_graph(commits.list[commits.nr]); for (parent = commits.list[commits.nr]->parents; parent; parent = parent->next) diff --git a/commit-graph.h b/commit-graph.h index ada7aea9ed..7dfb8c896f 100644 --- a/commit-graph.h +++ b/commit-graph.h @@ -7,6 +7,7 @@ #include "cache.h" #define GIT_TEST_COMMIT_GRAPH "GIT_TEST_COMMIT_GRAPH" +#define GIT_TEST_COMMIT_GRAPH_DIE_ON_LOAD "GIT_TEST_COMMIT_GRAPH_DIE_ON_LOAD" struct commit; diff --git a/commit.h b/commit.h index 42728c2906..5d33477e78 100644 --- a/commit.h +++ b/commit.h @@ -89,6 +89,12 @@ static inline int repo_parse_commit(struct repository *r, struct commit *item) { return repo_parse_commit_gently(r, item, 0); } + +static inline int parse_commit_no_graph(struct commit *commit) +{ + return repo_parse_commit_internal(the_repository, commit, 0, 0); +} + #ifndef NO_THE_REPOSITORY_COMPATIBILITY_MACROS #define parse_commit_internal(item, quiet, use) repo_parse_commit_internal(the_repository, item, quiet, use) #define parse_commit_gently(item, quiet) repo_parse_commit_gently(the_repository, item, quiet) diff --git a/t/t5318-commit-graph.sh b/t/t5318-commit-graph.sh index 4601732b99..e80c1cac02 100755 --- a/t/t5318-commit-graph.sh +++ b/t/t5318-commit-graph.sh @@ -377,7 +377,13 @@ corrupt_graph_verify() { test_must_fail git commit-graph verify 2>test_err && grep -v "^+" test_err >err && test_i18ngrep "$grepstr" err && - git status --short + if test "$2" != "no-copy" + then + cp $objdir/info/commit-graph commit-graph-pre-write-test + fi && + git status --short && + GIT_TEST_COMMIT_GRAPH_DIE_ON_LOAD=true git commit-graph write && + git commit-graph verify } # usage: corrupt_graph_and_verify [] @@ -403,7 +409,7 @@ corrupt_graph_and_verify() { test_expect_success POSIXPERM,SANITY 'detect permission problem' ' corrupt_graph_setup && chmod 000 $objdir/info/commit-graph && - corrupt_graph_verify "Could not open" + corrupt_graph_verify "Could not open" "no-copy" ' test_expect_success 'detect too small' ' @@ -522,6 +528,7 @@ test_expect_success 'git fsck (checks commit-graph)' ' git fsck && corrupt_graph_and_verify $GRAPH_BYTE_FOOTER "\00" \ "incorrect checksum" && + cp commit-graph-pre-write-test $objdir/info/commit-graph && test_must_fail git fsck ' -- cgit v1.2.3