diff options
Diffstat (limited to 'builtin/gc.c')
| -rw-r--r-- | builtin/gc.c | 782 |
1 files changed, 532 insertions, 250 deletions
diff --git a/builtin/gc.c b/builtin/gc.c index d52735354c..fab8f4dd4f 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -9,10 +9,14 @@ * * Copyright (c) 2006 Shawn O. Pearce */ + #define USE_THE_REPOSITORY_VARIABLE +#define DISABLE_SIGN_COMPARE_WARNINGS + #include "builtin.h" #include "abspath.h" #include "date.h" +#include "dir.h" #include "environment.h" #include "hex.h" #include "config.h" @@ -26,10 +30,11 @@ #include "commit-graph.h" #include "packfile.h" #include "object-file.h" -#include "object-store-ll.h" #include "pack.h" #include "pack-objects.h" #include "path.h" +#include "reflog.h" +#include "rerere.h" #include "blob.h" #include "tree.h" #include "promisor-remote.h" @@ -40,6 +45,7 @@ #include "hook.h" #include "setup.h" #include "trace2.h" +#include "worktree.h" #define FAILED_RUN "failed to run %s" @@ -49,16 +55,9 @@ static const char * const builtin_gc_usage[] = { }; static timestamp_t gc_log_expire_time; - -static struct strvec reflog = STRVEC_INIT; static struct strvec repack = STRVEC_INIT; -static struct strvec prune = STRVEC_INIT; -static struct strvec prune_worktrees = STRVEC_INIT; -static struct strvec rerere = STRVEC_INIT; - static struct tempfile *pidfile; static struct lock_file log_lock; - static struct string_list pack_garbage = STRING_LIST_INIT_DUP; static void clean_pack_garbage(void) @@ -96,9 +95,11 @@ static void process_log_file(void) /* There was some error recorded in the lock file */ commit_lock_file(&log_lock); } else { + char *path = repo_git_path(the_repository, "gc.log"); /* No error, clean up any old gc.log */ - unlink(git_path("gc.log")); + unlink(path); rollback_lock_file(&log_lock); + free(path); } } @@ -136,8 +137,14 @@ struct gc_config { char *prune_worktrees_expire; char *repack_filter; char *repack_filter_to; + char *repack_expire_to; unsigned long big_pack_threshold; unsigned long max_delta_cache_size; + /* + * Remove this member from gc_config once repo_settings is passed + * through the callchain. + */ + size_t delta_base_cache_limit; }; #define GC_CONFIG_INIT { \ @@ -153,6 +160,7 @@ struct gc_config { .prune_expire = xstrdup("2.weeks.ago"), \ .prune_worktrees_expire = xstrdup("3.months.ago"), \ .max_delta_cache_size = DEFAULT_DELTA_CACHE_SIZE, \ + .delta_base_cache_limit = DEFAULT_DELTA_BASE_CACHE_LIMIT, \ } static void gc_config_release(struct gc_config *cfg) @@ -168,6 +176,7 @@ static void gc_config(struct gc_config *cfg) { const char *value; char *owned = NULL; + unsigned long ulongval; if (!git_config_get_value("gc.packrefs", &value)) { if (value && !strcmp(value, "notbare")) @@ -206,6 +215,9 @@ static void gc_config(struct gc_config *cfg) git_config_get_ulong("gc.bigpackthreshold", &cfg->big_pack_threshold); git_config_get_ulong("pack.deltacachesize", &cfg->max_delta_cache_size); + if (!git_config_get_ulong("core.deltabasecachelimit", &ulongval)) + cfg->delta_base_cache_limit = ulongval; + if (!git_config_get_string("gc.repackfilter", &owned)) { free(cfg->repack_filter); cfg->repack_filter = owned; @@ -239,7 +251,24 @@ static enum schedule_priority parse_schedule(const char *value) return SCHEDULE_NONE; } +enum maintenance_task_label { + TASK_PREFETCH, + TASK_LOOSE_OBJECTS, + TASK_INCREMENTAL_REPACK, + TASK_GC, + TASK_COMMIT_GRAPH, + TASK_PACK_REFS, + TASK_REFLOG_EXPIRE, + TASK_WORKTREE_PRUNE, + TASK_RERERE_GC, + + /* Leave as final value */ + TASK__COUNT +}; + struct maintenance_run_opts { + enum maintenance_task_label *tasks; + size_t tasks_nr, tasks_alloc; int auto_flag; int detach; int quiet; @@ -249,6 +278,11 @@ struct maintenance_run_opts { .detach = -1, \ } +static void maintenance_run_opts_release(struct maintenance_run_opts *opts) +{ + free(opts->tasks); +} + static int pack_refs_condition(UNUSED struct gc_config *cfg) { /* @@ -272,6 +306,146 @@ static int maintenance_task_pack_refs(struct maintenance_run_opts *opts, return run_command(&cmd); } +struct count_reflog_entries_data { + struct expire_reflog_policy_cb policy; + size_t count; + size_t limit; +}; + +static int count_reflog_entries(struct object_id *old_oid, struct object_id *new_oid, + const char *committer, timestamp_t timestamp, + int tz, const char *msg, void *cb_data) +{ + struct count_reflog_entries_data *data = cb_data; + if (should_expire_reflog_ent(old_oid, new_oid, committer, timestamp, tz, msg, &data->policy)) + data->count++; + return data->count >= data->limit; +} + +static int reflog_expire_condition(struct gc_config *cfg UNUSED) +{ + timestamp_t now = time(NULL); + struct count_reflog_entries_data data = { + .policy = { + .opts = REFLOG_EXPIRE_OPTIONS_INIT(now), + }, + }; + int limit = 100; + + git_config_get_int("maintenance.reflog-expire.auto", &limit); + if (!limit) + return 0; + if (limit < 0) + return 1; + data.limit = limit; + + repo_config(the_repository, reflog_expire_config, &data.policy.opts); + + reflog_expire_options_set_refname(&data.policy.opts, "HEAD"); + refs_for_each_reflog_ent(get_main_ref_store(the_repository), "HEAD", + count_reflog_entries, &data); + + reflog_expiry_cleanup(&data.policy); + return data.count >= data.limit; +} + +static int maintenance_task_reflog_expire(struct maintenance_run_opts *opts UNUSED, + struct gc_config *cfg UNUSED) +{ + struct child_process cmd = CHILD_PROCESS_INIT; + cmd.git_cmd = 1; + strvec_pushl(&cmd.args, "reflog", "expire", "--all", NULL); + return run_command(&cmd); +} + +static int maintenance_task_worktree_prune(struct maintenance_run_opts *opts UNUSED, + struct gc_config *cfg) +{ + struct child_process prune_worktrees_cmd = CHILD_PROCESS_INIT; + + prune_worktrees_cmd.git_cmd = 1; + strvec_pushl(&prune_worktrees_cmd.args, "worktree", "prune", "--expire", NULL); + strvec_push(&prune_worktrees_cmd.args, cfg->prune_worktrees_expire); + + return run_command(&prune_worktrees_cmd); +} + +static int worktree_prune_condition(struct gc_config *cfg) +{ + struct strbuf buf = STRBUF_INIT; + int should_prune = 0, limit = 1; + timestamp_t expiry_date; + struct dirent *d; + DIR *dir = NULL; + + git_config_get_int("maintenance.worktree-prune.auto", &limit); + if (limit <= 0) { + should_prune = limit < 0; + goto out; + } + + if (parse_expiry_date(cfg->prune_worktrees_expire, &expiry_date)) + goto out; + + dir = opendir(repo_git_path_replace(the_repository, &buf, "worktrees")); + if (!dir) + goto out; + + while (limit && (d = readdir_skip_dot_and_dotdot(dir))) { + char *wtpath; + strbuf_reset(&buf); + if (should_prune_worktree(d->d_name, &buf, &wtpath, expiry_date)) + limit--; + free(wtpath); + } + + should_prune = !limit; + +out: + if (dir) + closedir(dir); + strbuf_release(&buf); + return should_prune; +} + +static int maintenance_task_rerere_gc(struct maintenance_run_opts *opts UNUSED, + struct gc_config *cfg UNUSED) +{ + struct child_process rerere_cmd = CHILD_PROCESS_INIT; + rerere_cmd.git_cmd = 1; + strvec_pushl(&rerere_cmd.args, "rerere", "gc", NULL); + return run_command(&rerere_cmd); +} + +static int rerere_gc_condition(struct gc_config *cfg UNUSED) +{ + struct strbuf path = STRBUF_INIT; + int should_gc = 0, limit = 1; + DIR *dir = NULL; + + git_config_get_int("maintenance.rerere-gc.auto", &limit); + if (limit <= 0) { + should_gc = limit < 0; + goto out; + } + + /* + * We skip garbage collection in case we either have no "rr-cache" + * directory or when it doesn't contain at least one entry. + */ + repo_git_path_replace(the_repository, &path, "rr-cache"); + dir = opendir(path.buf); + if (!dir) + goto out; + should_gc = !!readdir_skip_dot_and_dotdot(dir); + +out: + strbuf_release(&path); + if (dir) + closedir(dir); + return should_gc; +} + static int too_many_loose_objects(struct gc_config *cfg) { /* @@ -286,8 +460,11 @@ static int too_many_loose_objects(struct gc_config *cfg) int num_loose = 0; int needed = 0; const unsigned hexsz_loose = the_hash_algo->hexsz - 2; + char *path; - dir = opendir(git_path("objects/17")); + path = repo_git_path(the_repository, "objects/17"); + dir = opendir(path); + free(path); if (!dir) return 0; @@ -354,22 +531,36 @@ static uint64_t total_ram(void) #if defined(HAVE_SYSINFO) struct sysinfo si; - if (!sysinfo(&si)) - return si.totalram; -#elif defined(HAVE_BSD_SYSCTL) && (defined(HW_MEMSIZE) || defined(HW_PHYSMEM)) - int64_t physical_memory; + if (!sysinfo(&si)) { + uint64_t total = si.totalram; + + if (si.mem_unit > 1) + total *= (uint64_t)si.mem_unit; + return total; + } +#elif defined(HAVE_BSD_SYSCTL) && (defined(HW_MEMSIZE) || defined(HW_PHYSMEM) || defined(HW_PHYSMEM64)) + uint64_t physical_memory; int mib[2]; size_t length; mib[0] = CTL_HW; # if defined(HW_MEMSIZE) mib[1] = HW_MEMSIZE; +# elif defined(HW_PHYSMEM64) + mib[1] = HW_PHYSMEM64; # else mib[1] = HW_PHYSMEM; # endif - length = sizeof(int64_t); - if (!sysctl(mib, 2, &physical_memory, &length, NULL, 0)) + length = sizeof(physical_memory); + if (!sysctl(mib, 2, &physical_memory, &length, NULL, 0)) { + if (length == 4) { + uint32_t mem; + + if (!sysctl(mib, 2, &mem, &length, NULL, 0)) + physical_memory = mem; + } return physical_memory; + } #elif defined(GIT_WINDOWS_NATIVE) MEMORYSTATUSEX memInfo; @@ -416,7 +607,7 @@ static uint64_t estimate_repack_memory(struct gc_config *cfg, * read_sha1_file() (either at delta calculation phase, or * writing phase) also fills up the delta base cache */ - heap += delta_base_cache_limit; + heap += cfg->delta_base_cache_limit; /* and of course pack-objects has its own delta cache */ heap += cfg->max_delta_cache_size; @@ -432,7 +623,8 @@ static int keep_one_pack(struct string_list_item *item, void *data UNUSED) static void add_repack_all_option(struct gc_config *cfg, struct string_list *keep_pack) { - if (cfg->prune_expire && !strcmp(cfg->prune_expire, "now")) + if (cfg->prune_expire && !strcmp(cfg->prune_expire, "now") + && !(cfg->cruft_packs && cfg->repack_expire_to)) strvec_push(&repack, "-a"); else if (cfg->cruft_packs) { strvec_push(&repack, "--cruft"); @@ -441,6 +633,8 @@ static void add_repack_all_option(struct gc_config *cfg, if (cfg->max_cruft_size) strvec_pushf(&repack, "--max-cruft-size=%lu", cfg->max_cruft_size); + if (cfg->repack_expire_to) + strvec_pushf(&repack, "--expire-to=%s", cfg->repack_expire_to); } else { strvec_push(&repack, "-A"); if (cfg->prune_expire) @@ -533,7 +727,7 @@ static const char *lock_repo_for_gc(int force, pid_t* ret_pid) if (xgethostname(my_host, sizeof(my_host))) xsnprintf(my_host, sizeof(my_host), "unknown"); - pidfile_path = git_pathdup("gc.pid"); + pidfile_path = repo_git_path(the_repository, "gc.pid"); fd = hold_lock_file_for_update(&lock, pidfile_path, LOCK_DIE_ON_ERROR); if (!force) { @@ -594,7 +788,7 @@ static int report_last_gc_error(void) int ret = 0; ssize_t len; struct stat st; - char *gc_log_path = git_pathdup("gc.log"); + char *gc_log_path = repo_git_path(the_repository, "gc.log"); if (stat(gc_log_path, &st)) { if (errno == ENOENT) @@ -631,59 +825,48 @@ done: return ret; } -static void gc_before_repack(struct maintenance_run_opts *opts, - struct gc_config *cfg) +static int gc_foreground_tasks(struct maintenance_run_opts *opts, + struct gc_config *cfg) { - /* - * We may be called twice, as both the pre- and - * post-daemonized phases will call us, but running these - * commands more than once is pointless and wasteful. - */ - static int done = 0; - if (done++) - return; - if (cfg->pack_refs && maintenance_task_pack_refs(opts, cfg)) - die(FAILED_RUN, "pack-refs"); - - if (cfg->prune_reflogs) { - struct child_process cmd = CHILD_PROCESS_INIT; - - cmd.git_cmd = 1; - strvec_pushv(&cmd.args, reflog.v); - if (run_command(&cmd)) - die(FAILED_RUN, reflog.v[0]); - } + return error(FAILED_RUN, "pack-refs"); + if (cfg->prune_reflogs && maintenance_task_reflog_expire(opts, cfg)) + return error(FAILED_RUN, "reflog"); + return 0; } int cmd_gc(int argc, -const char **argv, -const char *prefix, -struct repository *repo UNUSED) + const char **argv, + const char *prefix, + struct repository *repo UNUSED) { int aggressive = 0; - int quiet = 0; int force = 0; const char *name; pid_t pid; int daemonized = 0; int keep_largest_pack = -1; + int skip_foreground_tasks = 0; timestamp_t dummy; - struct child_process rerere_cmd = CHILD_PROCESS_INIT; struct maintenance_run_opts opts = MAINTENANCE_RUN_OPTS_INIT; struct gc_config cfg = GC_CONFIG_INIT; const char *prune_expire_sentinel = "sentinel"; const char *prune_expire_arg = prune_expire_sentinel; int ret; - struct option builtin_gc_options[] = { - OPT__QUIET(&quiet, N_("suppress progress reporting")), - { OPTION_STRING, 0, "prune", &prune_expire_arg, N_("date"), - N_("prune unreferenced objects"), - PARSE_OPT_OPTARG, NULL, (intptr_t)prune_expire_arg }, + OPT__QUIET(&opts.quiet, N_("suppress progress reporting")), + { + .type = OPTION_STRING, + .long_name = "prune", + .value = &prune_expire_arg, + .argh = N_("date"), + .help = N_("prune unreferenced objects"), + .flags = PARSE_OPT_OPTARG, + .defval = (intptr_t)prune_expire_arg, + }, OPT_BOOL(0, "cruft", &cfg.cruft_packs, N_("pack unreferenced objects separately")), - OPT_MAGNITUDE(0, "max-cruft-size", &cfg.max_cruft_size, - N_("with --cruft, limit the size of new cruft packs")), + OPT_UNSIGNED(0, "max-cruft-size", &cfg.max_cruft_size, + N_("with --cruft, limit the size of new cruft packs")), OPT_BOOL(0, "aggressive", &aggressive, N_("be more thorough (increased runtime)")), OPT_BOOL_F(0, "auto", &opts.auto_flag, N_("enable auto-gc mode"), PARSE_OPT_NOCOMPLETE), @@ -694,17 +877,17 @@ struct repository *repo UNUSED) PARSE_OPT_NOCOMPLETE), OPT_BOOL(0, "keep-largest-pack", &keep_largest_pack, N_("repack all other packs except the largest pack")), + OPT_STRING(0, "expire-to", &cfg.repack_expire_to, N_("dir"), + N_("pack prefix to store a pack containing pruned objects")), + OPT_HIDDEN_BOOL(0, "skip-foreground-tasks", &skip_foreground_tasks, + N_("skip maintenance tasks typically done in the foreground")), OPT_END() }; - if (argc == 2 && !strcmp(argv[1], "-h")) - usage_with_options(builtin_gc_usage, builtin_gc_options); + show_usage_with_options_if_asked(argc, argv, + builtin_gc_usage, builtin_gc_options); - strvec_pushl(&reflog, "reflog", "expire", "--all", NULL); strvec_pushl(&repack, "repack", "-d", "-l", NULL); - strvec_pushl(&prune, "prune", "--expire", NULL); - strvec_pushl(&prune_worktrees, "worktree", "prune", "--expire", NULL); - strvec_pushl(&rerere, "rerere", "gc", NULL); gc_config(&cfg); @@ -733,7 +916,7 @@ struct repository *repo UNUSED) if (cfg.aggressive_window > 0) strvec_pushf(&repack, "--window=%d", cfg.aggressive_window); } - if (quiet) + if (opts.quiet) strvec_push(&repack, "-q"); if (opts.auto_flag) { @@ -748,7 +931,7 @@ struct repository *repo UNUSED) goto out; } - if (!quiet) { + if (!opts.quiet) { if (opts.detach > 0) fprintf(stderr, _("Auto packing the repository in background for optimum performance.\n")); else @@ -781,13 +964,16 @@ struct repository *repo UNUSED) goto out; } - if (lock_repo_for_gc(force, &pid)) { - ret = 0; - goto out; - } + if (!skip_foreground_tasks) { + if (lock_repo_for_gc(force, &pid)) { + ret = 0; + goto out; + } - gc_before_repack(&opts, &cfg); /* dies on failure */ - delete_tempfile(&pidfile); + if (gc_foreground_tasks(&opts, &cfg) < 0) + die(NULL); + delete_tempfile(&pidfile); + } /* * failure to daemonize is ok, we'll continue @@ -808,16 +994,18 @@ struct repository *repo UNUSED) } if (daemonized) { - hold_lock_file_for_update(&log_lock, - git_path("gc.log"), + char *path = repo_git_path(the_repository, "gc.log"); + hold_lock_file_for_update(&log_lock, path, LOCK_DIE_ON_ERROR); dup2(get_lock_file_fd(&log_lock), 2); atexit(process_log_file_at_exit); + free(path); } - gc_before_repack(&opts, &cfg); + if (opts.detach <= 0 && !skip_foreground_tasks) + gc_foreground_tasks(&opts, &cfg); - if (!repository_format_precious_objects) { + if (!the_repository->repository_format_precious_objects) { struct child_process repack_cmd = CHILD_PROCESS_INIT; repack_cmd.git_cmd = 1; @@ -829,34 +1017,27 @@ struct repository *repo UNUSED) if (cfg.prune_expire) { struct child_process prune_cmd = CHILD_PROCESS_INIT; + strvec_pushl(&prune_cmd.args, "prune", "--expire", NULL); /* run `git prune` even if using cruft packs */ - strvec_push(&prune, cfg.prune_expire); - if (quiet) - strvec_push(&prune, "--no-progress"); + strvec_push(&prune_cmd.args, cfg.prune_expire); + if (opts.quiet) + strvec_push(&prune_cmd.args, "--no-progress"); if (repo_has_promisor_remote(the_repository)) - strvec_push(&prune, + strvec_push(&prune_cmd.args, "--exclude-promisor-objects"); prune_cmd.git_cmd = 1; - strvec_pushv(&prune_cmd.args, prune.v); + if (run_command(&prune_cmd)) - die(FAILED_RUN, prune.v[0]); + die(FAILED_RUN, prune_cmd.args.v[0]); } } - if (cfg.prune_worktrees_expire) { - struct child_process prune_worktrees_cmd = CHILD_PROCESS_INIT; + if (cfg.prune_worktrees_expire && + maintenance_task_worktree_prune(&opts, &cfg)) + die(FAILED_RUN, "worktree"); - strvec_push(&prune_worktrees, cfg.prune_worktrees_expire); - prune_worktrees_cmd.git_cmd = 1; - strvec_pushv(&prune_worktrees_cmd.args, prune_worktrees.v); - if (run_command(&prune_worktrees_cmd)) - die(FAILED_RUN, prune_worktrees.v[0]); - } - - rerere_cmd.git_cmd = 1; - strvec_pushv(&rerere_cmd.args, rerere.v); - if (run_command(&rerere_cmd)) - die(FAILED_RUN, rerere.v[0]); + if (maintenance_task_rerere_gc(&opts, &cfg)) + die(FAILED_RUN, "rerere"); report_garbage = report_pack_garbage; reprepare_packed_git(the_repository); @@ -866,18 +1047,22 @@ struct repository *repo UNUSED) } if (the_repository->settings.gc_write_commit_graph == 1) - write_commit_graph_reachable(the_repository->objects->odb, - !quiet && !daemonized ? COMMIT_GRAPH_WRITE_PROGRESS : 0, + write_commit_graph_reachable(the_repository->objects->sources, + !opts.quiet && !daemonized ? COMMIT_GRAPH_WRITE_PROGRESS : 0, NULL); if (opts.auto_flag && too_many_loose_objects(&cfg)) warning(_("There are too many unreachable loose objects; " "run 'git prune' to remove them.")); - if (!daemonized) - unlink(git_path("gc.log")); + if (!daemonized) { + char *path = repo_git_path(the_repository, "gc.log"); + unlink(path); + free(path); + } out: + maintenance_run_opts_release(&opts); gc_config_release(&cfg); return 0; } @@ -925,7 +1110,7 @@ static int dfs_on_ref(const char *refname UNUSED, if (!peel_iterated_oid(the_repository, oid, &peeled)) oid = &peeled; - if (oid_object_info(the_repository, oid, NULL) != OBJ_COMMIT) + if (odb_read_object_info(the_repository->objects, oid, NULL) != OBJ_COMMIT) return 0; commit = lookup_commit(the_repository, oid); @@ -1002,6 +1187,8 @@ static int run_write_commit_graph(struct maintenance_run_opts *opts) if (opts->quiet) strvec_push(&child.args, "--no-progress"); + else + strvec_push(&child.args, "--progress"); return !!run_command(&child); } @@ -1052,8 +1239,14 @@ static int maintenance_task_prefetch(struct maintenance_run_opts *opts, return 0; } -static int maintenance_task_gc(struct maintenance_run_opts *opts, - struct gc_config *cfg UNUSED) +static int maintenance_task_gc_foreground(struct maintenance_run_opts *opts, + struct gc_config *cfg) +{ + return gc_foreground_tasks(opts, cfg); +} + +static int maintenance_task_gc_background(struct maintenance_run_opts *opts, + struct gc_config *cfg UNUSED) { struct child_process child = CHILD_PROCESS_INIT; @@ -1067,6 +1260,7 @@ static int maintenance_task_gc(struct maintenance_run_opts *opts, else strvec_push(&child.args, "--no-quiet"); strvec_push(&child.args, "--no-detach"); + strvec_push(&child.args, "--skip-foreground-tasks"); return run_command(&child); } @@ -1114,7 +1308,7 @@ static int loose_object_auto_condition(struct gc_config *cfg UNUSED) if (loose_object_auto_limit < 0) return 1; - return for_each_loose_file_in_objdir(the_repository->objects->odb->path, + return for_each_loose_file_in_objdir(the_repository->objects->sources->path, loose_object_count, NULL, NULL, &count); } @@ -1134,6 +1328,7 @@ static int write_loose_object_to_stdin(const struct object_id *oid, fprintf(d->in, "%s\n", oid_to_hex(oid)); + /* If batch_size is INT_MAX, then this will return 0 always. */ return ++(d->count) > d->batch_size; } @@ -1148,7 +1343,7 @@ static int pack_loose(struct maintenance_run_opts *opts) * Do not start pack-objects process * if there are no loose objects. */ - if (!for_each_loose_file_in_objdir(r->objects->odb->path, + if (!for_each_loose_file_in_objdir(r->objects->sources->path, bail_on_loose, NULL, NULL, NULL)) return 0; @@ -1158,7 +1353,9 @@ static int pack_loose(struct maintenance_run_opts *opts) strvec_push(&pack_proc.args, "pack-objects"); if (opts->quiet) strvec_push(&pack_proc.args, "--quiet"); - strvec_pushf(&pack_proc.args, "%s/pack/loose", r->objects->odb->path); + else + strvec_push(&pack_proc.args, "--no-quiet"); + strvec_pushf(&pack_proc.args, "%s/pack/loose", r->objects->sources->path); pack_proc.in = -1; @@ -1177,7 +1374,16 @@ static int pack_loose(struct maintenance_run_opts *opts) data.count = 0; data.batch_size = 50000; - for_each_loose_file_in_objdir(r->objects->odb->path, + repo_config_get_int(r, "maintenance.loose-objects.batchSize", + &data.batch_size); + + /* If configured as 0, then remove limit. */ + if (!data.batch_size) + data.batch_size = INT_MAX; + else if (data.batch_size > 0) + data.batch_size--; /* Decrease for equality on limit. */ + + for_each_loose_file_in_objdir(r->objects->sources->path, write_loose_object_to_stdin, NULL, NULL, @@ -1236,6 +1442,8 @@ static int multi_pack_index_write(struct maintenance_run_opts *opts) if (opts->quiet) strvec_push(&child.args, "--no-progress"); + else + strvec_push(&child.args, "--progress"); if (run_command(&child)) return error(_("failed to write multi-pack-index")); @@ -1252,6 +1460,8 @@ static int multi_pack_index_expire(struct maintenance_run_opts *opts) if (opts->quiet) strvec_push(&child.args, "--no-progress"); + else + strvec_push(&child.args, "--progress"); if (run_command(&child)) return error(_("'git multi-pack-index expire' failed")); @@ -1308,6 +1518,8 @@ static int multi_pack_index_repack(struct maintenance_run_opts *opts) if (opts->quiet) strvec_push(&child.args, "--no-progress"); + else + strvec_push(&child.args, "--progress"); strvec_pushf(&child.args, "--batch-size=%"PRIuMAX, (uintmax_t)get_auto_pack_size()); @@ -1336,89 +1548,120 @@ static int maintenance_task_incremental_repack(struct maintenance_run_opts *opts return 0; } -typedef int maintenance_task_fn(struct maintenance_run_opts *opts, - struct gc_config *cfg); - -/* - * An auto condition function returns 1 if the task should run - * and 0 if the task should NOT run. See needs_to_gc() for an - * example. - */ -typedef int maintenance_auto_fn(struct gc_config *cfg); +typedef int (*maintenance_task_fn)(struct maintenance_run_opts *opts, + struct gc_config *cfg); +typedef int (*maintenance_auto_fn)(struct gc_config *cfg); struct maintenance_task { const char *name; - maintenance_task_fn *fn; - maintenance_auto_fn *auto_condition; - unsigned enabled:1; - enum schedule_priority schedule; - - /* -1 if not selected. */ - int selected_order; -}; + /* + * Work that will be executed before detaching. This should not include + * tasks that may run for an extended amount of time as it does cause + * auto-maintenance to block until foreground tasks have been run. + */ + maintenance_task_fn foreground; -enum maintenance_task_label { - TASK_PREFETCH, - TASK_LOOSE_OBJECTS, - TASK_INCREMENTAL_REPACK, - TASK_GC, - TASK_COMMIT_GRAPH, - TASK_PACK_REFS, + /* + * Work that will be executed after detaching. When not detaching the + * work will be run in the foreground, as well. + */ + maintenance_task_fn background; - /* Leave as final value */ - TASK__COUNT + /* + * An auto condition function returns 1 if the task should run and 0 if + * the task should NOT run. See needs_to_gc() for an example. + */ + maintenance_auto_fn auto_condition; }; -static struct maintenance_task tasks[] = { +static const struct maintenance_task tasks[] = { [TASK_PREFETCH] = { - "prefetch", - maintenance_task_prefetch, + .name = "prefetch", + .background = maintenance_task_prefetch, }, [TASK_LOOSE_OBJECTS] = { - "loose-objects", - maintenance_task_loose_objects, - loose_object_auto_condition, + .name = "loose-objects", + .background = maintenance_task_loose_objects, + .auto_condition = loose_object_auto_condition, }, [TASK_INCREMENTAL_REPACK] = { - "incremental-repack", - maintenance_task_incremental_repack, - incremental_repack_auto_condition, + .name = "incremental-repack", + .background = maintenance_task_incremental_repack, + .auto_condition = incremental_repack_auto_condition, }, [TASK_GC] = { - "gc", - maintenance_task_gc, - need_to_gc, - 1, + .name = "gc", + .foreground = maintenance_task_gc_foreground, + .background = maintenance_task_gc_background, + .auto_condition = need_to_gc, }, [TASK_COMMIT_GRAPH] = { - "commit-graph", - maintenance_task_commit_graph, - should_write_commit_graph, + .name = "commit-graph", + .background = maintenance_task_commit_graph, + .auto_condition = should_write_commit_graph, }, [TASK_PACK_REFS] = { - "pack-refs", - maintenance_task_pack_refs, - pack_refs_condition, + .name = "pack-refs", + .foreground = maintenance_task_pack_refs, + .auto_condition = pack_refs_condition, + }, + [TASK_REFLOG_EXPIRE] = { + .name = "reflog-expire", + .foreground = maintenance_task_reflog_expire, + .auto_condition = reflog_expire_condition, }, + [TASK_WORKTREE_PRUNE] = { + .name = "worktree-prune", + .background = maintenance_task_worktree_prune, + .auto_condition = worktree_prune_condition, + }, + [TASK_RERERE_GC] = { + .name = "rerere-gc", + .background = maintenance_task_rerere_gc, + .auto_condition = rerere_gc_condition, + }, +}; + +enum task_phase { + TASK_PHASE_FOREGROUND, + TASK_PHASE_BACKGROUND, }; -static int compare_tasks_by_selection(const void *a_, const void *b_) +static int maybe_run_task(const struct maintenance_task *task, + struct repository *repo, + struct maintenance_run_opts *opts, + struct gc_config *cfg, + enum task_phase phase) { - const struct maintenance_task *a = a_; - const struct maintenance_task *b = b_; + int foreground = (phase == TASK_PHASE_FOREGROUND); + maintenance_task_fn fn = foreground ? task->foreground : task->background; + const char *region = foreground ? "maintenance foreground" : "maintenance"; + int ret = 0; - return b->selected_order - a->selected_order; + if (!fn) + return 0; + if (opts->auto_flag && + (!task->auto_condition || !task->auto_condition(cfg))) + return 0; + + trace2_region_enter(region, task->name, repo); + if (fn(opts, cfg)) { + error(_("task '%s' failed"), task->name); + ret = 1; + } + trace2_region_leave(region, task->name, repo); + + return ret; } static int maintenance_run_tasks(struct maintenance_run_opts *opts, struct gc_config *cfg) { - int i, found_selected = 0; int result = 0; struct lock_file lk; struct repository *r = the_repository; - char *lock_path = xstrfmt("%s/maintenance", r->objects->odb->path); + char *lock_path = xstrfmt("%s/maintenance", r->objects->sources->path); if (hold_lock_file_for_update(&lk, lock_path, LOCK_NO_DEREF) < 0) { /* @@ -1436,6 +1679,11 @@ static int maintenance_run_tasks(struct maintenance_run_opts *opts, } free(lock_path); + for (size_t i = 0; i < opts->tasks_nr; i++) + if (maybe_run_task(&tasks[opts->tasks[i]], r, opts, cfg, + TASK_PHASE_FOREGROUND)) + result = 1; + /* Failure to daemonize is ok, we'll continue in foreground. */ if (opts->detach > 0) { trace2_region_enter("maintenance", "detach", the_repository); @@ -1443,128 +1691,147 @@ static int maintenance_run_tasks(struct maintenance_run_opts *opts, trace2_region_leave("maintenance", "detach", the_repository); } - for (i = 0; !found_selected && i < TASK__COUNT; i++) - found_selected = tasks[i].selected_order >= 0; - - if (found_selected) - QSORT(tasks, TASK__COUNT, compare_tasks_by_selection); - - for (i = 0; i < TASK__COUNT; i++) { - if (found_selected && tasks[i].selected_order < 0) - continue; - - if (!found_selected && !tasks[i].enabled) - continue; - - if (opts->auto_flag && - (!tasks[i].auto_condition || - !tasks[i].auto_condition(cfg))) - continue; - - if (opts->schedule && tasks[i].schedule < opts->schedule) - continue; - - trace2_region_enter("maintenance", tasks[i].name, r); - if (tasks[i].fn(opts, cfg)) { - error(_("task '%s' failed"), tasks[i].name); + for (size_t i = 0; i < opts->tasks_nr; i++) + if (maybe_run_task(&tasks[opts->tasks[i]], r, opts, cfg, + TASK_PHASE_BACKGROUND)) result = 1; - } - trace2_region_leave("maintenance", tasks[i].name, r); - } rollback_lock_file(&lk); return result; } -static void initialize_maintenance_strategy(void) +struct maintenance_strategy { + struct { + int enabled; + enum schedule_priority schedule; + } tasks[TASK__COUNT]; +}; + +static const struct maintenance_strategy none_strategy = { 0 }; +static const struct maintenance_strategy default_strategy = { + .tasks = { + [TASK_GC].enabled = 1, + }, +}; +static const struct maintenance_strategy incremental_strategy = { + .tasks = { + [TASK_COMMIT_GRAPH].enabled = 1, + [TASK_COMMIT_GRAPH].schedule = SCHEDULE_HOURLY, + [TASK_PREFETCH].enabled = 1, + [TASK_PREFETCH].schedule = SCHEDULE_HOURLY, + [TASK_INCREMENTAL_REPACK].enabled = 1, + [TASK_INCREMENTAL_REPACK].schedule = SCHEDULE_DAILY, + [TASK_LOOSE_OBJECTS].enabled = 1, + [TASK_LOOSE_OBJECTS].schedule = SCHEDULE_DAILY, + [TASK_PACK_REFS].enabled = 1, + [TASK_PACK_REFS].schedule = SCHEDULE_WEEKLY, + }, +}; + +static void initialize_task_config(struct maintenance_run_opts *opts, + const struct string_list *selected_tasks) { + struct strbuf config_name = STRBUF_INIT; + struct maintenance_strategy strategy; const char *config_str; - if (git_config_get_string_tmp("maintenance.strategy", &config_str)) - return; + /* + * In case the user has asked us to run tasks explicitly we only use + * those specified tasks. Specifically, we do _not_ want to consult the + * config or maintenance strategy. + */ + if (selected_tasks->nr) { + for (size_t i = 0; i < selected_tasks->nr; i++) { + enum maintenance_task_label label = (intptr_t)selected_tasks->items[i].util;; + ALLOC_GROW(opts->tasks, opts->tasks_nr + 1, opts->tasks_alloc); + opts->tasks[opts->tasks_nr++] = label; + } - if (!strcasecmp(config_str, "incremental")) { - tasks[TASK_GC].schedule = SCHEDULE_NONE; - tasks[TASK_COMMIT_GRAPH].enabled = 1; - tasks[TASK_COMMIT_GRAPH].schedule = SCHEDULE_HOURLY; - tasks[TASK_PREFETCH].enabled = 1; - tasks[TASK_PREFETCH].schedule = SCHEDULE_HOURLY; - tasks[TASK_INCREMENTAL_REPACK].enabled = 1; - tasks[TASK_INCREMENTAL_REPACK].schedule = SCHEDULE_DAILY; - tasks[TASK_LOOSE_OBJECTS].enabled = 1; - tasks[TASK_LOOSE_OBJECTS].schedule = SCHEDULE_DAILY; - tasks[TASK_PACK_REFS].enabled = 1; - tasks[TASK_PACK_REFS].schedule = SCHEDULE_WEEKLY; + return; } -} -static void initialize_task_config(int schedule) -{ - int i; - struct strbuf config_name = STRBUF_INIT; + /* + * Otherwise, the strategy depends on whether we run as part of a + * scheduled job or not: + * + * - Scheduled maintenance does not perform any housekeeping by + * default, but requires the user to pick a maintenance strategy. + * + * - Unscheduled maintenance uses our default strategy. + * + * Both of these are affected by the gitconfig though, which may + * override specific aspects of our strategy. + */ + if (opts->schedule) { + strategy = none_strategy; - if (schedule) - initialize_maintenance_strategy(); + if (!git_config_get_string_tmp("maintenance.strategy", &config_str)) { + if (!strcasecmp(config_str, "incremental")) + strategy = incremental_strategy; + } + } else { + strategy = default_strategy; + } - for (i = 0; i < TASK__COUNT; i++) { + for (size_t i = 0; i < TASK__COUNT; i++) { int config_value; - char *config_str; strbuf_reset(&config_name); strbuf_addf(&config_name, "maintenance.%s.enabled", tasks[i].name); - if (!git_config_get_bool(config_name.buf, &config_value)) - tasks[i].enabled = config_value; - - strbuf_reset(&config_name); - strbuf_addf(&config_name, "maintenance.%s.schedule", - tasks[i].name); + strategy.tasks[i].enabled = config_value; + if (!strategy.tasks[i].enabled) + continue; - if (!git_config_get_string(config_name.buf, &config_str)) { - tasks[i].schedule = parse_schedule(config_str); - free(config_str); + if (opts->schedule) { + strbuf_reset(&config_name); + strbuf_addf(&config_name, "maintenance.%s.schedule", + tasks[i].name); + if (!git_config_get_string_tmp(config_name.buf, &config_str)) + strategy.tasks[i].schedule = parse_schedule(config_str); + if (strategy.tasks[i].schedule < opts->schedule) + continue; } + + ALLOC_GROW(opts->tasks, opts->tasks_nr + 1, opts->tasks_alloc); + opts->tasks[opts->tasks_nr++] = i; } strbuf_release(&config_name); } -static int task_option_parse(const struct option *opt UNUSED, +static int task_option_parse(const struct option *opt, const char *arg, int unset) { - int i, num_selected = 0; - struct maintenance_task *task = NULL; + struct string_list *selected_tasks = opt->value; + size_t i; BUG_ON_OPT_NEG(unset); - for (i = 0; i < TASK__COUNT; i++) { - if (tasks[i].selected_order >= 0) - num_selected++; - if (!strcasecmp(tasks[i].name, arg)) { - task = &tasks[i]; - } - } - - if (!task) { + for (i = 0; i < TASK__COUNT; i++) + if (!strcasecmp(tasks[i].name, arg)) + break; + if (i >= TASK__COUNT) { error(_("'%s' is not a valid task"), arg); return 1; } - if (task->selected_order >= 0) { + if (unsorted_string_list_has_string(selected_tasks, arg)) { error(_("task '%s' cannot be selected multiple times"), arg); return 1; } - task->selected_order = num_selected + 1; + string_list_append(selected_tasks, arg)->util = (void *)(intptr_t)i; return 0; } -static int maintenance_run(int argc, const char **argv, const char *prefix) +static int maintenance_run(int argc, const char **argv, const char *prefix, + struct repository *repo UNUSED) { - int i; struct maintenance_run_opts opts = MAINTENANCE_RUN_OPTS_INIT; + struct string_list selected_tasks = STRING_LIST_INIT_DUP; struct gc_config cfg = GC_CONFIG_INIT; struct option builtin_maintenance_run_options[] = { OPT_BOOL(0, "auto", &opts.auto_flag, @@ -1576,7 +1843,7 @@ static int maintenance_run(int argc, const char **argv, const char *prefix) maintenance_opt_schedule), OPT_BOOL(0, "quiet", &opts.quiet, N_("do not report progress or other information over stderr")), - OPT_CALLBACK_F(0, "task", NULL, N_("task"), + OPT_CALLBACK_F(0, "task", &selected_tasks, N_("task"), N_("run a specific task"), PARSE_OPT_NONEG, task_option_parse), OPT_END() @@ -1585,25 +1852,27 @@ static int maintenance_run(int argc, const char **argv, const char *prefix) opts.quiet = !isatty(2); - for (i = 0; i < TASK__COUNT; i++) - tasks[i].selected_order = -1; - argc = parse_options(argc, argv, prefix, builtin_maintenance_run_options, builtin_maintenance_run_usage, PARSE_OPT_STOP_AT_NON_OPTION); - if (opts.auto_flag && opts.schedule) - die(_("use at most one of --auto and --schedule=<frequency>")); + die_for_incompatible_opt2(opts.auto_flag, "--auto", + opts.schedule, "--schedule="); + die_for_incompatible_opt2(selected_tasks.nr, "--task=", + opts.schedule, "--schedule="); gc_config(&cfg); - initialize_task_config(opts.schedule); + initialize_task_config(&opts, &selected_tasks); if (argc != 0) usage_with_options(builtin_maintenance_run_usage, builtin_maintenance_run_options); ret = maintenance_run_tasks(&opts, &cfg); + + string_list_clear(&selected_tasks, 0); + maintenance_run_opts_release(&opts); gc_config_release(&cfg); return ret; } @@ -1623,7 +1892,8 @@ static char const * const builtin_maintenance_register_usage[] = { NULL }; -static int maintenance_register(int argc, const char **argv, const char *prefix) +static int maintenance_register(int argc, const char **argv, const char *prefix, + struct repository *repo UNUSED) { char *config_file = NULL; struct option options[] = { @@ -1687,7 +1957,8 @@ static char const * const builtin_maintenance_unregister_usage[] = { NULL }; -static int maintenance_unregister(int argc, const char **argv, const char *prefix) +static int maintenance_unregister(int argc, const char **argv, const char *prefix, + struct repository *repo UNUSED) { int force = 0; char *config_file = NULL; @@ -1893,7 +2164,7 @@ static int get_random_minute(void) if (getenv("GIT_TEST_MAINT_SCHEDULER")) return 13; - return git_rand() % 60; + return git_rand(0) % 60; } static int is_launchctl_available(void) @@ -2045,7 +2316,7 @@ static int launchctl_schedule_plist(const char *exec_path, enum schedule_priorit case SCHEDULE_DAILY: repeat = "<dict>\n" - "<key>Day</key><integer>%d</integer>\n" + "<key>Weekday</key><integer>%d</integer>\n" "<key>Hour</key><integer>0</integer>\n" "<key>Minute</key><integer>%d</integer>\n" "</dict>\n"; @@ -2056,7 +2327,7 @@ static int launchctl_schedule_plist(const char *exec_path, enum schedule_priorit case SCHEDULE_WEEKLY: strbuf_addf(&plist, "<dict>\n" - "<key>Day</key><integer>0</integer>\n" + "<key>Weekday</key><integer>0</integer>\n" "<key>Hour</key><integer>0</integer>\n" "<key>Minute</key><integer>%d</integer>\n" "</dict>\n", @@ -2069,7 +2340,7 @@ static int launchctl_schedule_plist(const char *exec_path, enum schedule_priorit } strbuf_addstr(&plist, "</array>\n</dict>\n</plist>\n"); - if (safe_create_leading_directories(filename)) + if (safe_create_leading_directories(the_repository, filename)) die(_("failed to create directories for '%s'"), filename); if ((long)lock_file_timeout_ms < 0 && @@ -2535,7 +2806,7 @@ static int systemd_timer_write_timer_file(enum schedule_priority schedule, filename = xdg_config_home_systemd(local_timer_name); - if (safe_create_leading_directories(filename)) { + if (safe_create_leading_directories(the_repository, filename)) { error(_("failed to create directories for '%s'"), filename); goto error; } @@ -2608,7 +2879,7 @@ static int systemd_timer_write_service_template(const char *exec_path) char *local_service_name = xstrfmt(SYSTEMD_UNIT_FORMAT, "", "service"); filename = xdg_config_home_systemd(local_service_name); - if (safe_create_leading_directories(filename)) { + if (safe_create_leading_directories(the_repository, filename)) { error(_("failed to create directories for '%s'"), filename); goto error; } @@ -2887,11 +3158,20 @@ static int update_background_schedule(const struct maintenance_start_opts *opts, unsigned int i; int result = 0; struct lock_file lk; - char *lock_path = xstrfmt("%s/schedule", the_repository->objects->odb->path); + char *lock_path = xstrfmt("%s/schedule", the_repository->objects->sources->path); if (hold_lock_file_for_update(&lk, lock_path, LOCK_NO_DEREF) < 0) { + if (errno == EEXIST) + error(_("unable to create '%s.lock': %s.\n\n" + "Another scheduled git-maintenance(1) process seems to be running in this\n" + "repository. Please make sure no other maintenance processes are running and\n" + "then try again. If it still fails, a git-maintenance(1) process may have\n" + "crashed in this repository earlier: remove the file manually to continue."), + absolute_path(lock_path), strerror(errno)); + else + error_errno(_("cannot acquire lock for scheduled background maintenance")); free(lock_path); - return error(_("another process is scheduling background maintenance")); + return -1; } for (i = 1; i < ARRAY_SIZE(scheduler_fn); i++) { @@ -2917,7 +3197,8 @@ static const char *const builtin_maintenance_start_usage[] = { NULL }; -static int maintenance_start(int argc, const char **argv, const char *prefix) +static int maintenance_start(int argc, const char **argv, const char *prefix, + struct repository *repo) { struct maintenance_start_opts opts = { 0 }; struct option options[] = { @@ -2940,7 +3221,7 @@ static int maintenance_start(int argc, const char **argv, const char *prefix) if (update_background_schedule(&opts, 1)) die(_("failed to set up maintenance schedule")); - if (maintenance_register(ARRAY_SIZE(register_args)-1, register_args, NULL)) + if (maintenance_register(ARRAY_SIZE(register_args)-1, register_args, NULL, repo)) warning(_("failed to add repo to global config")); return 0; } @@ -2950,7 +3231,8 @@ static const char *const builtin_maintenance_stop_usage[] = { NULL }; -static int maintenance_stop(int argc, const char **argv, const char *prefix) +static int maintenance_stop(int argc, const char **argv, const char *prefix, + struct repository *repo UNUSED) { struct option options[] = { OPT_END() @@ -2970,7 +3252,7 @@ static const char * const builtin_maintenance_usage[] = { int cmd_maintenance(int argc, const char **argv, const char *prefix, - struct repository *repo UNUSED) + struct repository *repo) { parse_opt_subcommand_fn *fn = NULL; struct option builtin_maintenance_options[] = { @@ -2984,5 +3266,5 @@ int cmd_maintenance(int argc, argc = parse_options(argc, argv, prefix, builtin_maintenance_options, builtin_maintenance_usage, 0); - return fn(argc, argv, prefix); + return fn(argc, argv, prefix, repo); } |
