diff options
Diffstat (limited to 'fetch-pack.c')
| -rw-r--r-- | fetch-pack.c | 960 |
1 files changed, 730 insertions, 230 deletions
diff --git a/fetch-pack.c b/fetch-pack.c index fbe52f0234..88a078e9be 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -1,10 +1,12 @@ #include "cache.h" +#include "repository.h" +#include "config.h" #include "lockfile.h" #include "refs.h" #include "pkt-line.h" #include "commit.h" #include "tag.h" -#include "exec_cmd.h" +#include "exec-cmd.h" #include "pack.h" #include "sideband.h" #include "fetch-pack.h" @@ -13,8 +15,13 @@ #include "connect.h" #include "transport.h" #include "version.h" -#include "prio-queue.h" #include "sha1-array.h" +#include "oidset.h" +#include "packfile.h" +#include "object-store.h" +#include "connected.h" +#include "fetch-negotiator.h" +#include "fsck.h" static int transfer_unpack_limit = -1; static int fetch_unpack_limit = -1; @@ -26,18 +33,15 @@ static int deepen_not_ok; static int fetch_fsck_objects = -1; static int transfer_fsck_objects = -1; static int agent_supported; +static int server_supports_filtering; static struct lock_file shallow_lock; static const char *alternate_shallow_file; +static char *negotiation_algorithm; +static struct strbuf fsck_msg_types = STRBUF_INIT; /* Remember to update object flag allocation in object.h */ #define COMPLETE (1U << 0) -#define COMMON (1U << 1) -#define COMMON_REF (1U << 2) -#define SEEN (1U << 3) -#define POPPED (1U << 4) -#define ALTERNATE (1U << 5) - -static int marked; +#define ALTERNATE (1U << 1) /* * After sending this many "have"s if we do not get any new ACK , we @@ -45,8 +49,7 @@ static int marked; */ #define MAX_IN_VAIN 256 -static struct prio_queue rev_list = { compare_commits_by_commit_date }; -static int non_common_revs, multi_ack, use_sideband; +static int multi_ack, use_sideband; /* Allow specifying sha1 if it is a ref tip. */ #define ALLOW_TIP_SHA1 01 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */ @@ -78,7 +81,7 @@ static void cache_one_alternate(const char *refname, void *vcache) { struct alternate_object_cache *cache = vcache; - struct object *obj = parse_object(oid->hash); + struct object *obj = parse_object(the_repository, oid); if (!obj || (obj->flags & ALTERNATE)) return; @@ -88,7 +91,9 @@ static void cache_one_alternate(const char *refname, cache->items[cache->nr++] = obj; } -static void for_each_cached_alternate(void (*cb)(struct object *)) +static void for_each_cached_alternate(struct fetch_negotiator *negotiator, + void (*cb)(struct fetch_negotiator *, + struct object *)) { static int initialized; static struct alternate_object_cache cache; @@ -100,30 +105,19 @@ static void for_each_cached_alternate(void (*cb)(struct object *)) } for (i = 0; i < cache.nr; i++) - cb(cache.items[i]); -} - -static void rev_list_push(struct commit *commit, int mark) -{ - if (!(commit->object.flags & mark)) { - commit->object.flags |= mark; - - if (parse_commit(commit)) - return; - - prio_queue_put(&rev_list, commit); - - if (!(commit->object.flags & COMMON)) - non_common_revs++; - } + cb(negotiator, cache.items[i]); } -static int rev_list_insert_ref(const char *refname, const unsigned char *sha1) +static int rev_list_insert_ref(struct fetch_negotiator *negotiator, + const char *refname, + const struct object_id *oid) { - struct object *o = deref_tag(parse_object(sha1), refname, 0); + struct object *o = deref_tag(the_repository, + parse_object(the_repository, oid), + refname, 0); if (o && o->type == OBJ_COMMIT) - rev_list_push((struct commit *)o, SEEN); + negotiator->add_tip(negotiator, (struct commit *)o); return 0; } @@ -131,98 +125,7 @@ static int rev_list_insert_ref(const char *refname, const unsigned char *sha1) static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid, int flag, void *cb_data) { - return rev_list_insert_ref(refname, oid->hash); -} - -static int clear_marks(const char *refname, const struct object_id *oid, - int flag, void *cb_data) -{ - struct object *o = deref_tag(parse_object(oid->hash), refname, 0); - - if (o && o->type == OBJ_COMMIT) - clear_commit_marks((struct commit *)o, - COMMON | COMMON_REF | SEEN | POPPED); - return 0; -} - -/* - This function marks a rev and its ancestors as common. - In some cases, it is desirable to mark only the ancestors (for example - when only the server does not yet know that they are common). -*/ - -static void mark_common(struct commit *commit, - int ancestors_only, int dont_parse) -{ - if (commit != NULL && !(commit->object.flags & COMMON)) { - struct object *o = (struct object *)commit; - - if (!ancestors_only) - o->flags |= COMMON; - - if (!(o->flags & SEEN)) - rev_list_push(commit, SEEN); - else { - struct commit_list *parents; - - if (!ancestors_only && !(o->flags & POPPED)) - non_common_revs--; - if (!o->parsed && !dont_parse) - if (parse_commit(commit)) - return; - - for (parents = commit->parents; - parents; - parents = parents->next) - mark_common(parents->item, 0, dont_parse); - } - } -} - -/* - Get the next rev to send, ignoring the common. -*/ - -static const unsigned char *get_rev(void) -{ - struct commit *commit = NULL; - - while (commit == NULL) { - unsigned int mark; - struct commit_list *parents; - - if (rev_list.nr == 0 || non_common_revs == 0) - return NULL; - - commit = prio_queue_get(&rev_list); - parse_commit(commit); - parents = commit->parents; - - commit->object.flags |= POPPED; - if (!(commit->object.flags & COMMON)) - non_common_revs--; - - if (commit->object.flags & COMMON) { - /* do not send "have", and ignore ancestors */ - commit = NULL; - mark = COMMON | SEEN; - } else if (commit->object.flags & COMMON_REF) - /* send "have", and ignore ancestors */ - mark = COMMON | SEEN; - else - /* send "have", also for its ancestors */ - mark = SEEN; - - while (parents) { - if (!(parents->item->object.flags & SEEN)) - rev_list_push(parents->item, mark); - if (mark & COMMON) - mark_common(parents->item, 1, 0); - parents = parents->next; - } - } - - return commit->object.oid.hash; + return rev_list_insert_ref(cb_data, refname, oid); } enum ack_type { @@ -251,18 +154,18 @@ static void consume_shallow_list(struct fetch_pack_args *args, int fd) } } -static enum ack_type get_ack(int fd, unsigned char *result_sha1) +static enum ack_type get_ack(int fd, struct object_id *result_oid) { int len; char *line = packet_read_line(fd, &len); const char *arg; - if (!len) - die(_("git fetch-pack: expected ACK/NAK, got EOF")); + if (!line) + die(_("git fetch-pack: expected ACK/NAK, got a flush packet")); if (!strcmp(line, "NAK")) return NAK; if (skip_prefix(line, "ACK ", &arg)) { - if (!get_sha1_hex(arg, result_sha1)) { + if (!get_oid_hex(arg, result_oid)) { arg += 40; len -= arg - line; if (len < 1) @@ -276,6 +179,8 @@ static enum ack_type get_ack(int fd, unsigned char *result_sha1) return ACK; } } + if (skip_prefix(line, "ERR ", &arg)) + die(_("remote error: %s"), arg); die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line); } @@ -289,18 +194,19 @@ static void send_request(struct fetch_pack_args *args, write_or_die(fd, buf->buf, buf->len); } -static void insert_one_alternate_object(struct object *obj) +static void insert_one_alternate_object(struct fetch_negotiator *negotiator, + struct object *obj) { - rev_list_insert_ref(NULL, obj->oid.hash); + rev_list_insert_ref(negotiator, NULL, &obj->oid); } #define INITIAL_FLUSH 16 #define PIPESAFE_FLUSH 32 #define LARGE_FLUSH 16384 -static int next_flush(struct fetch_pack_args *args, int count) +static int next_flush(int stateless_rpc, int count) { - if (args->stateless_rpc) { + if (stateless_rpc) { if (count < LARGE_FLUSH) count <<= 1; else @@ -314,13 +220,30 @@ static int next_flush(struct fetch_pack_args *args, int count) return count; } -static int find_common(struct fetch_pack_args *args, - int fd[2], unsigned char *result_sha1, +static void mark_tips(struct fetch_negotiator *negotiator, + const struct oid_array *negotiation_tips) +{ + int i; + + if (!negotiation_tips) { + for_each_ref(rev_list_insert_ref_oid, negotiator); + return; + } + + for (i = 0; i < negotiation_tips->nr; i++) + rev_list_insert_ref(negotiator, NULL, + &negotiation_tips->oid[i]); + return; +} + +static int find_common(struct fetch_negotiator *negotiator, + struct fetch_pack_args *args, + int fd[2], struct object_id *result_oid, struct ref *refs) { int fetching; int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval; - const unsigned char *sha1; + const struct object_id *oid; unsigned in_vain = 0; int got_continue = 0; int got_ready = 0; @@ -329,16 +252,13 @@ static int find_common(struct fetch_pack_args *args, if (args->stateless_rpc && multi_ack == 1) die(_("--stateless-rpc requires multi_ack_detailed")); - if (marked) - for_each_ref(clear_marks, NULL); - marked = 1; - for_each_ref(rev_list_insert_ref_oid, NULL); - for_each_cached_alternate(insert_one_alternate_object); + mark_tips(negotiator, args->negotiation_tips); + for_each_cached_alternate(negotiator, insert_one_alternate_object); fetching = 0; for ( ; refs ; refs = refs->next) { - unsigned char *remote = refs->old_oid.hash; + struct object_id *remote = &refs->old_oid; const char *remote_hex; struct object *o; @@ -352,12 +272,12 @@ static int find_common(struct fetch_pack_args *args, * interested in the case we *know* the object is * reachable and we have already scanned it. */ - if (((o = lookup_object(remote)) != NULL) && + if (((o = lookup_object(the_repository, remote->hash)) != NULL) && (o->flags & COMPLETE)) { continue; } - remote_hex = sha1_to_hex(remote); + remote_hex = oid_to_hex(remote); if (!fetching) { struct strbuf c = STRBUF_INIT; if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed"); @@ -374,6 +294,8 @@ static int find_common(struct fetch_pack_args *args, if (deepen_not_ok) strbuf_addstr(&c, " deepen-not"); if (agent_supported) strbuf_addf(&c, " agent=%s", git_user_agent_sanitized()); + if (args->filter_options.choice) + strbuf_addstr(&c, " filter"); packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf); strbuf_release(&c); } else @@ -387,7 +309,7 @@ static int find_common(struct fetch_pack_args *args, return 1; } - if (is_repository_shallow()) + if (is_repository_shallow(the_repository)) write_shallow_commits(&req_buf, 1, NULL); if (args->depth > 0) packet_buf_write(&req_buf, "deepen %d", args->depth); @@ -402,31 +324,34 @@ static int find_common(struct fetch_pack_args *args, packet_buf_write(&req_buf, "deepen-not %s", s->string); } } + if (server_supports_filtering && args->filter_options.choice) + packet_buf_write(&req_buf, "filter %s", + args->filter_options.filter_spec); packet_buf_flush(&req_buf); state_len = req_buf.len; if (args->deepen) { char *line; const char *arg; - unsigned char sha1[20]; + struct object_id oid; send_request(args, fd[1], &req_buf); while ((line = packet_read_line(fd[0], NULL))) { if (skip_prefix(line, "shallow ", &arg)) { - if (get_sha1_hex(arg, sha1)) + if (get_oid_hex(arg, &oid)) die(_("invalid shallow line: %s"), line); - register_shallow(sha1); + register_shallow(the_repository, &oid); continue; } if (skip_prefix(line, "unshallow ", &arg)) { - if (get_sha1_hex(arg, sha1)) + if (get_oid_hex(arg, &oid)) die(_("invalid unshallow line: %s"), line); - if (!lookup_object(sha1)) + if (!lookup_object(the_repository, oid.hash)) die(_("object not found: %s"), line); /* make sure that it is parsed as shallow */ - if (!parse_object(sha1)) + if (!parse_object(the_repository, &oid)) die(_("error in object: %s"), line); - if (unregister_shallow(sha1)) + if (unregister_shallow(&oid)) die(_("no shallow found: %s"), line); continue; } @@ -445,9 +370,11 @@ static int find_common(struct fetch_pack_args *args, flushes = 0; retval = -1; - while ((sha1 = get_rev())) { - packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1)); - print_verbose(args, "have %s", sha1_to_hex(sha1)); + if (args->no_dependents) + goto done; + while ((oid = negotiator->next(negotiator))) { + packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid)); + print_verbose(args, "have %s", oid_to_hex(oid)); in_vain++; if (flush_at <= ++count) { int ack; @@ -456,7 +383,7 @@ static int find_common(struct fetch_pack_args *args, send_request(args, fd[1], &req_buf); strbuf_setlen(&req_buf, state_len); flushes++; - flush_at = next_flush(args, count); + flush_at = next_flush(args->stateless_rpc, count); /* * We keep one window "ahead" of the other side, and @@ -467,10 +394,10 @@ static int find_common(struct fetch_pack_args *args, consume_shallow_list(args, fd[0]); do { - ack = get_ack(fd[0], result_sha1); + ack = get_ack(fd[0], result_oid); if (ack) print_verbose(args, _("got %s %d %s"), "ack", - ack, sha1_to_hex(result_sha1)); + ack, oid_to_hex(result_oid)); switch (ack) { case ACK: flushes = 0; @@ -481,17 +408,21 @@ static int find_common(struct fetch_pack_args *args, case ACK_ready: case ACK_continue: { struct commit *commit = - lookup_commit(result_sha1); + lookup_commit(the_repository, + result_oid); + int was_common; + if (!commit) - die(_("invalid commit %s"), sha1_to_hex(result_sha1)); + die(_("invalid commit %s"), oid_to_hex(result_oid)); + was_common = negotiator->ack(negotiator, commit); if (args->stateless_rpc && ack == ACK_common - && !(commit->object.flags & COMMON)) { + && !was_common) { /* We need to replay the have for this object * on the next RPC request so the peer knows * it is in common with us. */ - const char *hex = sha1_to_hex(result_sha1); + const char *hex = oid_to_hex(result_oid); packet_buf_write(&req_buf, "have %s\n", hex); state_len = req_buf.len; /* @@ -503,13 +434,10 @@ static int find_common(struct fetch_pack_args *args, } else if (!args->stateless_rpc || ack != ACK_common) in_vain = 0; - mark_common(commit, 0, 1); retval = 0; got_continue = 1; - if (ack == ACK_ready) { - clear_prio_queue(&rev_list); + if (ack == ACK_ready) got_ready = 1; - } break; } } @@ -519,6 +447,8 @@ static int find_common(struct fetch_pack_args *args, print_verbose(args, _("giving up")); break; /* give up */ } + if (got_ready) + break; } } done: @@ -536,10 +466,10 @@ done: if (!got_ready || !no_done) consume_shallow_list(args, fd[0]); while (flushes || multi_ack) { - int ack = get_ack(fd[0], result_sha1); + int ack = get_ack(fd[0], result_oid); if (ack) { print_verbose(args, _("got %s (%d) %s"), "ack", - ack, sha1_to_hex(result_sha1)); + ack, oid_to_hex(result_oid)); if (ack == ACK) return 0; multi_ack = 1; @@ -553,16 +483,16 @@ done: static struct commit_list *complete; -static int mark_complete(const unsigned char *sha1) +static int mark_complete(const struct object_id *oid) { - struct object *o = parse_object(sha1); + struct object *o = parse_object(the_repository, oid); while (o && o->type == OBJ_TAG) { struct tag *t = (struct tag *) o; if (!t->tagged) break; /* broken repository */ o->flags |= COMPLETE; - o = parse_object(t->tagged->oid.hash); + o = parse_object(the_repository, &t->tagged->oid); } if (o && o->type == OBJ_COMMIT) { struct commit *commit = (struct commit *)o; @@ -577,7 +507,7 @@ static int mark_complete(const unsigned char *sha1) static int mark_complete_oid(const char *refname, const struct object_id *oid, int flag, void *cb_data) { - return mark_complete(oid->hash); + return mark_complete(oid); } static void mark_recent_complete_commits(struct fetch_pack_args *args, @@ -590,13 +520,38 @@ static void mark_recent_complete_commits(struct fetch_pack_args *args, } } +static void add_refs_to_oidset(struct oidset *oids, struct ref *refs) +{ + for (; refs; refs = refs->next) + oidset_insert(oids, &refs->old_oid); +} + +static int tip_oids_contain(struct oidset *tip_oids, + struct ref *unmatched, struct ref *newlist, + const struct object_id *id) +{ + /* + * Note that this only looks at the ref lists the first time it's + * called. This works out in filter_refs() because even though it may + * add to "newlist" between calls, the additions will always be for + * oids that are already in the set. + */ + if (!tip_oids->map.map.tablesize) { + add_refs_to_oidset(tip_oids, unmatched); + add_refs_to_oidset(tip_oids, newlist); + } + return oidset_contains(tip_oids, id); +} + static void filter_refs(struct fetch_pack_args *args, struct ref **refs, struct ref **sought, int nr_sought) { struct ref *newlist = NULL; struct ref **newtail = &newlist; + struct ref *unmatched = NULL; struct ref *ref, *next; + struct oidset tip_oids = OIDSET_INIT; int i; i = 0; @@ -618,35 +573,39 @@ static void filter_refs(struct fetch_pack_args *args, } i++; } - } - if (!keep && args->fetch_all && - (!args->deepen || !starts_with(ref->name, "refs/tags/"))) - keep = 1; + if (!keep && args->fetch_all && + (!args->deepen || !starts_with(ref->name, "refs/tags/"))) + keep = 1; + } if (keep) { *newtail = ref; ref->next = NULL; newtail = &ref->next; } else { - free(ref); + ref->next = unmatched; + unmatched = ref; } } /* Append unmatched requests to the list */ for (i = 0; i < nr_sought; i++) { - unsigned char sha1[20]; + struct object_id oid; + const char *p; ref = sought[i]; if (ref->match_status != REF_NOT_MATCHED) continue; - if (get_sha1_hex(ref->name, sha1) || - ref->name[40] != '\0' || - hashcmp(sha1, ref->old_oid.hash)) + if (parse_oid_hex(ref->name, &oid, &p) || + *p != '\0' || + oidcmp(&oid, &ref->old_oid)) continue; if ((allow_unadvertised_object_request & - (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1))) { + (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1)) || + tip_oids_contain(&tip_oids, unmatched, newlist, + &ref->old_oid)) { ref->match_status = REF_MATCHED; *newtail = copy_ref(ref); newtail = &(*newtail)->next; @@ -654,31 +613,87 @@ static void filter_refs(struct fetch_pack_args *args, ref->match_status = REF_UNADVERTISED_NOT_ALLOWED; } } + + oidset_clear(&tip_oids); + for (ref = unmatched; ref; ref = next) { + next = ref->next; + free(ref); + } + *refs = newlist; } -static void mark_alternate_complete(struct object *obj) +static void mark_alternate_complete(struct fetch_negotiator *unused, + struct object *obj) { - mark_complete(obj->oid.hash); + mark_complete(&obj->oid); } -static int everything_local(struct fetch_pack_args *args, - struct ref **refs, - struct ref **sought, int nr_sought) +struct loose_object_iter { + struct oidset *loose_object_set; + struct ref *refs; +}; + +/* + * If the number of refs is not larger than the number of loose objects, + * this function stops inserting. + */ +static int add_loose_objects_to_set(const struct object_id *oid, + const char *path, + void *data) +{ + struct loose_object_iter *iter = data; + oidset_insert(iter->loose_object_set, oid); + if (iter->refs == NULL) + return 1; + + iter->refs = iter->refs->next; + return 0; +} + +/* + * Mark recent commits available locally and reachable from a local ref as + * COMPLETE. If args->no_dependents is false, also mark COMPLETE remote refs as + * COMMON_REF (otherwise, we are not planning to participate in negotiation, and + * thus do not need COMMON_REF marks). + * + * The cutoff time for recency is determined by this heuristic: it is the + * earliest commit time of the objects in refs that are commits and that we know + * the commit time of. + */ +static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator, + struct fetch_pack_args *args, + struct ref **refs) { struct ref *ref; - int retval; + int old_save_commit_buffer = save_commit_buffer; timestamp_t cutoff = 0; + struct oidset loose_oid_set = OIDSET_INIT; + int use_oidset = 0; + struct loose_object_iter iter = {&loose_oid_set, *refs}; + + /* Enumerate all loose objects or know refs are not so many. */ + use_oidset = !for_each_loose_object(add_loose_objects_to_set, + &iter, 0); save_commit_buffer = 0; for (ref = *refs; ref; ref = ref->next) { struct object *o; + unsigned int flags = OBJECT_INFO_QUICK; - if (!has_object_file(&ref->old_oid)) - continue; + if (use_oidset && + !oidset_contains(&loose_oid_set, &ref->old_oid)) { + /* + * I know this does not exist in the loose form, + * so check if it exists in a non-loose form. + */ + flags |= OBJECT_INFO_IGNORE_LOOSE; + } - o = parse_object(ref->old_oid.hash); + if (!has_object_file_with_flags(&ref->old_oid, flags)) + continue; + o = parse_object(the_repository, &ref->old_oid); if (!o) continue; @@ -693,48 +708,63 @@ static int everything_local(struct fetch_pack_args *args, } } - if (!args->deepen) { - for_each_ref(mark_complete_oid, NULL); - for_each_cached_alternate(mark_alternate_complete); - commit_list_sort_by_date(&complete); - if (cutoff) - mark_recent_complete_commits(args, cutoff); - } + oidset_clear(&loose_oid_set); - /* - * Mark all complete remote refs as common refs. - * Don't mark them common yet; the server has to be told so first. - */ - for (ref = *refs; ref; ref = ref->next) { - struct object *o = deref_tag(lookup_object(ref->old_oid.hash), - NULL, 0); + if (!args->no_dependents) { + if (!args->deepen) { + for_each_ref(mark_complete_oid, NULL); + for_each_cached_alternate(NULL, mark_alternate_complete); + commit_list_sort_by_date(&complete); + if (cutoff) + mark_recent_complete_commits(args, cutoff); + } - if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE)) - continue; + /* + * Mark all complete remote refs as common refs. + * Don't mark them common yet; the server has to be told so first. + */ + for (ref = *refs; ref; ref = ref->next) { + struct object *o = deref_tag(the_repository, + lookup_object(the_repository, + ref->old_oid.hash), + NULL, 0); - if (!(o->flags & SEEN)) { - rev_list_push((struct commit *)o, COMMON_REF | SEEN); + if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE)) + continue; - mark_common((struct commit *)o, 1, 1); + negotiator->known_common(negotiator, + (struct commit *)o); } } - filter_refs(args, refs, sought, nr_sought); + save_commit_buffer = old_save_commit_buffer; +} + +/* + * Returns 1 if every object pointed to by the given remote refs is available + * locally and reachable from a local ref, and 0 otherwise. + */ +static int everything_local(struct fetch_pack_args *args, + struct ref **refs) +{ + struct ref *ref; + int retval; for (retval = 1, ref = *refs; ref ; ref = ref->next) { - const unsigned char *remote = ref->old_oid.hash; + const struct object_id *remote = &ref->old_oid; struct object *o; - o = lookup_object(remote); + o = lookup_object(the_repository, remote->hash); if (!o || !(o->flags & COMPLETE)) { retval = 0; - print_verbose(args, "want %s (%s)", sha1_to_hex(remote), + print_verbose(args, "want %s (%s)", oid_to_hex(remote), ref->name); continue; } - print_verbose(args, _("already have %s (%s)"), sha1_to_hex(remote), + print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote), ref->name); } + return retval; } @@ -791,7 +821,7 @@ static int get_pack(struct fetch_pack_args *args, argv_array_push(&cmd.args, alternate_shallow_file); } - if (do_keep) { + if (do_keep || args->from_promisor) { if (pack_lockfile) cmd.out = -1; cmd_name = "index-pack"; @@ -801,9 +831,9 @@ static int get_pack(struct fetch_pack_args *args, argv_array_push(&cmd.args, "-v"); if (args->use_thin_pack) argv_array_push(&cmd.args, "--fix-thin"); - if (args->lock_pack || unpack_limit) { - char hostname[256]; - if (gethostname(hostname, sizeof(hostname))) + if (do_keep && (args->lock_pack || unpack_limit)) { + char hostname[HOST_NAME_MAX + 1]; + if (xgethostname(hostname, sizeof(hostname))) xsnprintf(hostname, sizeof(hostname), "localhost"); argv_array_pushf(&cmd.args, "--keep=fetch-pack %"PRIuMAX " on %s", @@ -811,6 +841,8 @@ static int get_pack(struct fetch_pack_args *args, } if (args->check_self_contained_and_connected) argv_array_push(&cmd.args, "--check-self-contained-and-connected"); + if (args->from_promisor) + argv_array_push(&cmd.args, "--promisor"); } else { cmd_name = "unpack-objects"; @@ -828,8 +860,18 @@ static int get_pack(struct fetch_pack_args *args, ? fetch_fsck_objects : transfer_fsck_objects >= 0 ? transfer_fsck_objects - : 0) - argv_array_push(&cmd.args, "--strict"); + : 0) { + if (args->from_promisor) + /* + * We cannot use --strict in index-pack because it + * checks both broken objects and links, but we only + * want to check for broken objects. + */ + argv_array_push(&cmd.args, "--fsck-objects"); + else + argv_array_pushf(&cmd.args, "--strict%s", + fsck_msg_types.buf); + } cmd.in = demux.out; cmd.git_cmd = 1; @@ -871,14 +913,16 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, char **pack_lockfile) { struct ref *ref = copy_ref_list(orig_ref); - unsigned char sha1[20]; + struct object_id oid; const char *agent_feature; int agent_len; + struct fetch_negotiator negotiator; + fetch_negotiator_init(&negotiator, negotiation_algorithm); sort_ref_list(&ref, ref_compare_name); QSORT(sought, nr_sought, cmp_ref_by_name); - if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow")) + if ((args->depth > 0 || is_repository_shallow(the_repository)) && !server_supports("shallow")) die(_("Server does not support shallow clients")); if (args->depth > 0 || args->deepen_since || args->deepen_not) args->deepen = 1; @@ -922,6 +966,13 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, else prefer_ofs_delta = 0; + if (server_supports("filter")) { + server_supports_filtering = 1; + print_verbose(args, _("Server supports filter")); + } else if (args->filter_options.choice) { + warning("filtering not recognized by server, ignoring"); + } + if ((agent_feature = server_feature_value("agent", &agent_len))) { agent_supported = 1; if (agent_len) @@ -939,11 +990,13 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, if (!server_supports("deepen-relative") && args->deepen_relative) die(_("Server does not support --deepen")); - if (everything_local(args, &ref, sought, nr_sought)) { + mark_complete_and_common_ref(&negotiator, args, &ref); + filter_refs(args, &ref, sought, nr_sought); + if (everything_local(args, &ref)) { packet_flush(fd[1]); goto all_done; } - if (find_common(args, fd, sha1, ref) < 0) + if (find_common(&negotiator, args, fd, &oid, ref) < 0) if (!args->keep_pack) /* When cloning, it is not unusual to have * no common commit. @@ -963,9 +1016,419 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, die(_("git fetch-pack: fetch failed.")); all_done: + negotiator.release(&negotiator); return ref; } +static void add_shallow_requests(struct strbuf *req_buf, + const struct fetch_pack_args *args) +{ + if (is_repository_shallow(the_repository)) + write_shallow_commits(req_buf, 1, NULL); + if (args->depth > 0) + packet_buf_write(req_buf, "deepen %d", args->depth); + if (args->deepen_since) { + timestamp_t max_age = approxidate(args->deepen_since); + packet_buf_write(req_buf, "deepen-since %"PRItime, max_age); + } + if (args->deepen_not) { + int i; + for (i = 0; i < args->deepen_not->nr; i++) { + struct string_list_item *s = args->deepen_not->items + i; + packet_buf_write(req_buf, "deepen-not %s", s->string); + } + } +} + +static void add_wants(const struct ref *wants, struct strbuf *req_buf) +{ + int use_ref_in_want = server_supports_feature("fetch", "ref-in-want", 0); + + for ( ; wants ; wants = wants->next) { + const struct object_id *remote = &wants->old_oid; + struct object *o; + + /* + * If that object is complete (i.e. it is an ancestor of a + * local ref), we tell them we have it but do not have to + * tell them about its ancestors, which they already know + * about. + * + * We use lookup_object here because we are only + * interested in the case we *know* the object is + * reachable and we have already scanned it. + */ + if (((o = lookup_object(the_repository, remote->hash)) != NULL) && + (o->flags & COMPLETE)) { + continue; + } + + if (!use_ref_in_want || wants->exact_oid) + packet_buf_write(req_buf, "want %s\n", oid_to_hex(remote)); + else + packet_buf_write(req_buf, "want-ref %s\n", wants->name); + } +} + +static void add_common(struct strbuf *req_buf, struct oidset *common) +{ + struct oidset_iter iter; + const struct object_id *oid; + oidset_iter_init(common, &iter); + + while ((oid = oidset_iter_next(&iter))) { + packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid)); + } +} + +static int add_haves(struct fetch_negotiator *negotiator, + struct strbuf *req_buf, + int *haves_to_send, int *in_vain) +{ + int ret = 0; + int haves_added = 0; + const struct object_id *oid; + + while ((oid = negotiator->next(negotiator))) { + packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid)); + if (++haves_added >= *haves_to_send) + break; + } + + *in_vain += haves_added; + if (!haves_added || *in_vain >= MAX_IN_VAIN) { + /* Send Done */ + packet_buf_write(req_buf, "done\n"); + ret = 1; + } + + /* Increase haves to send on next round */ + *haves_to_send = next_flush(1, *haves_to_send); + + return ret; +} + +static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out, + const struct fetch_pack_args *args, + const struct ref *wants, struct oidset *common, + int *haves_to_send, int *in_vain) +{ + int ret = 0; + struct strbuf req_buf = STRBUF_INIT; + + if (server_supports_v2("fetch", 1)) + packet_buf_write(&req_buf, "command=fetch"); + if (server_supports_v2("agent", 0)) + packet_buf_write(&req_buf, "agent=%s", git_user_agent_sanitized()); + if (args->server_options && args->server_options->nr && + server_supports_v2("server-option", 1)) { + int i; + for (i = 0; i < args->server_options->nr; i++) + packet_write_fmt(fd_out, "server-option=%s", + args->server_options->items[i].string); + } + + packet_buf_delim(&req_buf); + if (args->use_thin_pack) + packet_buf_write(&req_buf, "thin-pack"); + if (args->no_progress) + packet_buf_write(&req_buf, "no-progress"); + if (args->include_tag) + packet_buf_write(&req_buf, "include-tag"); + if (prefer_ofs_delta) + packet_buf_write(&req_buf, "ofs-delta"); + + /* Add shallow-info and deepen request */ + if (server_supports_feature("fetch", "shallow", 0)) + add_shallow_requests(&req_buf, args); + else if (is_repository_shallow(the_repository) || args->deepen) + die(_("Server does not support shallow requests")); + + /* Add filter */ + if (server_supports_feature("fetch", "filter", 0) && + args->filter_options.choice) { + print_verbose(args, _("Server supports filter")); + packet_buf_write(&req_buf, "filter %s", + args->filter_options.filter_spec); + } else if (args->filter_options.choice) { + warning("filtering not recognized by server, ignoring"); + } + + /* add wants */ + add_wants(wants, &req_buf); + + if (args->no_dependents) { + packet_buf_write(&req_buf, "done"); + ret = 1; + } else { + /* Add all of the common commits we've found in previous rounds */ + add_common(&req_buf, common); + + /* Add initial haves */ + ret = add_haves(negotiator, &req_buf, haves_to_send, in_vain); + } + + /* Send request */ + packet_buf_flush(&req_buf); + write_or_die(fd_out, req_buf.buf, req_buf.len); + + strbuf_release(&req_buf); + return ret; +} + +/* + * Processes a section header in a server's response and checks if it matches + * `section`. If the value of `peek` is 1, the header line will be peeked (and + * not consumed); if 0, the line will be consumed and the function will die if + * the section header doesn't match what was expected. + */ +static int process_section_header(struct packet_reader *reader, + const char *section, int peek) +{ + int ret; + + if (packet_reader_peek(reader) != PACKET_READ_NORMAL) + die(_("error reading section header '%s'"), section); + + ret = !strcmp(reader->line, section); + + if (!peek) { + if (!ret) + die(_("expected '%s', received '%s'"), + section, reader->line); + packet_reader_read(reader); + } + + return ret; +} + +static int process_acks(struct fetch_negotiator *negotiator, + struct packet_reader *reader, + struct oidset *common) +{ + /* received */ + int received_ready = 0; + int received_ack = 0; + + process_section_header(reader, "acknowledgments", 0); + while (packet_reader_read(reader) == PACKET_READ_NORMAL) { + const char *arg; + + if (!strcmp(reader->line, "NAK")) + continue; + + if (skip_prefix(reader->line, "ACK ", &arg)) { + struct object_id oid; + if (!get_oid_hex(arg, &oid)) { + struct commit *commit; + oidset_insert(common, &oid); + commit = lookup_commit(the_repository, &oid); + negotiator->ack(negotiator, commit); + } + continue; + } + + if (!strcmp(reader->line, "ready")) { + received_ready = 1; + continue; + } + + die(_("unexpected acknowledgment line: '%s'"), reader->line); + } + + if (reader->status != PACKET_READ_FLUSH && + reader->status != PACKET_READ_DELIM) + die(_("error processing acks: %d"), reader->status); + + /* return 0 if no common, 1 if there are common, or 2 if ready */ + return received_ready ? 2 : (received_ack ? 1 : 0); +} + +static void receive_shallow_info(struct fetch_pack_args *args, + struct packet_reader *reader) +{ + process_section_header(reader, "shallow-info", 0); + while (packet_reader_read(reader) == PACKET_READ_NORMAL) { + const char *arg; + struct object_id oid; + + if (skip_prefix(reader->line, "shallow ", &arg)) { + if (get_oid_hex(arg, &oid)) + die(_("invalid shallow line: %s"), reader->line); + register_shallow(the_repository, &oid); + continue; + } + if (skip_prefix(reader->line, "unshallow ", &arg)) { + if (get_oid_hex(arg, &oid)) + die(_("invalid unshallow line: %s"), reader->line); + if (!lookup_object(the_repository, oid.hash)) + die(_("object not found: %s"), reader->line); + /* make sure that it is parsed as shallow */ + if (!parse_object(the_repository, &oid)) + die(_("error in object: %s"), reader->line); + if (unregister_shallow(&oid)) + die(_("no shallow found: %s"), reader->line); + continue; + } + die(_("expected shallow/unshallow, got %s"), reader->line); + } + + if (reader->status != PACKET_READ_FLUSH && + reader->status != PACKET_READ_DELIM) + die(_("error processing shallow info: %d"), reader->status); + + setup_alternate_shallow(&shallow_lock, &alternate_shallow_file, NULL); + args->deepen = 1; +} + +static void receive_wanted_refs(struct packet_reader *reader, + struct ref **sought, int nr_sought) +{ + process_section_header(reader, "wanted-refs", 0); + while (packet_reader_read(reader) == PACKET_READ_NORMAL) { + struct object_id oid; + const char *end; + int i; + + if (parse_oid_hex(reader->line, &oid, &end) || *end++ != ' ') + die(_("expected wanted-ref, got '%s'"), reader->line); + + for (i = 0; i < nr_sought; i++) { + if (!strcmp(end, sought[i]->name)) { + oidcpy(&sought[i]->old_oid, &oid); + break; + } + } + + if (i == nr_sought) + die(_("unexpected wanted-ref: '%s'"), reader->line); + } + + if (reader->status != PACKET_READ_DELIM) + die(_("error processing wanted refs: %d"), reader->status); +} + +enum fetch_state { + FETCH_CHECK_LOCAL = 0, + FETCH_SEND_REQUEST, + FETCH_PROCESS_ACKS, + FETCH_GET_PACK, + FETCH_DONE, +}; + +static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, + int fd[2], + const struct ref *orig_ref, + struct ref **sought, int nr_sought, + char **pack_lockfile) +{ + struct ref *ref = copy_ref_list(orig_ref); + enum fetch_state state = FETCH_CHECK_LOCAL; + struct oidset common = OIDSET_INIT; + struct packet_reader reader; + int in_vain = 0; + int haves_to_send = INITIAL_FLUSH; + struct fetch_negotiator negotiator; + fetch_negotiator_init(&negotiator, negotiation_algorithm); + packet_reader_init(&reader, fd[0], NULL, 0, + PACKET_READ_CHOMP_NEWLINE); + + while (state != FETCH_DONE) { + switch (state) { + case FETCH_CHECK_LOCAL: + sort_ref_list(&ref, ref_compare_name); + QSORT(sought, nr_sought, cmp_ref_by_name); + + /* v2 supports these by default */ + allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1; + use_sideband = 2; + if (args->depth > 0 || args->deepen_since || args->deepen_not) + args->deepen = 1; + + /* Filter 'ref' by 'sought' and those that aren't local */ + mark_complete_and_common_ref(&negotiator, args, &ref); + filter_refs(args, &ref, sought, nr_sought); + if (everything_local(args, &ref)) + state = FETCH_DONE; + else + state = FETCH_SEND_REQUEST; + + mark_tips(&negotiator, args->negotiation_tips); + for_each_cached_alternate(&negotiator, + insert_one_alternate_object); + break; + case FETCH_SEND_REQUEST: + if (send_fetch_request(&negotiator, fd[1], args, ref, + &common, + &haves_to_send, &in_vain)) + state = FETCH_GET_PACK; + else + state = FETCH_PROCESS_ACKS; + break; + case FETCH_PROCESS_ACKS: + /* Process ACKs/NAKs */ + switch (process_acks(&negotiator, &reader, &common)) { + case 2: + state = FETCH_GET_PACK; + break; + case 1: + in_vain = 0; + /* fallthrough */ + default: + state = FETCH_SEND_REQUEST; + break; + } + break; + case FETCH_GET_PACK: + /* Check for shallow-info section */ + if (process_section_header(&reader, "shallow-info", 1)) + receive_shallow_info(args, &reader); + + if (process_section_header(&reader, "wanted-refs", 1)) + receive_wanted_refs(&reader, sought, nr_sought); + + /* get the pack */ + process_section_header(&reader, "packfile", 0); + if (get_pack(args, fd, pack_lockfile)) + die(_("git fetch-pack: fetch failed.")); + + state = FETCH_DONE; + break; + case FETCH_DONE: + continue; + } + } + + negotiator.release(&negotiator); + oidset_clear(&common); + return ref; +} + +static int fetch_pack_config_cb(const char *var, const char *value, void *cb) +{ + if (strcmp(var, "fetch.fsck.skiplist") == 0) { + const char *path; + + if (git_config_pathname(&path, var, value)) + return 1; + strbuf_addf(&fsck_msg_types, "%cskiplist=%s", + fsck_msg_types.len ? ',' : '=', path); + free((char *)path); + return 0; + } + + if (skip_prefix(var, "fetch.fsck.", &var)) { + if (is_valid_msg_type(var, value)) + strbuf_addf(&fsck_msg_types, "%c%s=%s", + fsck_msg_types.len ? ',' : '=', var, value); + else + warning("Skipping unknown msg id '%s'", var); + return 0; + } + + return git_default_config(var, value, cb); +} + static void fetch_pack_config(void) { git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit); @@ -973,8 +1436,10 @@ static void fetch_pack_config(void) git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta); git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects); git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects); + git_config_get_string("fetch.negotiationalgorithm", + &negotiation_algorithm); - git_config(git_default_config, NULL); + git_config(fetch_pack_config_cb, NULL); } static void fetch_pack_setup(void) @@ -1021,7 +1486,7 @@ static void update_shallow(struct fetch_pack_args *args, if (args->deepen && alternate_shallow_file) { if (*alternate_shallow_file == '\0') { /* --unshallow */ - unlink_or_warn(git_path_shallow()); + unlink_or_warn(git_path_shallow(the_repository)); rollback_lock_file(&shallow_lock); } else commit_lock_file(&shallow_lock); @@ -1105,13 +1570,26 @@ static void update_shallow(struct fetch_pack_args *args, oid_array_clear(&ref); } +static int iterate_ref_map(void *cb_data, struct object_id *oid) +{ + struct ref **rm = cb_data; + struct ref *ref = *rm; + + if (!ref) + return -1; /* end of the list */ + *rm = ref->next; + oidcpy(oid, &ref->old_oid); + return 0; +} + struct ref *fetch_pack(struct fetch_pack_args *args, int fd[], struct child_process *conn, const struct ref *ref, const char *dest, struct ref **sought, int nr_sought, struct oid_array *shallow, - char **pack_lockfile) + char **pack_lockfile, + enum protocol_version version) { struct ref *ref_cpy; struct shallow_info si; @@ -1125,10 +1603,32 @@ struct ref *fetch_pack(struct fetch_pack_args *args, die(_("no matching remote head")); } prepare_shallow_info(&si, shallow); - ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, - &si, pack_lockfile); - reprepare_packed_git(); + if (version == protocol_v2) + ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought, + pack_lockfile); + else + ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, + &si, pack_lockfile); + reprepare_packed_git(the_repository); + + if (!args->cloning && args->deepen) { + struct check_connected_options opt = CHECK_CONNECTED_INIT; + struct ref *iterator = ref_cpy; + opt.shallow_file = alternate_shallow_file; + if (args->deepen) + opt.is_deepening_fetch = 1; + if (check_connected(iterate_ref_map, &iterator, &opt)) { + error(_("remote did not send all necessary objects")); + free_refs(ref_cpy); + ref_cpy = NULL; + rollback_lock_file(&shallow_lock); + goto cleanup; + } + args->connectivity_checked = 1; + } + update_shallow(args, sought, nr_sought, &si); +cleanup: clear_shallow_info(&si); return ref_cpy; } |
