From f6486f07d25ab4f2f93483690acabb817b0729b8 Mon Sep 17 00:00:00 2001 From: Nguyễn Thái Ngọc Duy Date: Thu, 5 Dec 2013 20:02:38 +0700 Subject: fetch-pack.h: one statement per bitfield declaration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- fetch-pack.h | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'fetch-pack.h') diff --git a/fetch-pack.h b/fetch-pack.h index 461cbf39b2..9b08388edf 100644 --- a/fetch-pack.h +++ b/fetch-pack.h @@ -8,18 +8,18 @@ struct fetch_pack_args { const char *uploadpack; int unpacklimit; int depth; - unsigned quiet:1, - keep_pack:1, - lock_pack:1, - use_thin_pack:1, - fetch_all:1, - stdin_refs:1, - verbose:1, - no_progress:1, - include_tag:1, - stateless_rpc:1, - check_self_contained_and_connected:1, - self_contained_and_connected:1; + unsigned quiet:1; + unsigned keep_pack:1; + unsigned lock_pack:1; + unsigned use_thin_pack:1; + unsigned fetch_all:1; + unsigned stdin_refs:1; + unsigned verbose:1; + unsigned no_progress:1; + unsigned include_tag:1; + unsigned stateless_rpc:1; + unsigned check_self_contained_and_connected:1; + unsigned self_contained_and_connected:1; }; /* -- cgit v1.2.3 From beea4152d94cf7c77eeb6b226805b315d22b3a2f Mon Sep 17 00:00:00 2001 From: Nguyễn Thái Ngọc Duy Date: Thu, 5 Dec 2013 20:02:39 +0700 Subject: clone: support remote shallow repository MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cloning from a shallow repository does not follow the "8 steps for new .git/shallow" because if it does we need to get through step 6 for all refs. That means commit walking down to the bottom. Instead the rule to create .git/shallow is simpler and, more importantly, cheap: if a shallow commit is found in the pack, it's probably used (i.e. reachable from some refs), so we add it. Others are dropped. One may notice this method seems flawed by the word "probably". A shallow commit may not be reachable from any refs at all if it's attached to an object island (a group of objects that are not reachable by any refs). If that object island is not complete, a new fetch request may send more objects to connect it to some ref. At that time, because we incorrectly installed the shallow commit in this island, the user will not see anything after that commit (fsck is still ok). This is not desired. Given that object islands are rare (C Git never sends such islands for security reasons) and do not really harm the repository integrity, a tradeoff is made to surprise the user occasionally but work faster everyday. A new option --strict could be added later that follows exactly the 8 steps. "git prune" can also learn to remove dangling objects _and_ the shallow commits that are attached to them from .git/shallow. Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- builtin/clone.c | 1 + builtin/fetch-pack.c | 2 +- fetch-pack.c | 54 +++++++++++++++++++++++++++++++++++++++++++++++++--- fetch-pack.h | 4 ++++ transport.c | 11 ++++++++--- transport.h | 6 ++++++ 6 files changed, 71 insertions(+), 7 deletions(-) (limited to 'fetch-pack.h') diff --git a/builtin/clone.c b/builtin/clone.c index 900f56476a..0b182cefc2 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -889,6 +889,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix) remote = remote_get(option_origin); transport = transport_get(remote, remote->url[0]); + transport->cloning = 1; if (!transport->get_refs_list || (!is_local && !transport->fetch)) die(_("Don't know how to clone %s"), transport->url); diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index c1d918fe1b..927424b6b8 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -153,7 +153,7 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix) get_remote_heads(fd[0], NULL, 0, &ref, 0, NULL, NULL); ref = fetch_pack(&args, fd, conn, ref, dest, - sought, nr_sought, pack_lockfile_ptr); + sought, nr_sought, NULL, pack_lockfile_ptr); if (pack_lockfile) { printf("lock %s\n", pack_lockfile); fflush(stdout); diff --git a/fetch-pack.c b/fetch-pack.c index 35d097e1b1..6c980cd39f 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -13,6 +13,7 @@ #include "transport.h" #include "version.h" #include "prio-queue.h" +#include "sha1-array.h" static int transfer_unpack_limit = -1; static int fetch_unpack_limit = -1; @@ -774,6 +775,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, int fd[2], const struct ref *orig_ref, struct ref **sought, int nr_sought, + struct shallow_info *si, char **pack_lockfile) { struct ref *ref = copy_ref_list(orig_ref); @@ -852,6 +854,8 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args, if (args->depth > 0) setup_alternate_shallow(&shallow_lock, &alternate_shallow_file, NULL); + else if (args->cloning && si->shallow && si->shallow->nr) + alternate_shallow_file = setup_temporary_shallow(si->shallow); else alternate_shallow_file = NULL; if (get_pack(args, fd, pack_lockfile)) @@ -925,8 +929,11 @@ static int remove_duplicates_in_refs(struct ref **ref, int nr) return dst; } -static void update_shallow(struct fetch_pack_args *args) +static void update_shallow(struct fetch_pack_args *args, + struct shallow_info *si) { + int i; + if (args->depth > 0 && alternate_shallow_file) { if (*alternate_shallow_file == '\0') { /* --unshallow */ unlink_or_warn(git_path("shallow")); @@ -935,6 +942,42 @@ static void update_shallow(struct fetch_pack_args *args) commit_lock_file(&shallow_lock); return; } + + if (!si->shallow || !si->shallow->nr) + return; + + if (alternate_shallow_file) { + /* + * The temporary shallow file is only useful for + * index-pack and unpack-objects because it may + * contain more roots than we want. Delete it. + */ + if (*alternate_shallow_file) + unlink(alternate_shallow_file); + free((char *)alternate_shallow_file); + } + + if (args->cloning) { + /* + * remote is shallow, but this is a clone, there are + * no objects in repo to worry about. Accept any + * shallow points that exist in the pack (iow in repo + * after get_pack() and reprepare_packed_git()) + */ + struct sha1_array extra = SHA1_ARRAY_INIT; + unsigned char (*sha1)[20] = si->shallow->sha1; + for (i = 0; i < si->shallow->nr; i++) + if (has_sha1_file(sha1[i])) + sha1_array_append(&extra, sha1[i]); + if (extra.nr) { + setup_alternate_shallow(&shallow_lock, + &alternate_shallow_file, + &extra); + commit_lock_file(&shallow_lock); + } + sha1_array_clear(&extra); + return; + } } struct ref *fetch_pack(struct fetch_pack_args *args, @@ -942,9 +985,11 @@ struct ref *fetch_pack(struct fetch_pack_args *args, const struct ref *ref, const char *dest, struct ref **sought, int nr_sought, + struct sha1_array *shallow, char **pack_lockfile) { struct ref *ref_cpy; + struct shallow_info si; fetch_pack_setup(); if (nr_sought) @@ -954,8 +999,11 @@ struct ref *fetch_pack(struct fetch_pack_args *args, packet_flush(fd[1]); die("no matching remote head"); } - ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, pack_lockfile); - update_shallow(args); + prepare_shallow_info(&si, shallow); + ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, + &si, pack_lockfile); reprepare_packed_git(); + update_shallow(args, &si); + clear_shallow_info(&si); return ref_cpy; } diff --git a/fetch-pack.h b/fetch-pack.h index 9b08388edf..ce595376b7 100644 --- a/fetch-pack.h +++ b/fetch-pack.h @@ -4,6 +4,8 @@ #include "string-list.h" #include "run-command.h" +struct sha1_array; + struct fetch_pack_args { const char *uploadpack; int unpacklimit; @@ -20,6 +22,7 @@ struct fetch_pack_args { unsigned stateless_rpc:1; unsigned check_self_contained_and_connected:1; unsigned self_contained_and_connected:1; + unsigned cloning:1; }; /* @@ -33,6 +36,7 @@ struct ref *fetch_pack(struct fetch_pack_args *args, const char *dest, struct ref **sought, int nr_sought, + struct sha1_array *shallow, char **pack_lockfile); #endif diff --git a/transport.c b/transport.c index 90453df9c6..91c466742e 100644 --- a/transport.c +++ b/transport.c @@ -456,6 +456,7 @@ struct git_transport_data { int fd[2]; unsigned got_remote_heads : 1; struct sha1_array extra_have; + struct sha1_array shallow; }; static int set_git_option(struct git_transport_options *opts, @@ -512,7 +513,9 @@ static struct ref *get_refs_via_connect(struct transport *transport, int for_pus connect_setup(transport, for_push, 0); get_remote_heads(data->fd[0], NULL, 0, &refs, - for_push ? REF_NORMAL : 0, &data->extra_have, NULL); + for_push ? REF_NORMAL : 0, + &data->extra_have, + transport->cloning ? &data->shallow : NULL); data->got_remote_heads = 1; return refs; @@ -539,17 +542,19 @@ static int fetch_refs_via_pack(struct transport *transport, args.depth = data->options.depth; args.check_self_contained_and_connected = data->options.check_self_contained_and_connected; + args.cloning = transport->cloning; if (!data->got_remote_heads) { connect_setup(transport, 0, 0); get_remote_heads(data->fd[0], NULL, 0, &refs_tmp, 0, - NULL, NULL); + NULL, + transport->cloning ? &data->shallow : NULL); data->got_remote_heads = 1; } refs = fetch_pack(&args, data->fd, data->conn, refs_tmp ? refs_tmp : transport->remote_refs, - dest, to_fetch, nr_heads, + dest, to_fetch, nr_heads, &data->shallow, &transport->pack_lockfile); close(data->fd[0]); close(data->fd[1]); diff --git a/transport.h b/transport.h index b3679bbdc7..59842d4994 100644 --- a/transport.h +++ b/transport.h @@ -35,6 +35,12 @@ struct transport { */ unsigned cannot_reuse : 1; + /* + * A hint from caller that it will be performing a clone, not + * normal fetch. IOW the repository is guaranteed empty. + */ + unsigned cloning : 1; + /** * Returns 0 if successful, positive if the option is not * recognized or is inapplicable, and negative if the option -- cgit v1.2.3 From 48d25cae22667dfc2c31ad620172c0f0a3ac1490 Mon Sep 17 00:00:00 2001 From: Nguyễn Thái Ngọc Duy Date: Thu, 5 Dec 2013 20:02:42 +0700 Subject: fetch: add --update-shallow to accept refs that update .git/shallow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The same steps are done as in when --update-shallow is not given. The only difference is we now add all shallow commits in "ours" and "theirs" to .git/shallow (aka "step 8"). Signed-off-by: Nguyễn Thái Ngọc Duy Signed-off-by: Junio C Hamano --- Documentation/fetch-options.txt | 6 ++++++ builtin/fetch.c | 6 +++++- fetch-pack.c | 27 +++++++++++++++++++++++++++ fetch-pack.h | 1 + t/t5537-fetch-shallow.sh | 32 ++++++++++++++++++++++++++++++++ transport.c | 4 ++++ transport.h | 4 ++++ 7 files changed, 79 insertions(+), 1 deletion(-) (limited to 'fetch-pack.h') diff --git a/Documentation/fetch-options.txt b/Documentation/fetch-options.txt index a83d2b4778..54043e3633 100644 --- a/Documentation/fetch-options.txt +++ b/Documentation/fetch-options.txt @@ -21,6 +21,12 @@ If the source repository is shallow, fetch as much as possible so that the current repository has the same history as the source repository. +--update-shallow:: + By default when fetching from a shallow repository, + `git fetch` refuses refs that require updating + .git/shallow. This option updates .git/shallow and accept such + refs. + ifndef::git-pull[] --dry-run:: Show what would be done, without making any changes. diff --git a/builtin/fetch.c b/builtin/fetch.c index 7b41a7e388..d2e4fc03d8 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -36,7 +36,7 @@ static int prune = -1; /* unspecified */ static int all, append, dry_run, force, keep, multiple, update_head_ok, verbosity; static int progress = -1, recurse_submodules = RECURSE_SUBMODULES_DEFAULT; -static int tags = TAGS_DEFAULT, unshallow; +static int tags = TAGS_DEFAULT, unshallow, update_shallow; static const char *depth; static const char *upload_pack; static struct strbuf default_rla = STRBUF_INIT; @@ -104,6 +104,8 @@ static struct option builtin_fetch_options[] = { { OPTION_STRING, 0, "recurse-submodules-default", &recurse_submodules_default, NULL, N_("default mode for recursion"), PARSE_OPT_HIDDEN }, + OPT_BOOL(0, "update-shallow", &update_shallow, + N_("accept refs that update .git/shallow")), OPT_END() }; @@ -768,6 +770,8 @@ static struct transport *prepare_transport(struct remote *remote) set_option(transport, TRANS_OPT_KEEP, "yes"); if (depth) set_option(transport, TRANS_OPT_DEPTH, depth); + if (update_shallow) + set_option(transport, TRANS_OPT_UPDATE_SHALLOW, "yes"); return transport; } diff --git a/fetch-pack.c b/fetch-pack.c index 34c544d0ca..a2d1b4ab28 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -993,6 +993,33 @@ static void update_shallow(struct fetch_pack_args *args, sha1_array_append(&ref, sought[i]->old_sha1); si->ref = &ref; + if (args->update_shallow) { + /* + * remote is also shallow, .git/shallow may be updated + * so all refs can be accepted. Make sure we only add + * shallow roots that are actually reachable from new + * refs. + */ + struct sha1_array extra = SHA1_ARRAY_INIT; + unsigned char (*sha1)[20] = si->shallow->sha1; + assign_shallow_commits_to_refs(si, NULL, NULL); + if (!si->nr_ours && !si->nr_theirs) { + sha1_array_clear(&ref); + return; + } + for (i = 0; i < si->nr_ours; i++) + sha1_array_append(&extra, sha1[si->ours[i]]); + for (i = 0; i < si->nr_theirs; i++) + sha1_array_append(&extra, sha1[si->theirs[i]]); + setup_alternate_shallow(&shallow_lock, + &alternate_shallow_file, + &extra); + commit_lock_file(&shallow_lock); + sha1_array_clear(&extra); + sha1_array_clear(&ref); + return; + } + /* * remote is also shallow, check what ref is safe to update * without updating .git/shallow diff --git a/fetch-pack.h b/fetch-pack.h index ce595376b7..ada02f51c1 100644 --- a/fetch-pack.h +++ b/fetch-pack.h @@ -23,6 +23,7 @@ struct fetch_pack_args { unsigned check_self_contained_and_connected:1; unsigned self_contained_and_connected:1; unsigned cloning:1; + unsigned update_shallow:1; }; /* diff --git a/t/t5537-fetch-shallow.sh b/t/t5537-fetch-shallow.sh index 022cb2c990..3ae9092f5c 100755 --- a/t/t5537-fetch-shallow.sh +++ b/t/t5537-fetch-shallow.sh @@ -141,4 +141,36 @@ EOF ) ' +test_expect_success 'fetch --update-shallow' ' + ( + cd shallow && + git checkout master && + commit 7 && + git tag -m foo heavy-tag HEAD^ && + git tag light-tag HEAD^:tracked + ) && + ( + cd notshallow && + git fetch --update-shallow ../shallow/.git refs/heads/*:refs/remotes/shallow/* && + git fsck && + git for-each-ref --sort=refname --format="%(refname)" >actual.refs && + cat <expect.refs && +refs/remotes/shallow/master +refs/remotes/shallow/no-shallow +refs/tags/heavy-tag +refs/tags/light-tag +EOF + test_cmp expect.refs actual.refs && + git log --format=%s shallow/master >actual && + cat <expect && +7 +6 +5 +4 +3 +EOF + test_cmp expect actual + ) +' + test_done diff --git a/transport.c b/transport.c index 491360be2c..a09fdb6df2 100644 --- a/transport.c +++ b/transport.c @@ -477,6 +477,9 @@ static int set_git_option(struct git_transport_options *opts, } else if (!strcmp(name, TRANS_OPT_KEEP)) { opts->keep = !!value; return 0; + } else if (!strcmp(name, TRANS_OPT_UPDATE_SHALLOW)) { + opts->update_shallow = !!value; + return 0; } else if (!strcmp(name, TRANS_OPT_DEPTH)) { if (!value) opts->depth = 0; @@ -543,6 +546,7 @@ static int fetch_refs_via_pack(struct transport *transport, args.check_self_contained_and_connected = data->options.check_self_contained_and_connected; args.cloning = transport->cloning; + args.update_shallow = data->options.update_shallow; if (!data->got_remote_heads) { connect_setup(transport, 0, 0); diff --git a/transport.h b/transport.h index 59842d4994..02ea248db1 100644 --- a/transport.h +++ b/transport.h @@ -11,6 +11,7 @@ struct git_transport_options { unsigned followtags : 1; unsigned check_self_contained_and_connected : 1; unsigned self_contained_and_connected : 1; + unsigned update_shallow : 1; int depth; const char *uploadpack; const char *receivepack; @@ -152,6 +153,9 @@ struct transport *transport_get(struct remote *, const char *); /* Aggressively fetch annotated tags if possible */ #define TRANS_OPT_FOLLOWTAGS "followtags" +/* Accept refs that may update .git/shallow without --depth */ +#define TRANS_OPT_UPDATE_SHALLOW "updateshallow" + /** * Returns 0 if the option was used, non-zero otherwise. Prints a * message to stderr if the option is not used. -- cgit v1.2.3