diff options
Diffstat (limited to 'builtin/pack-objects.c')
| -rw-r--r-- | builtin/pack-objects.c | 620 |
1 files changed, 446 insertions, 174 deletions
diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 411aefd687..2a00358f34 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -26,13 +26,16 @@ #include "pack-bitmap.h" #include "delta-islands.h" #include "reachable.h" -#include "sha1-array.h" -#include "argv-array.h" +#include "oid-array.h" +#include "strvec.h" #include "list.h" #include "packfile.h" #include "object-store.h" #include "dir.h" #include "midx.h" +#include "trace2.h" +#include "shallow.h" +#include "promisor-remote.h" #define IN_PACK(obj) oe_in_pack(&to_pack, obj) #define SIZE(obj) oe_size(&to_pack, obj) @@ -84,18 +87,24 @@ static unsigned long pack_size_limit; static int depth = 50; static int delta_search_threads; static int pack_to_stdout; +static int sparse; static int thin; static int num_preferred_base; static struct progress *progress_state; static struct packed_git *reuse_packfile; static uint32_t reuse_packfile_objects; -static off_t reuse_packfile_offset; +static struct bitmap *reuse_packfile_bitmap; static int use_bitmap_index_default = 1; static int use_bitmap_index = -1; -static int write_bitmap_index; -static uint16_t write_bitmap_options; +static int allow_pack_reuse = 1; +static enum { + WRITE_BITMAP_FALSE = 0, + WRITE_BITMAP_QUIET, + WRITE_BITMAP_TRUE, +} write_bitmap_index; +static uint16_t write_bitmap_options = BITMAP_OPT_HASH_CACHE; static int exclude_promisor_objects; @@ -109,6 +118,8 @@ static unsigned long window_memory_limit = 0; static struct list_objects_filter_options filter_options; +static struct string_list uri_protocols = STRING_LIST_INIT_NODUP; + enum missing_action { MA_ERROR = 0, /* fail if any missing objects are encountered */ MA_ALLOW_ANY, /* silently allow ALL missing objects */ @@ -117,6 +128,15 @@ enum missing_action { static enum missing_action arg_missing_action; static show_object_fn fn_show_object; +struct configured_exclusion { + struct oidmap_entry e; + char *pack_hash_hex; + char *uri; +}; +static struct oidmap configured_exclusions; + +static struct oidset excluded_by_config; + /* * stats */ @@ -157,7 +177,7 @@ static void *get_delta(struct object_entry *entry) delta_buf = diff_delta(base_buf, base_size, buf, size, &delta_size, 0); /* - * We succesfully computed this delta once but dropped it for + * We successfully computed this delta once but dropped it for * memory reasons. Something is very wrong if this time we * recompute and create a different delta. */ @@ -297,7 +317,8 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent if (!usable_delta) { if (oe_type(entry) == OBJ_BLOB && oe_size_greater_than(&to_pack, entry, big_file_threshold) && - (st = open_istream(&entry->idx.oid, &type, &size, NULL)) != NULL) + (st = open_istream(the_repository, &entry->idx.oid, &type, + &size, NULL)) != NULL) buf = NULL; else { buf = read_object_file(&entry->idx.oid, &type, &size); @@ -604,12 +625,12 @@ static int mark_tagged(const char *path, const struct object_id *oid, int flag, void *cb_data) { struct object_id peeled; - struct object_entry *entry = packlist_find(&to_pack, oid->hash, NULL); + struct object_entry *entry = packlist_find(&to_pack, oid); if (entry) entry->tagged = 1; if (!peel_ref(path, &peeled)) { - entry = packlist_find(&to_pack, peeled.hash, NULL); + entry = packlist_find(&to_pack, &peeled); if (entry) entry->tagged = 1; } @@ -778,57 +799,205 @@ static struct object_entry **compute_write_order(void) return wo; } -static off_t write_reused_pack(struct hashfile *f) + +/* + * A reused set of objects. All objects in a chunk have the same + * relative position in the original packfile and the generated + * packfile. + */ + +static struct reused_chunk { + /* The offset of the first object of this chunk in the original + * packfile. */ + off_t original; + /* The offset of the first object of this chunk in the generated + * packfile minus "original". */ + off_t difference; +} *reused_chunks; +static int reused_chunks_nr; +static int reused_chunks_alloc; + +static void record_reused_object(off_t where, off_t offset) { - unsigned char buffer[8192]; - off_t to_write, total; - int fd; + if (reused_chunks_nr && reused_chunks[reused_chunks_nr-1].difference == offset) + return; - if (!is_pack_valid(reuse_packfile)) - die(_("packfile is invalid: %s"), reuse_packfile->pack_name); + ALLOC_GROW(reused_chunks, reused_chunks_nr + 1, + reused_chunks_alloc); + reused_chunks[reused_chunks_nr].original = where; + reused_chunks[reused_chunks_nr].difference = offset; + reused_chunks_nr++; +} - fd = git_open(reuse_packfile->pack_name); - if (fd < 0) - die_errno(_("unable to open packfile for reuse: %s"), - reuse_packfile->pack_name); +/* + * Binary search to find the chunk that "where" is in. Note + * that we're not looking for an exact match, just the first + * chunk that contains it (which implicitly ends at the start + * of the next chunk. + */ +static off_t find_reused_offset(off_t where) +{ + int lo = 0, hi = reused_chunks_nr; + while (lo < hi) { + int mi = lo + ((hi - lo) / 2); + if (where == reused_chunks[mi].original) + return reused_chunks[mi].difference; + if (where < reused_chunks[mi].original) + hi = mi; + else + lo = mi + 1; + } - if (lseek(fd, sizeof(struct pack_header), SEEK_SET) == -1) - die_errno(_("unable to seek in reused packfile")); + /* + * The first chunk starts at zero, so we can't have gone below + * there. + */ + assert(lo); + return reused_chunks[lo-1].difference; +} - if (reuse_packfile_offset < 0) - reuse_packfile_offset = reuse_packfile->pack_size - the_hash_algo->rawsz; +static void write_reused_pack_one(size_t pos, struct hashfile *out, + struct pack_window **w_curs) +{ + off_t offset, next, cur; + enum object_type type; + unsigned long size; - total = to_write = reuse_packfile_offset - sizeof(struct pack_header); + offset = reuse_packfile->revindex[pos].offset; + next = reuse_packfile->revindex[pos + 1].offset; - while (to_write) { - int read_pack = xread(fd, buffer, sizeof(buffer)); + record_reused_object(offset, offset - hashfile_total(out)); - if (read_pack <= 0) - die_errno(_("unable to read from reused packfile")); + cur = offset; + type = unpack_object_header(reuse_packfile, w_curs, &cur, &size); + assert(type >= 0); - if (read_pack > to_write) - read_pack = to_write; + if (type == OBJ_OFS_DELTA) { + off_t base_offset; + off_t fixup; - hashwrite(f, buffer, read_pack); - to_write -= read_pack; + unsigned char header[MAX_PACK_OBJECT_HEADER]; + unsigned len; + + base_offset = get_delta_base(reuse_packfile, w_curs, &cur, type, offset); + assert(base_offset != 0); + + /* Convert to REF_DELTA if we must... */ + if (!allow_ofs_delta) { + int base_pos = find_revindex_position(reuse_packfile, base_offset); + struct object_id base_oid; + + nth_packed_object_id(&base_oid, reuse_packfile, + reuse_packfile->revindex[base_pos].nr); + + len = encode_in_pack_object_header(header, sizeof(header), + OBJ_REF_DELTA, size); + hashwrite(out, header, len); + hashwrite(out, base_oid.hash, the_hash_algo->rawsz); + copy_pack_data(out, reuse_packfile, w_curs, cur, next - cur); + return; + } + + /* Otherwise see if we need to rewrite the offset... */ + fixup = find_reused_offset(offset) - + find_reused_offset(base_offset); + if (fixup) { + unsigned char ofs_header[10]; + unsigned i, ofs_len; + off_t ofs = offset - base_offset - fixup; + + len = encode_in_pack_object_header(header, sizeof(header), + OBJ_OFS_DELTA, size); + + i = sizeof(ofs_header) - 1; + ofs_header[i] = ofs & 127; + while (ofs >>= 7) + ofs_header[--i] = 128 | (--ofs & 127); + + ofs_len = sizeof(ofs_header) - i; + + hashwrite(out, header, len); + hashwrite(out, ofs_header + sizeof(ofs_header) - ofs_len, ofs_len); + copy_pack_data(out, reuse_packfile, w_curs, cur, next - cur); + return; + } + + /* ...otherwise we have no fixup, and can write it verbatim */ + } + + copy_pack_data(out, reuse_packfile, w_curs, offset, next - offset); +} + +static size_t write_reused_pack_verbatim(struct hashfile *out, + struct pack_window **w_curs) +{ + size_t pos = 0; + + while (pos < reuse_packfile_bitmap->word_alloc && + reuse_packfile_bitmap->words[pos] == (eword_t)~0) + pos++; + + if (pos) { + off_t to_write; + + written = (pos * BITS_IN_EWORD); + to_write = reuse_packfile->revindex[written].offset + - sizeof(struct pack_header); + + /* We're recording one chunk, not one object. */ + record_reused_object(sizeof(struct pack_header), 0); + hashflush(out); + copy_pack_data(out, reuse_packfile, w_curs, + sizeof(struct pack_header), to_write); - /* - * We don't know the actual number of objects written, - * only how many bytes written, how many bytes total, and - * how many objects total. So we can fake it by pretending all - * objects we are writing are the same size. This gives us a - * smooth progress meter, and at the end it matches the true - * answer. - */ - written = reuse_packfile_objects * - (((double)(total - to_write)) / total); display_progress(progress_state, written); } + return pos; +} - close(fd); - written = reuse_packfile_objects; - display_progress(progress_state, written); - return reuse_packfile_offset - sizeof(struct pack_header); +static void write_reused_pack(struct hashfile *f) +{ + size_t i = 0; + uint32_t offset; + struct pack_window *w_curs = NULL; + + if (allow_ofs_delta) + i = write_reused_pack_verbatim(f, &w_curs); + + for (; i < reuse_packfile_bitmap->word_alloc; ++i) { + eword_t word = reuse_packfile_bitmap->words[i]; + size_t pos = (i * BITS_IN_EWORD); + + for (offset = 0; offset < BITS_IN_EWORD; ++offset) { + if ((word >> offset) == 0) + break; + + offset += ewah_bit_ctz64(word >> offset); + write_reused_pack_one(pos + offset, f, &w_curs); + display_progress(progress_state, ++written); + } + } + + unuse_pack(&w_curs); +} + +static void write_excluded_by_configs(void) +{ + struct oidset_iter iter; + const struct object_id *oid; + + oidset_iter_init(&excluded_by_config, &iter); + while ((oid = oidset_iter_next(&iter))) { + struct configured_exclusion *ex = + oidmap_get(&configured_exclusions, oid); + + if (!ex) + BUG("configured exclusion wasn't configured"); + write_in_full(1, ex->pack_hash_hex, strlen(ex->pack_hash_hex)); + write_in_full(1, " ", 1); + write_in_full(1, ex->uri, strlen(ex->uri)); + write_in_full(1, "\n", 1); + } } static const char no_split_warning[] = N_( @@ -861,11 +1030,9 @@ static void write_pack_file(void) offset = write_pack_header(f, nr_remaining); if (reuse_packfile) { - off_t packfile_size; assert(pack_to_stdout); - - packfile_size = write_reused_pack(f); - offset += packfile_size; + write_reused_pack(f); + offset = hashfile_total(f); } nr_written = 0; @@ -890,7 +1057,8 @@ static void write_pack_file(void) nr_written, oid.hash, offset); close(fd); if (write_bitmap_index) { - warning(_(no_split_warning)); + if (write_bitmap_index != WRITE_BITMAP_QUIET) + warning(_(no_split_warning)); write_bitmap_index = 0; } } @@ -936,7 +1104,6 @@ static void write_pack_file(void) stop_progress(&progress_state); bitmap_writer_show_progress(progress); - bitmap_writer_reuse_bitmaps(&to_pack); bitmap_writer_select_commits(indexed_commits, indexed_commits_nr, -1); bitmap_writer_build(&to_pack); bitmap_writer_finish(written_list, nr_written, @@ -962,6 +1129,8 @@ static void write_pack_file(void) if (written != nr_result) die(_("wrote %"PRIu32" objects while expecting %"PRIu32), written, nr_result); + trace2_data_intmax("pack-objects", the_repository, + "write_pack_file/wrote", nr_result); } static int no_try_delta(const char *path) @@ -970,7 +1139,7 @@ static int no_try_delta(const char *path) if (!check) check = attr_check_initl("delta", NULL); - git_check_attr(&the_index, path, check); + git_check_attr(the_repository->index, path, check); if (ATTR_FALSE(check->items[0].value)) return 1; return 0; @@ -987,12 +1156,15 @@ static int no_try_delta(const char *path) * few lines later when we want to add the new entry. */ static int have_duplicate_entry(const struct object_id *oid, - int exclude, - uint32_t *index_pos) + int exclude) { struct object_entry *entry; - entry = packlist_find(&to_pack, oid->hash, index_pos); + if (reuse_packfile_bitmap && + bitmap_walk_contains(bitmap_git, reuse_packfile_bitmap, oid)) + return 1; + + entry = packlist_find(&to_pack, oid); if (!entry) return 0; @@ -1076,7 +1248,7 @@ static int want_object_in_pack(const struct object_id *oid, for (m = get_multi_pack_index(the_repository); m; m = m->next) { struct pack_entry e; - if (fill_midx_entry(oid, &e, m)) { + if (fill_midx_entry(the_repository, oid, &e, m)) { struct packed_git *p = e.p; off_t offset; @@ -1124,6 +1296,25 @@ static int want_object_in_pack(const struct object_id *oid, } } + if (uri_protocols.nr) { + struct configured_exclusion *ex = + oidmap_get(&configured_exclusions, oid); + int i; + const char *p; + + if (ex) { + for (i = 0; i < uri_protocols.nr; i++) { + if (skip_prefix(ex->uri, + uri_protocols.items[i].string, + &p) && + *p == ':') { + oidset_insert(&excluded_by_config, oid); + return 0; + } + } + } + } + return 1; } @@ -1132,13 +1323,12 @@ static void create_object_entry(const struct object_id *oid, uint32_t hash, int exclude, int no_try_delta, - uint32_t index_pos, struct packed_git *found_pack, off_t found_offset) { struct object_entry *entry; - entry = packlist_alloc(&to_pack, oid->hash, index_pos); + entry = packlist_alloc(&to_pack, oid); entry->hash = hash; oe_set_type(entry, type); if (exclude) @@ -1162,17 +1352,17 @@ static int add_object_entry(const struct object_id *oid, enum object_type type, { struct packed_git *found_pack = NULL; off_t found_offset = 0; - uint32_t index_pos; display_progress(progress_state, ++nr_seen); - if (have_duplicate_entry(oid, exclude, &index_pos)) + if (have_duplicate_entry(oid, exclude)) return 0; if (!want_object_in_pack(oid, exclude, &found_pack, &found_offset)) { /* The pack is missing an object, so it will not have closure */ if (write_bitmap_index) { - warning(_(no_closure_warning)); + if (write_bitmap_index != WRITE_BITMAP_QUIET) + warning(_(no_closure_warning)); write_bitmap_index = 0; } return 0; @@ -1180,7 +1370,7 @@ static int add_object_entry(const struct object_id *oid, enum object_type type, create_object_entry(oid, type, pack_name_hash(name), exclude, name && no_try_delta(name), - index_pos, found_pack, found_offset); + found_pack, found_offset); return 1; } @@ -1189,17 +1379,15 @@ static int add_object_entry_from_bitmap(const struct object_id *oid, int flags, uint32_t name_hash, struct packed_git *pack, off_t offset) { - uint32_t index_pos; - display_progress(progress_state, ++nr_seen); - if (have_duplicate_entry(oid, 0, &index_pos)) + if (have_duplicate_entry(oid, 0)) return 0; if (!want_object_in_pack(oid, 0, &pack, &offset)) return 0; - create_object_entry(oid, type, name_hash, 0, 0, index_pos, pack, offset); + create_object_entry(oid, type, name_hash, 0, 0, pack, offset); return 1; } @@ -1334,7 +1522,7 @@ static void add_pbase_object(struct tree_desc *tree, if (cmp < 0) return; if (name[cmplen] != '/') { - add_object_entry(entry.oid, + add_object_entry(&entry.oid, object_type(entry.mode), fullname, 1); return; @@ -1345,7 +1533,7 @@ static void add_pbase_object(struct tree_desc *tree, const char *down = name+cmplen+1; int downlen = name_cmp_len(down); - tree = pbase_tree_get(entry.oid); + tree = pbase_tree_get(&entry.oid); if (!tree) return; init_tree_desc(&sub, tree->tree_data, tree->tree_size); @@ -1424,7 +1612,8 @@ static void add_preferred_base(struct object_id *oid) if (window <= num_preferred_base++) return; - data = read_object_with_reference(oid, tree_type, &size, &tree_oid); + data = read_object_with_reference(the_repository, oid, + tree_type, &size, &tree_oid); if (!data) return; @@ -1480,20 +1669,17 @@ static void cleanup_preferred_base(void) * deltify other objects against, in order to avoid * circular deltas. */ -static int can_reuse_delta(const unsigned char *base_sha1, +static int can_reuse_delta(const struct object_id *base_oid, struct object_entry *delta, struct object_entry **base_out) { struct object_entry *base; - if (!base_sha1) - return 0; - /* * First see if we're already sending the base (or it's explicitly in * our "excluded" list). */ - base = packlist_find(&to_pack, base_sha1, NULL); + base = packlist_find(&to_pack, base_oid); if (base) { if (!in_same_island(&delta->idx.oid, &base->idx.oid)) return 0; @@ -1506,11 +1692,9 @@ static int can_reuse_delta(const unsigned char *base_sha1, * even if it was buried too deep in history to make it into the * packing list. */ - if (thin && bitmap_has_sha1_in_uninteresting(bitmap_git, base_sha1)) { + if (thin && bitmap_has_oid_in_uninteresting(bitmap_git, base_oid)) { if (use_delta_islands) { - struct object_id base_oid; - hashcpy(base_oid.hash, base_sha1); - if (!in_same_island(&delta->idx.oid, &base_oid)) + if (!in_same_island(&delta->idx.oid, base_oid)) return 0; } *base_out = NULL; @@ -1520,14 +1704,36 @@ static int can_reuse_delta(const unsigned char *base_sha1, return 0; } -static void check_object(struct object_entry *entry) +static void prefetch_to_pack(uint32_t object_index_start) { + struct oid_array to_fetch = OID_ARRAY_INIT; + uint32_t i; + + for (i = object_index_start; i < to_pack.nr_objects; i++) { + struct object_entry *entry = to_pack.objects + i; + + if (!oid_object_info_extended(the_repository, + &entry->idx.oid, + NULL, + OBJECT_INFO_FOR_PREFETCH)) + continue; + oid_array_append(&to_fetch, &entry->idx.oid); + } + promisor_remote_get_direct(the_repository, + to_fetch.oid, to_fetch.nr); + oid_array_clear(&to_fetch); +} + +static void check_object(struct object_entry *entry, uint32_t object_index) { unsigned long canonical_size; + enum object_type type; + struct object_info oi = {.typep = &type, .sizep = &canonical_size}; if (IN_PACK(entry)) { struct packed_git *p = IN_PACK(entry); struct pack_window *w_curs = NULL; - const unsigned char *base_ref = NULL; + int have_base = 0; + struct object_id base_ref; struct object_entry *base_entry; unsigned long used, used_0; unsigned long avail; @@ -1568,9 +1774,13 @@ static void check_object(struct object_entry *entry) unuse_pack(&w_curs); return; case OBJ_REF_DELTA: - if (reuse_delta && !entry->preferred_base) - base_ref = use_pack(p, &w_curs, - entry->in_pack_offset + used, NULL); + if (reuse_delta && !entry->preferred_base) { + oidread(&base_ref, + use_pack(p, &w_curs, + entry->in_pack_offset + used, + NULL)); + have_base = 1; + } entry->in_pack_header_size = used + the_hash_algo->rawsz; break; case OBJ_OFS_DELTA: @@ -1600,13 +1810,15 @@ static void check_object(struct object_entry *entry) revidx = find_pack_revindex(p, ofs); if (!revidx) goto give_up; - base_ref = nth_packed_object_sha1(p, revidx->nr); + if (!nth_packed_object_id(&base_ref, p, revidx->nr)) + have_base = 1; } entry->in_pack_header_size = used + used_0; break; } - if (can_reuse_delta(base_ref, entry, &base_entry)) { + if (have_base && + can_reuse_delta(&base_ref, entry, &base_entry)) { oe_set_type(entry, entry->in_pack_type); SET_SIZE(entry, in_pack_size); /* delta size */ SET_DELTA_SIZE(entry, in_pack_size); @@ -1616,7 +1828,7 @@ static void check_object(struct object_entry *entry) entry->delta_sibling_idx = base_entry->delta_child_idx; SET_DELTA_CHILD(base_entry, entry); } else { - SET_DELTA_EXT(entry, base_ref); + SET_DELTA_EXT(entry, &base_ref); } unuse_pack(&w_curs); @@ -1642,15 +1854,25 @@ static void check_object(struct object_entry *entry) /* * No choice but to fall back to the recursive delta walk - * with sha1_object_info() to find about the object type + * with oid_object_info() to find about the object type * at this point... */ give_up: unuse_pack(&w_curs); } - oe_set_type(entry, - oid_object_info(the_repository, &entry->idx.oid, &canonical_size)); + if (oid_object_info_extended(the_repository, &entry->idx.oid, &oi, + OBJECT_INFO_SKIP_FETCH_OBJECT | OBJECT_INFO_LOOKUP_REPLACE) < 0) { + if (has_promisor_remote()) { + prefetch_to_pack(object_index); + if (oid_object_info_extended(the_repository, &entry->idx.oid, &oi, + OBJECT_INFO_SKIP_FETCH_OBJECT | OBJECT_INFO_LOOKUP_REPLACE) < 0) + type = -1; + } else { + type = -1; + } + } + oe_set_type(entry, type); if (entry->type_valid) { SET_SIZE(entry, canonical_size); } else { @@ -1718,7 +1940,7 @@ static void drop_reused_delta(struct object_entry *entry) if (packed_object_info(the_repository, IN_PACK(entry), entry->in_pack_offset, &oi) < 0) { /* * We failed to get the info from this pack for some reason; - * fall back to sha1_object_info, which may find another copy. + * fall back to oid_object_info, which may find another copy. * And if that fails, the error will be recorded in oe_type(entry) * and dealt with in prepare_pack(). */ @@ -1870,7 +2092,7 @@ static void get_object_details(void) for (i = 0; i < to_pack.nr_objects; i++) { struct object_entry *entry = sorted_by_offset[i]; - check_object(entry); + check_object(entry, i); if (entry->type_valid && oe_size_greater_than(&to_pack, entry, big_file_threshold)) entry->no_try_delta = 1; @@ -1901,10 +2123,10 @@ static int type_size_sort(const void *_a, const void *_b) { const struct object_entry *a = *(struct object_entry **)_a; const struct object_entry *b = *(struct object_entry **)_b; - enum object_type a_type = oe_type(a); - enum object_type b_type = oe_type(b); - unsigned long a_size = SIZE(a); - unsigned long b_size = SIZE(b); + const enum object_type a_type = oe_type(a); + const enum object_type b_type = oe_type(b); + const unsigned long a_size = SIZE(a); + const unsigned long b_size = SIZE(b); if (a_type > b_type) return -1; @@ -1919,7 +2141,7 @@ static int type_size_sort(const void *_a, const void *_b) if (a->preferred_base < b->preferred_base) return 1; if (use_delta_islands) { - int island_cmp = island_delta_cmp(&a->idx.oid, &b->idx.oid); + const int island_cmp = island_delta_cmp(&a->idx.oid, &b->idx.oid); if (island_cmp) return island_cmp; } @@ -1953,11 +2175,6 @@ static int delta_cacheable(unsigned long src_size, unsigned long trg_size, return 0; } -/* Protect access to object database */ -static pthread_mutex_t read_mutex; -#define read_lock() pthread_mutex_lock(&read_mutex) -#define read_unlock() pthread_mutex_unlock(&read_mutex) - /* Protect delta_cache_size */ static pthread_mutex_t cache_mutex; #define cache_lock() pthread_mutex_lock(&cache_mutex) @@ -1993,11 +2210,11 @@ unsigned long oe_get_size_slow(struct packing_data *pack, unsigned long used, avail, size; if (e->type_ != OBJ_OFS_DELTA && e->type_ != OBJ_REF_DELTA) { - read_lock(); + packing_data_lock(&to_pack); if (oid_object_info(the_repository, &e->idx.oid, &size) < 0) die(_("unable to get size of %s"), oid_to_hex(&e->idx.oid)); - read_unlock(); + packing_data_unlock(&to_pack); return size; } @@ -2005,7 +2222,7 @@ unsigned long oe_get_size_slow(struct packing_data *pack, if (!p) BUG("when e->type is a delta, it must belong to a pack"); - read_lock(); + packing_data_lock(&to_pack); w_curs = NULL; buf = use_pack(p, &w_curs, e->in_pack_offset, &avail); used = unpack_object_header_buffer(buf, avail, &type, &size); @@ -2014,7 +2231,7 @@ unsigned long oe_get_size_slow(struct packing_data *pack, oid_to_hex(&e->idx.oid)); unuse_pack(&w_curs); - read_unlock(); + packing_data_unlock(&to_pack); return size; } @@ -2076,9 +2293,9 @@ static int try_delta(struct unpacked *trg, struct unpacked *src, /* Load data if not already done */ if (!trg->data) { - read_lock(); + packing_data_lock(&to_pack); trg->data = read_object_file(&trg_entry->idx.oid, &type, &sz); - read_unlock(); + packing_data_unlock(&to_pack); if (!trg->data) die(_("object %s cannot be read"), oid_to_hex(&trg_entry->idx.oid)); @@ -2089,9 +2306,9 @@ static int try_delta(struct unpacked *trg, struct unpacked *src, *mem_usage += sz; } if (!src->data) { - read_lock(); + packing_data_lock(&to_pack); src->data = read_object_file(&src_entry->idx.oid, &type, &sz); - read_unlock(); + packing_data_unlock(&to_pack); if (!src->data) { if (src_entry->preferred_base) { static int warned = 0; @@ -2171,7 +2388,7 @@ static unsigned int check_delta_limit(struct object_entry *me, unsigned int n) struct object_entry *child = DELTA_CHILD(me); unsigned int m = n; while (child) { - unsigned int c = check_delta_limit(child, n + 1); + const unsigned int c = check_delta_limit(child, n + 1); if (m < c) m = c; child = DELTA_SIBLING(child); @@ -2226,7 +2443,7 @@ static void find_deltas(struct object_entry **list, unsigned *list_size, while (window_memory_limit && mem_usage > window_memory_limit && count > 1) { - uint32_t tail = (idx + window - count) % window; + const uint32_t tail = (idx + window - count) % window; mem_usage -= free_unpacked(array + tail); count--; } @@ -2335,15 +2552,6 @@ static void find_deltas(struct object_entry **list, unsigned *list_size, free(array); } -static void try_to_free_from_threads(size_t size) -{ - read_lock(); - release_pack_memory(size); - read_unlock(); -} - -static try_to_free_t old_try_to_free_routine; - /* * The main object list is split into smaller lists, each is handed to * one worker. @@ -2381,18 +2589,14 @@ static pthread_cond_t progress_cond; */ static void init_threaded_search(void) { - init_recursive_mutex(&read_mutex); pthread_mutex_init(&cache_mutex, NULL); pthread_mutex_init(&progress_mutex, NULL); pthread_cond_init(&progress_cond, NULL); - old_try_to_free_routine = set_try_to_free_routine(try_to_free_from_threads); } static void cleanup_threaded_search(void) { - set_try_to_free_routine(old_try_to_free_routine); pthread_cond_destroy(&progress_cond); - pthread_mutex_destroy(&read_mutex); pthread_mutex_destroy(&cache_mutex); pthread_mutex_destroy(&progress_mutex); } @@ -2563,6 +2767,13 @@ static void ll_find_deltas(struct object_entry **list, unsigned list_size, free(p); } +static int obj_is_packed(const struct object_id *oid) +{ + return packlist_find(&to_pack, oid) || + (reuse_packfile_bitmap && + bitmap_walk_contains(bitmap_git, reuse_packfile_bitmap, oid)); +} + static void add_tag_chain(const struct object_id *oid) { struct tag *tag; @@ -2574,7 +2785,7 @@ static void add_tag_chain(const struct object_id *oid) * it was included via bitmaps, we would not have parsed it * previously). */ - if (packlist_find(&to_pack, oid->hash, NULL)) + if (obj_is_packed(oid)) return; tag = lookup_tag(the_repository, oid); @@ -2598,7 +2809,7 @@ static int add_ref_tag(const char *path, const struct object_id *oid, int flag, if (starts_with(path, "refs/tags/") && /* is a tag? */ !peel_ref(path, &peeled) && /* peelable? */ - packlist_find(&to_pack, peeled.hash, NULL)) /* object packed? */ + obj_is_packed(&peeled)) /* object packed? */ add_tag_chain(oid); return 0; } @@ -2610,7 +2821,7 @@ static void prepare_pack(int window, int depth) unsigned n; if (use_delta_islands) - resolve_tree_islands(progress, &to_pack); + resolve_tree_islands(the_repository, progress, &to_pack); get_object_details(); @@ -2666,6 +2877,7 @@ static void prepare_pack(int window, int depth) if (nr_deltas && n > 1) { unsigned nr_done = 0; + if (progress) progress_state = start_progress(_("Compressing objects"), nr_deltas); @@ -2710,6 +2922,10 @@ static int git_pack_config(const char *k, const char *v, void *cb) use_bitmap_index_default = git_config_bool(k, v); return 0; } + if (!strcmp(k, "pack.allowpackreuse")) { + allow_pack_reuse = git_config_bool(k, v); + return 0; + } if (!strcmp(k, "pack.threads")) { delta_search_threads = git_config_int(k, v); if (delta_search_threads < 0) @@ -2728,6 +2944,29 @@ static int git_pack_config(const char *k, const char *v, void *cb) pack_idx_opts.version); return 0; } + if (!strcmp(k, "uploadpack.blobpackfileuri")) { + struct configured_exclusion *ex = xmalloc(sizeof(*ex)); + const char *oid_end, *pack_end; + /* + * Stores the pack hash. This is not a true object ID, but is + * of the same form. + */ + struct object_id pack_hash; + + if (parse_oid_hex(v, &ex->e.oid, &oid_end) || + *oid_end != ' ' || + parse_oid_hex(oid_end + 1, &pack_hash, &pack_end) || + *pack_end != ' ') + die(_("value of uploadpack.blobpackfileuri must be " + "of the form '<object-hash> <pack-hash> <uri>' (got '%s')"), v); + if (oidmap_get(&configured_exclusions, &ex->e.oid)) + die(_("object already configured in another " + "uploadpack.blobpackfileuri (got '%s')"), v); + ex->pack_hash_hex = xcalloc(1, pack_end - oid_end); + memcpy(ex->pack_hash_hex, oid_end + 1, pack_end - oid_end - 1); + ex->uri = xstrdup(pack_end + 1); + oidmap_put(&configured_exclusions, ex); + } return git_default_config(k, v, cb); } @@ -2794,7 +3033,7 @@ static void show_object(struct object *obj, const char *name, void *data) for (p = strchr(name, '/'); p; p = strchr(p + 1, '/')) depth++; - ent = packlist_find(&to_pack, obj->oid.hash, NULL); + ent = packlist_find(&to_pack, &obj->oid); if (ent && depth > oe_tree_depth(&to_pack, ent)) oe_set_tree_depth(&to_pack, ent, depth); } @@ -2808,7 +3047,7 @@ static void show_object__ma_allow_any(struct object *obj, const char *name, void * Quietly ignore ALL missing objects. This avoids problems with * staging them now and getting an odd error later. */ - if (!has_object_file(&obj->oid)) + if (!has_object(the_repository, &obj->oid, 0)) return; show_object(obj, name, data); @@ -2822,7 +3061,7 @@ static void show_object__ma_allow_promisor(struct object *obj, const char *name, * Quietly ignore EXPECTED missing objects. This avoids problems with * staging them now and getting an odd error later. */ - if (!has_object_file(&obj->oid) && is_promisor_object(&obj->oid)) + if (!has_object(the_repository, &obj->oid, 0) && is_promisor_object(&obj->oid)) return; show_object(obj, name, data); @@ -2898,7 +3137,7 @@ static int ofscmp(const void *a_, const void *b_) return oidcmp(&a->object->oid, &b->object->oid); } -static void add_objects_in_unpacked_packs(struct rev_info *revs) +static void add_objects_in_unpacked_packs(void) { struct packed_git *p; struct in_pack in_pack; @@ -2920,8 +3159,8 @@ static void add_objects_in_unpacked_packs(struct rev_info *revs) in_pack.alloc); for (i = 0; i < p->num_objects; i++) { - nth_packed_object_oid(&oid, p, i); - o = lookup_unknown_object(oid.hash); + nth_packed_object_id(&oid, p, i); + o = lookup_unknown_object(&oid); if (!(o->flags & OBJECT_ADDED)) mark_in_pack_object(o, p, &in_pack); o->flags |= OBJECT_ADDED; @@ -3010,7 +3249,7 @@ static int loosened_object_can_be_discarded(const struct object_id *oid, return 1; } -static void loosen_unused_packed_objects(struct rev_info *revs) +static void loosen_unused_packed_objects(void) { struct packed_git *p; uint32_t i; @@ -3024,8 +3263,8 @@ static void loosen_unused_packed_objects(struct rev_info *revs) die(_("cannot open pack index")); for (i = 0; i < p->num_objects; i++) { - nth_packed_object_oid(&oid, p, i); - if (!packlist_find(&to_pack, oid.hash, NULL) && + nth_packed_object_id(&oid, p, i); + if (!packlist_find(&to_pack, &oid) && !has_sha1_pack_kept_or_nonlocal(&oid) && !loosened_object_can_be_discarded(&oid, p->mtime)) if (force_object_loose(&oid, p->mtime)) @@ -3041,8 +3280,8 @@ static void loosen_unused_packed_objects(struct rev_info *revs) */ static int pack_options_allow_reuse(void) { - return pack_to_stdout && - allow_ofs_delta && + return allow_pack_reuse && + pack_to_stdout && !ignore_packed_keep_on_disk && !ignore_packed_keep_in_core && (!local || !have_non_local_packs) && @@ -3051,7 +3290,7 @@ static int pack_options_allow_reuse(void) static int get_object_list_from_bitmap(struct rev_info *revs) { - if (!(bitmap_git = prepare_bitmap_walk(revs))) + if (!(bitmap_git = prepare_bitmap_walk(revs, &filter_options))) return -1; if (pack_options_allow_reuse() && @@ -3059,13 +3298,14 @@ static int get_object_list_from_bitmap(struct rev_info *revs) bitmap_git, &reuse_packfile, &reuse_packfile_objects, - &reuse_packfile_offset)) { + &reuse_packfile_bitmap)) { assert(reuse_packfile_objects); nr_result += reuse_packfile_objects; display_progress(progress_state, nr_result); } - traverse_bitmap_commit_list(bitmap_git, &add_object_entry_from_bitmap); + traverse_bitmap_commit_list(bitmap_git, revs, + &add_object_entry_from_bitmap); return 0; } @@ -3084,14 +3324,16 @@ static void record_recent_commit(struct commit *commit, void *data) static void get_object_list(int ac, const char **av) { struct rev_info revs; + struct setup_revision_opt s_r_opt = { + .allow_exclude_promisor_objects = 1, + }; char line[1000]; int flags = 0; int save_warning; repo_init_revisions(the_repository, &revs, NULL); save_commit_buffer = 0; - revs.allow_exclude_promisor_objects_opt = 1; - setup_revisions(ac, av, &revs, NULL); + setup_revisions(ac, av, &revs, &s_r_opt); /* make sure shallows are read */ is_repository_shallow(the_repository); @@ -3114,7 +3356,7 @@ static void get_object_list(int ac, const char **av) if (starts_with(line, "--shallow ")) { struct object_id oid; if (get_oid_hex(line + 10, &oid)) - die("not an SHA-1 '%s'", line + 10); + die("not an object name '%s'", line + 10); register_shallow(the_repository, &oid); use_bitmap_index = 0; continue; @@ -3131,11 +3373,11 @@ static void get_object_list(int ac, const char **av) return; if (use_delta_islands) - load_delta_islands(); + load_delta_islands(the_repository, progress); if (prepare_revision_walk(&revs)) die(_("revision walk setup failed")); - mark_edges_uninteresting(&revs, show_edge); + mark_edges_uninteresting(&revs, show_edge, sparse); if (!fn_show_object) fn_show_object = show_object; @@ -3155,11 +3397,11 @@ static void get_object_list(int ac, const char **av) } if (keep_unreachable) - add_objects_in_unpacked_packs(&revs); + add_objects_in_unpacked_packs(); if (pack_loose_unreachable) add_unreachable_loose_objects(); if (unpack_unreachable) - loosen_unused_packed_objects(&revs); + loosen_unused_packed_objects(); oid_array_clear(&recent_objects); } @@ -3228,7 +3470,7 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) int use_internal_rev_list = 0; int shallow = 0; int all_progress_implied = 0; - struct argv_array rp = ARGV_ARRAY_INIT; + struct strvec rp = STRVEC_INIT; int rev_list_unpacked = 0, rev_list_all = 0, rev_list_reflog = 0; int rev_list_index = 0; struct string_list keep_pack_list = STRING_LIST_INIT_NODUP; @@ -3242,9 +3484,9 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) OPT_BOOL(0, "all-progress-implied", &all_progress_implied, N_("similar to --all-progress when progress meter is shown")), - { OPTION_CALLBACK, 0, "index-version", NULL, N_("<version>[,<offset>]"), + OPT_CALLBACK_F(0, "index-version", NULL, N_("<version>[,<offset>]"), N_("write the pack index file in the specified idx format version"), - PARSE_OPT_NONEG, option_parse_index_version }, + PARSE_OPT_NONEG, option_parse_index_version), OPT_MAGNITUDE(0, "max-pack-size", &pack_size_limit, N_("maximum size of each output pack file")), OPT_BOOL(0, "local", &local, @@ -3289,9 +3531,11 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) N_("keep unreachable objects")), OPT_BOOL(0, "pack-loose-unreachable", &pack_loose_unreachable, N_("pack loose unreachable objects")), - { OPTION_CALLBACK, 0, "unpack-unreachable", NULL, N_("time"), + OPT_CALLBACK_F(0, "unpack-unreachable", NULL, N_("time"), N_("unpack unreachable objects newer than <time>"), - PARSE_OPT_OPTARG, option_parse_unpack_unreachable }, + PARSE_OPT_OPTARG, option_parse_unpack_unreachable), + OPT_BOOL(0, "sparse", &sparse, + N_("use the sparse reachability algorithm")), OPT_BOOL(0, "thin", &thin, N_("create thin packs")), OPT_BOOL(0, "shallow", &shallow, @@ -3306,16 +3550,24 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) N_("do not hide commits by grafts"), 0), OPT_BOOL(0, "use-bitmap-index", &use_bitmap_index, N_("use a bitmap index if available to speed up counting objects")), - OPT_BOOL(0, "write-bitmap-index", &write_bitmap_index, - N_("write a bitmap index together with the pack index")), + OPT_SET_INT(0, "write-bitmap-index", &write_bitmap_index, + N_("write a bitmap index together with the pack index"), + WRITE_BITMAP_TRUE), + OPT_SET_INT_F(0, "write-bitmap-index-quiet", + &write_bitmap_index, + N_("write a bitmap index if possible"), + WRITE_BITMAP_QUIET, PARSE_OPT_HIDDEN), OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options), - { OPTION_CALLBACK, 0, "missing", NULL, N_("action"), + OPT_CALLBACK_F(0, "missing", NULL, N_("action"), N_("handling for missing objects"), PARSE_OPT_NONEG, - option_parse_missing_action }, + option_parse_missing_action), OPT_BOOL(0, "exclude-promisor-objects", &exclude_promisor_objects, N_("do not pack objects in promisor packfiles")), OPT_BOOL(0, "delta-islands", &use_delta_islands, N_("respect islands during delta compression")), + OPT_STRING_LIST(0, "uri-protocol", &uri_protocols, + N_("protocol"), + N_("exclude any configured uploadpack.blobpackfileuri with this protocol")), OPT_END(), }; @@ -3324,6 +3576,11 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) read_replace_refs = 0; + sparse = git_env_bool("GIT_TEST_PACK_SPARSE", -1); + prepare_repo_settings(the_repository); + if (sparse < 0) + sparse = the_repository->settings.pack_use_sparse; + reset_pack_idx_option(&pack_idx_opts); git_config(git_pack_config, NULL); @@ -3349,36 +3606,36 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) cache_max_small_delta_size = (1U << OE_Z_DELTA_BITS) - 1; } - argv_array_push(&rp, "pack-objects"); + strvec_push(&rp, "pack-objects"); if (thin) { use_internal_rev_list = 1; - argv_array_push(&rp, shallow + strvec_push(&rp, shallow ? "--objects-edge-aggressive" : "--objects-edge"); } else - argv_array_push(&rp, "--objects"); + strvec_push(&rp, "--objects"); if (rev_list_all) { use_internal_rev_list = 1; - argv_array_push(&rp, "--all"); + strvec_push(&rp, "--all"); } if (rev_list_reflog) { use_internal_rev_list = 1; - argv_array_push(&rp, "--reflog"); + strvec_push(&rp, "--reflog"); } if (rev_list_index) { use_internal_rev_list = 1; - argv_array_push(&rp, "--indexed-objects"); + strvec_push(&rp, "--indexed-objects"); } if (rev_list_unpacked) { use_internal_rev_list = 1; - argv_array_push(&rp, "--unpacked"); + strvec_push(&rp, "--unpacked"); } if (exclude_promisor_objects) { use_internal_rev_list = 1; fetch_if_missing = 0; - argv_array_push(&rp, "--exclude-promisor-objects"); + strvec_push(&rp, "--exclude-promisor-objects"); } if (unpack_unreachable || keep_unreachable || pack_loose_unreachable) use_internal_rev_list = 1; @@ -3415,7 +3672,6 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) if (filter_options.choice) { if (!pack_to_stdout) die(_("cannot use --filter without --stdout")); - use_bitmap_index = 0; } /* @@ -3441,7 +3697,7 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) write_bitmap_index = 0; if (use_delta_islands) - argv_array_push(&rp, "--topo-order"); + strvec_push(&rp, "--topo-order"); if (progress && all_progress_implied) progress = 2; @@ -3470,30 +3726,46 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) } } - prepare_packing_data(&to_pack); + trace2_region_enter("pack-objects", "enumerate-objects", + the_repository); + prepare_packing_data(the_repository, &to_pack); if (progress) progress_state = start_progress(_("Enumerating objects"), 0); if (!use_internal_rev_list) read_object_list_from_stdin(); else { - get_object_list(rp.argc, rp.argv); - argv_array_clear(&rp); + get_object_list(rp.nr, rp.v); + strvec_clear(&rp); } cleanup_preferred_base(); if (include_tag && nr_result) for_each_ref(add_ref_tag, NULL); stop_progress(&progress_state); + trace2_region_leave("pack-objects", "enumerate-objects", + the_repository); if (non_empty && !nr_result) return 0; - if (nr_result) + if (nr_result) { + trace2_region_enter("pack-objects", "prepare-pack", + the_repository); prepare_pack(window, depth); + trace2_region_leave("pack-objects", "prepare-pack", + the_repository); + } + + trace2_region_enter("pack-objects", "write-pack-file", the_repository); + write_excluded_by_configs(); write_pack_file(); + trace2_region_leave("pack-objects", "write-pack-file", the_repository); + if (progress) fprintf_ln(stderr, _("Total %"PRIu32" (delta %"PRIu32")," - " reused %"PRIu32" (delta %"PRIu32")"), - written, written_delta, reused, reused_delta); + " reused %"PRIu32" (delta %"PRIu32")," + " pack-reused %"PRIu32), + written, written_delta, reused, reused_delta, + reuse_packfile_objects); return 0; } |