diff options
Diffstat (limited to 'builtin/pack-objects.c')
| -rw-r--r-- | builtin/pack-objects.c | 2121 |
1 files changed, 1634 insertions, 487 deletions
diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 6565c800ac..545b8bddc8 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -1,5 +1,6 @@ #include "builtin.h" -#include "cache.h" +#include "alloc.h" +#include "hex.h" #include "repository.h" #include "config.h" #include "attr.h" @@ -24,13 +25,148 @@ #include "streaming.h" #include "thread-utils.h" #include "pack-bitmap.h" +#include "delta-islands.h" #include "reachable.h" -#include "sha1-array.h" -#include "argv-array.h" +#include "oid-array.h" +#include "strvec.h" #include "list.h" #include "packfile.h" #include "object-store.h" +#include "replace-object.h" #include "dir.h" +#include "midx.h" +#include "trace2.h" +#include "shallow.h" +#include "promisor-remote.h" +#include "pack-mtimes.h" + +/* + * Objects we are going to pack are collected in the `to_pack` structure. + * It contains an array (dynamically expanded) of the object data, and a map + * that can resolve SHA1s to their position in the array. + */ +static struct packing_data to_pack; + +static inline struct object_entry *oe_delta( + const struct packing_data *pack, + const struct object_entry *e) +{ + if (!e->delta_idx) + return NULL; + if (e->ext_base) + return &pack->ext_bases[e->delta_idx - 1]; + else + return &pack->objects[e->delta_idx - 1]; +} + +static inline unsigned long oe_delta_size(struct packing_data *pack, + const struct object_entry *e) +{ + if (e->delta_size_valid) + return e->delta_size_; + + /* + * pack->delta_size[] can't be NULL because oe_set_delta_size() + * must have been called when a new delta is saved with + * oe_set_delta(). + * If oe_delta() returns NULL (i.e. default state, which means + * delta_size_valid is also false), then the caller must never + * call oe_delta_size(). + */ + return pack->delta_size[e - pack->objects]; +} + +unsigned long oe_get_size_slow(struct packing_data *pack, + const struct object_entry *e); + +static inline unsigned long oe_size(struct packing_data *pack, + const struct object_entry *e) +{ + if (e->size_valid) + return e->size_; + + return oe_get_size_slow(pack, e); +} + +static inline void oe_set_delta(struct packing_data *pack, + struct object_entry *e, + struct object_entry *delta) +{ + if (delta) + e->delta_idx = (delta - pack->objects) + 1; + else + e->delta_idx = 0; +} + +static inline struct object_entry *oe_delta_sibling( + const struct packing_data *pack, + const struct object_entry *e) +{ + if (e->delta_sibling_idx) + return &pack->objects[e->delta_sibling_idx - 1]; + return NULL; +} + +static inline struct object_entry *oe_delta_child( + const struct packing_data *pack, + const struct object_entry *e) +{ + if (e->delta_child_idx) + return &pack->objects[e->delta_child_idx - 1]; + return NULL; +} + +static inline void oe_set_delta_child(struct packing_data *pack, + struct object_entry *e, + struct object_entry *delta) +{ + if (delta) + e->delta_child_idx = (delta - pack->objects) + 1; + else + e->delta_child_idx = 0; +} + +static inline void oe_set_delta_sibling(struct packing_data *pack, + struct object_entry *e, + struct object_entry *delta) +{ + if (delta) + e->delta_sibling_idx = (delta - pack->objects) + 1; + else + e->delta_sibling_idx = 0; +} + +static inline void oe_set_size(struct packing_data *pack, + struct object_entry *e, + unsigned long size) +{ + if (size < pack->oe_size_limit) { + e->size_ = size; + e->size_valid = 1; + } else { + e->size_valid = 0; + if (oe_get_size_slow(pack, e) != size) + BUG("'size' is supposed to be the object size!"); + } +} + +static inline void oe_set_delta_size(struct packing_data *pack, + struct object_entry *e, + unsigned long size) +{ + if (size < pack->oe_delta_size_limit) { + e->delta_size_ = size; + e->delta_size_valid = 1; + } else { + packing_data_lock(pack); + if (!pack->delta_size) + ALLOC_ARRAY(pack->delta_size, pack->nr_alloc); + packing_data_unlock(pack); + + pack->delta_size[e - pack->objects] = size; + e->delta_size_valid = 0; + } +} #define IN_PACK(obj) oe_in_pack(&to_pack, obj) #define SIZE(obj) oe_size(&to_pack, obj) @@ -40,31 +176,29 @@ #define DELTA_CHILD(obj) oe_delta_child(&to_pack, obj) #define DELTA_SIBLING(obj) oe_delta_sibling(&to_pack, obj) #define SET_DELTA(obj, val) oe_set_delta(&to_pack, obj, val) +#define SET_DELTA_EXT(obj, oid) oe_set_delta_ext(&to_pack, obj, oid) #define SET_DELTA_SIZE(obj, val) oe_set_delta_size(&to_pack, obj, val) #define SET_DELTA_CHILD(obj, val) oe_set_delta_child(&to_pack, obj, val) #define SET_DELTA_SIBLING(obj, val) oe_set_delta_sibling(&to_pack, obj, val) static const char *pack_usage[] = { - N_("git pack-objects --stdout [<options>...] [< <ref-list> | < <object-list>]"), - N_("git pack-objects [<options>...] <base-name> [< <ref-list> | < <object-list>]"), + N_("git pack-objects --stdout [<options>] [< <ref-list> | < <object-list>]"), + N_("git pack-objects [<options>] <base-name> [< <ref-list> | < <object-list>]"), NULL }; -/* - * Objects we are going to pack are collected in the `to_pack` structure. - * It contains an array (dynamically expanded) of the object data, and a map - * that can resolve SHA1s to their position in the array. - */ -static struct packing_data to_pack; - static struct pack_idx_entry **written_list; static uint32_t nr_result, nr_written, nr_seen; +static struct bitmap_index *bitmap_git; +static uint32_t write_layer; static int non_empty; static int reuse_delta = 1, reuse_object = 1; static int keep_unreachable, unpack_unreachable, include_tag; static timestamp_t unpack_unreachable_expiration; static int pack_loose_unreachable; +static int cruft; +static timestamp_t cruft_expiration; static int local; static int have_non_local_packs; static int incremental; @@ -79,27 +213,36 @@ static unsigned long pack_size_limit; static int depth = 50; static int delta_search_threads; static int pack_to_stdout; +static int sparse; +static int thin; static int num_preferred_base; static struct progress *progress_state; static struct packed_git *reuse_packfile; static uint32_t reuse_packfile_objects; -static off_t reuse_packfile_offset; +static struct bitmap *reuse_packfile_bitmap; static int use_bitmap_index_default = 1; static int use_bitmap_index = -1; -static int write_bitmap_index; -static uint16_t write_bitmap_options; +static int allow_pack_reuse = 1; +static enum { + WRITE_BITMAP_FALSE = 0, + WRITE_BITMAP_QUIET, + WRITE_BITMAP_TRUE, +} write_bitmap_index; +static uint16_t write_bitmap_options = BITMAP_OPT_HASH_CACHE; static int exclude_promisor_objects; +static int use_delta_islands; + static unsigned long delta_cache_size = 0; static unsigned long max_delta_cache_size = DEFAULT_DELTA_CACHE_SIZE; static unsigned long cache_max_small_delta_size = 1000; static unsigned long window_memory_limit = 0; -static struct list_objects_filter_options filter_options; +static struct string_list uri_protocols = STRING_LIST_INIT_NODUP; enum missing_action { MA_ERROR = 0, /* fail if any missing objects are encountered */ @@ -109,6 +252,15 @@ enum missing_action { static enum missing_action arg_missing_action; static show_object_fn fn_show_object; +struct configured_exclusion { + struct oidmap_entry e; + char *pack_hash_hex; + char *uri; +}; +static struct oidmap configured_exclusions; + +static struct oidset excluded_by_config; + /* * stats */ @@ -140,7 +292,7 @@ static void *get_delta(struct object_entry *entry) buf = read_object_file(&entry->idx.oid, &type, &size); if (!buf) - die("unable to read %s", oid_to_hex(&entry->idx.oid)); + die(_("unable to read %s"), oid_to_hex(&entry->idx.oid)); base_buf = read_object_file(&DELTA(entry)->idx.oid, &type, &base_size); if (!base_buf) @@ -148,8 +300,13 @@ static void *get_delta(struct object_entry *entry) oid_to_hex(&DELTA(entry)->idx.oid)); delta_buf = diff_delta(base_buf, base_size, buf, size, &delta_size, 0); + /* + * We successfully computed this delta once but dropped it for + * memory reasons. Something is very wrong if this time we + * recompute and create a different delta. + */ if (!delta_buf || delta_size != DELTA_SIZE(entry)) - die("delta size changed"); + BUG("delta size changed"); free(buf); free(base_buf); return delta_buf; @@ -268,6 +425,17 @@ static void copy_pack_data(struct hashfile *f, } } +static inline int oe_size_greater_than(struct packing_data *pack, + const struct object_entry *lhs, + unsigned long rhs) +{ + if (lhs->size_valid) + return lhs->size_ > rhs; + if (rhs < pack->oe_size_limit) /* rhs < 2^x <= lhs ? */ + return 1; + return oe_get_size_slow(pack, lhs) > rhs; +} + /* Return 0 if we will bust the pack-size limit */ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_entry *entry, unsigned long limit, int usable_delta) @@ -284,7 +452,8 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent if (!usable_delta) { if (oe_type(entry) == OBJ_BLOB && oe_size_greater_than(&to_pack, entry, big_file_threshold) && - (st = open_istream(&entry->idx.oid, &type, &size, NULL)) != NULL) + (st = open_istream(the_repository, &entry->idx.oid, &type, + &size, NULL)) != NULL) buf = NULL; else { buf = read_object_file(&entry->idx.oid, &type, &size); @@ -385,7 +554,7 @@ static off_t write_reuse_object(struct hashfile *f, struct object_entry *entry, { struct packed_git *p = IN_PACK(entry); struct pack_window *w_curs = NULL; - struct revindex_entry *revidx; + uint32_t pos; off_t offset; enum object_type type = oe_type(entry); off_t datalen; @@ -402,11 +571,16 @@ static off_t write_reuse_object(struct hashfile *f, struct object_entry *entry, type, entry_size); offset = entry->in_pack_offset; - revidx = find_pack_revindex(p, offset); - datalen = revidx[1].offset - offset; + if (offset_to_pack_pos(p, offset, &pos) < 0) + die(_("write_reuse_object: could not locate %s, expected at " + "offset %"PRIuMAX" in pack %s"), + oid_to_hex(&entry->idx.oid), (uintmax_t)offset, + p->pack_name); + datalen = pack_pos_to_offset(p, pos + 1) - offset; if (!pack_to_stdout && p->index_version > 1 && - check_pack_crc(p, &w_curs, offset, datalen, revidx->nr)) { - error("bad packed object CRC for %s", + check_pack_crc(p, &w_curs, offset, datalen, + pack_pos_to_index(p, pos))) { + error(_("bad packed object CRC for %s"), oid_to_hex(&entry->idx.oid)); unuse_pack(&w_curs); return write_no_reuse_object(f, entry, limit, usable_delta); @@ -417,7 +591,7 @@ static off_t write_reuse_object(struct hashfile *f, struct object_entry *entry, if (!pack_to_stdout && p->index_version == 1 && check_pack_inflate(p, &w_curs, offset, datalen, entry_size)) { - error("corrupt packed object for %s", + error(_("corrupt packed object for %s"), oid_to_hex(&entry->idx.oid)); unuse_pack(&w_curs); return write_no_reuse_object(f, entry, limit, usable_delta); @@ -548,7 +722,7 @@ static enum write_one_status write_one(struct hashfile *f, */ recursing = (e->idx.offset == 1); if (recursing) { - warning("recursive delta detected for object %s", + warning(_("recursive delta detected for object %s"), oid_to_hex(&e->idx.oid)); return WRITE_ONE_RECURSIVE; } else if (e->idx.offset || e->preferred_base) { @@ -582,32 +756,40 @@ static enum write_one_status write_one(struct hashfile *f, /* make sure off_t is sufficiently large not to wrap */ if (signed_add_overflows(*offset, size)) - die("pack too large for current definition of off_t"); + die(_("pack too large for current definition of off_t")); *offset += size; return WRITE_ONE_WRITTEN; } -static int mark_tagged(const char *path, const struct object_id *oid, int flag, - void *cb_data) +static int mark_tagged(const char *path UNUSED, const struct object_id *oid, + int flag UNUSED, void *cb_data UNUSED) { struct object_id peeled; - struct object_entry *entry = packlist_find(&to_pack, oid->hash, NULL); + struct object_entry *entry = packlist_find(&to_pack, oid); if (entry) entry->tagged = 1; - if (!peel_ref(path, &peeled)) { - entry = packlist_find(&to_pack, peeled.hash, NULL); + if (!peel_iterated_oid(oid, &peeled)) { + entry = packlist_find(&to_pack, &peeled); if (entry) entry->tagged = 1; } return 0; } +static inline unsigned char oe_layer(struct packing_data *pack, + struct object_entry *e) +{ + if (!pack->layer) + return 0; + return pack->layer[e - pack->objects]; +} + static inline void add_to_write_order(struct object_entry **wo, unsigned int *endp, struct object_entry *e) { - if (e->filled) + if (e->filled || oe_layer(&to_pack, e) != write_layer) return; wo[(*endp)++] = e; e->filled = 1; @@ -667,48 +849,15 @@ static void add_family_to_write_order(struct object_entry **wo, add_descendants_to_write_order(wo, endp, root); } -static struct object_entry **compute_write_order(void) +static void compute_layer_order(struct object_entry **wo, unsigned int *wo_end) { - unsigned int i, wo_end, last_untagged; - - struct object_entry **wo; + unsigned int i, last_untagged; struct object_entry *objects = to_pack.objects; for (i = 0; i < to_pack.nr_objects; i++) { - objects[i].tagged = 0; - objects[i].filled = 0; - SET_DELTA_CHILD(&objects[i], NULL); - SET_DELTA_SIBLING(&objects[i], NULL); - } - - /* - * Fully connect delta_child/delta_sibling network. - * Make sure delta_sibling is sorted in the original - * recency order. - */ - for (i = to_pack.nr_objects; i > 0;) { - struct object_entry *e = &objects[--i]; - if (!DELTA(e)) - continue; - /* Mark me as the first child */ - e->delta_sibling_idx = DELTA(e)->delta_child_idx; - SET_DELTA_CHILD(DELTA(e), e); - } - - /* - * Mark objects that are at the tip of tags. - */ - for_each_tag_ref(mark_tagged, NULL); - - /* - * Give the objects in the original recency order until - * we see a tagged tip. - */ - ALLOC_ARRAY(wo, to_pack.nr_objects); - for (i = wo_end = 0; i < to_pack.nr_objects; i++) { if (objects[i].tagged) break; - add_to_write_order(wo, &wo_end, &objects[i]); + add_to_write_order(wo, wo_end, &objects[i]); } last_untagged = i; @@ -717,7 +866,7 @@ static struct object_entry **compute_write_order(void) */ for (; i < to_pack.nr_objects; i++) { if (objects[i].tagged) - add_to_write_order(wo, &wo_end, &objects[i]); + add_to_write_order(wo, wo_end, &objects[i]); } /* @@ -727,7 +876,7 @@ static struct object_entry **compute_write_order(void) if (oe_type(&objects[i]) != OBJ_COMMIT && oe_type(&objects[i]) != OBJ_TAG) continue; - add_to_write_order(wo, &wo_end, &objects[i]); + add_to_write_order(wo, wo_end, &objects[i]); } /* @@ -736,74 +885,280 @@ static struct object_entry **compute_write_order(void) for (i = last_untagged; i < to_pack.nr_objects; i++) { if (oe_type(&objects[i]) != OBJ_TREE) continue; - add_to_write_order(wo, &wo_end, &objects[i]); + add_to_write_order(wo, wo_end, &objects[i]); } /* * Finally all the rest in really tight order */ for (i = last_untagged; i < to_pack.nr_objects; i++) { - if (!objects[i].filled) - add_family_to_write_order(wo, &wo_end, &objects[i]); + if (!objects[i].filled && oe_layer(&to_pack, &objects[i]) == write_layer) + add_family_to_write_order(wo, wo_end, &objects[i]); + } +} + +static struct object_entry **compute_write_order(void) +{ + uint32_t max_layers = 1; + unsigned int i, wo_end; + + struct object_entry **wo; + struct object_entry *objects = to_pack.objects; + + for (i = 0; i < to_pack.nr_objects; i++) { + objects[i].tagged = 0; + objects[i].filled = 0; + SET_DELTA_CHILD(&objects[i], NULL); + SET_DELTA_SIBLING(&objects[i], NULL); + } + + /* + * Fully connect delta_child/delta_sibling network. + * Make sure delta_sibling is sorted in the original + * recency order. + */ + for (i = to_pack.nr_objects; i > 0;) { + struct object_entry *e = &objects[--i]; + if (!DELTA(e)) + continue; + /* Mark me as the first child */ + e->delta_sibling_idx = DELTA(e)->delta_child_idx; + SET_DELTA_CHILD(DELTA(e), e); } + /* + * Mark objects that are at the tip of tags. + */ + for_each_tag_ref(mark_tagged, NULL); + + if (use_delta_islands) { + max_layers = compute_pack_layers(&to_pack); + free_island_marks(); + } + + ALLOC_ARRAY(wo, to_pack.nr_objects); + wo_end = 0; + + for (; write_layer < max_layers; ++write_layer) + compute_layer_order(wo, &wo_end); + if (wo_end != to_pack.nr_objects) - die("ordered %u objects, expected %"PRIu32, wo_end, to_pack.nr_objects); + die(_("ordered %u objects, expected %"PRIu32), + wo_end, to_pack.nr_objects); return wo; } -static off_t write_reused_pack(struct hashfile *f) + +/* + * A reused set of objects. All objects in a chunk have the same + * relative position in the original packfile and the generated + * packfile. + */ + +static struct reused_chunk { + /* The offset of the first object of this chunk in the original + * packfile. */ + off_t original; + /* The difference for "original" minus the offset of the first object of + * this chunk in the generated packfile. */ + off_t difference; +} *reused_chunks; +static int reused_chunks_nr; +static int reused_chunks_alloc; + +static void record_reused_object(off_t where, off_t offset) +{ + if (reused_chunks_nr && reused_chunks[reused_chunks_nr-1].difference == offset) + return; + + ALLOC_GROW(reused_chunks, reused_chunks_nr + 1, + reused_chunks_alloc); + reused_chunks[reused_chunks_nr].original = where; + reused_chunks[reused_chunks_nr].difference = offset; + reused_chunks_nr++; +} + +/* + * Binary search to find the chunk that "where" is in. Note + * that we're not looking for an exact match, just the first + * chunk that contains it (which implicitly ends at the start + * of the next chunk. + */ +static off_t find_reused_offset(off_t where) +{ + int lo = 0, hi = reused_chunks_nr; + while (lo < hi) { + int mi = lo + ((hi - lo) / 2); + if (where == reused_chunks[mi].original) + return reused_chunks[mi].difference; + if (where < reused_chunks[mi].original) + hi = mi; + else + lo = mi + 1; + } + + /* + * The first chunk starts at zero, so we can't have gone below + * there. + */ + assert(lo); + return reused_chunks[lo-1].difference; +} + +static void write_reused_pack_one(size_t pos, struct hashfile *out, + struct pack_window **w_curs) { - unsigned char buffer[8192]; - off_t to_write, total; - int fd; + off_t offset, next, cur; + enum object_type type; + unsigned long size; - if (!is_pack_valid(reuse_packfile)) - die("packfile is invalid: %s", reuse_packfile->pack_name); + offset = pack_pos_to_offset(reuse_packfile, pos); + next = pack_pos_to_offset(reuse_packfile, pos + 1); - fd = git_open(reuse_packfile->pack_name); - if (fd < 0) - die_errno("unable to open packfile for reuse: %s", - reuse_packfile->pack_name); + record_reused_object(offset, offset - hashfile_total(out)); - if (lseek(fd, sizeof(struct pack_header), SEEK_SET) == -1) - die_errno("unable to seek in reused packfile"); + cur = offset; + type = unpack_object_header(reuse_packfile, w_curs, &cur, &size); + assert(type >= 0); - if (reuse_packfile_offset < 0) - reuse_packfile_offset = reuse_packfile->pack_size - the_hash_algo->rawsz; + if (type == OBJ_OFS_DELTA) { + off_t base_offset; + off_t fixup; + + unsigned char header[MAX_PACK_OBJECT_HEADER]; + unsigned len; + + base_offset = get_delta_base(reuse_packfile, w_curs, &cur, type, offset); + assert(base_offset != 0); + + /* Convert to REF_DELTA if we must... */ + if (!allow_ofs_delta) { + uint32_t base_pos; + struct object_id base_oid; + + if (offset_to_pack_pos(reuse_packfile, base_offset, &base_pos) < 0) + die(_("expected object at offset %"PRIuMAX" " + "in pack %s"), + (uintmax_t)base_offset, + reuse_packfile->pack_name); + + nth_packed_object_id(&base_oid, reuse_packfile, + pack_pos_to_index(reuse_packfile, base_pos)); + + len = encode_in_pack_object_header(header, sizeof(header), + OBJ_REF_DELTA, size); + hashwrite(out, header, len); + hashwrite(out, base_oid.hash, the_hash_algo->rawsz); + copy_pack_data(out, reuse_packfile, w_curs, cur, next - cur); + return; + } - total = to_write = reuse_packfile_offset - sizeof(struct pack_header); + /* Otherwise see if we need to rewrite the offset... */ + fixup = find_reused_offset(offset) - + find_reused_offset(base_offset); + if (fixup) { + unsigned char ofs_header[10]; + unsigned i, ofs_len; + off_t ofs = offset - base_offset - fixup; - while (to_write) { - int read_pack = xread(fd, buffer, sizeof(buffer)); + len = encode_in_pack_object_header(header, sizeof(header), + OBJ_OFS_DELTA, size); - if (read_pack <= 0) - die_errno("unable to read from reused packfile"); + i = sizeof(ofs_header) - 1; + ofs_header[i] = ofs & 127; + while (ofs >>= 7) + ofs_header[--i] = 128 | (--ofs & 127); - if (read_pack > to_write) - read_pack = to_write; + ofs_len = sizeof(ofs_header) - i; - hashwrite(f, buffer, read_pack); - to_write -= read_pack; + hashwrite(out, header, len); + hashwrite(out, ofs_header + sizeof(ofs_header) - ofs_len, ofs_len); + copy_pack_data(out, reuse_packfile, w_curs, cur, next - cur); + return; + } + + /* ...otherwise we have no fixup, and can write it verbatim */ + } + + copy_pack_data(out, reuse_packfile, w_curs, offset, next - offset); +} + +static size_t write_reused_pack_verbatim(struct hashfile *out, + struct pack_window **w_curs) +{ + size_t pos = 0; + + while (pos < reuse_packfile_bitmap->word_alloc && + reuse_packfile_bitmap->words[pos] == (eword_t)~0) + pos++; + + if (pos) { + off_t to_write; + + written = (pos * BITS_IN_EWORD); + to_write = pack_pos_to_offset(reuse_packfile, written) + - sizeof(struct pack_header); + + /* We're recording one chunk, not one object. */ + record_reused_object(sizeof(struct pack_header), 0); + hashflush(out); + copy_pack_data(out, reuse_packfile, w_curs, + sizeof(struct pack_header), to_write); - /* - * We don't know the actual number of objects written, - * only how many bytes written, how many bytes total, and - * how many objects total. So we can fake it by pretending all - * objects we are writing are the same size. This gives us a - * smooth progress meter, and at the end it matches the true - * answer. - */ - written = reuse_packfile_objects * - (((double)(total - to_write)) / total); display_progress(progress_state, written); } + return pos; +} + +static void write_reused_pack(struct hashfile *f) +{ + size_t i = 0; + uint32_t offset; + struct pack_window *w_curs = NULL; + + if (allow_ofs_delta) + i = write_reused_pack_verbatim(f, &w_curs); + + for (; i < reuse_packfile_bitmap->word_alloc; ++i) { + eword_t word = reuse_packfile_bitmap->words[i]; + size_t pos = (i * BITS_IN_EWORD); + + for (offset = 0; offset < BITS_IN_EWORD; ++offset) { + if ((word >> offset) == 0) + break; + + offset += ewah_bit_ctz64(word >> offset); + /* + * Can use bit positions directly, even for MIDX + * bitmaps. See comment in try_partial_reuse() + * for why. + */ + write_reused_pack_one(pos + offset, f, &w_curs); + display_progress(progress_state, ++written); + } + } + + unuse_pack(&w_curs); +} - close(fd); - written = reuse_packfile_objects; - display_progress(progress_state, written); - return reuse_packfile_offset - sizeof(struct pack_header); +static void write_excluded_by_configs(void) +{ + struct oidset_iter iter; + const struct object_id *oid; + + oidset_iter_init(&excluded_by_config, &iter); + while ((oid = oidset_iter_next(&iter))) { + struct configured_exclusion *ex = + oidmap_get(&configured_exclusions, oid); + + if (!ex) + BUG("configured exclusion wasn't configured"); + write_in_full(1, ex->pack_hash_hex, strlen(ex->pack_hash_hex)); + write_in_full(1, " ", 1); + write_in_full(1, ex->uri, strlen(ex->uri)); + write_in_full(1, "\n", 1); + } } static const char no_split_warning[] = N_( @@ -825,7 +1180,7 @@ static void write_pack_file(void) write_order = compute_write_order(); do { - struct object_id oid; + unsigned char hash[GIT_MAX_RAWSZ]; char *pack_tmp_name = NULL; if (pack_to_stdout) @@ -836,11 +1191,9 @@ static void write_pack_file(void) offset = write_pack_header(f, nr_remaining); if (reuse_packfile) { - off_t packfile_size; assert(pack_to_stdout); - - packfile_size = write_reused_pack(f); - offset += packfile_size; + write_reused_pack(f); + offset = hashfile_total(f); } nr_written = 0; @@ -851,21 +1204,32 @@ static void write_pack_file(void) display_progress(progress_state, written); } - /* - * Did we write the wrong # entries in the header? - * If so, rewrite it like in fast-import - */ if (pack_to_stdout) { - finalize_hashfile(f, oid.hash, CSUM_HASH_IN_STREAM | CSUM_CLOSE); + /* + * We never fsync when writing to stdout since we may + * not be writing to an actual pack file. For instance, + * the upload-pack code passes a pipe here. Calling + * fsync on a pipe results in unnecessary + * synchronization with the reader on some platforms. + */ + finalize_hashfile(f, hash, FSYNC_COMPONENT_NONE, + CSUM_HASH_IN_STREAM | CSUM_CLOSE); } else if (nr_written == nr_remaining) { - finalize_hashfile(f, oid.hash, CSUM_HASH_IN_STREAM | CSUM_FSYNC | CSUM_CLOSE); + finalize_hashfile(f, hash, FSYNC_COMPONENT_PACK, + CSUM_HASH_IN_STREAM | CSUM_FSYNC | CSUM_CLOSE); } else { - int fd = finalize_hashfile(f, oid.hash, 0); - fixup_pack_header_footer(fd, oid.hash, pack_tmp_name, - nr_written, oid.hash, offset); + /* + * If we wrote the wrong number of entries in the + * header, rewrite it like in fast-import. + */ + + int fd = finalize_hashfile(f, hash, FSYNC_COMPONENT_PACK, 0); + fixup_pack_header_footer(fd, hash, pack_tmp_name, + nr_written, hash, offset); close(fd); if (write_bitmap_index) { - warning(_(no_split_warning)); + if (write_bitmap_index != WRITE_BITMAP_QUIET) + warning(_(no_split_warning)); write_bitmap_index = 0; } } @@ -873,6 +1237,7 @@ static void write_pack_file(void) if (!pack_to_stdout) { struct stat st; struct strbuf tmpname = STRBUF_INIT; + char *idx_tmp_name = NULL; /* * Packs are runtime accessed in their mtime @@ -882,7 +1247,7 @@ static void write_pack_file(void) * to preserve this property. */ if (stat(pack_tmp_name, &st) < 0) { - warning_errno("failed to stat %s", pack_tmp_name); + warning_errno(_("failed to stat %s"), pack_tmp_name); } else if (!last_mtime) { last_mtime = st.st_mtime; } else { @@ -890,38 +1255,48 @@ static void write_pack_file(void) utb.actime = st.st_atime; utb.modtime = --last_mtime; if (utime(pack_tmp_name, &utb) < 0) - warning_errno("failed utime() on %s", pack_tmp_name); + warning_errno(_("failed utime() on %s"), pack_tmp_name); } - strbuf_addf(&tmpname, "%s-", base_name); + strbuf_addf(&tmpname, "%s-%s.", base_name, + hash_to_hex(hash)); if (write_bitmap_index) { - bitmap_writer_set_checksum(oid.hash); + bitmap_writer_set_checksum(hash); bitmap_writer_build_type_index( &to_pack, written_list, nr_written); } - finish_tmp_packfile(&tmpname, pack_tmp_name, + if (cruft) + pack_idx_opts.flags |= WRITE_MTIMES; + + stage_tmp_packfiles(&tmpname, pack_tmp_name, written_list, nr_written, - &pack_idx_opts, oid.hash); + &to_pack, &pack_idx_opts, hash, + &idx_tmp_name); if (write_bitmap_index) { - strbuf_addf(&tmpname, "%s.bitmap", oid_to_hex(&oid)); + size_t tmpname_len = tmpname.len; + strbuf_addstr(&tmpname, "bitmap"); stop_progress(&progress_state); bitmap_writer_show_progress(progress); - bitmap_writer_reuse_bitmaps(&to_pack); bitmap_writer_select_commits(indexed_commits, indexed_commits_nr, -1); - bitmap_writer_build(&to_pack); + if (bitmap_writer_build(&to_pack) < 0) + die(_("failed to write bitmap index")); bitmap_writer_finish(written_list, nr_written, tmpname.buf, write_bitmap_options); write_bitmap_index = 0; + strbuf_setlen(&tmpname, tmpname_len); } + rename_tmp_packfile_idx(&tmpname, &idx_tmp_name); + + free(idx_tmp_name); strbuf_release(&tmpname); free(pack_tmp_name); - puts(oid_to_hex(&oid)); + puts(hash_to_hex(hash)); } /* mark written objects as written to previous pack */ @@ -935,8 +1310,10 @@ static void write_pack_file(void) free(write_order); stop_progress(&progress_state); if (written != nr_result) - die("wrote %"PRIu32" objects while expecting %"PRIu32, - written, nr_result); + die(_("wrote %"PRIu32" objects while expecting %"PRIu32), + written, nr_result); + trace2_data_intmax("pack-objects", the_repository, + "write_pack_file/wrote", nr_result); } static int no_try_delta(const char *path) @@ -945,8 +1322,7 @@ static int no_try_delta(const char *path) if (!check) check = attr_check_initl("delta", NULL); - if (git_check_attr(path, check)) - return 0; + git_check_attr(the_repository->index, NULL, path, check); if (ATTR_FALSE(check->items[0].value)) return 1; return 0; @@ -963,12 +1339,15 @@ static int no_try_delta(const char *path) * few lines later when we want to add the new entry. */ static int have_duplicate_entry(const struct object_id *oid, - int exclude, - uint32_t *index_pos) + int exclude) { struct object_entry *entry; - entry = packlist_find(&to_pack, oid->hash, index_pos); + if (reuse_packfile_bitmap && + bitmap_walk_contains(bitmap_git, reuse_packfile_bitmap, oid)) + return 1; + + entry = packlist_find(&to_pack, oid); if (!entry) return 0; @@ -981,13 +1360,17 @@ static int have_duplicate_entry(const struct object_id *oid, return 1; } -static int want_found_object(int exclude, struct packed_git *p) +static int want_found_object(const struct object_id *oid, int exclude, + struct packed_git *p) { if (exclude) return 1; if (incremental) return 0; + if (!is_pack_valid(p)) + return -1; + /* * When asked to do --local (do not include an object that appears in a * pack we borrow from elsewhere) or --honor-pack-keep (do not include @@ -997,27 +1380,82 @@ static int want_found_object(int exclude, struct packed_git *p) * make sure no copy of this object appears in _any_ pack that makes us * to omit the object, so we need to check all the packs. * - * We can however first check whether these options can possible matter; + * We can however first check whether these options can possibly matter; * if they do not matter we know we want the object in generated pack. * Otherwise, we signal "-1" at the end to tell the caller that we do * not know either way, and it needs to check more packs. */ - if (!ignore_packed_keep_on_disk && - !ignore_packed_keep_in_core && - (!local || !have_non_local_packs)) - return 1; + /* + * Objects in packs borrowed from elsewhere are discarded regardless of + * if they appear in other packs that weren't borrowed. + */ if (local && !p->pack_local) return 0; - if (p->pack_local && - ((ignore_packed_keep_on_disk && p->pack_keep) || - (ignore_packed_keep_in_core && p->pack_keep_in_core))) - return 0; + + /* + * Then handle .keep first, as we have a fast(er) path there. + */ + if (ignore_packed_keep_on_disk || ignore_packed_keep_in_core) { + /* + * Set the flags for the kept-pack cache to be the ones we want + * to ignore. + * + * That is, if we are ignoring objects in on-disk keep packs, + * then we want to search through the on-disk keep and ignore + * the in-core ones. + */ + unsigned flags = 0; + if (ignore_packed_keep_on_disk) + flags |= ON_DISK_KEEP_PACKS; + if (ignore_packed_keep_in_core) + flags |= IN_CORE_KEEP_PACKS; + + if (ignore_packed_keep_on_disk && p->pack_keep) + return 0; + if (ignore_packed_keep_in_core && p->pack_keep_in_core) + return 0; + if (has_object_kept_pack(oid, flags)) + return 0; + } + + /* + * At this point we know definitively that either we don't care about + * keep-packs, or the object is not in one. Keep checking other + * conditions... + */ + if (!local || !have_non_local_packs) + return 1; /* we don't know yet; keep looking for more packs */ return -1; } +static int want_object_in_pack_one(struct packed_git *p, + const struct object_id *oid, + int exclude, + struct packed_git **found_pack, + off_t *found_offset) +{ + off_t offset; + + if (p == *found_pack) + offset = *found_offset; + else + offset = find_pack_entry_one(oid->hash, p); + + if (offset) { + if (!*found_pack) { + if (!is_pack_valid(p)) + return -1; + *found_offset = offset; + *found_pack = p; + } + return want_found_object(oid, exclude, p); + } + return -1; +} + /* * Check whether we want the object in the pack (e.g., we do not want * objects found in non-local stores if the "--local" option was used). @@ -1034,6 +1472,7 @@ static int want_object_in_pack(const struct object_id *oid, { int want; struct list_head *pos; + struct multi_pack_index *m; if (!exclude && local && has_loose_object_nonlocal(oid)) return 0; @@ -1044,50 +1483,66 @@ static int want_object_in_pack(const struct object_id *oid, * are present we will determine the answer right now. */ if (*found_pack) { - want = want_found_object(exclude, *found_pack); + want = want_found_object(oid, exclude, *found_pack); if (want != -1) return want; + + *found_pack = NULL; + *found_offset = 0; + } + + for (m = get_multi_pack_index(the_repository); m; m = m->next) { + struct pack_entry e; + if (fill_midx_entry(the_repository, oid, &e, m)) { + want = want_object_in_pack_one(e.p, oid, exclude, found_pack, found_offset); + if (want != -1) + return want; + } } + list_for_each(pos, get_packed_git_mru(the_repository)) { struct packed_git *p = list_entry(pos, struct packed_git, mru); - off_t offset; + want = want_object_in_pack_one(p, oid, exclude, found_pack, found_offset); + if (!exclude && want > 0) + list_move(&p->mru, + get_packed_git_mru(the_repository)); + if (want != -1) + return want; + } - if (p == *found_pack) - offset = *found_offset; - else - offset = find_pack_entry_one(oid->hash, p); - - if (offset) { - if (!*found_pack) { - if (!is_pack_valid(p)) - continue; - *found_offset = offset; - *found_pack = p; + if (uri_protocols.nr) { + struct configured_exclusion *ex = + oidmap_get(&configured_exclusions, oid); + int i; + const char *p; + + if (ex) { + for (i = 0; i < uri_protocols.nr; i++) { + if (skip_prefix(ex->uri, + uri_protocols.items[i].string, + &p) && + *p == ':') { + oidset_insert(&excluded_by_config, oid); + return 0; + } } - want = want_found_object(exclude, p); - if (!exclude && want > 0) - list_move(&p->mru, - get_packed_git_mru(the_repository)); - if (want != -1) - return want; } } return 1; } -static void create_object_entry(const struct object_id *oid, - enum object_type type, - uint32_t hash, - int exclude, - int no_try_delta, - uint32_t index_pos, - struct packed_git *found_pack, - off_t found_offset) +static struct object_entry *create_object_entry(const struct object_id *oid, + enum object_type type, + uint32_t hash, + int exclude, + int no_try_delta, + struct packed_git *found_pack, + off_t found_offset) { struct object_entry *entry; - entry = packlist_alloc(&to_pack, oid->hash, index_pos); + entry = packlist_alloc(&to_pack, oid); entry->hash = hash; oe_set_type(entry, type); if (exclude) @@ -1100,6 +1555,8 @@ static void create_object_entry(const struct object_id *oid, } entry->no_try_delta = no_try_delta; + + return entry; } static const char no_closure_warning[] = N_( @@ -1111,17 +1568,17 @@ static int add_object_entry(const struct object_id *oid, enum object_type type, { struct packed_git *found_pack = NULL; off_t found_offset = 0; - uint32_t index_pos; display_progress(progress_state, ++nr_seen); - if (have_duplicate_entry(oid, exclude, &index_pos)) + if (have_duplicate_entry(oid, exclude)) return 0; if (!want_object_in_pack(oid, exclude, &found_pack, &found_offset)) { /* The pack is missing an object, so it will not have closure */ if (write_bitmap_index) { - warning(_(no_closure_warning)); + if (write_bitmap_index != WRITE_BITMAP_QUIET) + warning(_(no_closure_warning)); write_bitmap_index = 0; } return 0; @@ -1129,26 +1586,24 @@ static int add_object_entry(const struct object_id *oid, enum object_type type, create_object_entry(oid, type, pack_name_hash(name), exclude, name && no_try_delta(name), - index_pos, found_pack, found_offset); + found_pack, found_offset); return 1; } static int add_object_entry_from_bitmap(const struct object_id *oid, enum object_type type, - int flags, uint32_t name_hash, + int flags UNUSED, uint32_t name_hash, struct packed_git *pack, off_t offset) { - uint32_t index_pos; - display_progress(progress_state, ++nr_seen); - if (have_duplicate_entry(oid, 0, &index_pos)) + if (have_duplicate_entry(oid, 0)) return 0; if (!want_object_in_pack(oid, 0, &pack, &offset)) return 0; - create_object_entry(oid, type, name_hash, 0, 0, index_pos, pack, offset); + create_object_entry(oid, type, name_hash, 0, 0, pack, offset); return 1; } @@ -1196,7 +1651,7 @@ static struct pbase_tree_cache *pbase_tree_get(const struct object_id *oid) */ for (neigh = 0; neigh < 8; neigh++) { ent = pbase_tree_cache[my_ix]; - if (ent && !oidcmp(&ent->oid, oid)) { + if (ent && oideq(&ent->oid, oid)) { ent->ref++; return ent; } @@ -1257,17 +1712,14 @@ static void pbase_tree_put(struct pbase_tree_cache *cache) free(cache); } -static int name_cmp_len(const char *name) +static size_t name_cmp_len(const char *name) { - int i; - for (i = 0; name[i] && name[i] != '\n' && name[i] != '/'; i++) - ; - return i; + return strcspn(name, "\n/"); } static void add_pbase_object(struct tree_desc *tree, const char *name, - int cmplen, + size_t cmplen, const char *fullname) { struct name_entry entry; @@ -1283,7 +1735,7 @@ static void add_pbase_object(struct tree_desc *tree, if (cmp < 0) return; if (name[cmplen] != '/') { - add_object_entry(entry.oid, + add_object_entry(&entry.oid, object_type(entry.mode), fullname, 1); return; @@ -1292,9 +1744,9 @@ static void add_pbase_object(struct tree_desc *tree, struct tree_desc sub; struct pbase_tree_cache *tree; const char *down = name+cmplen+1; - int downlen = name_cmp_len(down); + size_t downlen = name_cmp_len(down); - tree = pbase_tree_get(entry.oid); + tree = pbase_tree_get(&entry.oid); if (!tree) return; init_tree_desc(&sub, tree->tree_data, tree->tree_size); @@ -1344,7 +1796,7 @@ static int check_pbase_path(unsigned hash) static void add_preferred_base_object(const char *name) { struct pbase_tree *it; - int cmplen; + size_t cmplen; unsigned hash = pack_name_hash(name); if (!num_preferred_base || check_pbase_path(hash)) @@ -1373,18 +1825,19 @@ static void add_preferred_base(struct object_id *oid) if (window <= num_preferred_base++) return; - data = read_object_with_reference(oid, tree_type, &size, &tree_oid); + data = read_object_with_reference(the_repository, oid, + OBJ_TREE, &size, &tree_oid); if (!data) return; for (it = pbase_tree; it; it = it->next) { - if (!oidcmp(&it->pcache.oid, &tree_oid)) { + if (oideq(&it->pcache.oid, &tree_oid)) { free(data); return; } } - it = xcalloc(1, sizeof(*it)); + CALLOC_ARRAY(it, 1); it->next = pbase_tree; pbase_tree = it; @@ -1418,14 +1871,82 @@ static void cleanup_preferred_base(void) done_pbase_paths_num = done_pbase_paths_alloc = 0; } -static void check_object(struct object_entry *entry) +/* + * Return 1 iff the object specified by "delta" can be sent + * literally as a delta against the base in "base_sha1". If + * so, then *base_out will point to the entry in our packing + * list, or NULL if we must use the external-base list. + * + * Depth value does not matter - find_deltas() will + * never consider reused delta as the base object to + * deltify other objects against, in order to avoid + * circular deltas. + */ +static int can_reuse_delta(const struct object_id *base_oid, + struct object_entry *delta, + struct object_entry **base_out) +{ + struct object_entry *base; + + /* + * First see if we're already sending the base (or it's explicitly in + * our "excluded" list). + */ + base = packlist_find(&to_pack, base_oid); + if (base) { + if (!in_same_island(&delta->idx.oid, &base->idx.oid)) + return 0; + *base_out = base; + return 1; + } + + /* + * Otherwise, reachability bitmaps may tell us if the receiver has it, + * even if it was buried too deep in history to make it into the + * packing list. + */ + if (thin && bitmap_has_oid_in_uninteresting(bitmap_git, base_oid)) { + if (use_delta_islands) { + if (!in_same_island(&delta->idx.oid, base_oid)) + return 0; + } + *base_out = NULL; + return 1; + } + + return 0; +} + +static void prefetch_to_pack(uint32_t object_index_start) { + struct oid_array to_fetch = OID_ARRAY_INIT; + uint32_t i; + + for (i = object_index_start; i < to_pack.nr_objects; i++) { + struct object_entry *entry = to_pack.objects + i; + + if (!oid_object_info_extended(the_repository, + &entry->idx.oid, + NULL, + OBJECT_INFO_FOR_PREFETCH)) + continue; + oid_array_append(&to_fetch, &entry->idx.oid); + } + promisor_remote_get_direct(the_repository, + to_fetch.oid, to_fetch.nr); + oid_array_clear(&to_fetch); +} + +static void check_object(struct object_entry *entry, uint32_t object_index) { unsigned long canonical_size; + enum object_type type; + struct object_info oi = {.typep = &type, .sizep = &canonical_size}; if (IN_PACK(entry)) { struct packed_git *p = IN_PACK(entry); struct pack_window *w_curs = NULL; - const unsigned char *base_ref = NULL; + int have_base = 0; + struct object_id base_ref; struct object_entry *base_entry; unsigned long used, used_0; unsigned long avail; @@ -1466,9 +1987,13 @@ static void check_object(struct object_entry *entry) unuse_pack(&w_curs); return; case OBJ_REF_DELTA: - if (reuse_delta && !entry->preferred_base) - base_ref = use_pack(p, &w_curs, - entry->in_pack_offset + used, NULL); + if (reuse_delta && !entry->preferred_base) { + oidread(&base_ref, + use_pack(p, &w_curs, + entry->in_pack_offset + used, + NULL)); + have_base = 1; + } entry->in_pack_header_size = used + the_hash_algo->rawsz; break; case OBJ_OFS_DELTA: @@ -1480,7 +2005,7 @@ static void check_object(struct object_entry *entry) while (c & 128) { ofs += 1; if (!ofs || MSB(ofs, 7)) { - error("delta base offset overflow in pack for %s", + error(_("delta base offset overflow in pack for %s"), oid_to_hex(&entry->idx.oid)); goto give_up; } @@ -1489,38 +2014,36 @@ static void check_object(struct object_entry *entry) } ofs = entry->in_pack_offset - ofs; if (ofs <= 0 || ofs >= entry->in_pack_offset) { - error("delta base offset out of bound for %s", + error(_("delta base offset out of bound for %s"), oid_to_hex(&entry->idx.oid)); goto give_up; } if (reuse_delta && !entry->preferred_base) { - struct revindex_entry *revidx; - revidx = find_pack_revindex(p, ofs); - if (!revidx) + uint32_t pos; + if (offset_to_pack_pos(p, ofs, &pos) < 0) goto give_up; - base_ref = nth_packed_object_sha1(p, revidx->nr); + if (!nth_packed_object_id(&base_ref, p, + pack_pos_to_index(p, pos))) + have_base = 1; } entry->in_pack_header_size = used + used_0; break; } - if (base_ref && (base_entry = packlist_find(&to_pack, base_ref, NULL))) { - /* - * If base_ref was set above that means we wish to - * reuse delta data, and we even found that base - * in the list of objects we want to pack. Goodie! - * - * Depth value does not matter - find_deltas() will - * never consider reused delta as the base object to - * deltify other objects against, in order to avoid - * circular deltas. - */ + if (have_base && + can_reuse_delta(&base_ref, entry, &base_entry)) { oe_set_type(entry, entry->in_pack_type); SET_SIZE(entry, in_pack_size); /* delta size */ - SET_DELTA(entry, base_entry); SET_DELTA_SIZE(entry, in_pack_size); - entry->delta_sibling_idx = base_entry->delta_child_idx; - SET_DELTA_CHILD(base_entry, entry); + + if (base_entry) { + SET_DELTA(entry, base_entry); + entry->delta_sibling_idx = base_entry->delta_child_idx; + SET_DELTA_CHILD(base_entry, entry); + } else { + SET_DELTA_EXT(entry, &base_ref); + } + unuse_pack(&w_curs); return; } @@ -1544,15 +2067,25 @@ static void check_object(struct object_entry *entry) /* * No choice but to fall back to the recursive delta walk - * with sha1_object_info() to find about the object type + * with oid_object_info() to find about the object type * at this point... */ give_up: unuse_pack(&w_curs); } - oe_set_type(entry, - oid_object_info(the_repository, &entry->idx.oid, &canonical_size)); + if (oid_object_info_extended(the_repository, &entry->idx.oid, &oi, + OBJECT_INFO_SKIP_FETCH_OBJECT | OBJECT_INFO_LOOKUP_REPLACE) < 0) { + if (has_promisor_remote()) { + prefetch_to_pack(object_index); + if (oid_object_info_extended(the_repository, &entry->idx.oid, &oi, + OBJECT_INFO_SKIP_FETCH_OBJECT | OBJECT_INFO_LOOKUP_REPLACE) < 0) + type = -1; + } else { + type = -1; + } + } + oe_set_type(entry, type); if (entry->type_valid) { SET_SIZE(entry, canonical_size); } else { @@ -1620,7 +2153,7 @@ static void drop_reused_delta(struct object_entry *entry) if (packed_object_info(the_repository, IN_PACK(entry), entry->in_pack_offset, &oi) < 0) { /* * We failed to get the info from this pack for some reason; - * fall back to sha1_object_info, which may find another copy. + * fall back to oid_object_info, which may find another copy. * And if that fails, the error will be recorded in oe_type(entry) * and dealt with in prepare_pack(). */ @@ -1765,14 +2298,14 @@ static void get_object_details(void) progress_state = start_progress(_("Counting objects"), to_pack.nr_objects); - sorted_by_offset = xcalloc(to_pack.nr_objects, sizeof(struct object_entry *)); + CALLOC_ARRAY(sorted_by_offset, to_pack.nr_objects); for (i = 0; i < to_pack.nr_objects; i++) sorted_by_offset[i] = to_pack.objects + i; QSORT(sorted_by_offset, to_pack.nr_objects, pack_offset_sort); for (i = 0; i < to_pack.nr_objects; i++) { struct object_entry *entry = sorted_by_offset[i]; - check_object(entry); + check_object(entry, i); if (entry->type_valid && oe_size_greater_than(&to_pack, entry, big_file_threshold)) entry->no_try_delta = 1; @@ -1803,10 +2336,10 @@ static int type_size_sort(const void *_a, const void *_b) { const struct object_entry *a = *(struct object_entry **)_a; const struct object_entry *b = *(struct object_entry **)_b; - enum object_type a_type = oe_type(a); - enum object_type b_type = oe_type(b); - unsigned long a_size = SIZE(a); - unsigned long b_size = SIZE(b); + const enum object_type a_type = oe_type(a); + const enum object_type b_type = oe_type(b); + const unsigned long a_size = SIZE(a); + const unsigned long b_size = SIZE(b); if (a_type > b_type) return -1; @@ -1820,6 +2353,11 @@ static int type_size_sort(const void *_a, const void *_b) return -1; if (a->preferred_base < b->preferred_base) return 1; + if (use_delta_islands) { + const int island_cmp = island_delta_cmp(&a->idx.oid, &b->idx.oid); + if (island_cmp) + return island_cmp; + } if (a_size > b_size) return -1; if (a_size < b_size) @@ -1850,30 +2388,45 @@ static int delta_cacheable(unsigned long src_size, unsigned long trg_size, return 0; } -#ifndef NO_PTHREADS - -static pthread_mutex_t read_mutex; -#define read_lock() pthread_mutex_lock(&read_mutex) -#define read_unlock() pthread_mutex_unlock(&read_mutex) - +/* Protect delta_cache_size */ static pthread_mutex_t cache_mutex; #define cache_lock() pthread_mutex_lock(&cache_mutex) #define cache_unlock() pthread_mutex_unlock(&cache_mutex) +/* + * Protect object list partitioning (e.g. struct thread_param) and + * progress_state + */ static pthread_mutex_t progress_mutex; #define progress_lock() pthread_mutex_lock(&progress_mutex) #define progress_unlock() pthread_mutex_unlock(&progress_mutex) -#else +/* + * Access to struct object_entry is unprotected since each thread owns + * a portion of the main object list. Just don't access object entries + * ahead in the list because they can be stolen and would need + * progress_mutex for protection. + */ -#define read_lock() (void)0 -#define read_unlock() (void)0 -#define cache_lock() (void)0 -#define cache_unlock() (void)0 -#define progress_lock() (void)0 -#define progress_unlock() (void)0 +static inline int oe_size_less_than(struct packing_data *pack, + const struct object_entry *lhs, + unsigned long rhs) +{ + if (lhs->size_valid) + return lhs->size_ < rhs; + if (rhs < pack->oe_size_limit) /* rhs < 2^x <= lhs ? */ + return 0; + return oe_get_size_slow(pack, lhs) < rhs; +} -#endif +static inline void oe_set_tree_depth(struct packing_data *pack, + struct object_entry *e, + unsigned int tree_depth) +{ + if (!pack->tree_depth) + CALLOC_ARRAY(pack->tree_depth, pack->nr_alloc); + pack->tree_depth[e - pack->objects] = tree_depth; +} /* * Return the size of the object without doing any delta @@ -1890,11 +2443,11 @@ unsigned long oe_get_size_slow(struct packing_data *pack, unsigned long used, avail, size; if (e->type_ != OBJ_OFS_DELTA && e->type_ != OBJ_REF_DELTA) { - read_lock(); + packing_data_lock(&to_pack); if (oid_object_info(the_repository, &e->idx.oid, &size) < 0) die(_("unable to get size of %s"), oid_to_hex(&e->idx.oid)); - read_unlock(); + packing_data_unlock(&to_pack); return size; } @@ -1902,7 +2455,7 @@ unsigned long oe_get_size_slow(struct packing_data *pack, if (!p) BUG("when e->type is a delta, it must belong to a pack"); - read_lock(); + packing_data_lock(&to_pack); w_curs = NULL; buf = use_pack(p, &w_curs, e->in_pack_offset, &avail); used = unpack_object_header_buffer(buf, avail, &type, &size); @@ -1911,7 +2464,7 @@ unsigned long oe_get_size_slow(struct packing_data *pack, oid_to_hex(&e->idx.oid)); unuse_pack(&w_curs); - read_unlock(); + packing_data_unlock(&to_pack); return size; } @@ -1968,29 +2521,32 @@ static int try_delta(struct unpacked *trg, struct unpacked *src, if (trg_size < src_size / 32) return 0; + if (!in_same_island(&trg->entry->idx.oid, &src->entry->idx.oid)) + return 0; + /* Load data if not already done */ if (!trg->data) { - read_lock(); + packing_data_lock(&to_pack); trg->data = read_object_file(&trg_entry->idx.oid, &type, &sz); - read_unlock(); + packing_data_unlock(&to_pack); if (!trg->data) - die("object %s cannot be read", + die(_("object %s cannot be read"), oid_to_hex(&trg_entry->idx.oid)); if (sz != trg_size) - die("object %s inconsistent object length (%lu vs %lu)", - oid_to_hex(&trg_entry->idx.oid), sz, - trg_size); + die(_("object %s inconsistent object length (%"PRIuMAX" vs %"PRIuMAX")"), + oid_to_hex(&trg_entry->idx.oid), (uintmax_t)sz, + (uintmax_t)trg_size); *mem_usage += sz; } if (!src->data) { - read_lock(); + packing_data_lock(&to_pack); src->data = read_object_file(&src_entry->idx.oid, &type, &sz); - read_unlock(); + packing_data_unlock(&to_pack); if (!src->data) { if (src_entry->preferred_base) { static int warned = 0; if (!warned++) - warning("object %s cannot be read", + warning(_("object %s cannot be read"), oid_to_hex(&src_entry->idx.oid)); /* * Those objects are not included in the @@ -2000,13 +2556,13 @@ static int try_delta(struct unpacked *trg, struct unpacked *src, */ return 0; } - die("object %s cannot be read", + die(_("object %s cannot be read"), oid_to_hex(&src_entry->idx.oid)); } if (sz != src_size) - die("object %s inconsistent object length (%lu vs %lu)", - oid_to_hex(&src_entry->idx.oid), sz, - src_size); + die(_("object %s inconsistent object length (%"PRIuMAX" vs %"PRIuMAX")"), + oid_to_hex(&src_entry->idx.oid), (uintmax_t)sz, + (uintmax_t)src_size); *mem_usage += sz; } if (!src->index) { @@ -2014,7 +2570,7 @@ static int try_delta(struct unpacked *trg, struct unpacked *src, if (!src->index) { static int warned = 0; if (!warned++) - warning("suboptimal pack - out of memory"); + warning(_("suboptimal pack - out of memory")); return 0; } *mem_usage += sizeof_delta_index(src->index); @@ -2023,10 +2579,6 @@ static int try_delta(struct unpacked *trg, struct unpacked *src, delta_buf = create_delta(src->index, trg->data, trg_size, &delta_size, max_size); if (!delta_buf) return 0; - if (delta_size >= (1U << OE_DELTA_SIZE_BITS)) { - free(delta_buf); - return 0; - } if (DELTA(trg_entry)) { /* Prefer only shallower same-sized deltas. */ @@ -2069,7 +2621,7 @@ static unsigned int check_delta_limit(struct object_entry *me, unsigned int n) struct object_entry *child = DELTA_CHILD(me); unsigned int m = n; while (child) { - unsigned int c = check_delta_limit(child, n + 1); + const unsigned int c = check_delta_limit(child, n + 1); if (m < c) m = c; child = DELTA_SIBLING(child); @@ -2098,7 +2650,7 @@ static void find_deltas(struct object_entry **list, unsigned *list_size, struct unpacked *array; unsigned long mem_usage = 0; - array = xcalloc(window, sizeof(struct unpacked)); + CALLOC_ARRAY(array, window); for (;;) { struct object_entry *entry; @@ -2124,7 +2676,7 @@ static void find_deltas(struct object_entry **list, unsigned *list_size, while (window_memory_limit && mem_usage > window_memory_limit && count > 1) { - uint32_t tail = (idx + window - count) % window; + const uint32_t tail = (idx + window - count) % window; mem_usage -= free_unpacked(array + tail); count--; } @@ -2233,24 +2785,20 @@ static void find_deltas(struct object_entry **list, unsigned *list_size, free(array); } -#ifndef NO_PTHREADS - -static void try_to_free_from_threads(size_t size) -{ - read_lock(); - release_pack_memory(size); - read_unlock(); -} - -static try_to_free_t old_try_to_free_routine; - /* + * The main object list is split into smaller lists, each is handed to + * one worker. + * * The main thread waits on the condition that (at least) one of the workers * has stopped working (which is indicated in the .working member of * struct thread_params). + * * When a work thread has completed its work, it sets .working to 0 and * signals the main thread and waits on the condition that .data_ready * becomes 1. + * + * The main thread steals half of the work from the worker that has + * most work left to hand it to the idle worker. */ struct thread_params { @@ -2274,18 +2822,14 @@ static pthread_cond_t progress_cond; */ static void init_threaded_search(void) { - init_recursive_mutex(&read_mutex); pthread_mutex_init(&cache_mutex, NULL); pthread_mutex_init(&progress_mutex, NULL); pthread_cond_init(&progress_cond, NULL); - old_try_to_free_routine = set_try_to_free_routine(try_to_free_from_threads); } static void cleanup_threaded_search(void) { - set_try_to_free_routine(old_try_to_free_routine); pthread_cond_destroy(&progress_cond); - pthread_mutex_destroy(&read_mutex); pthread_mutex_destroy(&cache_mutex); pthread_mutex_destroy(&progress_mutex); } @@ -2341,9 +2885,9 @@ static void ll_find_deltas(struct object_entry **list, unsigned list_size, return; } if (progress > pack_to_stdout) - fprintf(stderr, "Delta compression using up to %d threads.\n", - delta_search_threads); - p = xcalloc(delta_search_threads, sizeof(*p)); + fprintf_ln(stderr, _("Delta compression using up to %d threads"), + delta_search_threads); + CALLOC_ARRAY(p, delta_search_threads); /* Partition the work amongst work threads. */ for (i = 0; i < delta_search_threads; i++) { @@ -2382,7 +2926,7 @@ static void ll_find_deltas(struct object_entry **list, unsigned list_size, ret = pthread_create(&p[i].thread, NULL, threaded_find_deltas, &p[i]); if (ret) - die("unable to create thread: %s", strerror(ret)); + die(_("unable to create thread: %s"), strerror(ret)); active_threads++; } @@ -2456,9 +3000,12 @@ static void ll_find_deltas(struct object_entry **list, unsigned list_size, free(p); } -#else -#define ll_find_deltas(l, s, w, d, p) find_deltas(l, &s, w, d, p) -#endif +static int obj_is_packed(const struct object_id *oid) +{ + return packlist_find(&to_pack, oid) || + (reuse_packfile_bitmap && + bitmap_walk_contains(bitmap_git, reuse_packfile_bitmap, oid)); +} static void add_tag_chain(const struct object_id *oid) { @@ -2471,13 +3018,13 @@ static void add_tag_chain(const struct object_id *oid) * it was included via bitmaps, we would not have parsed it * previously). */ - if (packlist_find(&to_pack, oid->hash, NULL)) + if (obj_is_packed(oid)) return; tag = lookup_tag(the_repository, oid); while (1) { if (!tag || parse_tag(tag) || !tag->tagged) - die("unable to pack objects reachable from tag %s", + die(_("unable to pack objects reachable from tag %s"), oid_to_hex(oid)); add_object_entry(&tag->object.oid, OBJ_TAG, NULL, 0); @@ -2489,13 +3036,12 @@ static void add_tag_chain(const struct object_id *oid) } } -static int add_ref_tag(const char *path, const struct object_id *oid, int flag, void *cb_data) +static int add_ref_tag(const char *tag UNUSED, const struct object_id *oid, + int flag UNUSED, void *cb_data UNUSED) { struct object_id peeled; - if (starts_with(path, "refs/tags/") && /* is a tag? */ - !peel_ref(path, &peeled) && /* peelable? */ - packlist_find(&to_pack, peeled.hash, NULL)) /* object packed? */ + if (!peel_iterated_oid(oid, &peeled) && obj_is_packed(&peeled)) add_tag_chain(oid); return 0; } @@ -2506,6 +3052,9 @@ static void prepare_pack(int window, int depth) uint32_t i, nr_deltas; unsigned n; + if (use_delta_islands) + resolve_tree_islands(the_repository, progress, &to_pack); + get_object_details(); /* @@ -2543,7 +3092,7 @@ static void prepare_pack(int window, int depth) if (!entry->preferred_base) { nr_deltas++; if (oe_type(entry) < 0) - die("unable to get type of object %s", + die(_("unable to get type of object %s"), oid_to_hex(&entry->idx.oid)); } else { if (oe_type(entry) < 0) { @@ -2560,6 +3109,7 @@ static void prepare_pack(int window, int depth) if (nr_deltas && n > 1) { unsigned nr_done = 0; + if (progress) progress_state = start_progress(_("Compressing objects"), nr_deltas); @@ -2567,7 +3117,7 @@ static void prepare_pack(int window, int depth) ll_find_deltas(delta_list, n, window+1, depth, &nr_done); stop_progress(&progress_state); if (nr_done != nr_deltas) - die("inconsistency with delta count"); + die(_("inconsistency with delta count")); } free(delta_list); } @@ -2600,33 +3150,490 @@ static int git_pack_config(const char *k, const char *v, void *cb) else write_bitmap_options &= ~BITMAP_OPT_HASH_CACHE; } + + if (!strcmp(k, "pack.writebitmaplookuptable")) { + if (git_config_bool(k, v)) + write_bitmap_options |= BITMAP_OPT_LOOKUP_TABLE; + else + write_bitmap_options &= ~BITMAP_OPT_LOOKUP_TABLE; + } + if (!strcmp(k, "pack.usebitmaps")) { use_bitmap_index_default = git_config_bool(k, v); return 0; } + if (!strcmp(k, "pack.allowpackreuse")) { + allow_pack_reuse = git_config_bool(k, v); + return 0; + } if (!strcmp(k, "pack.threads")) { delta_search_threads = git_config_int(k, v); if (delta_search_threads < 0) - die("invalid number of threads specified (%d)", + die(_("invalid number of threads specified (%d)"), delta_search_threads); -#ifdef NO_PTHREADS - if (delta_search_threads != 1) { - warning("no threads support, ignoring %s", k); + if (!HAVE_THREADS && delta_search_threads != 1) { + warning(_("no threads support, ignoring %s"), k); delta_search_threads = 0; } -#endif return 0; } if (!strcmp(k, "pack.indexversion")) { pack_idx_opts.version = git_config_int(k, v); if (pack_idx_opts.version > 2) - die("bad pack.indexversion=%"PRIu32, + die(_("bad pack.indexVersion=%"PRIu32), pack_idx_opts.version); return 0; } + if (!strcmp(k, "pack.writereverseindex")) { + if (git_config_bool(k, v)) + pack_idx_opts.flags |= WRITE_REV; + else + pack_idx_opts.flags &= ~WRITE_REV; + return 0; + } + if (!strcmp(k, "uploadpack.blobpackfileuri")) { + struct configured_exclusion *ex = xmalloc(sizeof(*ex)); + const char *oid_end, *pack_end; + /* + * Stores the pack hash. This is not a true object ID, but is + * of the same form. + */ + struct object_id pack_hash; + + if (parse_oid_hex(v, &ex->e.oid, &oid_end) || + *oid_end != ' ' || + parse_oid_hex(oid_end + 1, &pack_hash, &pack_end) || + *pack_end != ' ') + die(_("value of uploadpack.blobpackfileuri must be " + "of the form '<object-hash> <pack-hash> <uri>' (got '%s')"), v); + if (oidmap_get(&configured_exclusions, &ex->e.oid)) + die(_("object already configured in another " + "uploadpack.blobpackfileuri (got '%s')"), v); + ex->pack_hash_hex = xcalloc(1, pack_end - oid_end); + memcpy(ex->pack_hash_hex, oid_end + 1, pack_end - oid_end - 1); + ex->uri = xstrdup(pack_end + 1); + oidmap_put(&configured_exclusions, ex); + } return git_default_config(k, v, cb); } +/* Counters for trace2 output when in --stdin-packs mode. */ +static int stdin_packs_found_nr; +static int stdin_packs_hints_nr; + +static int add_object_entry_from_pack(const struct object_id *oid, + struct packed_git *p, + uint32_t pos, + void *_data) +{ + off_t ofs; + enum object_type type = OBJ_NONE; + + display_progress(progress_state, ++nr_seen); + + if (have_duplicate_entry(oid, 0)) + return 0; + + ofs = nth_packed_object_offset(p, pos); + if (!want_object_in_pack(oid, 0, &p, &ofs)) + return 0; + + if (p) { + struct rev_info *revs = _data; + struct object_info oi = OBJECT_INFO_INIT; + + oi.typep = &type; + if (packed_object_info(the_repository, p, ofs, &oi) < 0) { + die(_("could not get type of object %s in pack %s"), + oid_to_hex(oid), p->pack_name); + } else if (type == OBJ_COMMIT) { + /* + * commits in included packs are used as starting points for the + * subsequent revision walk + */ + add_pending_oid(revs, NULL, oid, 0); + } + + stdin_packs_found_nr++; + } + + create_object_entry(oid, type, 0, 0, 0, p, ofs); + + return 0; +} + +static void show_commit_pack_hint(struct commit *commit UNUSED, + void *data UNUSED) +{ + /* nothing to do; commits don't have a namehash */ +} + +static void show_object_pack_hint(struct object *object, const char *name, + void *data UNUSED) +{ + struct object_entry *oe = packlist_find(&to_pack, &object->oid); + if (!oe) + return; + + /* + * Our 'to_pack' list was constructed by iterating all objects packed in + * included packs, and so doesn't have a non-zero hash field that you + * would typically pick up during a reachability traversal. + * + * Make a best-effort attempt to fill in the ->hash and ->no_try_delta + * here using a now in order to perhaps improve the delta selection + * process. + */ + oe->hash = pack_name_hash(name); + oe->no_try_delta = name && no_try_delta(name); + + stdin_packs_hints_nr++; +} + +static int pack_mtime_cmp(const void *_a, const void *_b) +{ + struct packed_git *a = ((const struct string_list_item*)_a)->util; + struct packed_git *b = ((const struct string_list_item*)_b)->util; + + /* + * order packs by descending mtime so that objects are laid out + * roughly as newest-to-oldest + */ + if (a->mtime < b->mtime) + return 1; + else if (b->mtime < a->mtime) + return -1; + else + return 0; +} + +static void read_packs_list_from_stdin(void) +{ + struct strbuf buf = STRBUF_INIT; + struct string_list include_packs = STRING_LIST_INIT_DUP; + struct string_list exclude_packs = STRING_LIST_INIT_DUP; + struct string_list_item *item = NULL; + + struct packed_git *p; + struct rev_info revs; + + repo_init_revisions(the_repository, &revs, NULL); + /* + * Use a revision walk to fill in the namehash of objects in the include + * packs. To save time, we'll avoid traversing through objects that are + * in excluded packs. + * + * That may cause us to avoid populating all of the namehash fields of + * all included objects, but our goal is best-effort, since this is only + * an optimization during delta selection. + */ + revs.no_kept_objects = 1; + revs.keep_pack_cache_flags |= IN_CORE_KEEP_PACKS; + revs.blob_objects = 1; + revs.tree_objects = 1; + revs.tag_objects = 1; + revs.ignore_missing_links = 1; + + while (strbuf_getline(&buf, stdin) != EOF) { + if (!buf.len) + continue; + + if (*buf.buf == '^') + string_list_append(&exclude_packs, buf.buf + 1); + else + string_list_append(&include_packs, buf.buf); + + strbuf_reset(&buf); + } + + string_list_sort(&include_packs); + string_list_sort(&exclude_packs); + + for (p = get_all_packs(the_repository); p; p = p->next) { + const char *pack_name = pack_basename(p); + + item = string_list_lookup(&include_packs, pack_name); + if (!item) + item = string_list_lookup(&exclude_packs, pack_name); + + if (item) + item->util = p; + } + + /* + * Arguments we got on stdin may not even be packs. First + * check that to avoid segfaulting later on in + * e.g. pack_mtime_cmp(), excluded packs are handled below. + * + * Since we first parsed our STDIN and then sorted the input + * lines the pack we error on will be whatever line happens to + * sort first. This is lazy, it's enough that we report one + * bad case here, we don't need to report the first/last one, + * or all of them. + */ + for_each_string_list_item(item, &include_packs) { + struct packed_git *p = item->util; + if (!p) + die(_("could not find pack '%s'"), item->string); + if (!is_pack_valid(p)) + die(_("packfile %s cannot be accessed"), p->pack_name); + } + + /* + * Then, handle all of the excluded packs, marking them as + * kept in-core so that later calls to add_object_entry() + * discards any objects that are also found in excluded packs. + */ + for_each_string_list_item(item, &exclude_packs) { + struct packed_git *p = item->util; + if (!p) + die(_("could not find pack '%s'"), item->string); + p->pack_keep_in_core = 1; + } + + /* + * Order packs by ascending mtime; use QSORT directly to access the + * string_list_item's ->util pointer, which string_list_sort() does not + * provide. + */ + QSORT(include_packs.items, include_packs.nr, pack_mtime_cmp); + + for_each_string_list_item(item, &include_packs) { + struct packed_git *p = item->util; + for_each_object_in_pack(p, + add_object_entry_from_pack, + &revs, + FOR_EACH_OBJECT_PACK_ORDER); + } + + if (prepare_revision_walk(&revs)) + die(_("revision walk setup failed")); + traverse_commit_list(&revs, + show_commit_pack_hint, + show_object_pack_hint, + NULL); + + trace2_data_intmax("pack-objects", the_repository, "stdin_packs_found", + stdin_packs_found_nr); + trace2_data_intmax("pack-objects", the_repository, "stdin_packs_hints", + stdin_packs_hints_nr); + + strbuf_release(&buf); + string_list_clear(&include_packs, 0); + string_list_clear(&exclude_packs, 0); +} + +static void add_cruft_object_entry(const struct object_id *oid, enum object_type type, + struct packed_git *pack, off_t offset, + const char *name, uint32_t mtime) +{ + struct object_entry *entry; + + display_progress(progress_state, ++nr_seen); + + entry = packlist_find(&to_pack, oid); + if (entry) { + if (name) { + entry->hash = pack_name_hash(name); + entry->no_try_delta = no_try_delta(name); + } + } else { + if (!want_object_in_pack(oid, 0, &pack, &offset)) + return; + if (!pack && type == OBJ_BLOB && !has_loose_object(oid)) { + /* + * If a traversed tree has a missing blob then we want + * to avoid adding that missing object to our pack. + * + * This only applies to missing blobs, not trees, + * because the traversal needs to parse sub-trees but + * not blobs. + * + * Note we only perform this check when we couldn't + * already find the object in a pack, so we're really + * limited to "ensure non-tip blobs which don't exist in + * packs do exist via loose objects". Confused? + */ + return; + } + + entry = create_object_entry(oid, type, pack_name_hash(name), + 0, name && no_try_delta(name), + pack, offset); + } + + if (mtime > oe_cruft_mtime(&to_pack, entry)) + oe_set_cruft_mtime(&to_pack, entry, mtime); + return; +} + +static void show_cruft_object(struct object *obj, const char *name, void *data UNUSED) +{ + /* + * if we did not record it earlier, it's at least as old as our + * expiration value. Rather than find it exactly, just use that + * value. This may bump it forward from its real mtime, but it + * will still be "too old" next time we run with the same + * expiration. + * + * if obj does appear in the packing list, this call is a noop (or may + * set the namehash). + */ + add_cruft_object_entry(&obj->oid, obj->type, NULL, 0, name, cruft_expiration); +} + +static void show_cruft_commit(struct commit *commit, void *data) +{ + show_cruft_object((struct object*)commit, NULL, data); +} + +static int cruft_include_check_obj(struct object *obj, void *data UNUSED) +{ + return !has_object_kept_pack(&obj->oid, IN_CORE_KEEP_PACKS); +} + +static int cruft_include_check(struct commit *commit, void *data) +{ + return cruft_include_check_obj((struct object*)commit, data); +} + +static void set_cruft_mtime(const struct object *object, + struct packed_git *pack, + off_t offset, time_t mtime) +{ + add_cruft_object_entry(&object->oid, object->type, pack, offset, NULL, + mtime); +} + +static void mark_pack_kept_in_core(struct string_list *packs, unsigned keep) +{ + struct string_list_item *item = NULL; + for_each_string_list_item(item, packs) { + struct packed_git *p = item->util; + if (!p) + die(_("could not find pack '%s'"), item->string); + p->pack_keep_in_core = keep; + } +} + +static void add_unreachable_loose_objects(void); +static void add_objects_in_unpacked_packs(void); + +static void enumerate_cruft_objects(void) +{ + if (progress) + progress_state = start_progress(_("Enumerating cruft objects"), 0); + + add_objects_in_unpacked_packs(); + add_unreachable_loose_objects(); + + stop_progress(&progress_state); +} + +static void enumerate_and_traverse_cruft_objects(struct string_list *fresh_packs) +{ + struct packed_git *p; + struct rev_info revs; + int ret; + + repo_init_revisions(the_repository, &revs, NULL); + + revs.tag_objects = 1; + revs.tree_objects = 1; + revs.blob_objects = 1; + + revs.include_check = cruft_include_check; + revs.include_check_obj = cruft_include_check_obj; + + revs.ignore_missing_links = 1; + + if (progress) + progress_state = start_progress(_("Enumerating cruft objects"), 0); + ret = add_unseen_recent_objects_to_traversal(&revs, cruft_expiration, + set_cruft_mtime, 1); + stop_progress(&progress_state); + + if (ret) + die(_("unable to add cruft objects")); + + /* + * Re-mark only the fresh packs as kept so that objects in + * unknown packs do not halt the reachability traversal early. + */ + for (p = get_all_packs(the_repository); p; p = p->next) + p->pack_keep_in_core = 0; + mark_pack_kept_in_core(fresh_packs, 1); + + if (prepare_revision_walk(&revs)) + die(_("revision walk setup failed")); + if (progress) + progress_state = start_progress(_("Traversing cruft objects"), 0); + nr_seen = 0; + traverse_commit_list(&revs, show_cruft_commit, show_cruft_object, NULL); + + stop_progress(&progress_state); +} + +static void read_cruft_objects(void) +{ + struct strbuf buf = STRBUF_INIT; + struct string_list discard_packs = STRING_LIST_INIT_DUP; + struct string_list fresh_packs = STRING_LIST_INIT_DUP; + struct packed_git *p; + + ignore_packed_keep_in_core = 1; + + while (strbuf_getline(&buf, stdin) != EOF) { + if (!buf.len) + continue; + + if (*buf.buf == '-') + string_list_append(&discard_packs, buf.buf + 1); + else + string_list_append(&fresh_packs, buf.buf); + strbuf_reset(&buf); + } + + string_list_sort(&discard_packs); + string_list_sort(&fresh_packs); + + for (p = get_all_packs(the_repository); p; p = p->next) { + const char *pack_name = pack_basename(p); + struct string_list_item *item; + + item = string_list_lookup(&fresh_packs, pack_name); + if (!item) + item = string_list_lookup(&discard_packs, pack_name); + + if (item) { + item->util = p; + } else { + /* + * This pack wasn't mentioned in either the "fresh" or + * "discard" list, so the caller didn't know about it. + * + * Mark it as kept so that its objects are ignored by + * add_unseen_recent_objects_to_traversal(). We'll + * unmark it before starting the traversal so it doesn't + * halt the traversal early. + */ + p->pack_keep_in_core = 1; + } + } + + mark_pack_kept_in_core(&fresh_packs, 1); + mark_pack_kept_in_core(&discard_packs, 0); + + if (cruft_expiration) + enumerate_and_traverse_cruft_objects(&fresh_packs); + else + enumerate_cruft_objects(); + + strbuf_release(&buf); + string_list_clear(&discard_packs, 0); + string_list_clear(&fresh_packs, 0); +} + static void read_object_list_from_stdin(void) { char line[GIT_MAX_HEXSZ + 1 + PATH_MAX + 2]; @@ -2638,7 +3645,7 @@ static void read_object_list_from_stdin(void) if (feof(stdin)) break; if (!ferror(stdin)) - die("fgets returned NULL, not EOF, not error!"); + BUG("fgets returned NULL, not EOF, not error!"); if (errno != EINTR) die_errno("fgets"); clearerr(stdin); @@ -2646,36 +3653,50 @@ static void read_object_list_from_stdin(void) } if (line[0] == '-') { if (get_oid_hex(line+1, &oid)) - die("expected edge object ID, got garbage:\n %s", + die(_("expected edge object ID, got garbage:\n %s"), line); add_preferred_base(&oid); continue; } if (parse_oid_hex(line, &oid, &p)) - die("expected object ID, got garbage:\n %s", line); + die(_("expected object ID, got garbage:\n %s"), line); add_preferred_base_object(p + 1); add_object_entry(&oid, OBJ_NONE, p + 1, 0); } } -/* Remember to update object flag allocation in object.h */ -#define OBJECT_ADDED (1u<<20) - -static void show_commit(struct commit *commit, void *data) +static void show_commit(struct commit *commit, void *data UNUSED) { add_object_entry(&commit->object.oid, OBJ_COMMIT, NULL, 0); - commit->object.flags |= OBJECT_ADDED; if (write_bitmap_index) index_commit_for_bitmap(commit); + + if (use_delta_islands) + propagate_island_marks(commit); } -static void show_object(struct object *obj, const char *name, void *data) +static void show_object(struct object *obj, const char *name, + void *data UNUSED) { add_preferred_base_object(name); add_object_entry(&obj->oid, obj->type, name, 0); - obj->flags |= OBJECT_ADDED; + + if (use_delta_islands) { + const char *p; + unsigned depth; + struct object_entry *ent; + + /* the empty string is a root tree, which is depth 0 */ + depth = *name ? 1 : 0; + for (p = strchr(name, '/'); p; p = strchr(p + 1, '/')) + depth++; + + ent = packlist_find(&to_pack, &obj->oid); + if (ent && depth > oe_tree_depth(&to_pack, ent)) + oe_set_tree_depth(&to_pack, ent, depth); + } } static void show_object__ma_allow_any(struct object *obj, const char *name, void *data) @@ -2686,7 +3707,7 @@ static void show_object__ma_allow_any(struct object *obj, const char *name, void * Quietly ignore ALL missing objects. This avoids problems with * staging them now and getting an odd error later. */ - if (!has_object_file(&obj->oid)) + if (!has_object(the_repository, &obj->oid, 0)) return; show_object(obj, name, data); @@ -2700,7 +3721,7 @@ static void show_object__ma_allow_promisor(struct object *obj, const char *name, * Quietly ignore EXPECTED missing objects. This avoids problems with * staging them now and getting an odd error later. */ - if (!has_object_file(&obj->oid) && is_promisor_object(&obj->oid)) + if (!has_object(the_repository, &obj->oid, 0) && is_promisor_object(&obj->oid)) return; show_object(obj, name, data); @@ -2732,7 +3753,7 @@ static int option_parse_missing_action(const struct option *opt, return 0; } - die(_("invalid value for --missing")); + die(_("invalid value for '%s': '%s'"), "--missing", arg); return 0; } @@ -2741,92 +3762,65 @@ static void show_edge(struct commit *commit) add_preferred_base(&commit->object.oid); } -struct in_pack_object { - off_t offset; - struct object *object; -}; - -struct in_pack { - unsigned int alloc; - unsigned int nr; - struct in_pack_object *array; -}; - -static void mark_in_pack_object(struct object *object, struct packed_git *p, struct in_pack *in_pack) +static int add_object_in_unpacked_pack(const struct object_id *oid, + struct packed_git *pack, + uint32_t pos, + void *data UNUSED) { - in_pack->array[in_pack->nr].offset = find_pack_entry_one(object->oid.hash, p); - in_pack->array[in_pack->nr].object = object; - in_pack->nr++; -} + if (cruft) { + off_t offset; + time_t mtime; -/* - * Compare the objects in the offset order, in order to emulate the - * "git rev-list --objects" output that produced the pack originally. - */ -static int ofscmp(const void *a_, const void *b_) -{ - struct in_pack_object *a = (struct in_pack_object *)a_; - struct in_pack_object *b = (struct in_pack_object *)b_; + if (pack->is_cruft) { + if (load_pack_mtimes(pack) < 0) + die(_("could not load cruft pack .mtimes")); + mtime = nth_packed_mtime(pack, pos); + } else { + mtime = pack->mtime; + } + offset = nth_packed_object_offset(pack, pos); - if (a->offset < b->offset) - return -1; - else if (a->offset > b->offset) - return 1; - else - return oidcmp(&a->object->oid, &b->object->oid); + add_cruft_object_entry(oid, OBJ_NONE, pack, offset, + NULL, mtime); + } else { + add_object_entry(oid, OBJ_NONE, "", 0); + } + return 0; } -static void add_objects_in_unpacked_packs(struct rev_info *revs) +static void add_objects_in_unpacked_packs(void) { - struct packed_git *p; - struct in_pack in_pack; - uint32_t i; - - memset(&in_pack, 0, sizeof(in_pack)); - - for (p = get_packed_git(the_repository); p; p = p->next) { - struct object_id oid; - struct object *o; - - if (!p->pack_local || p->pack_keep || p->pack_keep_in_core) - continue; - if (open_pack_index(p)) - die("cannot open pack index"); - - ALLOC_GROW(in_pack.array, - in_pack.nr + p->num_objects, - in_pack.alloc); - - for (i = 0; i < p->num_objects; i++) { - nth_packed_object_oid(&oid, p, i); - o = lookup_unknown_object(oid.hash); - if (!(o->flags & OBJECT_ADDED)) - mark_in_pack_object(o, p, &in_pack); - o->flags |= OBJECT_ADDED; - } - } - - if (in_pack.nr) { - QSORT(in_pack.array, in_pack.nr, ofscmp); - for (i = 0; i < in_pack.nr; i++) { - struct object *o = in_pack.array[i].object; - add_object_entry(&o->oid, o->type, "", 0); - } - } - free(in_pack.array); + if (for_each_packed_object(add_object_in_unpacked_pack, NULL, + FOR_EACH_OBJECT_PACK_ORDER | + FOR_EACH_OBJECT_LOCAL_ONLY | + FOR_EACH_OBJECT_SKIP_IN_CORE_KEPT_PACKS | + FOR_EACH_OBJECT_SKIP_ON_DISK_KEPT_PACKS)) + die(_("cannot open pack index")); } static int add_loose_object(const struct object_id *oid, const char *path, - void *data) + void *data UNUSED) { enum object_type type = oid_object_info(the_repository, oid, NULL); if (type < 0) { - warning("loose object at %s could not be examined", path); + warning(_("loose object at %s could not be examined"), path); return 0; } - add_object_entry(oid, type, "", 0); + if (cruft) { + struct stat st; + if (stat(path, &st) < 0) { + if (errno == ENOENT) + return 0; + return error_errno("unable to stat %s", oid_to_hex(oid)); + } + + add_cruft_object_entry(oid, type, NULL, 0, NULL, + st.st_mtime); + } else { + add_object_entry(oid, type, "", 0); + } return 0; } @@ -2848,7 +3842,7 @@ static int has_sha1_pack_kept_or_nonlocal(const struct object_id *oid) struct packed_git *p; p = (last_found != (void *)1) ? last_found : - get_packed_git(the_repository); + get_all_packs(the_repository); while (p) { if ((!p->pack_local || p->pack_keep || @@ -2858,7 +3852,7 @@ static int has_sha1_pack_kept_or_nonlocal(const struct object_id *oid) return 1; } if (p == last_found) - p = get_packed_git(the_repository); + p = get_all_packs(the_repository); else p = p->next; if (p == last_found) @@ -2888,28 +3882,34 @@ static int loosened_object_can_be_discarded(const struct object_id *oid, return 1; } -static void loosen_unused_packed_objects(struct rev_info *revs) +static void loosen_unused_packed_objects(void) { struct packed_git *p; uint32_t i; + uint32_t loosened_objects_nr = 0; struct object_id oid; - for (p = get_packed_git(the_repository); p; p = p->next) { + for (p = get_all_packs(the_repository); p; p = p->next) { if (!p->pack_local || p->pack_keep || p->pack_keep_in_core) continue; if (open_pack_index(p)) - die("cannot open pack index"); + die(_("cannot open pack index")); for (i = 0; i < p->num_objects; i++) { - nth_packed_object_oid(&oid, p, i); - if (!packlist_find(&to_pack, oid.hash, NULL) && + nth_packed_object_id(&oid, p, i); + if (!packlist_find(&to_pack, &oid) && !has_sha1_pack_kept_or_nonlocal(&oid) && - !loosened_object_can_be_discarded(&oid, p->mtime)) + !loosened_object_can_be_discarded(&oid, p->mtime)) { if (force_object_loose(&oid, p->mtime)) - die("unable to force loose object"); + die(_("unable to force loose object")); + loosened_objects_nr++; + } } } + + trace2_data_intmax("pack-objects", the_repository, + "loosen_unused_packed_objects/loosened", loosened_objects_nr); } /* @@ -2919,8 +3919,8 @@ static void loosen_unused_packed_objects(struct rev_info *revs) */ static int pack_options_allow_reuse(void) { - return pack_to_stdout && - allow_ofs_delta && + return allow_pack_reuse && + pack_to_stdout && !ignore_packed_keep_on_disk && !ignore_packed_keep_in_core && (!local || !have_non_local_packs) && @@ -2929,48 +3929,88 @@ static int pack_options_allow_reuse(void) static int get_object_list_from_bitmap(struct rev_info *revs) { - if (prepare_bitmap_walk(revs) < 0) + if (!(bitmap_git = prepare_bitmap_walk(revs, 0))) return -1; if (pack_options_allow_reuse() && !reuse_partial_packfile_from_bitmap( + bitmap_git, &reuse_packfile, &reuse_packfile_objects, - &reuse_packfile_offset)) { + &reuse_packfile_bitmap)) { assert(reuse_packfile_objects); nr_result += reuse_packfile_objects; - display_progress(progress_state, nr_result); + nr_seen += reuse_packfile_objects; + display_progress(progress_state, nr_seen); } - traverse_bitmap_commit_list(&add_object_entry_from_bitmap); + traverse_bitmap_commit_list(bitmap_git, revs, + &add_object_entry_from_bitmap); return 0; } static void record_recent_object(struct object *obj, - const char *name, - void *data) + const char *name UNUSED, + void *data UNUSED) { oid_array_append(&recent_objects, &obj->oid); } -static void record_recent_commit(struct commit *commit, void *data) +static void record_recent_commit(struct commit *commit, void *data UNUSED) { oid_array_append(&recent_objects, &commit->object.oid); } -static void get_object_list(int ac, const char **av) +static int mark_bitmap_preferred_tip(const char *refname, + const struct object_id *oid, + int flags UNUSED, + void *data UNUSED) { - struct rev_info revs; + struct object_id peeled; + struct object *object; + + if (!peel_iterated_oid(oid, &peeled)) + oid = &peeled; + + object = parse_object_or_die(oid, refname); + if (object->type == OBJ_COMMIT) + object->flags |= NEEDS_BITMAP; + + return 0; +} + +static void mark_bitmap_preferred_tips(void) +{ + struct string_list_item *item; + const struct string_list *preferred_tips; + + preferred_tips = bitmap_preferred_tips(the_repository); + if (!preferred_tips) + return; + + for_each_string_list_item(item, preferred_tips) { + for_each_ref_in(item->string, mark_bitmap_preferred_tip, NULL); + } +} + +static void get_object_list(struct rev_info *revs, int ac, const char **av) +{ + struct setup_revision_opt s_r_opt = { + .allow_exclude_promisor_objects = 1, + }; char line[1000]; int flags = 0; + int save_warning; - init_revisions(&revs, NULL); save_commit_buffer = 0; - setup_revisions(ac, av, &revs, NULL); + setup_revisions(ac, av, revs, &s_r_opt); /* make sure shallows are read */ is_repository_shallow(the_repository); + save_warning = warn_on_object_refname_ambiguity; + warn_on_object_refname_ambiguity = 0; + while (fgets(line, sizeof(line), stdin) != NULL) { int len = strlen(line); if (len && line[len - 1] == '\n') @@ -2986,47 +4026,55 @@ static void get_object_list(int ac, const char **av) if (starts_with(line, "--shallow ")) { struct object_id oid; if (get_oid_hex(line + 10, &oid)) - die("not an SHA-1 '%s'", line + 10); + die("not an object name '%s'", line + 10); register_shallow(the_repository, &oid); use_bitmap_index = 0; continue; } - die("not a rev '%s'", line); + die(_("not a rev '%s'"), line); } - if (handle_revision_arg(line, &revs, flags, REVARG_CANNOT_BE_FILENAME)) - die("bad revision '%s'", line); + if (handle_revision_arg(line, revs, flags, REVARG_CANNOT_BE_FILENAME)) + die(_("bad revision '%s'"), line); } - if (use_bitmap_index && !get_object_list_from_bitmap(&revs)) + warn_on_object_refname_ambiguity = save_warning; + + if (use_bitmap_index && !get_object_list_from_bitmap(revs)) return; - if (prepare_revision_walk(&revs)) - die("revision walk setup failed"); - mark_edges_uninteresting(&revs, show_edge); + if (use_delta_islands) + load_delta_islands(the_repository, progress); + + if (write_bitmap_index) + mark_bitmap_preferred_tips(); + + if (prepare_revision_walk(revs)) + die(_("revision walk setup failed")); + mark_edges_uninteresting(revs, show_edge, sparse); if (!fn_show_object) fn_show_object = show_object; - traverse_commit_list_filtered(&filter_options, &revs, - show_commit, fn_show_object, NULL, - NULL); + traverse_commit_list(revs, + show_commit, fn_show_object, + NULL); if (unpack_unreachable_expiration) { - revs.ignore_missing_links = 1; - if (add_unseen_recent_objects_to_traversal(&revs, - unpack_unreachable_expiration)) - die("unable to add recent objects"); - if (prepare_revision_walk(&revs)) - die("revision walk setup failed"); - traverse_commit_list(&revs, record_recent_commit, + revs->ignore_missing_links = 1; + if (add_unseen_recent_objects_to_traversal(revs, + unpack_unreachable_expiration, NULL, 0)) + die(_("unable to add recent objects")); + if (prepare_revision_walk(revs)) + die(_("revision walk setup failed")); + traverse_commit_list(revs, record_recent_commit, record_recent_object, NULL); } if (keep_unreachable) - add_objects_in_unpacked_packs(&revs); + add_objects_in_unpacked_packs(); if (pack_loose_unreachable) add_unreachable_loose_objects(); if (unpack_unreachable) - loosen_unused_packed_objects(&revs); + loosen_unused_packed_objects(); oid_array_clear(&recent_objects); } @@ -3038,7 +4086,7 @@ static void add_extra_kept_packs(const struct string_list *names) if (!names->nr) return; - for (p = get_packed_git(the_repository); p; p = p->next) { + for (p = get_all_packs(the_repository); p; p = p->next) { const char *name = basename(p->pack_name); int i; @@ -3062,6 +4110,9 @@ static int option_parse_index_version(const struct option *opt, { char *c; const char *val = arg; + + BUG_ON_OPT_NEG(unset); + pack_idx_opts.version = strtoul(val, &c, 10); if (pack_idx_opts.version > 2) die(_("unsupported index version %s"), val); @@ -3087,16 +4138,33 @@ static int option_parse_unpack_unreachable(const struct option *opt, return 0; } +static int option_parse_cruft_expiration(const struct option *opt, + const char *arg, int unset) +{ + if (unset) { + cruft = 0; + cruft_expiration = 0; + } else { + cruft = 1; + if (arg) + cruft_expiration = approxidate(arg); + } + return 0; +} + int cmd_pack_objects(int argc, const char **argv, const char *prefix) { int use_internal_rev_list = 0; - int thin = 0; int shallow = 0; int all_progress_implied = 0; - struct argv_array rp = ARGV_ARRAY_INIT; + struct strvec rp = STRVEC_INIT; int rev_list_unpacked = 0, rev_list_all = 0, rev_list_reflog = 0; int rev_list_index = 0; + int stdin_packs = 0; struct string_list keep_pack_list = STRING_LIST_INIT_NODUP; + struct list_objects_filter_options filter_options = + LIST_OBJECTS_FILTER_INIT; + struct option pack_objects_options[] = { OPT_SET_INT('q', "quiet", &progress, N_("do not show progress meter"), 0), @@ -3107,9 +4175,9 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) OPT_BOOL(0, "all-progress-implied", &all_progress_implied, N_("similar to --all-progress when progress meter is shown")), - { OPTION_CALLBACK, 0, "index-version", NULL, N_("version[,offset]"), + OPT_CALLBACK_F(0, "index-version", NULL, N_("<version>[,<offset>]"), N_("write the pack index file in the specified idx format version"), - 0, option_parse_index_version }, + PARSE_OPT_NONEG, option_parse_index_version), OPT_MAGNITUDE(0, "max-pack-size", &pack_size_limit, N_("maximum size of each output pack file")), OPT_BOOL(0, "local", &local, @@ -3146,6 +4214,8 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) OPT_SET_INT_F(0, "indexed-objects", &rev_list_index, N_("include objects referred to by the index"), 1, PARSE_OPT_NONEG), + OPT_BOOL(0, "stdin-packs", &stdin_packs, + N_("read packs from stdin")), OPT_BOOL(0, "stdout", &pack_to_stdout, N_("output pack to stdout")), OPT_BOOL(0, "include-tag", &include_tag, @@ -3154,9 +4224,15 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) N_("keep unreachable objects")), OPT_BOOL(0, "pack-loose-unreachable", &pack_loose_unreachable, N_("pack loose unreachable objects")), - { OPTION_CALLBACK, 0, "unpack-unreachable", NULL, N_("time"), + OPT_CALLBACK_F(0, "unpack-unreachable", NULL, N_("time"), N_("unpack unreachable objects newer than <time>"), - PARSE_OPT_OPTARG, option_parse_unpack_unreachable }, + PARSE_OPT_OPTARG, option_parse_unpack_unreachable), + OPT_BOOL(0, "cruft", &cruft, N_("create a cruft pack")), + OPT_CALLBACK_F(0, "cruft-expiration", NULL, N_("time"), + N_("expire cruft objects older than <time>"), + PARSE_OPT_OPTARG, option_parse_cruft_expiration), + OPT_BOOL(0, "sparse", &sparse, + N_("use the sparse reachability algorithm")), OPT_BOOL(0, "thin", &thin, N_("create thin packs")), OPT_BOOL(0, "shallow", &shallow, @@ -3171,24 +4247,43 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) N_("do not hide commits by grafts"), 0), OPT_BOOL(0, "use-bitmap-index", &use_bitmap_index, N_("use a bitmap index if available to speed up counting objects")), - OPT_BOOL(0, "write-bitmap-index", &write_bitmap_index, - N_("write a bitmap index together with the pack index")), + OPT_SET_INT(0, "write-bitmap-index", &write_bitmap_index, + N_("write a bitmap index together with the pack index"), + WRITE_BITMAP_TRUE), + OPT_SET_INT_F(0, "write-bitmap-index-quiet", + &write_bitmap_index, + N_("write a bitmap index if possible"), + WRITE_BITMAP_QUIET, PARSE_OPT_HIDDEN), OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options), - { OPTION_CALLBACK, 0, "missing", NULL, N_("action"), + OPT_CALLBACK_F(0, "missing", NULL, N_("action"), N_("handling for missing objects"), PARSE_OPT_NONEG, - option_parse_missing_action }, + option_parse_missing_action), OPT_BOOL(0, "exclude-promisor-objects", &exclude_promisor_objects, N_("do not pack objects in promisor packfiles")), + OPT_BOOL(0, "delta-islands", &use_delta_islands, + N_("respect islands during delta compression")), + OPT_STRING_LIST(0, "uri-protocol", &uri_protocols, + N_("protocol"), + N_("exclude any configured uploadpack.blobpackfileuri with this protocol")), OPT_END(), }; if (DFS_NUM_STATES > (1 << OE_DFS_STATE_BITS)) BUG("too many dfs states, increase OE_DFS_STATE_BITS"); - check_replace_refs = 0; + read_replace_refs = 0; + + sparse = git_env_bool("GIT_TEST_PACK_SPARSE", -1); + if (the_repository->gitdir) { + prepare_repo_settings(the_repository); + if (sparse < 0) + sparse = the_repository->settings.pack_use_sparse; + } reset_pack_idx_option(&pack_idx_opts); git_config(git_pack_config, NULL); + if (git_env_bool(GIT_TEST_WRITE_REV_INDEX, 0)) + pack_idx_opts.flags |= WRITE_REV; progress = isatty(2); argc = parse_options(argc, argv, prefix, pack_objects_options, @@ -3201,6 +4296,8 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) if (pack_to_stdout != !base_name || argc) usage_with_options(pack_usage, pack_objects_options); + if (depth < 0) + depth = 0; if (depth >= (1 << OE_DEPTH_BITS)) { warning(_("delta chain depth %d is too deep, forcing %d"), depth, (1 << OE_DEPTH_BITS) - 1); @@ -3211,37 +4308,39 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) (1U << OE_Z_DELTA_BITS) - 1); cache_max_small_delta_size = (1U << OE_Z_DELTA_BITS) - 1; } + if (window < 0) + window = 0; - argv_array_push(&rp, "pack-objects"); + strvec_push(&rp, "pack-objects"); if (thin) { use_internal_rev_list = 1; - argv_array_push(&rp, shallow + strvec_push(&rp, shallow ? "--objects-edge-aggressive" : "--objects-edge"); } else - argv_array_push(&rp, "--objects"); + strvec_push(&rp, "--objects"); if (rev_list_all) { use_internal_rev_list = 1; - argv_array_push(&rp, "--all"); + strvec_push(&rp, "--all"); } if (rev_list_reflog) { use_internal_rev_list = 1; - argv_array_push(&rp, "--reflog"); + strvec_push(&rp, "--reflog"); } if (rev_list_index) { use_internal_rev_list = 1; - argv_array_push(&rp, "--indexed-objects"); + strvec_push(&rp, "--indexed-objects"); } - if (rev_list_unpacked) { + if (rev_list_unpacked && !stdin_packs) { use_internal_rev_list = 1; - argv_array_push(&rp, "--unpacked"); + strvec_push(&rp, "--unpacked"); } if (exclude_promisor_objects) { use_internal_rev_list = 1; fetch_if_missing = 0; - argv_array_push(&rp, "--exclude-promisor-objects"); + strvec_push(&rp, "--exclude-promisor-objects"); } if (unpack_unreachable || keep_unreachable || pack_loose_unreachable) use_internal_rev_list = 1; @@ -3251,36 +4350,47 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) if (pack_compression_level == -1) pack_compression_level = Z_DEFAULT_COMPRESSION; else if (pack_compression_level < 0 || pack_compression_level > Z_BEST_COMPRESSION) - die("bad pack compression level %d", pack_compression_level); + die(_("bad pack compression level %d"), pack_compression_level); if (!delta_search_threads) /* --threads=0 means autodetect */ delta_search_threads = online_cpus(); -#ifdef NO_PTHREADS - if (delta_search_threads != 1) - warning("no threads support, ignoring --threads"); -#endif - if (!pack_to_stdout && !pack_size_limit) + if (!HAVE_THREADS && delta_search_threads != 1) + warning(_("no threads support, ignoring --threads")); + if (!pack_to_stdout && !pack_size_limit && !cruft) pack_size_limit = pack_size_limit_cfg; if (pack_to_stdout && pack_size_limit) - die("--max-pack-size cannot be used to build a pack for transfer."); + die(_("--max-pack-size cannot be used to build a pack for transfer")); if (pack_size_limit && pack_size_limit < 1024*1024) { - warning("minimum pack size limit is 1 MiB"); + warning(_("minimum pack size limit is 1 MiB")); pack_size_limit = 1024*1024; } if (!pack_to_stdout && thin) - die("--thin cannot be used to build an indexable pack."); + die(_("--thin cannot be used to build an indexable pack")); if (keep_unreachable && unpack_unreachable) - die("--keep-unreachable and --unpack-unreachable are incompatible."); + die(_("options '%s' and '%s' cannot be used together"), "--keep-unreachable", "--unpack-unreachable"); if (!rev_list_all || !rev_list_reflog || !rev_list_index) unpack_unreachable_expiration = 0; if (filter_options.choice) { if (!pack_to_stdout) - die("cannot use --filter without --stdout."); - use_bitmap_index = 0; + die(_("cannot use --filter without --stdout")); + if (stdin_packs) + die(_("cannot use --filter with --stdin-packs")); + } + + if (stdin_packs && use_internal_rev_list) + die(_("cannot use internal rev list with --stdin-packs")); + + if (cruft) { + if (use_internal_rev_list) + die(_("cannot use internal rev list with --cruft")); + if (stdin_packs) + die(_("cannot use --stdin-packs with --cruft")); + if (pack_size_limit) + die(_("cannot use --max-pack-size with --cruft")); } /* @@ -3305,13 +4415,16 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) if (pack_to_stdout || !rev_list_all) write_bitmap_index = 0; + if (use_delta_islands) + strvec_push(&rp, "--topo-order"); + if (progress && all_progress_implied) progress = 2; add_extra_kept_packs(&keep_pack_list); if (ignore_packed_keep_on_disk) { struct packed_git *p; - for (p = get_packed_git(the_repository); p; p = p->next) + for (p = get_all_packs(the_repository); p; p = p->next) if (p->pack_local && p->pack_keep) break; if (!p) /* no keep-able packs found */ @@ -3324,7 +4437,7 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) * it also covers non-local objects */ struct packed_git *p; - for (p = get_packed_git(the_repository); p; p = p->next) { + for (p = get_all_packs(the_repository); p; p = p->next) { if (!p->pack_local) { have_non_local_packs = 1; break; @@ -3332,29 +4445,63 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) } } - prepare_packing_data(&to_pack); + trace2_region_enter("pack-objects", "enumerate-objects", + the_repository); + prepare_packing_data(the_repository, &to_pack); - if (progress) + if (progress && !cruft) progress_state = start_progress(_("Enumerating objects"), 0); - if (!use_internal_rev_list) + if (stdin_packs) { + /* avoids adding objects in excluded packs */ + ignore_packed_keep_in_core = 1; + read_packs_list_from_stdin(); + if (rev_list_unpacked) + add_unreachable_loose_objects(); + } else if (cruft) { + read_cruft_objects(); + } else if (!use_internal_rev_list) { read_object_list_from_stdin(); - else { - get_object_list(rp.argc, rp.argv); - argv_array_clear(&rp); + } else { + struct rev_info revs; + + repo_init_revisions(the_repository, &revs, NULL); + list_objects_filter_copy(&revs.filter, &filter_options); + get_object_list(&revs, rp.nr, rp.v); + release_revisions(&revs); } cleanup_preferred_base(); if (include_tag && nr_result) - for_each_ref(add_ref_tag, NULL); + for_each_tag_ref(add_ref_tag, NULL); stop_progress(&progress_state); + trace2_region_leave("pack-objects", "enumerate-objects", + the_repository); if (non_empty && !nr_result) - return 0; - if (nr_result) + goto cleanup; + if (nr_result) { + trace2_region_enter("pack-objects", "prepare-pack", + the_repository); prepare_pack(window, depth); + trace2_region_leave("pack-objects", "prepare-pack", + the_repository); + } + + trace2_region_enter("pack-objects", "write-pack-file", the_repository); + write_excluded_by_configs(); write_pack_file(); + trace2_region_leave("pack-objects", "write-pack-file", the_repository); + if (progress) - fprintf(stderr, "Total %"PRIu32" (delta %"PRIu32")," - " reused %"PRIu32" (delta %"PRIu32")\n", - written, written_delta, reused, reused_delta); + fprintf_ln(stderr, + _("Total %"PRIu32" (delta %"PRIu32")," + " reused %"PRIu32" (delta %"PRIu32")," + " pack-reused %"PRIu32), + written, written_delta, reused, reused_delta, + reuse_packfile_objects); + +cleanup: + list_objects_filter_release(&filter_options); + strvec_clear(&rp); + return 0; } |