diff options
Diffstat (limited to 'reftable')
46 files changed, 2219 insertions, 4483 deletions
diff --git a/reftable/basics.c b/reftable/basics.c index 0058619ca6..7d84a5d62d 100644 --- a/reftable/basics.c +++ b/reftable/basics.c @@ -6,7 +6,142 @@ license that can be found in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd */ +#define REFTABLE_ALLOW_BANNED_ALLOCATORS #include "basics.h" +#include "reftable-basics.h" +#include "reftable-error.h" + +static void *(*reftable_malloc_ptr)(size_t sz); +static void *(*reftable_realloc_ptr)(void *, size_t); +static void (*reftable_free_ptr)(void *); + +void *reftable_malloc(size_t sz) +{ + if (reftable_malloc_ptr) + return (*reftable_malloc_ptr)(sz); + return malloc(sz); +} + +void *reftable_realloc(void *p, size_t sz) +{ + if (reftable_realloc_ptr) + return (*reftable_realloc_ptr)(p, sz); + return realloc(p, sz); +} + +void reftable_free(void *p) +{ + if (reftable_free_ptr) + reftable_free_ptr(p); + else + free(p); +} + +void *reftable_calloc(size_t nelem, size_t elsize) +{ + void *p; + + if (nelem && elsize > SIZE_MAX / nelem) + return NULL; + + p = reftable_malloc(nelem * elsize); + if (!p) + return NULL; + + memset(p, 0, nelem * elsize); + return p; +} + +char *reftable_strdup(const char *str) +{ + size_t len = strlen(str); + char *result = reftable_malloc(len + 1); + if (!result) + return NULL; + memcpy(result, str, len + 1); + return result; +} + +void reftable_set_alloc(void *(*malloc)(size_t), + void *(*realloc)(void *, size_t), void (*free)(void *)) +{ + reftable_malloc_ptr = malloc; + reftable_realloc_ptr = realloc; + reftable_free_ptr = free; +} + +void reftable_buf_init(struct reftable_buf *buf) +{ + struct reftable_buf empty = REFTABLE_BUF_INIT; + *buf = empty; +} + +void reftable_buf_release(struct reftable_buf *buf) +{ + reftable_free(buf->buf); + reftable_buf_init(buf); +} + +void reftable_buf_reset(struct reftable_buf *buf) +{ + if (buf->alloc) { + buf->len = 0; + buf->buf[0] = '\0'; + } +} + +int reftable_buf_setlen(struct reftable_buf *buf, size_t len) +{ + if (len > buf->len) + return -1; + if (len == buf->len) + return 0; + buf->buf[len] = '\0'; + buf->len = len; + return 0; +} + +int reftable_buf_cmp(const struct reftable_buf *a, const struct reftable_buf *b) +{ + size_t len = a->len < b->len ? a->len : b->len; + if (len) { + int cmp = memcmp(a->buf, b->buf, len); + if (cmp) + return cmp; + } + return a->len < b->len ? -1 : a->len != b->len; +} + +int reftable_buf_add(struct reftable_buf *buf, const void *data, size_t len) +{ + size_t newlen = buf->len + len; + + if (newlen + 1 > buf->alloc) { + char *reallocated = buf->buf; + REFTABLE_ALLOC_GROW(reallocated, newlen + 1, buf->alloc); + if (!reallocated) + return REFTABLE_OUT_OF_MEMORY_ERROR; + buf->buf = reallocated; + } + + memcpy(buf->buf + buf->len, data, len); + buf->buf[newlen] = '\0'; + buf->len = newlen; + + return 0; +} + +int reftable_buf_addstr(struct reftable_buf *buf, const char *s) +{ + return reftable_buf_add(buf, s, strlen(s)); +} + +char *reftable_buf_detach(struct reftable_buf *buf) +{ + char *result = buf->buf; + reftable_buf_init(buf); + return result; +} void put_be24(uint8_t *out, uint32_t i) { @@ -75,14 +210,14 @@ size_t names_length(const char **names) return p - names; } -void parse_names(char *buf, int size, char ***namesp) +char **parse_names(char *buf, int size) { char **names = NULL; size_t names_cap = 0; size_t names_len = 0; - char *p = buf; char *end = buf + size; + while (p < end) { char *next = strchr(p, '\n'); if (next && next < end) { @@ -91,15 +226,29 @@ void parse_names(char *buf, int size, char ***namesp) next = end; } if (p < next) { - REFTABLE_ALLOC_GROW(names, names_len + 1, names_cap); - names[names_len++] = xstrdup(p); + char **names_grown = names; + REFTABLE_ALLOC_GROW(names_grown, names_len + 1, names_cap); + if (!names_grown) + goto err; + names = names_grown; + + names[names_len] = reftable_strdup(p); + if (!names[names_len++]) + goto err; } p = next + 1; } REFTABLE_REALLOC_ARRAY(names, names_len + 1); names[names_len] = NULL; - *namesp = names; + + return names; + +err: + for (size_t i = 0; i < names_len; i++) + reftable_free(names[i]); + reftable_free(names); + return NULL; } int names_equal(const char **a, const char **b) @@ -111,7 +260,7 @@ int names_equal(const char **a, const char **b) return a[i] == b[i]; } -int common_prefix_size(struct strbuf *a, struct strbuf *b) +int common_prefix_size(struct reftable_buf *a, struct reftable_buf *b) { int p = 0; for (; p < a->len && p < b->len; p++) { @@ -121,3 +270,16 @@ int common_prefix_size(struct strbuf *a, struct strbuf *b) return p; } + +int hash_size(enum reftable_hash id) +{ + if (!id) + return REFTABLE_HASH_SIZE_SHA1; + switch (id) { + case REFTABLE_HASH_SHA1: + return REFTABLE_HASH_SIZE_SHA1; + case REFTABLE_HASH_SHA256: + return REFTABLE_HASH_SIZE_SHA256; + } + abort(); +} diff --git a/reftable/basics.h b/reftable/basics.h index c8fec68d4e..36beda2c25 100644 --- a/reftable/basics.h +++ b/reftable/basics.h @@ -14,6 +14,65 @@ https://developers.google.com/open-source/licenses/bsd */ #include "system.h" +#include "reftable-basics.h" + +struct reftable_buf { + size_t alloc; + size_t len; + char *buf; +}; +#define REFTABLE_BUF_INIT { 0 } + +/* + * Initialize the buffer such that it is ready for use. This is equivalent to + * using REFTABLE_BUF_INIT for stack-allocated variables. + */ +void reftable_buf_init(struct reftable_buf *buf); + +/* + * Release memory associated with the buffer. The buffer is reinitialized such + * that it can be reused for subsequent operations. + */ +void reftable_buf_release(struct reftable_buf *buf); + +/* + * Reset the buffer such that it is effectively empty, without releasing the + * memory that this structure holds on to. This is equivalent to calling + * `reftable_buf_setlen(buf, 0)`. + */ +void reftable_buf_reset(struct reftable_buf *buf); + +/* + * Trim the buffer to a shorter length by updating the `len` member and writing + * a NUL byte to `buf[len]`. Returns 0 on success, -1 when `len` points outside + * of the array. + */ +int reftable_buf_setlen(struct reftable_buf *buf, size_t len); + +/* + * Lexicographically compare the two buffers. Returns 0 when both buffers have + * the same contents, -1 when `a` is lexicographically smaller than `b`, and 1 + * otherwise. + */ +int reftable_buf_cmp(const struct reftable_buf *a, const struct reftable_buf *b); + +/* + * Append `len` bytes from `data` to the buffer. This function works with + * arbitrary byte sequences, including ones that contain embedded NUL + * characters. As such, we use `void *` as input type. Returns 0 on success, + * REFTABLE_OUT_OF_MEMORY_ERROR on allocation failure. + */ +int reftable_buf_add(struct reftable_buf *buf, const void *data, size_t len); + +/* Equivalent to `reftable_buf_add(buf, s, strlen(s))`. */ +int reftable_buf_addstr(struct reftable_buf *buf, const char *s); + +/* + * Detach the buffer from the structure such that the underlying memory is now + * owned by the caller. The buffer is reinitialized such that it can be reused + * for subsequent operations. + */ +char *reftable_buf_detach(struct reftable_buf *buf); /* Bigendian en/decoding of integers */ @@ -37,9 +96,12 @@ size_t binsearch(size_t sz, int (*f)(size_t k, void *args), void *args); */ void free_names(char **a); -/* parse a newline separated list of names. `size` is the length of the buffer, - * without terminating '\0'. Empty names are discarded. */ -void parse_names(char *buf, int size, char ***namesp); +/* + * Parse a newline separated list of names. `size` is the length of the buffer, + * without terminating '\0'. Empty names are discarded. Returns a `NULL` + * pointer when allocations fail. + */ +char **parse_names(char *buf, int size); /* compares two NULL-terminated arrays of strings. */ int names_equal(const char **a, const char **b); @@ -53,6 +115,7 @@ void *reftable_malloc(size_t sz); void *reftable_realloc(void *p, size_t sz); void reftable_free(void *p); void *reftable_calloc(size_t nelem, size_t elsize); +char *reftable_strdup(const char *str); #define REFTABLE_ALLOC_ARRAY(x, alloc) (x) = reftable_malloc(st_mult(sizeof(*(x)), (alloc))) #define REFTABLE_CALLOC_ARRAY(x, alloc) (x) = reftable_calloc((alloc), sizeof(*(x))) @@ -66,9 +129,33 @@ void *reftable_calloc(size_t nelem, size_t elsize); REFTABLE_REALLOC_ARRAY(x, alloc); \ } \ } while (0) +#define REFTABLE_FREE_AND_NULL(p) do { reftable_free(p); (p) = NULL; } while (0) + +#ifndef REFTABLE_ALLOW_BANNED_ALLOCATORS +# define REFTABLE_BANNED(func) use_reftable_##func##_instead +# undef malloc +# define malloc(sz) REFTABLE_BANNED(malloc) +# undef realloc +# define realloc(ptr, sz) REFTABLE_BANNED(realloc) +# undef free +# define free(ptr) REFTABLE_BANNED(free) +# undef calloc +# define calloc(nelem, elsize) REFTABLE_BANNED(calloc) +# undef strdup +# define strdup(str) REFTABLE_BANNED(strdup) +#endif /* Find the longest shared prefix size of `a` and `b` */ -struct strbuf; -int common_prefix_size(struct strbuf *a, struct strbuf *b); +int common_prefix_size(struct reftable_buf *a, struct reftable_buf *b); + +int hash_size(enum reftable_hash id); + +/* + * Format IDs that identify the hash function used by a reftable. Note that + * these constants end up on disk and thus mustn't change. The format IDs are + * "sha1" and "s256" in big endian, respectively. + */ +#define REFTABLE_FORMAT_ID_SHA1 ((uint32_t) 0x73686131) +#define REFTABLE_FORMAT_ID_SHA256 ((uint32_t) 0x73323536) #endif diff --git a/reftable/block.c b/reftable/block.c index 00030eee06..0198078485 100644 --- a/reftable/block.c +++ b/reftable/block.c @@ -38,9 +38,11 @@ int footer_size(int version) } static int block_writer_register_restart(struct block_writer *w, int n, - int is_restart, struct strbuf *key) + int is_restart, struct reftable_buf *key) { - int rlen = w->restart_len; + int rlen, err; + + rlen = w->restart_len; if (rlen >= MAX_RESTARTS) { is_restart = 0; } @@ -52,25 +54,30 @@ static int block_writer_register_restart(struct block_writer *w, int n, return -1; if (is_restart) { REFTABLE_ALLOC_GROW(w->restarts, w->restart_len + 1, w->restart_cap); + if (!w->restarts) + return REFTABLE_OUT_OF_MEMORY_ERROR; w->restarts[w->restart_len++] = w->next; } w->next += n; - strbuf_reset(&w->last_key); - strbuf_addbuf(&w->last_key, key); + reftable_buf_reset(&w->last_key); + err = reftable_buf_add(&w->last_key, key->buf, key->len); + if (err < 0) + return err; + w->entries++; return 0; } -void block_writer_init(struct block_writer *bw, uint8_t typ, uint8_t *buf, - uint32_t block_size, uint32_t header_off, int hash_size) +int block_writer_init(struct block_writer *bw, uint8_t typ, uint8_t *block, + uint32_t block_size, uint32_t header_off, int hash_size) { - bw->buf = buf; + bw->block = block; bw->hash_size = hash_size; bw->block_size = block_size; bw->header_off = header_off; - bw->buf[header_off] = typ; + bw->block[header_off] = typ; bw->next = header_off + 4; bw->restart_interval = 16; bw->entries = 0; @@ -78,13 +85,17 @@ void block_writer_init(struct block_writer *bw, uint8_t typ, uint8_t *buf, bw->last_key.len = 0; if (!bw->zstream) { REFTABLE_CALLOC_ARRAY(bw->zstream, 1); + if (!bw->zstream) + return REFTABLE_OUT_OF_MEMORY_ERROR; deflateInit(bw->zstream, 9); } + + return 0; } uint8_t block_writer_type(struct block_writer *bw) { - return bw->buf[bw->header_off]; + return bw->block[bw->header_off]; } /* Adds the reftable_record to the block. Returns -1 if it does not fit, 0 on @@ -92,42 +103,45 @@ uint8_t block_writer_type(struct block_writer *bw) empty key. */ int block_writer_add(struct block_writer *w, struct reftable_record *rec) { - struct strbuf empty = STRBUF_INIT; - struct strbuf last = + struct reftable_buf empty = REFTABLE_BUF_INIT; + struct reftable_buf last = w->entries % w->restart_interval == 0 ? empty : w->last_key; struct string_view out = { - .buf = w->buf + w->next, + .buf = w->block + w->next, .len = w->block_size - w->next, }; - struct string_view start = out; - int is_restart = 0; - struct strbuf key = STRBUF_INIT; int n = 0; - int err = -1; + int err; + + err = reftable_record_key(rec, &w->scratch); + if (err < 0) + goto done; - reftable_record_key(rec, &key); - if (!key.len) { + if (!w->scratch.len) { err = REFTABLE_API_ERROR; goto done; } - n = reftable_encode_key(&is_restart, out, last, key, + n = reftable_encode_key(&is_restart, out, last, w->scratch, reftable_record_val_type(rec)); - if (n < 0) + if (n < 0) { + err = -1; goto done; + } string_view_consume(&out, n); n = reftable_record_encode(rec, out, w->hash_size); - if (n < 0) + if (n < 0) { + err = -1; goto done; + } string_view_consume(&out, n); err = block_writer_register_restart(w, start.len - out.len, is_restart, - &key); + &w->scratch); done: - strbuf_release(&key); return err; } @@ -135,13 +149,13 @@ int block_writer_finish(struct block_writer *w) { int i; for (i = 0; i < w->restart_len; i++) { - put_be24(w->buf + w->next, w->restarts[i]); + put_be24(w->block + w->next, w->restarts[i]); w->next += 3; } - put_be16(w->buf + w->next, w->restart_len); + put_be16(w->block + w->next, w->restart_len); w->next += 2; - put_be24(w->buf + 1 + w->header_off, w->next); + put_be24(w->block + 1 + w->header_off, w->next); /* * Log records are stored zlib-compressed. Note that the compression @@ -163,10 +177,14 @@ int block_writer_finish(struct block_writer *w) */ compressed_len = deflateBound(w->zstream, src_len); REFTABLE_ALLOC_GROW(w->compressed, compressed_len, w->compressed_cap); + if (!w->compressed) { + ret = REFTABLE_OUT_OF_MEMORY_ERROR; + return ret; + } w->zstream->next_out = w->compressed; w->zstream->avail_out = compressed_len; - w->zstream->next_in = w->buf + block_header_skip; + w->zstream->next_in = w->block + block_header_skip; w->zstream->avail_in = src_len; /* @@ -184,7 +202,7 @@ int block_writer_finish(struct block_writer *w) * adjust the `next` pointer to point right after the * compressed data. */ - memcpy(w->buf + block_header_skip, w->compressed, + memcpy(w->block + block_header_skip, w->compressed, w->zstream->total_out); w->next = w->zstream->total_out + block_header_skip; } @@ -219,12 +237,21 @@ int block_reader_init(struct block_reader *br, struct reftable_block *block, /* Log blocks specify the *uncompressed* size in their header. */ REFTABLE_ALLOC_GROW(br->uncompressed_data, sz, br->uncompressed_cap); + if (!br->uncompressed_data) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto done; + } /* Copy over the block header verbatim. It's not compressed. */ memcpy(br->uncompressed_data, block->data, block_header_skip); if (!br->zstream) { REFTABLE_CALLOC_ARRAY(br->zstream, 1); + if (!br->zstream) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto done; + } + err = inflateInit(br->zstream); } else { err = inflateReset(br->zstream); @@ -306,7 +333,7 @@ uint8_t block_reader_type(const struct block_reader *r) return r->block.data[r->header_off]; } -int block_reader_first_key(const struct block_reader *br, struct strbuf *key) +int block_reader_first_key(const struct block_reader *br, struct reftable_buf *key) { int off = br->header_off + 4, n; struct string_view in = { @@ -315,7 +342,7 @@ int block_reader_first_key(const struct block_reader *br, struct strbuf *key) }; uint8_t extra = 0; - strbuf_reset(key); + reftable_buf_reset(key); n = reftable_decode_key(key, &extra, in); if (n < 0) @@ -336,13 +363,13 @@ void block_iter_seek_start(struct block_iter *it, const struct block_reader *br) it->block = br->block.data; it->block_len = br->block_len; it->hash_size = br->hash_size; - strbuf_reset(&it->last_key); + reftable_buf_reset(&it->last_key); it->next_off = br->header_off + 4; } struct restart_needle_less_args { int error; - struct strbuf needle; + struct reftable_buf needle; const struct block_reader *reader; }; @@ -414,7 +441,7 @@ int block_iter_next(struct block_iter *it, struct reftable_record *rec) void block_iter_reset(struct block_iter *it) { - strbuf_reset(&it->last_key); + reftable_buf_reset(&it->last_key); it->next_off = 0; it->block = NULL; it->block_len = 0; @@ -423,12 +450,12 @@ void block_iter_reset(struct block_iter *it) void block_iter_close(struct block_iter *it) { - strbuf_release(&it->last_key); - strbuf_release(&it->scratch); + reftable_buf_release(&it->last_key); + reftable_buf_release(&it->scratch); } int block_iter_seek_key(struct block_iter *it, const struct block_reader *br, - struct strbuf *want) + struct reftable_buf *want) { struct restart_needle_less_args args = { .needle = *want, @@ -503,6 +530,10 @@ int block_iter_seek_key(struct block_iter *it, const struct block_reader *br, goto done; } + err = reftable_record_key(&rec, &it->last_key); + if (err < 0) + goto done; + /* * Check whether the current key is greater or equal to the * sought-after key. In case it is greater we know that the @@ -517,8 +548,7 @@ int block_iter_seek_key(struct block_iter *it, const struct block_reader *br, * to `last_key` now, and naturally all keys share a prefix * with themselves. */ - reftable_record_key(&rec, &it->last_key); - if (strbuf_cmp(&it->last_key, want) >= 0) { + if (reftable_buf_cmp(&it->last_key, want) >= 0) { it->next_off = prev_off; goto done; } @@ -532,10 +562,11 @@ done: void block_writer_release(struct block_writer *bw) { deflateEnd(bw->zstream); - FREE_AND_NULL(bw->zstream); - FREE_AND_NULL(bw->restarts); - FREE_AND_NULL(bw->compressed); - strbuf_release(&bw->last_key); + REFTABLE_FREE_AND_NULL(bw->zstream); + REFTABLE_FREE_AND_NULL(bw->restarts); + REFTABLE_FREE_AND_NULL(bw->compressed); + reftable_buf_release(&bw->scratch); + reftable_buf_release(&bw->last_key); /* the block is not owned. */ } diff --git a/reftable/block.h b/reftable/block.h index 1c8f25ee6e..0431e8591f 100644 --- a/reftable/block.h +++ b/reftable/block.h @@ -22,7 +22,7 @@ struct block_writer { unsigned char *compressed; size_t compressed_cap; - uint8_t *buf; + uint8_t *block; uint32_t block_size; /* Offset of the global header. Nonzero in the first block only. */ @@ -38,15 +38,17 @@ struct block_writer { uint32_t restart_len; uint32_t restart_cap; - struct strbuf last_key; + struct reftable_buf last_key; + /* Scratch buffer used to avoid allocations. */ + struct reftable_buf scratch; int entries; }; /* - * initializes the blockwriter to write `typ` entries, using `buf` as temporary - * storage. `buf` is not owned by the block_writer. */ -void block_writer_init(struct block_writer *bw, uint8_t typ, uint8_t *buf, - uint32_t block_size, uint32_t header_off, int hash_size); + * initializes the blockwriter to write `typ` entries, using `block` as temporary + * storage. `block` is not owned by the block_writer. */ +int block_writer_init(struct block_writer *bw, uint8_t typ, uint8_t *block, + uint32_t block_size, uint32_t header_off, int hash_size); /* returns the block type (eg. 'r' for ref records. */ uint8_t block_writer_type(struct block_writer *bw); @@ -98,7 +100,7 @@ void block_reader_release(struct block_reader *br); uint8_t block_reader_type(const struct block_reader *r); /* Decodes the first key in the block */ -int block_reader_first_key(const struct block_reader *br, struct strbuf *key); +int block_reader_first_key(const struct block_reader *br, struct reftable_buf *key); /* Iterate over entries in a block */ struct block_iter { @@ -109,13 +111,13 @@ struct block_iter { int hash_size; /* key for last entry we read. */ - struct strbuf last_key; - struct strbuf scratch; + struct reftable_buf last_key; + struct reftable_buf scratch; }; #define BLOCK_ITER_INIT { \ - .last_key = STRBUF_INIT, \ - .scratch = STRBUF_INIT, \ + .last_key = REFTABLE_BUF_INIT, \ + .scratch = REFTABLE_BUF_INIT, \ } /* Position `it` at start of the block */ @@ -123,7 +125,7 @@ void block_iter_seek_start(struct block_iter *it, const struct block_reader *br) /* Position `it` to the `want` key in the block */ int block_iter_seek_key(struct block_iter *it, const struct block_reader *br, - struct strbuf *want); + struct reftable_buf *want); /* return < 0 for error, 0 for OK, > 0 for EOF. */ int block_iter_next(struct block_iter *it, struct reftable_record *rec); diff --git a/reftable/block_test.c b/reftable/block_test.c deleted file mode 100644 index 90aecd5a7c..0000000000 --- a/reftable/block_test.c +++ /dev/null @@ -1,123 +0,0 @@ -/* -Copyright 2020 Google LLC - -Use of this source code is governed by a BSD-style -license that can be found in the LICENSE file or at -https://developers.google.com/open-source/licenses/bsd -*/ - -#include "block.h" - -#include "system.h" -#include "blocksource.h" -#include "basics.h" -#include "constants.h" -#include "record.h" -#include "test_framework.h" -#include "reftable-tests.h" - -static void test_block_read_write(void) -{ - const int header_off = 21; /* random */ - char *names[30]; - const int N = ARRAY_SIZE(names); - const int block_size = 1024; - struct reftable_block block = { NULL }; - struct block_writer bw = { - .last_key = STRBUF_INIT, - }; - struct reftable_record rec = { - .type = BLOCK_TYPE_REF, - }; - int i = 0; - int n; - struct block_reader br = { 0 }; - struct block_iter it = BLOCK_ITER_INIT; - int j = 0; - struct strbuf want = STRBUF_INIT; - - REFTABLE_CALLOC_ARRAY(block.data, block_size); - block.len = block_size; - block.source = malloc_block_source(); - block_writer_init(&bw, BLOCK_TYPE_REF, block.data, block_size, - header_off, hash_size(GIT_SHA1_FORMAT_ID)); - - rec.u.ref.refname = (char *) ""; - rec.u.ref.value_type = REFTABLE_REF_DELETION; - n = block_writer_add(&bw, &rec); - EXPECT(n == REFTABLE_API_ERROR); - - for (i = 0; i < N; i++) { - char name[100]; - snprintf(name, sizeof(name), "branch%02d", i); - - rec.u.ref.refname = name; - rec.u.ref.value_type = REFTABLE_REF_VAL1; - memset(rec.u.ref.value.val1, i, GIT_SHA1_RAWSZ); - - names[i] = xstrdup(name); - n = block_writer_add(&bw, &rec); - rec.u.ref.refname = NULL; - rec.u.ref.value_type = REFTABLE_REF_DELETION; - EXPECT(n == 0); - } - - n = block_writer_finish(&bw); - EXPECT(n > 0); - - block_writer_release(&bw); - - block_reader_init(&br, &block, header_off, block_size, GIT_SHA1_RAWSZ); - - block_iter_seek_start(&it, &br); - - while (1) { - int r = block_iter_next(&it, &rec); - EXPECT(r >= 0); - if (r > 0) { - break; - } - EXPECT_STREQ(names[j], rec.u.ref.refname); - j++; - } - - reftable_record_release(&rec); - block_iter_close(&it); - - for (i = 0; i < N; i++) { - struct block_iter it = BLOCK_ITER_INIT; - strbuf_reset(&want); - strbuf_addstr(&want, names[i]); - - n = block_iter_seek_key(&it, &br, &want); - EXPECT(n == 0); - - n = block_iter_next(&it, &rec); - EXPECT(n == 0); - - EXPECT_STREQ(names[i], rec.u.ref.refname); - - want.len--; - n = block_iter_seek_key(&it, &br, &want); - EXPECT(n == 0); - - n = block_iter_next(&it, &rec); - EXPECT(n == 0); - EXPECT_STREQ(names[10 * (i / 10)], rec.u.ref.refname); - - block_iter_close(&it); - } - - reftable_record_release(&rec); - reftable_block_done(&br.block); - strbuf_release(&want); - for (i = 0; i < N; i++) { - reftable_free(names[i]); - } -} - -int block_test_main(int argc, const char *argv[]) -{ - RUN_TEST(test_block_read_write); - return 0; -} diff --git a/reftable/blocksource.c b/reftable/blocksource.c index eeed254ba9..52e0915a67 100644 --- a/reftable/blocksource.c +++ b/reftable/blocksource.c @@ -13,68 +13,50 @@ https://developers.google.com/open-source/licenses/bsd #include "reftable-blocksource.h" #include "reftable-error.h" -static void strbuf_return_block(void *b, struct reftable_block *dest) +static void reftable_buf_return_block(void *b UNUSED, struct reftable_block *dest) { if (dest->len) memset(dest->data, 0xff, dest->len); reftable_free(dest->data); } -static void strbuf_close(void *b) +static void reftable_buf_close(void *b UNUSED) { } -static int strbuf_read_block(void *v, struct reftable_block *dest, uint64_t off, - uint32_t size) +static int reftable_buf_read_block(void *v, struct reftable_block *dest, + uint64_t off, uint32_t size) { - struct strbuf *b = v; + struct reftable_buf *b = v; assert(off + size <= b->len); REFTABLE_CALLOC_ARRAY(dest->data, size); + if (!dest->data) + return -1; memcpy(dest->data, b->buf + off, size); dest->len = size; return size; } -static uint64_t strbuf_size(void *b) +static uint64_t reftable_buf_size(void *b) { - return ((struct strbuf *)b)->len; + return ((struct reftable_buf *)b)->len; } -static struct reftable_block_source_vtable strbuf_vtable = { - .size = &strbuf_size, - .read_block = &strbuf_read_block, - .return_block = &strbuf_return_block, - .close = &strbuf_close, +static struct reftable_block_source_vtable reftable_buf_vtable = { + .size = &reftable_buf_size, + .read_block = &reftable_buf_read_block, + .return_block = &reftable_buf_return_block, + .close = &reftable_buf_close, }; -void block_source_from_strbuf(struct reftable_block_source *bs, - struct strbuf *buf) +void block_source_from_buf(struct reftable_block_source *bs, + struct reftable_buf *buf) { assert(!bs->ops); - bs->ops = &strbuf_vtable; + bs->ops = &reftable_buf_vtable; bs->arg = buf; } -static void malloc_return_block(void *b, struct reftable_block *dest) -{ - if (dest->len) - memset(dest->data, 0xff, dest->len); - reftable_free(dest->data); -} - -static struct reftable_block_source_vtable malloc_vtable = { - .return_block = &malloc_return_block, -}; - -static struct reftable_block_source malloc_block_source_instance = { - .ops = &malloc_vtable, -}; - -struct reftable_block_source malloc_block_source(void) -{ - return malloc_block_source_instance; -} - struct file_block_source { uint64_t size; unsigned char *data; @@ -85,7 +67,7 @@ static uint64_t file_size(void *b) return ((struct file_block_source *)b)->size; } -static void file_return_block(void *b, struct reftable_block *dest) +static void file_return_block(void *b UNUSED, struct reftable_block *dest UNUSED) { } @@ -118,27 +100,40 @@ int reftable_block_source_from_file(struct reftable_block_source *bs, { struct file_block_source *p; struct stat st; - int fd; + int fd, err; fd = open(name, O_RDONLY); if (fd < 0) { if (errno == ENOENT) return REFTABLE_NOT_EXIST_ERROR; - return -1; + err = -1; + goto out; } if (fstat(fd, &st) < 0) { - close(fd); - return REFTABLE_IO_ERROR; + err = REFTABLE_IO_ERROR; + goto out; } REFTABLE_CALLOC_ARRAY(p, 1); + if (!p) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto out; + } + p->size = st.st_size; p->data = xmmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); - close(fd); assert(!bs->ops); bs->ops = &file_vtable; bs->arg = p; + + err = 0; + +out: + if (fd >= 0) + close(fd); + if (err < 0) + reftable_free(p); return 0; } diff --git a/reftable/blocksource.h b/reftable/blocksource.h index 072e2727ad..a84a3ccd89 100644 --- a/reftable/blocksource.h +++ b/reftable/blocksource.h @@ -12,11 +12,10 @@ https://developers.google.com/open-source/licenses/bsd #include "system.h" struct reftable_block_source; +struct reftable_buf; /* Create an in-memory block source for reading reftables */ -void block_source_from_strbuf(struct reftable_block_source *bs, - struct strbuf *buf); - -struct reftable_block_source malloc_block_source(void); +void block_source_from_buf(struct reftable_block_source *bs, + struct reftable_buf *buf); #endif diff --git a/reftable/dump.c b/reftable/dump.c deleted file mode 100644 index dd65d9e8bb..0000000000 --- a/reftable/dump.c +++ /dev/null @@ -1,111 +0,0 @@ -/* -Copyright 2020 Google LLC - -Use of this source code is governed by a BSD-style -license that can be found in the LICENSE file or at -https://developers.google.com/open-source/licenses/bsd -*/ - -#include "git-compat-util.h" -#include "hash.h" - -#include "reftable-blocksource.h" -#include "reftable-error.h" -#include "reftable-record.h" -#include "reftable-tests.h" -#include "reftable-writer.h" -#include "reftable-iterator.h" -#include "reftable-reader.h" -#include "reftable-stack.h" - -#include <stddef.h> -#include <stdio.h> -#include <stdlib.h> -#include <unistd.h> -#include <string.h> - -static int compact_stack(const char *stackdir) -{ - struct reftable_stack *stack = NULL; - struct reftable_write_options opts = { 0 }; - - int err = reftable_new_stack(&stack, stackdir, &opts); - if (err < 0) - goto done; - - err = reftable_stack_compact_all(stack, NULL); - if (err < 0) - goto done; -done: - if (stack) { - reftable_stack_destroy(stack); - } - return err; -} - -static void print_help(void) -{ - printf("usage: dump [-cst] arg\n\n" - "options: \n" - " -c compact\n" - " -b dump blocks\n" - " -t dump table\n" - " -s dump stack\n" - " -6 sha256 hash format\n" - " -h this help\n" - "\n"); -} - -int reftable_dump_main(int argc, char *const *argv) -{ - int err = 0; - int opt_dump_blocks = 0; - int opt_dump_table = 0; - int opt_dump_stack = 0; - int opt_compact = 0; - uint32_t opt_hash_id = GIT_SHA1_FORMAT_ID; - const char *arg = NULL, *argv0 = argv[0]; - - for (; argc > 1; argv++, argc--) - if (*argv[1] != '-') - break; - else if (!strcmp("-b", argv[1])) - opt_dump_blocks = 1; - else if (!strcmp("-t", argv[1])) - opt_dump_table = 1; - else if (!strcmp("-6", argv[1])) - opt_hash_id = GIT_SHA256_FORMAT_ID; - else if (!strcmp("-s", argv[1])) - opt_dump_stack = 1; - else if (!strcmp("-c", argv[1])) - opt_compact = 1; - else if (!strcmp("-?", argv[1]) || !strcmp("-h", argv[1])) { - print_help(); - return 2; - } - - if (argc != 2) { - fprintf(stderr, "need argument\n"); - print_help(); - return 2; - } - - arg = argv[1]; - - if (opt_dump_blocks) { - err = reftable_reader_print_blocks(arg); - } else if (opt_dump_table) { - err = reftable_reader_print_file(arg); - } else if (opt_dump_stack) { - err = reftable_stack_print_directory(arg, opt_hash_id); - } else if (opt_compact) { - err = compact_stack(arg); - } - - if (err < 0) { - fprintf(stderr, "%s: %s: %s\n", argv0, arg, - reftable_error_str(err)); - return 1; - } - return 0; -} diff --git a/reftable/error.c b/reftable/error.c index a25f28a43e..660d029617 100644 --- a/reftable/error.c +++ b/reftable/error.c @@ -35,6 +35,8 @@ const char *reftable_error_str(int err) return "entry too large"; case REFTABLE_OUTDATED_ERROR: return "data concurrently modified"; + case REFTABLE_OUT_OF_MEMORY_ERROR: + return "out of memory"; case -1: return "general error"; default: diff --git a/reftable/generic.c b/reftable/generic.c deleted file mode 100644 index 28ae26145e..0000000000 --- a/reftable/generic.c +++ /dev/null @@ -1,229 +0,0 @@ -/* -Copyright 2020 Google LLC - -Use of this source code is governed by a BSD-style -license that can be found in the LICENSE file or at -https://developers.google.com/open-source/licenses/bsd -*/ - -#include "constants.h" -#include "record.h" -#include "generic.h" -#include "reftable-iterator.h" -#include "reftable-generic.h" - -void table_init_iter(struct reftable_table *tab, - struct reftable_iterator *it, - uint8_t typ) -{ - - tab->ops->init_iter(tab->table_arg, it, typ); -} - -void reftable_table_init_ref_iter(struct reftable_table *tab, - struct reftable_iterator *it) -{ - table_init_iter(tab, it, BLOCK_TYPE_REF); -} - -void reftable_table_init_log_iter(struct reftable_table *tab, - struct reftable_iterator *it) -{ - table_init_iter(tab, it, BLOCK_TYPE_LOG); -} - -int reftable_iterator_seek_ref(struct reftable_iterator *it, - const char *name) -{ - struct reftable_record want = { - .type = BLOCK_TYPE_REF, - .u.ref = { - .refname = (char *)name, - }, - }; - return it->ops->seek(it->iter_arg, &want); -} - -int reftable_iterator_seek_log_at(struct reftable_iterator *it, - const char *name, uint64_t update_index) -{ - struct reftable_record want = { - .type = BLOCK_TYPE_LOG, - .u.log = { - .refname = (char *)name, - .update_index = update_index, - }, - }; - return it->ops->seek(it->iter_arg, &want); -} - -int reftable_iterator_seek_log(struct reftable_iterator *it, - const char *name) -{ - return reftable_iterator_seek_log_at(it, name, ~((uint64_t) 0)); -} - -int reftable_table_read_ref(struct reftable_table *tab, const char *name, - struct reftable_ref_record *ref) -{ - struct reftable_iterator it = { NULL }; - int err; - - reftable_table_init_ref_iter(tab, &it); - - err = reftable_iterator_seek_ref(&it, name); - if (err) - goto done; - - err = reftable_iterator_next_ref(&it, ref); - if (err) - goto done; - - if (strcmp(ref->refname, name) || - reftable_ref_record_is_deletion(ref)) { - reftable_ref_record_release(ref); - err = 1; - goto done; - } - -done: - reftable_iterator_destroy(&it); - return err; -} - -int reftable_table_print(struct reftable_table *tab) { - struct reftable_iterator it = { NULL }; - struct reftable_ref_record ref = { NULL }; - struct reftable_log_record log = { NULL }; - uint32_t hash_id = reftable_table_hash_id(tab); - int err; - - reftable_table_init_ref_iter(tab, &it); - - err = reftable_iterator_seek_ref(&it, ""); - if (err < 0) - return err; - - while (1) { - err = reftable_iterator_next_ref(&it, &ref); - if (err > 0) { - break; - } - if (err < 0) { - return err; - } - reftable_ref_record_print(&ref, hash_id); - } - reftable_iterator_destroy(&it); - reftable_ref_record_release(&ref); - - reftable_table_init_log_iter(tab, &it); - - err = reftable_iterator_seek_log(&it, ""); - if (err < 0) - return err; - - while (1) { - err = reftable_iterator_next_log(&it, &log); - if (err > 0) { - break; - } - if (err < 0) { - return err; - } - reftable_log_record_print(&log, hash_id); - } - reftable_iterator_destroy(&it); - reftable_log_record_release(&log); - return 0; -} - -uint64_t reftable_table_max_update_index(struct reftable_table *tab) -{ - return tab->ops->max_update_index(tab->table_arg); -} - -uint64_t reftable_table_min_update_index(struct reftable_table *tab) -{ - return tab->ops->min_update_index(tab->table_arg); -} - -uint32_t reftable_table_hash_id(struct reftable_table *tab) -{ - return tab->ops->hash_id(tab->table_arg); -} - -void reftable_iterator_destroy(struct reftable_iterator *it) -{ - if (!it->ops) { - return; - } - it->ops->close(it->iter_arg); - it->ops = NULL; - FREE_AND_NULL(it->iter_arg); -} - -int reftable_iterator_next_ref(struct reftable_iterator *it, - struct reftable_ref_record *ref) -{ - struct reftable_record rec = { - .type = BLOCK_TYPE_REF, - .u = { - .ref = *ref - }, - }; - int err = iterator_next(it, &rec); - *ref = rec.u.ref; - return err; -} - -int reftable_iterator_next_log(struct reftable_iterator *it, - struct reftable_log_record *log) -{ - struct reftable_record rec = { - .type = BLOCK_TYPE_LOG, - .u = { - .log = *log, - }, - }; - int err = iterator_next(it, &rec); - *log = rec.u.log; - return err; -} - -int iterator_seek(struct reftable_iterator *it, struct reftable_record *want) -{ - return it->ops->seek(it->iter_arg, want); -} - -int iterator_next(struct reftable_iterator *it, struct reftable_record *rec) -{ - return it->ops->next(it->iter_arg, rec); -} - -static int empty_iterator_seek(void *arg, struct reftable_record *want) -{ - return 0; -} - -static int empty_iterator_next(void *arg, struct reftable_record *rec) -{ - return 1; -} - -static void empty_iterator_close(void *arg) -{ -} - -static struct reftable_iterator_vtable empty_vtable = { - .seek = &empty_iterator_seek, - .next = &empty_iterator_next, - .close = &empty_iterator_close, -}; - -void iterator_set_empty(struct reftable_iterator *it) -{ - assert(!it->ops); - it->iter_arg = NULL; - it->ops = &empty_vtable; -} diff --git a/reftable/generic.h b/reftable/generic.h deleted file mode 100644 index 8341fa570e..0000000000 --- a/reftable/generic.h +++ /dev/null @@ -1,37 +0,0 @@ -/* -Copyright 2020 Google LLC - -Use of this source code is governed by a BSD-style -license that can be found in the LICENSE file or at -https://developers.google.com/open-source/licenses/bsd -*/ - -#ifndef GENERIC_H -#define GENERIC_H - -#include "record.h" -#include "reftable-generic.h" - -/* generic interface to reftables */ -struct reftable_table_vtable { - void (*init_iter)(void *tab, struct reftable_iterator *it, uint8_t typ); - uint32_t (*hash_id)(void *tab); - uint64_t (*min_update_index)(void *tab); - uint64_t (*max_update_index)(void *tab); -}; - -void table_init_iter(struct reftable_table *tab, - struct reftable_iterator *it, - uint8_t typ); - -struct reftable_iterator_vtable { - int (*seek)(void *iter_arg, struct reftable_record *want); - int (*next)(void *iter_arg, struct reftable_record *rec); - void (*close)(void *iter_arg); -}; - -void iterator_set_empty(struct reftable_iterator *it); -int iterator_seek(struct reftable_iterator *it, struct reftable_record *want); -int iterator_next(struct reftable_iterator *it, struct reftable_record *rec); - -#endif diff --git a/reftable/iter.c b/reftable/iter.c index fddea31e51..86e801ca9f 100644 --- a/reftable/iter.c +++ b/reftable/iter.c @@ -11,15 +11,51 @@ https://developers.google.com/open-source/licenses/bsd #include "system.h" #include "block.h" -#include "generic.h" #include "constants.h" #include "reader.h" #include "reftable-error.h" +int iterator_seek(struct reftable_iterator *it, struct reftable_record *want) +{ + return it->ops->seek(it->iter_arg, want); +} + +int iterator_next(struct reftable_iterator *it, struct reftable_record *rec) +{ + return it->ops->next(it->iter_arg, rec); +} + +static int empty_iterator_seek(void *arg UNUSED, struct reftable_record *want UNUSED) +{ + return 0; +} + +static int empty_iterator_next(void *arg UNUSED, struct reftable_record *rec UNUSED) +{ + return 1; +} + +static void empty_iterator_close(void *arg UNUSED) +{ +} + +static struct reftable_iterator_vtable empty_vtable = { + .seek = &empty_iterator_seek, + .next = &empty_iterator_next, + .close = &empty_iterator_close, +}; + +void iterator_set_empty(struct reftable_iterator *it) +{ + assert(!it->ops); + it->iter_arg = NULL; + it->ops = &empty_vtable; +} + static void filtering_ref_iterator_close(void *iter_arg) { struct filtering_ref_iterator *fri = iter_arg; - strbuf_release(&fri->oid); + reftable_buf_release(&fri->oid); reftable_iterator_destroy(&fri->it); } @@ -42,26 +78,6 @@ static int filtering_ref_iterator_next(void *iter_arg, break; } - if (fri->double_check) { - struct reftable_iterator it = { NULL }; - - reftable_table_init_ref_iter(&fri->tab, &it); - - err = reftable_iterator_seek_ref(&it, ref->refname); - if (err == 0) - err = reftable_iterator_next_ref(&it, ref); - - reftable_iterator_destroy(&it); - - if (err < 0) { - break; - } - - if (err > 0) { - continue; - } - } - if (ref->value_type == REFTABLE_REF_VAL2 && (!memcmp(fri->oid.buf, ref->value.val2.target_value, fri->oid.len) || @@ -99,7 +115,7 @@ static void indexed_table_ref_iter_close(void *p) block_iter_close(&it->cur); reftable_block_done(&it->block_reader.block); reftable_free(it->offsets); - strbuf_release(&it->oid); + reftable_buf_release(&it->oid); } static int indexed_table_ref_iter_next_block(struct indexed_table_ref_iter *it) @@ -127,7 +143,8 @@ static int indexed_table_ref_iter_next_block(struct indexed_table_ref_iter *it) return 0; } -static int indexed_table_ref_iter_seek(void *p, struct reftable_record *want) +static int indexed_table_ref_iter_seek(void *p UNUSED, + struct reftable_record *want UNUSED) { BUG("seeking indexed table is not supported"); return -1; @@ -164,26 +181,41 @@ static int indexed_table_ref_iter_next(void *p, struct reftable_record *rec) } } -int new_indexed_table_ref_iter(struct indexed_table_ref_iter **dest, +int indexed_table_ref_iter_new(struct indexed_table_ref_iter **dest, struct reftable_reader *r, uint8_t *oid, int oid_len, uint64_t *offsets, int offset_len) { struct indexed_table_ref_iter empty = INDEXED_TABLE_REF_ITER_INIT; - struct indexed_table_ref_iter *itr = reftable_calloc(1, sizeof(*itr)); + struct indexed_table_ref_iter *itr; int err = 0; + itr = reftable_calloc(1, sizeof(*itr)); + if (!itr) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto out; + } + *itr = empty; itr->r = r; - strbuf_add(&itr->oid, oid, oid_len); + + err = reftable_buf_add(&itr->oid, oid, oid_len); + if (err < 0) + goto out; itr->offsets = offsets; itr->offset_len = offset_len; err = indexed_table_ref_iter_next_block(itr); + if (err < 0) + goto out; + + *dest = itr; + err = 0; + +out: if (err < 0) { + *dest = NULL; reftable_free(itr); - } else { - *dest = itr; } return err; } @@ -201,3 +233,71 @@ void iterator_from_indexed_table_ref_iter(struct reftable_iterator *it, it->iter_arg = itr; it->ops = &indexed_table_ref_iter_vtable; } + +void reftable_iterator_destroy(struct reftable_iterator *it) +{ + if (!it->ops) + return; + it->ops->close(it->iter_arg); + it->ops = NULL; + REFTABLE_FREE_AND_NULL(it->iter_arg); +} + +int reftable_iterator_seek_ref(struct reftable_iterator *it, + const char *name) +{ + struct reftable_record want = { + .type = BLOCK_TYPE_REF, + .u.ref = { + .refname = (char *)name, + }, + }; + return it->ops->seek(it->iter_arg, &want); +} + +int reftable_iterator_next_ref(struct reftable_iterator *it, + struct reftable_ref_record *ref) +{ + struct reftable_record rec = { + .type = BLOCK_TYPE_REF, + .u = { + .ref = *ref + }, + }; + int err = iterator_next(it, &rec); + *ref = rec.u.ref; + return err; +} + +int reftable_iterator_seek_log_at(struct reftable_iterator *it, + const char *name, uint64_t update_index) +{ + struct reftable_record want = { + .type = BLOCK_TYPE_LOG, + .u.log = { + .refname = (char *)name, + .update_index = update_index, + }, + }; + return it->ops->seek(it->iter_arg, &want); +} + +int reftable_iterator_seek_log(struct reftable_iterator *it, + const char *name) +{ + return reftable_iterator_seek_log_at(it, name, ~((uint64_t) 0)); +} + +int reftable_iterator_next_log(struct reftable_iterator *it, + struct reftable_log_record *log) +{ + struct reftable_record rec = { + .type = BLOCK_TYPE_LOG, + .u = { + .log = *log, + }, + }; + int err = iterator_next(it, &rec); + *log = rec.u.log; + return err; +} diff --git a/reftable/iter.h b/reftable/iter.h index 537431baba..40f98893b8 100644 --- a/reftable/iter.h +++ b/reftable/iter.h @@ -14,18 +14,42 @@ https://developers.google.com/open-source/licenses/bsd #include "record.h" #include "reftable-iterator.h" -#include "reftable-generic.h" + +/* + * The virtual function table for implementing generic reftable iterators. + */ +struct reftable_iterator_vtable { + int (*seek)(void *iter_arg, struct reftable_record *want); + int (*next)(void *iter_arg, struct reftable_record *rec); + void (*close)(void *iter_arg); +}; + +/* + * Position the iterator at the wanted record such that a call to + * `iterator_next()` would return that record, if it exists. + */ +int iterator_seek(struct reftable_iterator *it, struct reftable_record *want); + +/* + * Yield the next record and advance the iterator. Returns <0 on error, 0 when + * a record was yielded, and >0 when the iterator hit an error. + */ +int iterator_next(struct reftable_iterator *it, struct reftable_record *rec); + +/* + * Set up the iterator such that it behaves the same as an iterator with no + * entries. + */ +void iterator_set_empty(struct reftable_iterator *it); /* iterator that produces only ref records that point to `oid` */ struct filtering_ref_iterator { - int double_check; - struct reftable_table tab; - struct strbuf oid; + struct reftable_buf oid; struct reftable_iterator it; }; #define FILTERING_REF_ITERATOR_INIT \ { \ - .oid = STRBUF_INIT \ + .oid = REFTABLE_BUF_INIT \ } void iterator_from_filtering_ref_iterator(struct reftable_iterator *, @@ -36,7 +60,7 @@ void iterator_from_filtering_ref_iterator(struct reftable_iterator *, */ struct indexed_table_ref_iter { struct reftable_reader *r; - struct strbuf oid; + struct reftable_buf oid; /* mutable */ uint64_t *offsets; @@ -51,14 +75,14 @@ struct indexed_table_ref_iter { #define INDEXED_TABLE_REF_ITER_INIT { \ .cur = BLOCK_ITER_INIT, \ - .oid = STRBUF_INIT, \ + .oid = REFTABLE_BUF_INIT, \ } void iterator_from_indexed_table_ref_iter(struct reftable_iterator *it, struct indexed_table_ref_iter *itr); /* Takes ownership of `offsets` */ -int new_indexed_table_ref_iter(struct indexed_table_ref_iter **dest, +int indexed_table_ref_iter_new(struct indexed_table_ref_iter **dest, struct reftable_reader *r, uint8_t *oid, int oid_len, uint64_t *offsets, int offset_len); diff --git a/reftable/merged.c b/reftable/merged.c index 6adce44f4b..bb0836e344 100644 --- a/reftable/merged.c +++ b/reftable/merged.c @@ -11,8 +11,8 @@ https://developers.google.com/open-source/licenses/bsd #include "constants.h" #include "iter.h" #include "pq.h" +#include "reader.h" #include "record.h" -#include "generic.h" #include "reftable-merged.h" #include "reftable-error.h" #include "system.h" @@ -25,33 +25,17 @@ struct merged_subiter { struct merged_iter { struct merged_subiter *subiters; struct merged_iter_pqueue pq; - size_t stack_len; + size_t subiters_len; int suppress_deletions; ssize_t advance_index; }; -static void merged_iter_init(struct merged_iter *mi, - struct reftable_merged_table *mt, - uint8_t typ) -{ - memset(mi, 0, sizeof(*mi)); - mi->advance_index = -1; - mi->suppress_deletions = mt->suppress_deletions; - - REFTABLE_CALLOC_ARRAY(mi->subiters, mt->stack_len); - for (size_t i = 0; i < mt->stack_len; i++) { - reftable_record_init(&mi->subiters[i].rec, typ); - table_init_iter(&mt->stack[i], &mi->subiters[i].iter, typ); - } - mi->stack_len = mt->stack_len; -} - static void merged_iter_close(void *p) { struct merged_iter *mi = p; merged_iter_pqueue_release(&mi->pq); - for (size_t i = 0; i < mi->stack_len; i++) { + for (size_t i = 0; i < mi->subiters_len; i++) { reftable_iterator_destroy(&mi->subiters[i].iter); reftable_record_release(&mi->subiters[i].rec); } @@ -70,7 +54,10 @@ static int merged_iter_advance_subiter(struct merged_iter *mi, size_t idx) if (err) return err; - merged_iter_pqueue_add(&mi->pq, &e); + err = merged_iter_pqueue_add(&mi->pq, &e); + if (err) + return err; + return 0; } @@ -79,8 +66,10 @@ static int merged_iter_seek(struct merged_iter *mi, struct reftable_record *want int err; mi->advance_index = -1; + while (!merged_iter_pqueue_is_empty(mi->pq)) + merged_iter_pqueue_remove(&mi->pq); - for (size_t i = 0; i < mi->stack_len; i++) { + for (size_t i = 0; i < mi->subiters_len; i++) { err = iterator_seek(&mi->subiters[i].iter, want); if (err < 0) return err; @@ -192,19 +181,19 @@ static void iterator_from_merged_iter(struct reftable_iterator *it, it->ops = &merged_iter_vtable; } -int reftable_new_merged_table(struct reftable_merged_table **dest, - struct reftable_table *stack, size_t n, - uint32_t hash_id) +int reftable_merged_table_new(struct reftable_merged_table **dest, + struct reftable_reader **readers, size_t n, + enum reftable_hash hash_id) { struct reftable_merged_table *m = NULL; uint64_t last_max = 0; uint64_t first_min = 0; for (size_t i = 0; i < n; i++) { - uint64_t min = reftable_table_min_update_index(&stack[i]); - uint64_t max = reftable_table_max_update_index(&stack[i]); + uint64_t min = reftable_reader_min_update_index(readers[i]); + uint64_t max = reftable_reader_max_update_index(readers[i]); - if (reftable_table_hash_id(&stack[i]) != hash_id) { + if (reftable_reader_hash_id(readers[i]) != hash_id) { return REFTABLE_FORMAT_ERROR; } if (i == 0 || min < first_min) { @@ -216,8 +205,11 @@ int reftable_new_merged_table(struct reftable_merged_table **dest, } REFTABLE_CALLOC_ARRAY(m, 1); - m->stack = stack; - m->stack_len = n; + if (!m) + return REFTABLE_OUT_OF_MEMORY_ERROR; + + m->readers = readers; + m->readers_len = n; m->min = first_min; m->max = last_max; m->hash_id = hash_id; @@ -229,7 +221,6 @@ void reftable_merged_table_free(struct reftable_merged_table *mt) { if (!mt) return; - FREE_AND_NULL(mt->stack); reftable_free(mt); } @@ -245,53 +236,66 @@ reftable_merged_table_min_update_index(struct reftable_merged_table *mt) return mt->min; } -void merged_table_init_iter(struct reftable_merged_table *mt, - struct reftable_iterator *it, - uint8_t typ) +int merged_table_init_iter(struct reftable_merged_table *mt, + struct reftable_iterator *it, + uint8_t typ) { - struct merged_iter *mi = reftable_malloc(sizeof(*mi)); - merged_iter_init(mi, mt, typ); - iterator_from_merged_iter(it, mi); -} + struct merged_subiter *subiters; + struct merged_iter *mi = NULL; + int ret; -uint32_t reftable_merged_table_hash_id(struct reftable_merged_table *mt) -{ - return mt->hash_id; -} + REFTABLE_CALLOC_ARRAY(subiters, mt->readers_len); + if (!subiters) { + ret = REFTABLE_OUT_OF_MEMORY_ERROR; + goto out; + } -static void reftable_merged_table_init_iter_void(void *tab, - struct reftable_iterator *it, - uint8_t typ) -{ - merged_table_init_iter(tab, it, typ); -} + for (size_t i = 0; i < mt->readers_len; i++) { + reftable_record_init(&subiters[i].rec, typ); + ret = reader_init_iter(mt->readers[i], &subiters[i].iter, typ); + if (ret < 0) + goto out; + } -static uint32_t reftable_merged_table_hash_id_void(void *tab) -{ - return reftable_merged_table_hash_id(tab); + REFTABLE_CALLOC_ARRAY(mi, 1); + if (!mi) { + ret = REFTABLE_OUT_OF_MEMORY_ERROR; + goto out; + } + mi->advance_index = -1; + mi->suppress_deletions = mt->suppress_deletions; + mi->subiters = subiters; + mi->subiters_len = mt->readers_len; + + iterator_from_merged_iter(it, mi); + ret = 0; + +out: + if (ret < 0) { + for (size_t i = 0; subiters && i < mt->readers_len; i++) { + reftable_iterator_destroy(&subiters[i].iter); + reftable_record_release(&subiters[i].rec); + } + reftable_free(subiters); + reftable_free(mi); + } + + return ret; } -static uint64_t reftable_merged_table_min_update_index_void(void *tab) +int reftable_merged_table_init_ref_iterator(struct reftable_merged_table *mt, + struct reftable_iterator *it) { - return reftable_merged_table_min_update_index(tab); + return merged_table_init_iter(mt, it, BLOCK_TYPE_REF); } -static uint64_t reftable_merged_table_max_update_index_void(void *tab) +int reftable_merged_table_init_log_iterator(struct reftable_merged_table *mt, + struct reftable_iterator *it) { - return reftable_merged_table_max_update_index(tab); + return merged_table_init_iter(mt, it, BLOCK_TYPE_LOG); } -static struct reftable_table_vtable merged_table_vtable = { - .init_iter = reftable_merged_table_init_iter_void, - .hash_id = reftable_merged_table_hash_id_void, - .min_update_index = reftable_merged_table_min_update_index_void, - .max_update_index = reftable_merged_table_max_update_index_void, -}; - -void reftable_table_from_merged_table(struct reftable_table *tab, - struct reftable_merged_table *merged) +enum reftable_hash reftable_merged_table_hash_id(struct reftable_merged_table *mt) { - assert(!tab->ops); - tab->ops = &merged_table_vtable; - tab->table_arg = merged; + return mt->hash_id; } diff --git a/reftable/merged.h b/reftable/merged.h index 2efe571da6..0b7d939e92 100644 --- a/reftable/merged.h +++ b/reftable/merged.h @@ -10,11 +10,12 @@ https://developers.google.com/open-source/licenses/bsd #define MERGED_H #include "system.h" +#include "reftable-basics.h" struct reftable_merged_table { - struct reftable_table *stack; - size_t stack_len; - uint32_t hash_id; + struct reftable_reader **readers; + size_t readers_len; + enum reftable_hash hash_id; /* If unset, produce deletions. This is useful for compaction. For the * full stack, deletions should be produced. */ @@ -26,8 +27,8 @@ struct reftable_merged_table { struct reftable_iterator; -void merged_table_init_iter(struct reftable_merged_table *mt, - struct reftable_iterator *it, - uint8_t typ); +int merged_table_init_iter(struct reftable_merged_table *mt, + struct reftable_iterator *it, + uint8_t typ); #endif diff --git a/reftable/merged_test.c b/reftable/merged_test.c deleted file mode 100644 index a9d6661c13..0000000000 --- a/reftable/merged_test.c +++ /dev/null @@ -1,461 +0,0 @@ -/* -Copyright 2020 Google LLC - -Use of this source code is governed by a BSD-style -license that can be found in the LICENSE file or at -https://developers.google.com/open-source/licenses/bsd -*/ - -#include "merged.h" - -#include "system.h" - -#include "basics.h" -#include "blocksource.h" -#include "constants.h" -#include "reader.h" -#include "record.h" -#include "test_framework.h" -#include "reftable-merged.h" -#include "reftable-tests.h" -#include "reftable-generic.h" -#include "reftable-writer.h" - -static void write_test_table(struct strbuf *buf, - struct reftable_ref_record refs[], int n) -{ - uint64_t min = 0xffffffff; - uint64_t max = 0; - int i = 0; - int err; - - struct reftable_write_options opts = { - .block_size = 256, - }; - struct reftable_writer *w = NULL; - for (i = 0; i < n; i++) { - uint64_t ui = refs[i].update_index; - if (ui > max) { - max = ui; - } - if (ui < min) { - min = ui; - } - } - - w = reftable_new_writer(&strbuf_add_void, &noop_flush, buf, &opts); - reftable_writer_set_limits(w, min, max); - - for (i = 0; i < n; i++) { - uint64_t before = refs[i].update_index; - int n = reftable_writer_add_ref(w, &refs[i]); - EXPECT(n == 0); - EXPECT(before == refs[i].update_index); - } - - err = reftable_writer_close(w); - EXPECT_ERR(err); - - reftable_writer_free(w); -} - -static void write_test_log_table(struct strbuf *buf, - struct reftable_log_record logs[], int n, - uint64_t update_index) -{ - int i = 0; - int err; - - struct reftable_write_options opts = { - .block_size = 256, - .exact_log_message = 1, - }; - struct reftable_writer *w = NULL; - w = reftable_new_writer(&strbuf_add_void, &noop_flush, buf, &opts); - reftable_writer_set_limits(w, update_index, update_index); - - for (i = 0; i < n; i++) { - int err = reftable_writer_add_log(w, &logs[i]); - EXPECT_ERR(err); - } - - err = reftable_writer_close(w); - EXPECT_ERR(err); - - reftable_writer_free(w); -} - -static struct reftable_merged_table * -merged_table_from_records(struct reftable_ref_record **refs, - struct reftable_block_source **source, - struct reftable_reader ***readers, int *sizes, - struct strbuf *buf, size_t n) -{ - struct reftable_merged_table *mt = NULL; - struct reftable_table *tabs; - int err; - - REFTABLE_CALLOC_ARRAY(tabs, n); - REFTABLE_CALLOC_ARRAY(*readers, n); - REFTABLE_CALLOC_ARRAY(*source, n); - - for (size_t i = 0; i < n; i++) { - write_test_table(&buf[i], refs[i], sizes[i]); - block_source_from_strbuf(&(*source)[i], &buf[i]); - - err = reftable_new_reader(&(*readers)[i], &(*source)[i], - "name"); - EXPECT_ERR(err); - reftable_table_from_reader(&tabs[i], (*readers)[i]); - } - - err = reftable_new_merged_table(&mt, tabs, n, GIT_SHA1_FORMAT_ID); - EXPECT_ERR(err); - return mt; -} - -static void readers_destroy(struct reftable_reader **readers, size_t n) -{ - int i = 0; - for (; i < n; i++) - reftable_reader_free(readers[i]); - reftable_free(readers); -} - -static void test_merged_between(void) -{ - struct reftable_ref_record r1[] = { { - .refname = (char *) "b", - .update_index = 1, - .value_type = REFTABLE_REF_VAL1, - .value.val1 = { 1, 2, 3, 0 }, - } }; - struct reftable_ref_record r2[] = { { - .refname = (char *) "a", - .update_index = 2, - .value_type = REFTABLE_REF_DELETION, - } }; - - struct reftable_ref_record *refs[] = { r1, r2 }; - int sizes[] = { 1, 1 }; - struct strbuf bufs[2] = { STRBUF_INIT, STRBUF_INIT }; - struct reftable_block_source *bs = NULL; - struct reftable_reader **readers = NULL; - struct reftable_merged_table *mt = - merged_table_from_records(refs, &bs, &readers, sizes, bufs, 2); - int i; - struct reftable_ref_record ref = { NULL }; - struct reftable_iterator it = { NULL }; - int err; - - merged_table_init_iter(mt, &it, BLOCK_TYPE_REF); - err = reftable_iterator_seek_ref(&it, "a"); - EXPECT_ERR(err); - - err = reftable_iterator_next_ref(&it, &ref); - EXPECT_ERR(err); - EXPECT(ref.update_index == 2); - reftable_ref_record_release(&ref); - reftable_iterator_destroy(&it); - readers_destroy(readers, 2); - reftable_merged_table_free(mt); - for (i = 0; i < ARRAY_SIZE(bufs); i++) { - strbuf_release(&bufs[i]); - } - reftable_free(bs); -} - -static void test_merged(void) -{ - struct reftable_ref_record r1[] = { - { - .refname = (char *) "a", - .update_index = 1, - .value_type = REFTABLE_REF_VAL1, - .value.val1 = { 1 }, - }, - { - .refname = (char *) "b", - .update_index = 1, - .value_type = REFTABLE_REF_VAL1, - .value.val1 = { 1 }, - }, - { - .refname = (char *) "c", - .update_index = 1, - .value_type = REFTABLE_REF_VAL1, - .value.val1 = { 1 }, - } - }; - struct reftable_ref_record r2[] = { { - .refname = (char *) "a", - .update_index = 2, - .value_type = REFTABLE_REF_DELETION, - } }; - struct reftable_ref_record r3[] = { - { - .refname = (char *) "c", - .update_index = 3, - .value_type = REFTABLE_REF_VAL1, - .value.val1 = { 2 }, - }, - { - .refname = (char *) "d", - .update_index = 3, - .value_type = REFTABLE_REF_VAL1, - .value.val1 = { 1 }, - }, - }; - - struct reftable_ref_record *want[] = { - &r2[0], - &r1[1], - &r3[0], - &r3[1], - }; - - struct reftable_ref_record *refs[] = { r1, r2, r3 }; - int sizes[3] = { 3, 1, 2 }; - struct strbuf bufs[3] = { STRBUF_INIT, STRBUF_INIT, STRBUF_INIT }; - struct reftable_block_source *bs = NULL; - struct reftable_reader **readers = NULL; - struct reftable_merged_table *mt = - merged_table_from_records(refs, &bs, &readers, sizes, bufs, 3); - struct reftable_iterator it = { NULL }; - int err; - struct reftable_ref_record *out = NULL; - size_t len = 0; - size_t cap = 0; - int i = 0; - - merged_table_init_iter(mt, &it, BLOCK_TYPE_REF); - err = reftable_iterator_seek_ref(&it, "a"); - EXPECT_ERR(err); - EXPECT(reftable_merged_table_hash_id(mt) == GIT_SHA1_FORMAT_ID); - EXPECT(reftable_merged_table_min_update_index(mt) == 1); - - while (len < 100) { /* cap loops/recursion. */ - struct reftable_ref_record ref = { NULL }; - int err = reftable_iterator_next_ref(&it, &ref); - if (err > 0) - break; - - REFTABLE_ALLOC_GROW(out, len + 1, cap); - out[len++] = ref; - } - reftable_iterator_destroy(&it); - - EXPECT(ARRAY_SIZE(want) == len); - for (i = 0; i < len; i++) { - EXPECT(reftable_ref_record_equal(want[i], &out[i], - GIT_SHA1_RAWSZ)); - } - for (i = 0; i < len; i++) { - reftable_ref_record_release(&out[i]); - } - reftable_free(out); - - for (i = 0; i < 3; i++) { - strbuf_release(&bufs[i]); - } - readers_destroy(readers, 3); - reftable_merged_table_free(mt); - reftable_free(bs); -} - -static struct reftable_merged_table * -merged_table_from_log_records(struct reftable_log_record **logs, - struct reftable_block_source **source, - struct reftable_reader ***readers, int *sizes, - struct strbuf *buf, size_t n) -{ - struct reftable_merged_table *mt = NULL; - struct reftable_table *tabs; - int err; - - REFTABLE_CALLOC_ARRAY(tabs, n); - REFTABLE_CALLOC_ARRAY(*readers, n); - REFTABLE_CALLOC_ARRAY(*source, n); - - for (size_t i = 0; i < n; i++) { - write_test_log_table(&buf[i], logs[i], sizes[i], i + 1); - block_source_from_strbuf(&(*source)[i], &buf[i]); - - err = reftable_new_reader(&(*readers)[i], &(*source)[i], - "name"); - EXPECT_ERR(err); - reftable_table_from_reader(&tabs[i], (*readers)[i]); - } - - err = reftable_new_merged_table(&mt, tabs, n, GIT_SHA1_FORMAT_ID); - EXPECT_ERR(err); - return mt; -} - -static void test_merged_logs(void) -{ - struct reftable_log_record r1[] = { - { - .refname = (char *) "a", - .update_index = 2, - .value_type = REFTABLE_LOG_UPDATE, - .value.update = { - .old_hash = { 2 }, - /* deletion */ - .name = (char *) "jane doe", - .email = (char *) "jane@invalid", - .message = (char *) "message2", - } - }, - { - .refname = (char *) "a", - .update_index = 1, - .value_type = REFTABLE_LOG_UPDATE, - .value.update = { - .old_hash = { 1 }, - .new_hash = { 2 }, - .name = (char *) "jane doe", - .email = (char *) "jane@invalid", - .message = (char *) "message1", - } - }, - }; - struct reftable_log_record r2[] = { - { - .refname = (char *) "a", - .update_index = 3, - .value_type = REFTABLE_LOG_UPDATE, - .value.update = { - .new_hash = { 3 }, - .name = (char *) "jane doe", - .email = (char *) "jane@invalid", - .message = (char *) "message3", - } - }, - }; - struct reftable_log_record r3[] = { - { - .refname = (char *) "a", - .update_index = 2, - .value_type = REFTABLE_LOG_DELETION, - }, - }; - struct reftable_log_record *want[] = { - &r2[0], - &r3[0], - &r1[1], - }; - - struct reftable_log_record *logs[] = { r1, r2, r3 }; - int sizes[3] = { 2, 1, 1 }; - struct strbuf bufs[3] = { STRBUF_INIT, STRBUF_INIT, STRBUF_INIT }; - struct reftable_block_source *bs = NULL; - struct reftable_reader **readers = NULL; - struct reftable_merged_table *mt = merged_table_from_log_records( - logs, &bs, &readers, sizes, bufs, 3); - struct reftable_iterator it = { NULL }; - int err; - struct reftable_log_record *out = NULL; - size_t len = 0; - size_t cap = 0; - int i = 0; - - merged_table_init_iter(mt, &it, BLOCK_TYPE_LOG); - err = reftable_iterator_seek_log(&it, "a"); - EXPECT_ERR(err); - EXPECT(reftable_merged_table_hash_id(mt) == GIT_SHA1_FORMAT_ID); - EXPECT(reftable_merged_table_min_update_index(mt) == 1); - - while (len < 100) { /* cap loops/recursion. */ - struct reftable_log_record log = { NULL }; - int err = reftable_iterator_next_log(&it, &log); - if (err > 0) - break; - - REFTABLE_ALLOC_GROW(out, len + 1, cap); - out[len++] = log; - } - reftable_iterator_destroy(&it); - - EXPECT(ARRAY_SIZE(want) == len); - for (i = 0; i < len; i++) { - EXPECT(reftable_log_record_equal(want[i], &out[i], - GIT_SHA1_RAWSZ)); - } - - merged_table_init_iter(mt, &it, BLOCK_TYPE_LOG); - err = reftable_iterator_seek_log_at(&it, "a", 2); - EXPECT_ERR(err); - reftable_log_record_release(&out[0]); - err = reftable_iterator_next_log(&it, &out[0]); - EXPECT_ERR(err); - EXPECT(reftable_log_record_equal(&out[0], &r3[0], GIT_SHA1_RAWSZ)); - reftable_iterator_destroy(&it); - - for (i = 0; i < len; i++) { - reftable_log_record_release(&out[i]); - } - reftable_free(out); - - for (i = 0; i < 3; i++) { - strbuf_release(&bufs[i]); - } - readers_destroy(readers, 3); - reftable_merged_table_free(mt); - reftable_free(bs); -} - -static void test_default_write_opts(void) -{ - struct reftable_write_options opts = { 0 }; - struct strbuf buf = STRBUF_INIT; - struct reftable_writer *w = - reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts); - - struct reftable_ref_record rec = { - .refname = (char *) "master", - .update_index = 1, - }; - int err; - struct reftable_block_source source = { NULL }; - struct reftable_table *tab = reftable_calloc(1, sizeof(*tab)); - uint32_t hash_id; - struct reftable_reader *rd = NULL; - struct reftable_merged_table *merged = NULL; - - reftable_writer_set_limits(w, 1, 1); - - err = reftable_writer_add_ref(w, &rec); - EXPECT_ERR(err); - - err = reftable_writer_close(w); - EXPECT_ERR(err); - reftable_writer_free(w); - - block_source_from_strbuf(&source, &buf); - - err = reftable_new_reader(&rd, &source, "filename"); - EXPECT_ERR(err); - - hash_id = reftable_reader_hash_id(rd); - EXPECT(hash_id == GIT_SHA1_FORMAT_ID); - - reftable_table_from_reader(&tab[0], rd); - err = reftable_new_merged_table(&merged, tab, 1, GIT_SHA1_FORMAT_ID); - EXPECT_ERR(err); - - reftable_reader_free(rd); - reftable_merged_table_free(merged); - strbuf_release(&buf); -} - -/* XXX test refs_for(oid) */ - -int merged_test_main(int argc, const char *argv[]) -{ - RUN_TEST(test_merged_logs); - RUN_TEST(test_merged_between); - RUN_TEST(test_merged); - RUN_TEST(test_default_write_opts); - return 0; -} diff --git a/reftable/pq.c b/reftable/pq.c index 7fb45d8c60..6ee1164dd3 100644 --- a/reftable/pq.c +++ b/reftable/pq.c @@ -8,6 +8,7 @@ https://developers.google.com/open-source/licenses/bsd #include "pq.h" +#include "reftable-error.h" #include "reftable-record.h" #include "system.h" #include "basics.h" @@ -22,27 +23,21 @@ int pq_less(struct pq_entry *a, struct pq_entry *b) struct pq_entry merged_iter_pqueue_remove(struct merged_iter_pqueue *pq) { - int i = 0; + size_t i = 0; struct pq_entry e = pq->heap[0]; pq->heap[0] = pq->heap[pq->len - 1]; pq->len--; - i = 0; while (i < pq->len) { - int min = i; - int j = 2 * i + 1; - int k = 2 * i + 2; - if (j < pq->len && pq_less(&pq->heap[j], &pq->heap[i])) { + size_t min = i; + size_t j = 2 * i + 1; + size_t k = 2 * i + 2; + if (j < pq->len && pq_less(&pq->heap[j], &pq->heap[i])) min = j; - } - if (k < pq->len && pq_less(&pq->heap[k], &pq->heap[min])) { + if (k < pq->len && pq_less(&pq->heap[k], &pq->heap[min])) min = k; - } - - if (min == i) { + if (min == i) break; - } - SWAP(pq->heap[i], pq->heap[min]); i = min; } @@ -50,28 +45,29 @@ struct pq_entry merged_iter_pqueue_remove(struct merged_iter_pqueue *pq) return e; } -void merged_iter_pqueue_add(struct merged_iter_pqueue *pq, const struct pq_entry *e) +int merged_iter_pqueue_add(struct merged_iter_pqueue *pq, const struct pq_entry *e) { - int i = 0; + size_t i = 0; REFTABLE_ALLOC_GROW(pq->heap, pq->len + 1, pq->cap); + if (!pq->heap) + return REFTABLE_OUT_OF_MEMORY_ERROR; pq->heap[pq->len++] = *e; i = pq->len - 1; while (i > 0) { - int j = (i - 1) / 2; - if (pq_less(&pq->heap[j], &pq->heap[i])) { + size_t j = (i - 1) / 2; + if (pq_less(&pq->heap[j], &pq->heap[i])) break; - } - SWAP(pq->heap[j], pq->heap[i]); - i = j; } + + return 0; } void merged_iter_pqueue_release(struct merged_iter_pqueue *pq) { - FREE_AND_NULL(pq->heap); + REFTABLE_FREE_AND_NULL(pq->heap); memset(pq, 0, sizeof(*pq)); } diff --git a/reftable/pq.h b/reftable/pq.h index f796c23179..83c062eeca 100644 --- a/reftable/pq.h +++ b/reftable/pq.h @@ -22,9 +22,8 @@ struct merged_iter_pqueue { size_t cap; }; -void merged_iter_pqueue_check(struct merged_iter_pqueue pq); struct pq_entry merged_iter_pqueue_remove(struct merged_iter_pqueue *pq); -void merged_iter_pqueue_add(struct merged_iter_pqueue *pq, const struct pq_entry *e); +int merged_iter_pqueue_add(struct merged_iter_pqueue *pq, const struct pq_entry *e); void merged_iter_pqueue_release(struct merged_iter_pqueue *pq); int pq_less(struct pq_entry *a, struct pq_entry *b); diff --git a/reftable/pq_test.c b/reftable/pq_test.c deleted file mode 100644 index b7d3c80cc7..0000000000 --- a/reftable/pq_test.c +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2020 Google LLC - -Use of this source code is governed by a BSD-style -license that can be found in the LICENSE file or at -https://developers.google.com/open-source/licenses/bsd -*/ - -#include "system.h" - -#include "basics.h" -#include "constants.h" -#include "pq.h" -#include "record.h" -#include "reftable-tests.h" -#include "test_framework.h" - -void merged_iter_pqueue_check(struct merged_iter_pqueue pq) -{ - int i; - for (i = 1; i < pq.len; i++) { - int parent = (i - 1) / 2; - - EXPECT(pq_less(&pq.heap[parent], &pq.heap[i])); - } -} - -static void test_pq(void) -{ - struct merged_iter_pqueue pq = { NULL }; - struct reftable_record recs[54]; - int N = ARRAY_SIZE(recs) - 1, i; - char *last = NULL; - - for (i = 0; i < N; i++) { - struct strbuf refname = STRBUF_INIT; - strbuf_addf(&refname, "%02d", i); - - reftable_record_init(&recs[i], BLOCK_TYPE_REF); - recs[i].u.ref.refname = strbuf_detach(&refname, NULL); - } - - i = 1; - do { - struct pq_entry e = { - .rec = &recs[i], - }; - - merged_iter_pqueue_add(&pq, &e); - merged_iter_pqueue_check(pq); - - i = (i * 7) % N; - } while (i != 1); - - while (!merged_iter_pqueue_is_empty(pq)) { - struct pq_entry e = merged_iter_pqueue_remove(&pq); - merged_iter_pqueue_check(pq); - - EXPECT(reftable_record_type(e.rec) == BLOCK_TYPE_REF); - if (last) - EXPECT(strcmp(last, e.rec->u.ref.refname) < 0); - last = e.rec->u.ref.refname; - } - - for (i = 0; i < N; i++) - reftable_record_release(&recs[i]); - merged_iter_pqueue_release(&pq); -} - -int pq_test_main(int argc, const char *argv[]) -{ - RUN_TEST(test_pq); - return 0; -} diff --git a/reftable/publicbasics.c b/reftable/publicbasics.c deleted file mode 100644 index 44b84a125e..0000000000 --- a/reftable/publicbasics.c +++ /dev/null @@ -1,66 +0,0 @@ -/* -Copyright 2020 Google LLC - -Use of this source code is governed by a BSD-style -license that can be found in the LICENSE file or at -https://developers.google.com/open-source/licenses/bsd -*/ - -#include "system.h" -#include "reftable-malloc.h" - -#include "basics.h" - -static void *(*reftable_malloc_ptr)(size_t sz); -static void *(*reftable_realloc_ptr)(void *, size_t); -static void (*reftable_free_ptr)(void *); - -void *reftable_malloc(size_t sz) -{ - if (reftable_malloc_ptr) - return (*reftable_malloc_ptr)(sz); - return malloc(sz); -} - -void *reftable_realloc(void *p, size_t sz) -{ - if (reftable_realloc_ptr) - return (*reftable_realloc_ptr)(p, sz); - return realloc(p, sz); -} - -void reftable_free(void *p) -{ - if (reftable_free_ptr) - reftable_free_ptr(p); - else - free(p); -} - -void *reftable_calloc(size_t nelem, size_t elsize) -{ - size_t sz = st_mult(nelem, elsize); - void *p = reftable_malloc(sz); - memset(p, 0, sz); - return p; -} - -void reftable_set_alloc(void *(*malloc)(size_t), - void *(*realloc)(void *, size_t), void (*free)(void *)) -{ - reftable_malloc_ptr = malloc; - reftable_realloc_ptr = realloc; - reftable_free_ptr = free; -} - -int hash_size(uint32_t id) -{ - switch (id) { - case 0: - case GIT_SHA1_FORMAT_ID: - return GIT_SHA1_RAWSZ; - case GIT_SHA256_FORMAT_ID: - return GIT_SHA256_RAWSZ; - } - abort(); -} diff --git a/reftable/reader.c b/reftable/reader.c index 29c99e2269..ea82955c9b 100644 --- a/reftable/reader.c +++ b/reftable/reader.c @@ -11,11 +11,9 @@ https://developers.google.com/open-source/licenses/bsd #include "system.h" #include "block.h" #include "constants.h" -#include "generic.h" #include "iter.h" #include "record.h" #include "reftable-error.h" -#include "reftable-generic.h" uint64_t block_source_size(struct reftable_block_source *source) { @@ -69,7 +67,7 @@ static int reader_get_block(struct reftable_reader *r, return block_source_read_block(&r->source, dest, off, sz); } -uint32_t reftable_reader_hash_id(struct reftable_reader *r) +enum reftable_hash reftable_reader_hash_id(struct reftable_reader *r) { return r->hash_id; } @@ -109,18 +107,20 @@ static int parse_footer(struct reftable_reader *r, uint8_t *footer, f += 8; if (r->version == 1) { - r->hash_id = GIT_SHA1_FORMAT_ID; + r->hash_id = REFTABLE_HASH_SHA1; } else { - r->hash_id = get_be32(f); - switch (r->hash_id) { - case GIT_SHA1_FORMAT_ID: + switch (get_be32(f)) { + case REFTABLE_FORMAT_ID_SHA1: + r->hash_id = REFTABLE_HASH_SHA1; break; - case GIT_SHA256_FORMAT_ID: + case REFTABLE_FORMAT_ID_SHA256: + r->hash_id = REFTABLE_HASH_SHA256; break; default: err = REFTABLE_FORMAT_ERROR; goto done; } + f += 4; } @@ -164,58 +164,6 @@ done: return err; } -int init_reader(struct reftable_reader *r, struct reftable_block_source *source, - const char *name) -{ - struct reftable_block footer = { NULL }; - struct reftable_block header = { NULL }; - int err = 0; - uint64_t file_size = block_source_size(source); - - /* Need +1 to read type of first block. */ - uint32_t read_size = header_size(2) + 1; /* read v2 because it's larger. */ - memset(r, 0, sizeof(struct reftable_reader)); - - if (read_size > file_size) { - err = REFTABLE_FORMAT_ERROR; - goto done; - } - - err = block_source_read_block(source, &header, 0, read_size); - if (err != read_size) { - err = REFTABLE_IO_ERROR; - goto done; - } - - if (memcmp(header.data, "REFT", 4)) { - err = REFTABLE_FORMAT_ERROR; - goto done; - } - r->version = header.data[4]; - if (r->version != 1 && r->version != 2) { - err = REFTABLE_FORMAT_ERROR; - goto done; - } - - r->size = file_size - footer_size(r->version); - r->source = *source; - r->name = xstrdup(name); - r->hash_id = 0; - - err = block_source_read_block(source, &footer, r->size, - footer_size(r->version)); - if (err != footer_size(r->version)) { - err = REFTABLE_IO_ERROR; - goto done; - } - - err = parse_footer(r, footer.data, header.data); -done: - reftable_block_done(&footer); - reftable_block_done(&header); - return err; -} - struct table_iter { struct reftable_reader *r; uint8_t typ; @@ -229,6 +177,7 @@ static int table_iter_init(struct table_iter *ti, struct reftable_reader *r) { struct block_iter bi = BLOCK_ITER_INIT; memset(ti, 0, sizeof(*ti)); + reftable_reader_incref(r); ti->r = r; ti->bi = bi; return 0; @@ -316,6 +265,7 @@ static void table_iter_close(struct table_iter *ti) { table_iter_block_done(ti); block_iter_close(&ti->bi); + reftable_reader_decref(ti->r); } static int table_iter_next_block(struct table_iter *ti) @@ -380,6 +330,7 @@ static int table_iter_seek_to(struct table_iter *ti, uint64_t off, uint8_t typ) ti->typ = block_reader_type(&ti->br); ti->block_off = off; block_iter_seek_start(&ti->bi, &ti->br); + ti->is_finished = 0; return 0; } @@ -401,13 +352,15 @@ static int table_iter_seek_start(struct table_iter *ti, uint8_t typ, int index) static int table_iter_seek_linear(struct table_iter *ti, struct reftable_record *want) { - struct strbuf want_key = STRBUF_INIT; - struct strbuf got_key = STRBUF_INIT; + struct reftable_buf want_key = REFTABLE_BUF_INIT; + struct reftable_buf got_key = REFTABLE_BUF_INIT; struct reftable_record rec; int err; reftable_record_init(&rec, reftable_record_type(want)); - reftable_record_key(want, &want_key); + err = reftable_record_key(want, &want_key); + if (err < 0) + goto done; /* * First we need to locate the block that must contain our record. To @@ -452,7 +405,7 @@ static int table_iter_seek_linear(struct table_iter *ti, if (err < 0) goto done; - if (strbuf_cmp(&got_key, &want_key) > 0) { + if (reftable_buf_cmp(&got_key, &want_key) > 0) { table_iter_block_done(&next); break; } @@ -473,8 +426,8 @@ static int table_iter_seek_linear(struct table_iter *ti, done: reftable_record_release(&rec); - strbuf_release(&want_key); - strbuf_release(&got_key); + reftable_buf_release(&want_key); + reftable_buf_release(&got_key); return err; } @@ -482,15 +435,17 @@ static int table_iter_seek_indexed(struct table_iter *ti, struct reftable_record *rec) { struct reftable_record want_index = { - .type = BLOCK_TYPE_INDEX, .u.idx = { .last_key = STRBUF_INIT } + .type = BLOCK_TYPE_INDEX, .u.idx = { .last_key = REFTABLE_BUF_INIT } }; struct reftable_record index_result = { .type = BLOCK_TYPE_INDEX, - .u.idx = { .last_key = STRBUF_INIT }, + .u.idx = { .last_key = REFTABLE_BUF_INIT }, }; int err; - reftable_record_key(rec, &want_index.u.idx.last_key); + err = reftable_record_key(rec, &want_index.u.idx.last_key); + if (err < 0) + goto done; /* * The index may consist of multiple levels, where each level may have @@ -605,59 +560,131 @@ static void iterator_from_table_iter(struct reftable_iterator *it, it->ops = &table_iter_vtable; } -static void reader_init_iter(struct reftable_reader *r, - struct reftable_iterator *it, - uint8_t typ) +int reader_init_iter(struct reftable_reader *r, + struct reftable_iterator *it, + uint8_t typ) { struct reftable_reader_offsets *offs = reader_offsets_for(r, typ); if (offs->is_present) { struct table_iter *ti; REFTABLE_ALLOC_ARRAY(ti, 1); + if (!ti) + return REFTABLE_OUT_OF_MEMORY_ERROR; + table_iter_init(ti, r); iterator_from_table_iter(it, ti); } else { iterator_set_empty(it); } -} -void reftable_reader_init_ref_iterator(struct reftable_reader *r, - struct reftable_iterator *it) -{ - reader_init_iter(r, it, BLOCK_TYPE_REF); + return 0; } -void reftable_reader_init_log_iterator(struct reftable_reader *r, - struct reftable_iterator *it) +int reftable_reader_init_ref_iterator(struct reftable_reader *r, + struct reftable_iterator *it) { - reader_init_iter(r, it, BLOCK_TYPE_LOG); + return reader_init_iter(r, it, BLOCK_TYPE_REF); } -void reader_close(struct reftable_reader *r) +int reftable_reader_init_log_iterator(struct reftable_reader *r, + struct reftable_iterator *it) { - block_source_close(&r->source); - FREE_AND_NULL(r->name); + return reader_init_iter(r, it, BLOCK_TYPE_LOG); } -int reftable_new_reader(struct reftable_reader **p, - struct reftable_block_source *src, char const *name) +int reftable_reader_new(struct reftable_reader **out, + struct reftable_block_source *source, char const *name) { - struct reftable_reader *rd = reftable_calloc(1, sizeof(*rd)); - int err = init_reader(rd, src, name); - if (err == 0) { - *p = rd; - } else { - block_source_close(src); - reftable_free(rd); + struct reftable_block footer = { 0 }; + struct reftable_block header = { 0 }; + struct reftable_reader *r; + uint64_t file_size = block_source_size(source); + uint32_t read_size; + int err; + + REFTABLE_CALLOC_ARRAY(r, 1); + if (!r) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto done; + } + + /* + * We need one extra byte to read the type of first block. We also + * pretend to always be reading v2 of the format because it is larger. + */ + read_size = header_size(2) + 1; + if (read_size > file_size) { + err = REFTABLE_FORMAT_ERROR; + goto done; + } + + err = block_source_read_block(source, &header, 0, read_size); + if (err != read_size) { + err = REFTABLE_IO_ERROR; + goto done; + } + + if (memcmp(header.data, "REFT", 4)) { + err = REFTABLE_FORMAT_ERROR; + goto done; + } + r->version = header.data[4]; + if (r->version != 1 && r->version != 2) { + err = REFTABLE_FORMAT_ERROR; + goto done; + } + + r->size = file_size - footer_size(r->version); + r->source = *source; + r->name = reftable_strdup(name); + if (!r->name) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto done; + } + r->hash_id = 0; + r->refcount = 1; + + err = block_source_read_block(source, &footer, r->size, + footer_size(r->version)); + if (err != footer_size(r->version)) { + err = REFTABLE_IO_ERROR; + goto done; + } + + err = parse_footer(r, footer.data, header.data); + if (err) + goto done; + + *out = r; + +done: + reftable_block_done(&footer); + reftable_block_done(&header); + if (err) { + reftable_free(r); + block_source_close(source); } return err; } -void reftable_reader_free(struct reftable_reader *r) +void reftable_reader_incref(struct reftable_reader *r) +{ + if (!r->refcount) + BUG("cannot increment ref counter of dead reader"); + r->refcount++; +} + +void reftable_reader_decref(struct reftable_reader *r) { if (!r) return; - reader_close(r); + if (!r->refcount) + BUG("cannot decrement ref counter of dead reader"); + if (--r->refcount) + return; + block_source_close(&r->source); + REFTABLE_FREE_AND_NULL(r->name); reftable_free(r); } @@ -681,7 +708,10 @@ static int reftable_reader_refs_for_indexed(struct reftable_reader *r, struct indexed_table_ref_iter *itr = NULL; /* Look through the reverse index. */ - reader_init_iter(r, &oit, BLOCK_TYPE_OBJ); + err = reader_init_iter(r, &oit, BLOCK_TYPE_OBJ); + if (err < 0) + goto done; + err = iterator_seek(&oit, &want); if (err != 0) goto done; @@ -699,7 +729,7 @@ static int reftable_reader_refs_for_indexed(struct reftable_reader *r, goto done; } - err = new_indexed_table_ref_iter(&itr, r, oid, hash_size(r->hash_id), + err = indexed_table_ref_iter_new(&itr, r, oid, hash_size(r->hash_id), got.u.obj.offsets, got.u.obj.offset_len); if (err < 0) @@ -724,23 +754,40 @@ static int reftable_reader_refs_for_unindexed(struct reftable_reader *r, int err; REFTABLE_ALLOC_ARRAY(ti, 1); + if (!ti) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto out; + } + table_iter_init(ti, r); err = table_iter_seek_start(ti, BLOCK_TYPE_REF, 0); - if (err < 0) { - reftable_free(ti); - return err; - } + if (err < 0) + goto out; - filter = reftable_malloc(sizeof(struct filtering_ref_iterator)); + filter = reftable_malloc(sizeof(*filter)); + if (!filter) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto out; + } *filter = empty; - strbuf_add(&filter->oid, oid, oid_len); - reftable_table_from_reader(&filter->tab, r); - filter->double_check = 0; + err = reftable_buf_add(&filter->oid, oid, oid_len); + if (err < 0) + goto out; + iterator_from_table_iter(&filter->it, ti); iterator_from_filtering_ref_iterator(it, filter); - return 0; + + err = 0; + +out: + if (err < 0) { + if (ti) + table_iter_close(ti); + reftable_free(ti); + } + return err; } int reftable_reader_refs_for(struct reftable_reader *r, @@ -761,66 +808,6 @@ uint64_t reftable_reader_min_update_index(struct reftable_reader *r) return r->min_update_index; } -/* generic table interface. */ - -static void reftable_reader_init_iter_void(void *tab, - struct reftable_iterator *it, - uint8_t typ) -{ - reader_init_iter(tab, it, typ); -} - -static uint32_t reftable_reader_hash_id_void(void *tab) -{ - return reftable_reader_hash_id(tab); -} - -static uint64_t reftable_reader_min_update_index_void(void *tab) -{ - return reftable_reader_min_update_index(tab); -} - -static uint64_t reftable_reader_max_update_index_void(void *tab) -{ - return reftable_reader_max_update_index(tab); -} - -static struct reftable_table_vtable reader_vtable = { - .init_iter = reftable_reader_init_iter_void, - .hash_id = reftable_reader_hash_id_void, - .min_update_index = reftable_reader_min_update_index_void, - .max_update_index = reftable_reader_max_update_index_void, -}; - -void reftable_table_from_reader(struct reftable_table *tab, - struct reftable_reader *reader) -{ - assert(!tab->ops); - tab->ops = &reader_vtable; - tab->table_arg = reader; -} - - -int reftable_reader_print_file(const char *tablename) -{ - struct reftable_block_source src = { NULL }; - int err = reftable_block_source_from_file(&src, tablename); - struct reftable_reader *r = NULL; - struct reftable_table tab = { NULL }; - if (err < 0) - goto done; - - err = reftable_new_reader(&r, &src, tablename); - if (err < 0) - goto done; - - reftable_table_from_reader(&tab, r); - err = reftable_table_print(&tab); -done: - reftable_reader_free(r); - return err; -} - int reftable_reader_print_blocks(const char *tablename) { struct { @@ -850,7 +837,7 @@ int reftable_reader_print_blocks(const char *tablename) if (err < 0) goto done; - err = reftable_new_reader(&r, &src, tablename); + err = reftable_reader_new(&r, &src, tablename); if (err < 0) goto done; @@ -881,7 +868,7 @@ int reftable_reader_print_blocks(const char *tablename) } done: - reftable_reader_free(r); + reftable_reader_decref(r); table_iter_close(&ti); return err; } diff --git a/reftable/reader.h b/reftable/reader.h index e869165f23..d2b48a4849 100644 --- a/reftable/reader.h +++ b/reftable/reader.h @@ -30,15 +30,15 @@ struct reftable_reader_offsets { /* The state for reading a reftable file. */ struct reftable_reader { - /* for convience, associate a name with the instance. */ + /* for convenience, associate a name with the instance. */ char *name; struct reftable_block_source source; /* Size of the file, excluding the footer. */ uint64_t size; - /* 'sha1' for SHA1, 's256' for SHA-256 */ - uint32_t hash_id; + /* The hash function used for ref records. */ + enum reftable_hash hash_id; uint32_t block_size; uint64_t min_update_index; @@ -50,13 +50,16 @@ struct reftable_reader { struct reftable_reader_offsets ref_offsets; struct reftable_reader_offsets obj_offsets; struct reftable_reader_offsets log_offsets; + + uint64_t refcount; }; -int init_reader(struct reftable_reader *r, struct reftable_block_source *source, - const char *name); -void reader_close(struct reftable_reader *r); const char *reader_name(struct reftable_reader *r); +int reader_init_iter(struct reftable_reader *r, + struct reftable_iterator *it, + uint8_t typ); + /* initialize a block reader to read from `r` */ int reader_init_block_reader(struct reftable_reader *r, struct block_reader *br, uint64_t next_off, uint8_t want_typ); diff --git a/reftable/readwrite_test.c b/reftable/readwrite_test.c deleted file mode 100644 index f411abfe9c..0000000000 --- a/reftable/readwrite_test.c +++ /dev/null @@ -1,979 +0,0 @@ -/* -Copyright 2020 Google LLC - -Use of this source code is governed by a BSD-style -license that can be found in the LICENSE file or at -https://developers.google.com/open-source/licenses/bsd -*/ - -#include "system.h" - -#include "basics.h" -#include "block.h" -#include "blocksource.h" -#include "reader.h" -#include "record.h" -#include "test_framework.h" -#include "reftable-tests.h" -#include "reftable-writer.h" - -static const int update_index = 5; - -static void test_buffer(void) -{ - struct strbuf buf = STRBUF_INIT; - struct reftable_block_source source = { NULL }; - struct reftable_block out = { NULL }; - int n; - uint8_t in[] = "hello"; - strbuf_add(&buf, in, sizeof(in)); - block_source_from_strbuf(&source, &buf); - EXPECT(block_source_size(&source) == 6); - n = block_source_read_block(&source, &out, 0, sizeof(in)); - EXPECT(n == sizeof(in)); - EXPECT(!memcmp(in, out.data, n)); - reftable_block_done(&out); - - n = block_source_read_block(&source, &out, 1, 2); - EXPECT(n == 2); - EXPECT(!memcmp(out.data, "el", 2)); - - reftable_block_done(&out); - block_source_close(&source); - strbuf_release(&buf); -} - -static void write_table(char ***names, struct strbuf *buf, int N, - int block_size, uint32_t hash_id) -{ - struct reftable_write_options opts = { - .block_size = block_size, - .hash_id = hash_id, - }; - struct reftable_writer *w = - reftable_new_writer(&strbuf_add_void, &noop_flush, buf, &opts); - struct reftable_ref_record ref = { NULL }; - int i = 0, n; - struct reftable_log_record log = { NULL }; - const struct reftable_stats *stats = NULL; - - REFTABLE_CALLOC_ARRAY(*names, N + 1); - - reftable_writer_set_limits(w, update_index, update_index); - for (i = 0; i < N; i++) { - char name[100]; - int n; - - snprintf(name, sizeof(name), "refs/heads/branch%02d", i); - - ref.refname = name; - ref.update_index = update_index; - ref.value_type = REFTABLE_REF_VAL1; - set_test_hash(ref.value.val1, i); - (*names)[i] = xstrdup(name); - - n = reftable_writer_add_ref(w, &ref); - EXPECT(n == 0); - } - - for (i = 0; i < N; i++) { - char name[100]; - int n; - - snprintf(name, sizeof(name), "refs/heads/branch%02d", i); - - log.refname = name; - log.update_index = update_index; - log.value_type = REFTABLE_LOG_UPDATE; - set_test_hash(log.value.update.new_hash, i); - log.value.update.message = (char *) "message"; - - n = reftable_writer_add_log(w, &log); - EXPECT(n == 0); - } - - n = reftable_writer_close(w); - EXPECT(n == 0); - - stats = reftable_writer_stats(w); - for (i = 0; i < stats->ref_stats.blocks; i++) { - int off = i * opts.block_size; - if (off == 0) { - off = header_size( - (hash_id == GIT_SHA256_FORMAT_ID) ? 2 : 1); - } - EXPECT(buf->buf[off] == 'r'); - } - - EXPECT(stats->log_stats.blocks > 0); - reftable_writer_free(w); -} - -static void test_log_buffer_size(void) -{ - struct strbuf buf = STRBUF_INIT; - struct reftable_write_options opts = { - .block_size = 4096, - }; - int err; - int i; - struct reftable_log_record - log = { .refname = (char *) "refs/heads/master", - .update_index = 0xa, - .value_type = REFTABLE_LOG_UPDATE, - .value = { .update = { - .name = (char *) "Han-Wen Nienhuys", - .email = (char *) "hanwen@google.com", - .tz_offset = 100, - .time = 0x5e430672, - .message = (char *) "commit: 9\n", - } } }; - struct reftable_writer *w = - reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts); - - /* This tests buffer extension for log compression. Must use a random - hash, to ensure that the compressed part is larger than the original. - */ - for (i = 0; i < GIT_SHA1_RAWSZ; i++) { - log.value.update.old_hash[i] = (uint8_t)(git_rand() % 256); - log.value.update.new_hash[i] = (uint8_t)(git_rand() % 256); - } - reftable_writer_set_limits(w, update_index, update_index); - err = reftable_writer_add_log(w, &log); - EXPECT_ERR(err); - err = reftable_writer_close(w); - EXPECT_ERR(err); - reftable_writer_free(w); - strbuf_release(&buf); -} - -static void test_log_overflow(void) -{ - struct strbuf buf = STRBUF_INIT; - char msg[256] = { 0 }; - struct reftable_write_options opts = { - .block_size = ARRAY_SIZE(msg), - }; - int err; - struct reftable_log_record log = { - .refname = (char *) "refs/heads/master", - .update_index = 0xa, - .value_type = REFTABLE_LOG_UPDATE, - .value = { - .update = { - .old_hash = { 1 }, - .new_hash = { 2 }, - .name = (char *) "Han-Wen Nienhuys", - .email = (char *) "hanwen@google.com", - .tz_offset = 100, - .time = 0x5e430672, - .message = msg, - }, - }, - }; - struct reftable_writer *w = - reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts); - - memset(msg, 'x', sizeof(msg) - 1); - reftable_writer_set_limits(w, update_index, update_index); - err = reftable_writer_add_log(w, &log); - EXPECT(err == REFTABLE_ENTRY_TOO_BIG_ERROR); - reftable_writer_free(w); - strbuf_release(&buf); -} - -static void test_log_write_read(void) -{ - int N = 2; - char **names = reftable_calloc(N + 1, sizeof(*names)); - int err; - struct reftable_write_options opts = { - .block_size = 256, - }; - struct reftable_ref_record ref = { NULL }; - int i = 0; - struct reftable_log_record log = { NULL }; - int n; - struct reftable_iterator it = { NULL }; - struct reftable_reader rd = { NULL }; - struct reftable_block_source source = { NULL }; - struct strbuf buf = STRBUF_INIT; - struct reftable_writer *w = - reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts); - const struct reftable_stats *stats = NULL; - reftable_writer_set_limits(w, 0, N); - for (i = 0; i < N; i++) { - char name[256]; - struct reftable_ref_record ref = { NULL }; - snprintf(name, sizeof(name), "b%02d%0*d", i, 130, 7); - names[i] = xstrdup(name); - ref.refname = name; - ref.update_index = i; - - err = reftable_writer_add_ref(w, &ref); - EXPECT_ERR(err); - } - for (i = 0; i < N; i++) { - struct reftable_log_record log = { NULL }; - - log.refname = names[i]; - log.update_index = i; - log.value_type = REFTABLE_LOG_UPDATE; - set_test_hash(log.value.update.old_hash, i); - set_test_hash(log.value.update.new_hash, i + 1); - - err = reftable_writer_add_log(w, &log); - EXPECT_ERR(err); - } - - n = reftable_writer_close(w); - EXPECT(n == 0); - - stats = reftable_writer_stats(w); - EXPECT(stats->log_stats.blocks > 0); - reftable_writer_free(w); - w = NULL; - - block_source_from_strbuf(&source, &buf); - - err = init_reader(&rd, &source, "file.log"); - EXPECT_ERR(err); - - reftable_reader_init_ref_iterator(&rd, &it); - - err = reftable_iterator_seek_ref(&it, names[N - 1]); - EXPECT_ERR(err); - - err = reftable_iterator_next_ref(&it, &ref); - EXPECT_ERR(err); - - /* end of iteration. */ - err = reftable_iterator_next_ref(&it, &ref); - EXPECT(0 < err); - - reftable_iterator_destroy(&it); - reftable_ref_record_release(&ref); - - reftable_reader_init_log_iterator(&rd, &it); - - err = reftable_iterator_seek_log(&it, ""); - EXPECT_ERR(err); - - i = 0; - while (1) { - int err = reftable_iterator_next_log(&it, &log); - if (err > 0) { - break; - } - - EXPECT_ERR(err); - EXPECT_STREQ(names[i], log.refname); - EXPECT(i == log.update_index); - i++; - reftable_log_record_release(&log); - } - - EXPECT(i == N); - reftable_iterator_destroy(&it); - - /* cleanup. */ - strbuf_release(&buf); - free_names(names); - reader_close(&rd); -} - -static void test_log_zlib_corruption(void) -{ - struct reftable_write_options opts = { - .block_size = 256, - }; - struct reftable_iterator it = { 0 }; - struct reftable_reader rd = { 0 }; - struct reftable_block_source source = { 0 }; - struct strbuf buf = STRBUF_INIT; - struct reftable_writer *w = - reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts); - const struct reftable_stats *stats = NULL; - char message[100] = { 0 }; - int err, i, n; - struct reftable_log_record log = { - .refname = (char *) "refname", - .value_type = REFTABLE_LOG_UPDATE, - .value = { - .update = { - .new_hash = { 1 }, - .old_hash = { 2 }, - .name = (char *) "My Name", - .email = (char *) "myname@invalid", - .message = message, - }, - }, - }; - - for (i = 0; i < sizeof(message) - 1; i++) - message[i] = (uint8_t)(git_rand() % 64 + ' '); - - reftable_writer_set_limits(w, 1, 1); - - err = reftable_writer_add_log(w, &log); - EXPECT_ERR(err); - - n = reftable_writer_close(w); - EXPECT(n == 0); - - stats = reftable_writer_stats(w); - EXPECT(stats->log_stats.blocks > 0); - reftable_writer_free(w); - w = NULL; - - /* corrupt the data. */ - buf.buf[50] ^= 0x99; - - block_source_from_strbuf(&source, &buf); - - err = init_reader(&rd, &source, "file.log"); - EXPECT_ERR(err); - - reftable_reader_init_log_iterator(&rd, &it); - err = reftable_iterator_seek_log(&it, "refname"); - EXPECT(err == REFTABLE_ZLIB_ERROR); - - reftable_iterator_destroy(&it); - - /* cleanup. */ - strbuf_release(&buf); - reader_close(&rd); -} - -static void test_table_read_write_sequential(void) -{ - char **names; - struct strbuf buf = STRBUF_INIT; - int N = 50; - struct reftable_iterator it = { NULL }; - struct reftable_block_source source = { NULL }; - struct reftable_reader rd = { NULL }; - int err = 0; - int j = 0; - - write_table(&names, &buf, N, 256, GIT_SHA1_FORMAT_ID); - - block_source_from_strbuf(&source, &buf); - - err = init_reader(&rd, &source, "file.ref"); - EXPECT_ERR(err); - - reftable_reader_init_ref_iterator(&rd, &it); - err = reftable_iterator_seek_ref(&it, ""); - EXPECT_ERR(err); - - while (1) { - struct reftable_ref_record ref = { NULL }; - int r = reftable_iterator_next_ref(&it, &ref); - EXPECT(r >= 0); - if (r > 0) { - break; - } - EXPECT(0 == strcmp(names[j], ref.refname)); - EXPECT(update_index == ref.update_index); - - j++; - reftable_ref_record_release(&ref); - } - EXPECT(j == N); - reftable_iterator_destroy(&it); - strbuf_release(&buf); - free_names(names); - - reader_close(&rd); -} - -static void test_table_write_small_table(void) -{ - char **names; - struct strbuf buf = STRBUF_INIT; - int N = 1; - write_table(&names, &buf, N, 4096, GIT_SHA1_FORMAT_ID); - EXPECT(buf.len < 200); - strbuf_release(&buf); - free_names(names); -} - -static void test_table_read_api(void) -{ - char **names; - struct strbuf buf = STRBUF_INIT; - int N = 50; - struct reftable_reader rd = { NULL }; - struct reftable_block_source source = { NULL }; - int err; - int i; - struct reftable_log_record log = { NULL }; - struct reftable_iterator it = { NULL }; - - write_table(&names, &buf, N, 256, GIT_SHA1_FORMAT_ID); - - block_source_from_strbuf(&source, &buf); - - err = init_reader(&rd, &source, "file.ref"); - EXPECT_ERR(err); - - reftable_reader_init_ref_iterator(&rd, &it); - err = reftable_iterator_seek_ref(&it, names[0]); - EXPECT_ERR(err); - - err = reftable_iterator_next_log(&it, &log); - EXPECT(err == REFTABLE_API_ERROR); - - strbuf_release(&buf); - for (i = 0; i < N; i++) { - reftable_free(names[i]); - } - reftable_iterator_destroy(&it); - reftable_free(names); - reader_close(&rd); - strbuf_release(&buf); -} - -static void test_table_read_write_seek(int index, int hash_id) -{ - char **names; - struct strbuf buf = STRBUF_INIT; - int N = 50; - struct reftable_reader rd = { NULL }; - struct reftable_block_source source = { NULL }; - int err; - int i = 0; - - struct reftable_iterator it = { NULL }; - struct strbuf pastLast = STRBUF_INIT; - struct reftable_ref_record ref = { NULL }; - - write_table(&names, &buf, N, 256, hash_id); - - block_source_from_strbuf(&source, &buf); - - err = init_reader(&rd, &source, "file.ref"); - EXPECT_ERR(err); - EXPECT(hash_id == reftable_reader_hash_id(&rd)); - - if (!index) { - rd.ref_offsets.index_offset = 0; - } else { - EXPECT(rd.ref_offsets.index_offset > 0); - } - - for (i = 1; i < N; i++) { - reftable_reader_init_ref_iterator(&rd, &it); - err = reftable_iterator_seek_ref(&it, names[i]); - EXPECT_ERR(err); - err = reftable_iterator_next_ref(&it, &ref); - EXPECT_ERR(err); - EXPECT(0 == strcmp(names[i], ref.refname)); - EXPECT(REFTABLE_REF_VAL1 == ref.value_type); - EXPECT(i == ref.value.val1[0]); - - reftable_ref_record_release(&ref); - reftable_iterator_destroy(&it); - } - - strbuf_addstr(&pastLast, names[N - 1]); - strbuf_addstr(&pastLast, "/"); - - reftable_reader_init_ref_iterator(&rd, &it); - err = reftable_iterator_seek_ref(&it, pastLast.buf); - if (err == 0) { - struct reftable_ref_record ref = { NULL }; - int err = reftable_iterator_next_ref(&it, &ref); - EXPECT(err > 0); - } else { - EXPECT(err > 0); - } - - strbuf_release(&pastLast); - reftable_iterator_destroy(&it); - - strbuf_release(&buf); - for (i = 0; i < N; i++) { - reftable_free(names[i]); - } - reftable_free(names); - reader_close(&rd); -} - -static void test_table_read_write_seek_linear(void) -{ - test_table_read_write_seek(0, GIT_SHA1_FORMAT_ID); -} - -static void test_table_read_write_seek_linear_sha256(void) -{ - test_table_read_write_seek(0, GIT_SHA256_FORMAT_ID); -} - -static void test_table_read_write_seek_index(void) -{ - test_table_read_write_seek(1, GIT_SHA1_FORMAT_ID); -} - -static void test_table_refs_for(int indexed) -{ - int N = 50; - char **want_names = reftable_calloc(N + 1, sizeof(*want_names)); - int want_names_len = 0; - uint8_t want_hash[GIT_SHA1_RAWSZ]; - - struct reftable_write_options opts = { - .block_size = 256, - }; - struct reftable_ref_record ref = { NULL }; - int i = 0; - int n; - int err; - struct reftable_reader rd; - struct reftable_block_source source = { NULL }; - - struct strbuf buf = STRBUF_INIT; - struct reftable_writer *w = - reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts); - - struct reftable_iterator it = { NULL }; - int j; - - set_test_hash(want_hash, 4); - - for (i = 0; i < N; i++) { - uint8_t hash[GIT_SHA1_RAWSZ]; - char fill[51] = { 0 }; - char name[100]; - struct reftable_ref_record ref = { NULL }; - - memset(hash, i, sizeof(hash)); - memset(fill, 'x', 50); - /* Put the variable part in the start */ - snprintf(name, sizeof(name), "br%02d%s", i, fill); - name[40] = 0; - ref.refname = name; - - ref.value_type = REFTABLE_REF_VAL2; - set_test_hash(ref.value.val2.value, i / 4); - set_test_hash(ref.value.val2.target_value, 3 + i / 4); - - /* 80 bytes / entry, so 3 entries per block. Yields 17 - */ - /* blocks. */ - n = reftable_writer_add_ref(w, &ref); - EXPECT(n == 0); - - if (!memcmp(ref.value.val2.value, want_hash, GIT_SHA1_RAWSZ) || - !memcmp(ref.value.val2.target_value, want_hash, GIT_SHA1_RAWSZ)) { - want_names[want_names_len++] = xstrdup(name); - } - } - - n = reftable_writer_close(w); - EXPECT(n == 0); - - reftable_writer_free(w); - w = NULL; - - block_source_from_strbuf(&source, &buf); - - err = init_reader(&rd, &source, "file.ref"); - EXPECT_ERR(err); - if (!indexed) { - rd.obj_offsets.is_present = 0; - } - - reftable_reader_init_ref_iterator(&rd, &it); - err = reftable_iterator_seek_ref(&it, ""); - EXPECT_ERR(err); - reftable_iterator_destroy(&it); - - err = reftable_reader_refs_for(&rd, &it, want_hash); - EXPECT_ERR(err); - - j = 0; - while (1) { - int err = reftable_iterator_next_ref(&it, &ref); - EXPECT(err >= 0); - if (err > 0) { - break; - } - - EXPECT(j < want_names_len); - EXPECT(0 == strcmp(ref.refname, want_names[j])); - j++; - reftable_ref_record_release(&ref); - } - EXPECT(j == want_names_len); - - strbuf_release(&buf); - free_names(want_names); - reftable_iterator_destroy(&it); - reader_close(&rd); -} - -static void test_table_refs_for_no_index(void) -{ - test_table_refs_for(0); -} - -static void test_table_refs_for_obj_index(void) -{ - test_table_refs_for(1); -} - -static void test_write_empty_table(void) -{ - struct reftable_write_options opts = { 0 }; - struct strbuf buf = STRBUF_INIT; - struct reftable_writer *w = - reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts); - struct reftable_block_source source = { NULL }; - struct reftable_reader *rd = NULL; - struct reftable_ref_record rec = { NULL }; - struct reftable_iterator it = { NULL }; - int err; - - reftable_writer_set_limits(w, 1, 1); - - err = reftable_writer_close(w); - EXPECT(err == REFTABLE_EMPTY_TABLE_ERROR); - reftable_writer_free(w); - - EXPECT(buf.len == header_size(1) + footer_size(1)); - - block_source_from_strbuf(&source, &buf); - - err = reftable_new_reader(&rd, &source, "filename"); - EXPECT_ERR(err); - - reftable_reader_init_ref_iterator(rd, &it); - err = reftable_iterator_seek_ref(&it, ""); - EXPECT_ERR(err); - - err = reftable_iterator_next_ref(&it, &rec); - EXPECT(err > 0); - - reftable_iterator_destroy(&it); - reftable_reader_free(rd); - strbuf_release(&buf); -} - -static void test_write_object_id_min_length(void) -{ - struct reftable_write_options opts = { - .block_size = 75, - }; - struct strbuf buf = STRBUF_INIT; - struct reftable_writer *w = - reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts); - struct reftable_ref_record ref = { - .update_index = 1, - .value_type = REFTABLE_REF_VAL1, - .value.val1 = {42}, - }; - int err; - int i; - - reftable_writer_set_limits(w, 1, 1); - - /* Write the same hash in many refs. If there is only 1 hash, the - * disambiguating prefix is length 0 */ - for (i = 0; i < 256; i++) { - char name[256]; - snprintf(name, sizeof(name), "ref%05d", i); - ref.refname = name; - err = reftable_writer_add_ref(w, &ref); - EXPECT_ERR(err); - } - - err = reftable_writer_close(w); - EXPECT_ERR(err); - EXPECT(reftable_writer_stats(w)->object_id_len == 2); - reftable_writer_free(w); - strbuf_release(&buf); -} - -static void test_write_object_id_length(void) -{ - struct reftable_write_options opts = { - .block_size = 75, - }; - struct strbuf buf = STRBUF_INIT; - struct reftable_writer *w = - reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts); - struct reftable_ref_record ref = { - .update_index = 1, - .value_type = REFTABLE_REF_VAL1, - .value.val1 = {42}, - }; - int err; - int i; - - reftable_writer_set_limits(w, 1, 1); - - /* Write the same hash in many refs. If there is only 1 hash, the - * disambiguating prefix is length 0 */ - for (i = 0; i < 256; i++) { - char name[256]; - snprintf(name, sizeof(name), "ref%05d", i); - ref.refname = name; - ref.value.val1[15] = i; - err = reftable_writer_add_ref(w, &ref); - EXPECT_ERR(err); - } - - err = reftable_writer_close(w); - EXPECT_ERR(err); - EXPECT(reftable_writer_stats(w)->object_id_len == 16); - reftable_writer_free(w); - strbuf_release(&buf); -} - -static void test_write_empty_key(void) -{ - struct reftable_write_options opts = { 0 }; - struct strbuf buf = STRBUF_INIT; - struct reftable_writer *w = - reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts); - struct reftable_ref_record ref = { - .refname = (char *) "", - .update_index = 1, - .value_type = REFTABLE_REF_DELETION, - }; - int err; - - reftable_writer_set_limits(w, 1, 1); - err = reftable_writer_add_ref(w, &ref); - EXPECT(err == REFTABLE_API_ERROR); - - err = reftable_writer_close(w); - EXPECT(err == REFTABLE_EMPTY_TABLE_ERROR); - reftable_writer_free(w); - strbuf_release(&buf); -} - -static void test_write_key_order(void) -{ - struct reftable_write_options opts = { 0 }; - struct strbuf buf = STRBUF_INIT; - struct reftable_writer *w = - reftable_new_writer(&strbuf_add_void, &noop_flush, &buf, &opts); - struct reftable_ref_record refs[2] = { - { - .refname = (char *) "b", - .update_index = 1, - .value_type = REFTABLE_REF_SYMREF, - .value = { - .symref = (char *) "target", - }, - }, { - .refname = (char *) "a", - .update_index = 1, - .value_type = REFTABLE_REF_SYMREF, - .value = { - .symref = (char *) "target", - }, - } - }; - int err; - - reftable_writer_set_limits(w, 1, 1); - err = reftable_writer_add_ref(w, &refs[0]); - EXPECT_ERR(err); - err = reftable_writer_add_ref(w, &refs[1]); - EXPECT(err == REFTABLE_API_ERROR); - reftable_writer_close(w); - reftable_writer_free(w); - strbuf_release(&buf); -} - -static void test_write_multiple_indices(void) -{ - struct reftable_write_options opts = { - .block_size = 100, - }; - struct strbuf writer_buf = STRBUF_INIT, buf = STRBUF_INIT; - struct reftable_block_source source = { 0 }; - struct reftable_iterator it = { 0 }; - const struct reftable_stats *stats; - struct reftable_writer *writer; - struct reftable_reader *reader; - int err, i; - - writer = reftable_new_writer(&strbuf_add_void, &noop_flush, &writer_buf, &opts); - reftable_writer_set_limits(writer, 1, 1); - for (i = 0; i < 100; i++) { - struct reftable_ref_record ref = { - .update_index = 1, - .value_type = REFTABLE_REF_VAL1, - .value.val1 = {i}, - }; - - strbuf_reset(&buf); - strbuf_addf(&buf, "refs/heads/%04d", i); - ref.refname = buf.buf, - - err = reftable_writer_add_ref(writer, &ref); - EXPECT_ERR(err); - } - - for (i = 0; i < 100; i++) { - struct reftable_log_record log = { - .update_index = 1, - .value_type = REFTABLE_LOG_UPDATE, - .value.update = { - .old_hash = { i }, - .new_hash = { i }, - }, - }; - - strbuf_reset(&buf); - strbuf_addf(&buf, "refs/heads/%04d", i); - log.refname = buf.buf, - - err = reftable_writer_add_log(writer, &log); - EXPECT_ERR(err); - } - - reftable_writer_close(writer); - - /* - * The written data should be sufficiently large to result in indices - * for each of the block types. - */ - stats = reftable_writer_stats(writer); - EXPECT(stats->ref_stats.index_offset > 0); - EXPECT(stats->obj_stats.index_offset > 0); - EXPECT(stats->log_stats.index_offset > 0); - - block_source_from_strbuf(&source, &writer_buf); - err = reftable_new_reader(&reader, &source, "filename"); - EXPECT_ERR(err); - - /* - * Seeking the log uses the log index now. In case there is any - * confusion regarding indices we would notice here. - */ - reftable_reader_init_log_iterator(reader, &it); - err = reftable_iterator_seek_log(&it, ""); - EXPECT_ERR(err); - - reftable_iterator_destroy(&it); - reftable_writer_free(writer); - reftable_reader_free(reader); - strbuf_release(&writer_buf); - strbuf_release(&buf); -} - -static void test_write_multi_level_index(void) -{ - struct reftable_write_options opts = { - .block_size = 100, - }; - struct strbuf writer_buf = STRBUF_INIT, buf = STRBUF_INIT; - struct reftable_block_source source = { 0 }; - struct reftable_iterator it = { 0 }; - const struct reftable_stats *stats; - struct reftable_writer *writer; - struct reftable_reader *reader; - int err; - - writer = reftable_new_writer(&strbuf_add_void, &noop_flush, &writer_buf, &opts); - reftable_writer_set_limits(writer, 1, 1); - for (size_t i = 0; i < 200; i++) { - struct reftable_ref_record ref = { - .update_index = 1, - .value_type = REFTABLE_REF_VAL1, - .value.val1 = {i}, - }; - - strbuf_reset(&buf); - strbuf_addf(&buf, "refs/heads/%03" PRIuMAX, (uintmax_t)i); - ref.refname = buf.buf, - - err = reftable_writer_add_ref(writer, &ref); - EXPECT_ERR(err); - } - reftable_writer_close(writer); - - /* - * The written refs should be sufficiently large to result in a - * multi-level index. - */ - stats = reftable_writer_stats(writer); - EXPECT(stats->ref_stats.max_index_level == 2); - - block_source_from_strbuf(&source, &writer_buf); - err = reftable_new_reader(&reader, &source, "filename"); - EXPECT_ERR(err); - - /* - * Seeking the last ref should work as expected. - */ - reftable_reader_init_ref_iterator(reader, &it); - err = reftable_iterator_seek_ref(&it, "refs/heads/199"); - EXPECT_ERR(err); - - reftable_iterator_destroy(&it); - reftable_writer_free(writer); - reftable_reader_free(reader); - strbuf_release(&writer_buf); - strbuf_release(&buf); -} - -static void test_corrupt_table_empty(void) -{ - struct strbuf buf = STRBUF_INIT; - struct reftable_block_source source = { NULL }; - struct reftable_reader rd = { NULL }; - int err; - - block_source_from_strbuf(&source, &buf); - err = init_reader(&rd, &source, "file.log"); - EXPECT(err == REFTABLE_FORMAT_ERROR); -} - -static void test_corrupt_table(void) -{ - uint8_t zeros[1024] = { 0 }; - struct strbuf buf = STRBUF_INIT; - struct reftable_block_source source = { NULL }; - struct reftable_reader rd = { NULL }; - int err; - strbuf_add(&buf, zeros, sizeof(zeros)); - - block_source_from_strbuf(&source, &buf); - err = init_reader(&rd, &source, "file.log"); - EXPECT(err == REFTABLE_FORMAT_ERROR); - strbuf_release(&buf); -} - -int readwrite_test_main(int argc, const char *argv[]) -{ - RUN_TEST(test_log_zlib_corruption); - RUN_TEST(test_corrupt_table); - RUN_TEST(test_corrupt_table_empty); - RUN_TEST(test_log_write_read); - RUN_TEST(test_write_key_order); - RUN_TEST(test_table_read_write_seek_linear_sha256); - RUN_TEST(test_log_buffer_size); - RUN_TEST(test_table_write_small_table); - RUN_TEST(test_buffer); - RUN_TEST(test_table_read_api); - RUN_TEST(test_table_read_write_sequential); - RUN_TEST(test_table_read_write_seek_linear); - RUN_TEST(test_table_read_write_seek_index); - RUN_TEST(test_table_refs_for_no_index); - RUN_TEST(test_table_refs_for_obj_index); - RUN_TEST(test_write_empty_key); - RUN_TEST(test_write_empty_table); - RUN_TEST(test_log_overflow); - RUN_TEST(test_write_object_id_length); - RUN_TEST(test_write_object_id_min_length); - RUN_TEST(test_write_multiple_indices); - RUN_TEST(test_write_multi_level_index); - return 0; -} diff --git a/reftable/record.c b/reftable/record.c index a2cba5ef74..fb5652ed57 100644 --- a/reftable/record.c +++ b/reftable/record.c @@ -98,19 +98,24 @@ const unsigned char *reftable_ref_record_val2(const struct reftable_ref_record * } } -static int decode_string(struct strbuf *dest, struct string_view in) +static int decode_string(struct reftable_buf *dest, struct string_view in) { int start_len = in.len; uint64_t tsize = 0; - int n = get_var_int(&tsize, &in); + int n, err; + + n = get_var_int(&tsize, &in); if (n <= 0) return -1; string_view_consume(&in, n); if (in.len < tsize) return -1; - strbuf_reset(dest); - strbuf_add(dest, in.buf, tsize); + reftable_buf_reset(dest); + err = reftable_buf_add(dest, in.buf, tsize); + if (err < 0) + return err; + string_view_consume(&in, tsize); return start_len - in.len; @@ -133,7 +138,7 @@ static int encode_string(const char *str, struct string_view s) } int reftable_encode_key(int *restart, struct string_view dest, - struct strbuf prev_key, struct strbuf key, + struct reftable_buf prev_key, struct reftable_buf key, uint8_t extra) { struct string_view start = dest; @@ -183,13 +188,13 @@ int reftable_decode_keylen(struct string_view in, return start_len - in.len; } -int reftable_decode_key(struct strbuf *last_key, uint8_t *extra, +int reftable_decode_key(struct reftable_buf *last_key, uint8_t *extra, struct string_view in) { int start_len = in.len; uint64_t prefix_len = 0; uint64_t suffix_len = 0; - int n; + int err, n; n = reftable_decode_keylen(in, &prefix_len, &suffix_len, extra); if (n < 0) @@ -200,28 +205,35 @@ int reftable_decode_key(struct strbuf *last_key, uint8_t *extra, prefix_len > last_key->len) return -1; - strbuf_setlen(last_key, prefix_len); - strbuf_add(last_key, in.buf, suffix_len); + err = reftable_buf_setlen(last_key, prefix_len); + if (err < 0) + return err; + + err = reftable_buf_add(last_key, in.buf, suffix_len); + if (err < 0) + return err; + string_view_consume(&in, suffix_len); return start_len - in.len; } -static void reftable_ref_record_key(const void *r, struct strbuf *dest) +static int reftable_ref_record_key(const void *r, struct reftable_buf *dest) { const struct reftable_ref_record *rec = (const struct reftable_ref_record *)r; - strbuf_reset(dest); - strbuf_addstr(dest, rec->refname); + reftable_buf_reset(dest); + return reftable_buf_addstr(dest, rec->refname); } -static void reftable_ref_record_copy_from(void *rec, const void *src_rec, - int hash_size) +static int reftable_ref_record_copy_from(void *rec, const void *src_rec, + int hash_size) { struct reftable_ref_record *ref = rec; const struct reftable_ref_record *src = src_rec; char *refname = NULL; size_t refname_cap = 0; + int err; assert(hash_size > 0); @@ -236,6 +248,11 @@ static void reftable_ref_record_copy_from(void *rec, const void *src_rec, REFTABLE_ALLOC_GROW(ref->refname, refname_len + 1, ref->refname_cap); + if (!ref->refname) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto out; + } + memcpy(ref->refname, src->refname, refname_len); ref->refname[refname_len] = 0; } @@ -254,61 +271,17 @@ static void reftable_ref_record_copy_from(void *rec, const void *src_rec, src->value.val2.target_value, hash_size); break; case REFTABLE_REF_SYMREF: - ref->value.symref = xstrdup(src->value.symref); - break; - } -} - -static char hexdigit(int c) -{ - if (c <= 9) - return '0' + c; - return 'a' + (c - 10); -} - -static void hex_format(char *dest, const unsigned char *src, int hash_size) -{ - assert(hash_size > 0); - if (src) { - int i = 0; - for (i = 0; i < hash_size; i++) { - dest[2 * i] = hexdigit(src[i] >> 4); - dest[2 * i + 1] = hexdigit(src[i] & 0xf); + ref->value.symref = reftable_strdup(src->value.symref); + if (!ref->value.symref) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto out; } - dest[2 * hash_size] = 0; - } -} - -static void reftable_ref_record_print_sz(const struct reftable_ref_record *ref, - int hash_size) -{ - char hex[GIT_MAX_HEXSZ + 1] = { 0 }; /* BUG */ - printf("ref{%s(%" PRIu64 ") ", ref->refname, ref->update_index); - switch (ref->value_type) { - case REFTABLE_REF_SYMREF: - printf("=> %s", ref->value.symref); - break; - case REFTABLE_REF_VAL2: - hex_format(hex, ref->value.val2.value, hash_size); - printf("val 2 %s", hex); - hex_format(hex, ref->value.val2.target_value, - hash_size); - printf("(T %s)", hex); - break; - case REFTABLE_REF_VAL1: - hex_format(hex, ref->value.val1, hash_size); - printf("val 1 %s", hex); - break; - case REFTABLE_REF_DELETION: - printf("delete"); break; } - printf("}\n"); -} -void reftable_ref_record_print(const struct reftable_ref_record *ref, - uint32_t hash_id) { - reftable_ref_record_print_sz(ref, hash_size(hash_id)); + err = 0; +out: + return err; } static void reftable_ref_record_release_void(void *rec) @@ -388,16 +361,16 @@ static int reftable_ref_record_encode(const void *rec, struct string_view s, return start.len - s.len; } -static int reftable_ref_record_decode(void *rec, struct strbuf key, +static int reftable_ref_record_decode(void *rec, struct reftable_buf key, uint8_t val_type, struct string_view in, - int hash_size, struct strbuf *scratch) + int hash_size, struct reftable_buf *scratch) { struct reftable_ref_record *r = rec; struct string_view start = in; uint64_t update_index = 0; const char *refname = NULL; size_t refname_cap = 0; - int n; + int n, err; assert(hash_size > 0); @@ -413,6 +386,10 @@ static int reftable_ref_record_decode(void *rec, struct strbuf key, SWAP(r->refname_cap, refname_cap); REFTABLE_ALLOC_GROW(r->refname, key.len + 1, r->refname_cap); + if (!r->refname) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto done; + } memcpy(r->refname, key.buf, key.len); r->refname[key.len] = 0; @@ -421,7 +398,8 @@ static int reftable_ref_record_decode(void *rec, struct strbuf key, switch (val_type) { case REFTABLE_REF_VAL1: if (in.len < hash_size) { - return -1; + err = REFTABLE_FORMAT_ERROR; + goto done; } memcpy(r->value.val1, in.buf, hash_size); @@ -430,7 +408,8 @@ static int reftable_ref_record_decode(void *rec, struct strbuf key, case REFTABLE_REF_VAL2: if (in.len < 2 * hash_size) { - return -1; + err = REFTABLE_FORMAT_ERROR; + goto done; } memcpy(r->value.val2.value, in.buf, hash_size); @@ -443,10 +422,11 @@ static int reftable_ref_record_decode(void *rec, struct strbuf key, case REFTABLE_REF_SYMREF: { int n = decode_string(scratch, in); if (n < 0) { - return -1; + err = REFTABLE_FORMAT_ERROR; + goto done; } string_view_consume(&in, n); - r->value.symref = strbuf_detach(scratch, NULL); + r->value.symref = reftable_buf_detach(scratch); } break; case REFTABLE_REF_DELETION: @@ -457,6 +437,9 @@ static int reftable_ref_record_decode(void *rec, struct strbuf key, } return start.len - in.len; + +done: + return err; } static int reftable_ref_record_is_deletion_void(const void *p) @@ -480,12 +463,6 @@ static int reftable_ref_record_cmp_void(const void *_a, const void *_b) return strcmp(a->refname, b->refname); } -static void reftable_ref_record_print_void(const void *rec, - int hash_size) -{ - reftable_ref_record_print_sz((struct reftable_ref_record *) rec, hash_size); -} - static struct reftable_record_vtable reftable_ref_record_vtable = { .key = &reftable_ref_record_key, .type = BLOCK_TYPE_REF, @@ -497,57 +474,46 @@ static struct reftable_record_vtable reftable_ref_record_vtable = { .is_deletion = &reftable_ref_record_is_deletion_void, .equal = &reftable_ref_record_equal_void, .cmp = &reftable_ref_record_cmp_void, - .print = &reftable_ref_record_print_void, }; -static void reftable_obj_record_key(const void *r, struct strbuf *dest) +static int reftable_obj_record_key(const void *r, struct reftable_buf *dest) { const struct reftable_obj_record *rec = (const struct reftable_obj_record *)r; - strbuf_reset(dest); - strbuf_add(dest, rec->hash_prefix, rec->hash_prefix_len); + reftable_buf_reset(dest); + return reftable_buf_add(dest, rec->hash_prefix, rec->hash_prefix_len); } static void reftable_obj_record_release(void *rec) { struct reftable_obj_record *obj = rec; - FREE_AND_NULL(obj->hash_prefix); - FREE_AND_NULL(obj->offsets); + REFTABLE_FREE_AND_NULL(obj->hash_prefix); + REFTABLE_FREE_AND_NULL(obj->offsets); memset(obj, 0, sizeof(struct reftable_obj_record)); } -static void reftable_obj_record_print(const void *rec, int hash_size) -{ - const struct reftable_obj_record *obj = rec; - char hex[GIT_MAX_HEXSZ + 1] = { 0 }; - struct strbuf offset_str = STRBUF_INIT; - int i; - - for (i = 0; i < obj->offset_len; i++) - strbuf_addf(&offset_str, "%" PRIu64 " ", obj->offsets[i]); - hex_format(hex, obj->hash_prefix, obj->hash_prefix_len); - printf("prefix %s (len %d), offsets [%s]\n", - hex, obj->hash_prefix_len, offset_str.buf); - strbuf_release(&offset_str); -} - -static void reftable_obj_record_copy_from(void *rec, const void *src_rec, - int hash_size) +static int reftable_obj_record_copy_from(void *rec, const void *src_rec, + int hash_size UNUSED) { struct reftable_obj_record *obj = rec; - const struct reftable_obj_record *src = - (const struct reftable_obj_record *)src_rec; + const struct reftable_obj_record *src = src_rec; reftable_obj_record_release(obj); REFTABLE_ALLOC_ARRAY(obj->hash_prefix, src->hash_prefix_len); + if (!obj->hash_prefix) + return REFTABLE_OUT_OF_MEMORY_ERROR; obj->hash_prefix_len = src->hash_prefix_len; if (src->hash_prefix_len) memcpy(obj->hash_prefix, src->hash_prefix, obj->hash_prefix_len); REFTABLE_ALLOC_ARRAY(obj->offsets, src->offset_len); + if (!obj->offsets) + return REFTABLE_OUT_OF_MEMORY_ERROR; obj->offset_len = src->offset_len; COPY_ARRAY(obj->offsets, src->offsets, src->offset_len); + + return 0; } static uint8_t reftable_obj_record_val_type(const void *rec) @@ -559,7 +525,7 @@ static uint8_t reftable_obj_record_val_type(const void *rec) } static int reftable_obj_record_encode(const void *rec, struct string_view s, - int hash_size) + int hash_size UNUSED) { const struct reftable_obj_record *r = rec; struct string_view start = s; @@ -592,9 +558,10 @@ static int reftable_obj_record_encode(const void *rec, struct string_view s, return start.len - s.len; } -static int reftable_obj_record_decode(void *rec, struct strbuf key, +static int reftable_obj_record_decode(void *rec, struct reftable_buf key, uint8_t val_type, struct string_view in, - int hash_size, struct strbuf *scratch UNUSED) + int hash_size UNUSED, + struct reftable_buf *scratch UNUSED) { struct string_view start = in; struct reftable_obj_record *r = rec; @@ -606,6 +573,8 @@ static int reftable_obj_record_decode(void *rec, struct strbuf key, reftable_obj_record_release(r); REFTABLE_ALLOC_ARRAY(r->hash_prefix, key.len); + if (!r->hash_prefix) + return REFTABLE_OUT_OF_MEMORY_ERROR; memcpy(r->hash_prefix, key.buf, key.len); r->hash_prefix_len = key.len; @@ -624,6 +593,8 @@ static int reftable_obj_record_decode(void *rec, struct strbuf key, return start.len - in.len; REFTABLE_ALLOC_ARRAY(r->offsets, count); + if (!r->offsets) + return REFTABLE_OUT_OF_MEMORY_ERROR; r->offset_len = count; n = get_var_int(&r->offsets[0], &in); @@ -647,12 +618,13 @@ static int reftable_obj_record_decode(void *rec, struct strbuf key, return start.len - in.len; } -static int not_a_deletion(const void *p) +static int not_a_deletion(const void *p UNUSED) { return 0; } -static int reftable_obj_record_equal_void(const void *a, const void *b, int hash_size) +static int reftable_obj_record_equal_void(const void *a, const void *b, + int hash_size UNUSED) { struct reftable_obj_record *ra = (struct reftable_obj_record *) a; struct reftable_obj_record *rb = (struct reftable_obj_record *) b; @@ -701,83 +673,69 @@ static struct reftable_record_vtable reftable_obj_record_vtable = { .is_deletion = ¬_a_deletion, .equal = &reftable_obj_record_equal_void, .cmp = &reftable_obj_record_cmp_void, - .print = &reftable_obj_record_print, }; -static void reftable_log_record_print_sz(struct reftable_log_record *log, - int hash_size) -{ - char hex[GIT_MAX_HEXSZ + 1] = { 0 }; - - switch (log->value_type) { - case REFTABLE_LOG_DELETION: - printf("log{%s(%" PRIu64 ") delete\n", log->refname, - log->update_index); - break; - case REFTABLE_LOG_UPDATE: - printf("log{%s(%" PRIu64 ") %s <%s> %" PRIu64 " %04d\n", - log->refname, log->update_index, - log->value.update.name ? log->value.update.name : "", - log->value.update.email ? log->value.update.email : "", - log->value.update.time, - log->value.update.tz_offset); - hex_format(hex, log->value.update.old_hash, hash_size); - printf("%s => ", hex); - hex_format(hex, log->value.update.new_hash, hash_size); - printf("%s\n\n%s\n}\n", hex, - log->value.update.message ? log->value.update.message : ""); - break; - } -} - -void reftable_log_record_print(struct reftable_log_record *log, - uint32_t hash_id) -{ - reftable_log_record_print_sz(log, hash_size(hash_id)); -} - -static void reftable_log_record_key(const void *r, struct strbuf *dest) +static int reftable_log_record_key(const void *r, struct reftable_buf *dest) { const struct reftable_log_record *rec = (const struct reftable_log_record *)r; - int len = strlen(rec->refname); + int len = strlen(rec->refname), err; uint8_t i64[8]; uint64_t ts = 0; - strbuf_reset(dest); - strbuf_add(dest, (uint8_t *)rec->refname, len + 1); + + reftable_buf_reset(dest); + err = reftable_buf_add(dest, (uint8_t *)rec->refname, len + 1); + if (err < 0) + return err; ts = (~ts) - rec->update_index; put_be64(&i64[0], ts); - strbuf_add(dest, i64, sizeof(i64)); + + err = reftable_buf_add(dest, i64, sizeof(i64)); + if (err < 0) + return err; + + return 0; } -static void reftable_log_record_copy_from(void *rec, const void *src_rec, - int hash_size) +static int reftable_log_record_copy_from(void *rec, const void *src_rec, + int hash_size) { struct reftable_log_record *dst = rec; const struct reftable_log_record *src = (const struct reftable_log_record *)src_rec; + int ret; reftable_log_record_release(dst); *dst = *src; + if (dst->refname) { - dst->refname = xstrdup(dst->refname); + dst->refname = reftable_strdup(dst->refname); + if (!dst->refname) { + ret = REFTABLE_OUT_OF_MEMORY_ERROR; + goto out; + } } + switch (dst->value_type) { case REFTABLE_LOG_DELETION: break; case REFTABLE_LOG_UPDATE: - if (dst->value.update.email) { + if (dst->value.update.email) dst->value.update.email = - xstrdup(dst->value.update.email); - } - if (dst->value.update.name) { + reftable_strdup(dst->value.update.email); + if (dst->value.update.name) dst->value.update.name = - xstrdup(dst->value.update.name); - } - if (dst->value.update.message) { + reftable_strdup(dst->value.update.name); + if (dst->value.update.message) dst->value.update.message = - xstrdup(dst->value.update.message); + reftable_strdup(dst->value.update.message); + + if (!dst->value.update.email || + !dst->value.update.name || + !dst->value.update.message) { + ret = REFTABLE_OUT_OF_MEMORY_ERROR; + goto out; } memcpy(dst->value.update.new_hash, @@ -786,6 +744,10 @@ static void reftable_log_record_copy_from(void *rec, const void *src_rec, src->value.update.old_hash, hash_size); break; } + + ret = 0; +out: + return ret; } static void reftable_log_record_release_void(void *rec) @@ -864,20 +826,25 @@ static int reftable_log_record_encode(const void *rec, struct string_view s, return start.len - s.len; } -static int reftable_log_record_decode(void *rec, struct strbuf key, +static int reftable_log_record_decode(void *rec, struct reftable_buf key, uint8_t val_type, struct string_view in, - int hash_size, struct strbuf *scratch) + int hash_size, struct reftable_buf *scratch) { struct string_view start = in; struct reftable_log_record *r = rec; uint64_t max = 0; uint64_t ts = 0; - int n; + int err, n; if (key.len <= 9 || key.buf[key.len - 9] != 0) return REFTABLE_FORMAT_ERROR; REFTABLE_ALLOC_GROW(r->refname, key.len - 8, r->refname_cap); + if (!r->refname) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto done; + } + memcpy(r->refname, key.buf, key.len - 8); ts = get_be64(key.buf + key.len - 8); @@ -886,10 +853,10 @@ static int reftable_log_record_decode(void *rec, struct strbuf key, if (val_type != r->value_type) { switch (r->value_type) { case REFTABLE_LOG_UPDATE: - FREE_AND_NULL(r->value.update.message); + REFTABLE_FREE_AND_NULL(r->value.update.message); r->value.update.message_cap = 0; - FREE_AND_NULL(r->value.update.email); - FREE_AND_NULL(r->value.update.name); + REFTABLE_FREE_AND_NULL(r->value.update.email); + REFTABLE_FREE_AND_NULL(r->value.update.name); break; case REFTABLE_LOG_DELETION: break; @@ -900,8 +867,10 @@ static int reftable_log_record_decode(void *rec, struct strbuf key, if (val_type == REFTABLE_LOG_DELETION) return 0; - if (in.len < 2 * hash_size) - return REFTABLE_FORMAT_ERROR; + if (in.len < 2 * hash_size) { + err = REFTABLE_FORMAT_ERROR; + goto done; + } memcpy(r->value.update.old_hash, in.buf, hash_size); memcpy(r->value.update.new_hash, in.buf + hash_size, hash_size); @@ -909,8 +878,10 @@ static int reftable_log_record_decode(void *rec, struct strbuf key, string_view_consume(&in, 2 * hash_size); n = decode_string(scratch, in); - if (n < 0) + if (n < 0) { + err = REFTABLE_FORMAT_ERROR; goto done; + } string_view_consume(&in, n); /* @@ -921,52 +892,75 @@ static int reftable_log_record_decode(void *rec, struct strbuf key, */ if (!r->value.update.name || strcmp(r->value.update.name, scratch->buf)) { - r->value.update.name = - reftable_realloc(r->value.update.name, scratch->len + 1); + char *name = reftable_realloc(r->value.update.name, scratch->len + 1); + if (!name) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto done; + } + + r->value.update.name = name; memcpy(r->value.update.name, scratch->buf, scratch->len); r->value.update.name[scratch->len] = 0; } n = decode_string(scratch, in); - if (n < 0) + if (n < 0) { + err = REFTABLE_FORMAT_ERROR; goto done; + } string_view_consume(&in, n); /* Same as above, but for the reflog email. */ if (!r->value.update.email || strcmp(r->value.update.email, scratch->buf)) { - r->value.update.email = - reftable_realloc(r->value.update.email, scratch->len + 1); + char *email = reftable_realloc(r->value.update.email, scratch->len + 1); + if (!email) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto done; + } + + r->value.update.email = email; memcpy(r->value.update.email, scratch->buf, scratch->len); r->value.update.email[scratch->len] = 0; } ts = 0; n = get_var_int(&ts, &in); - if (n < 0) + if (n < 0) { + err = REFTABLE_FORMAT_ERROR; goto done; + } string_view_consume(&in, n); r->value.update.time = ts; - if (in.len < 2) + if (in.len < 2) { + err = REFTABLE_FORMAT_ERROR; goto done; + } r->value.update.tz_offset = get_be16(in.buf); string_view_consume(&in, 2); n = decode_string(scratch, in); - if (n < 0) + if (n < 0) { + err = REFTABLE_FORMAT_ERROR; goto done; + } string_view_consume(&in, n); REFTABLE_ALLOC_GROW(r->value.update.message, scratch->len + 1, r->value.update.message_cap); + if (!r->value.update.message) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto done; + } + memcpy(r->value.update.message, scratch->buf, scratch->len); r->value.update.message[scratch->len] = 0; return start.len - in.len; done: - return REFTABLE_FORMAT_ERROR; + return err; } static int null_streq(const char *a, const char *b) @@ -1039,11 +1033,6 @@ static int reftable_log_record_is_deletion_void(const void *p) (const struct reftable_log_record *)p); } -static void reftable_log_record_print_void(const void *rec, int hash_size) -{ - reftable_log_record_print_sz((struct reftable_log_record*)rec, hash_size); -} - static struct reftable_record_vtable reftable_log_record_vtable = { .key = &reftable_log_record_key, .type = BLOCK_TYPE_LOG, @@ -1055,40 +1044,44 @@ static struct reftable_record_vtable reftable_log_record_vtable = { .is_deletion = &reftable_log_record_is_deletion_void, .equal = &reftable_log_record_equal_void, .cmp = &reftable_log_record_cmp_void, - .print = &reftable_log_record_print_void, }; -static void reftable_index_record_key(const void *r, struct strbuf *dest) +static int reftable_index_record_key(const void *r, struct reftable_buf *dest) { const struct reftable_index_record *rec = r; - strbuf_reset(dest); - strbuf_addbuf(dest, &rec->last_key); + reftable_buf_reset(dest); + return reftable_buf_add(dest, rec->last_key.buf, rec->last_key.len); } -static void reftable_index_record_copy_from(void *rec, const void *src_rec, - int hash_size) +static int reftable_index_record_copy_from(void *rec, const void *src_rec, + int hash_size UNUSED) { struct reftable_index_record *dst = rec; const struct reftable_index_record *src = src_rec; + int err; - strbuf_reset(&dst->last_key); - strbuf_addbuf(&dst->last_key, &src->last_key); + reftable_buf_reset(&dst->last_key); + err = reftable_buf_add(&dst->last_key, src->last_key.buf, src->last_key.len); + if (err < 0) + return err; dst->offset = src->offset; + + return 0; } static void reftable_index_record_release(void *rec) { struct reftable_index_record *idx = rec; - strbuf_release(&idx->last_key); + reftable_buf_release(&idx->last_key); } -static uint8_t reftable_index_record_val_type(const void *rec) +static uint8_t reftable_index_record_val_type(const void *rec UNUSED) { return 0; } static int reftable_index_record_encode(const void *rec, struct string_view out, - int hash_size) + int hash_size UNUSED) { const struct reftable_index_record *r = (const struct reftable_index_record *)rec; @@ -1103,16 +1096,20 @@ static int reftable_index_record_encode(const void *rec, struct string_view out, return start.len - out.len; } -static int reftable_index_record_decode(void *rec, struct strbuf key, - uint8_t val_type, struct string_view in, - int hash_size, struct strbuf *scratch UNUSED) +static int reftable_index_record_decode(void *rec, struct reftable_buf key, + uint8_t val_type UNUSED, + struct string_view in, + int hash_size UNUSED, + struct reftable_buf *scratch UNUSED) { struct string_view start = in; struct reftable_index_record *r = rec; - int n = 0; + int err, n = 0; - strbuf_reset(&r->last_key); - strbuf_addbuf(&r->last_key, &key); + reftable_buf_reset(&r->last_key); + err = reftable_buf_add(&r->last_key, key.buf, key.len); + if (err < 0) + return err; n = get_var_int(&r->offset, &in); if (n < 0) @@ -1122,26 +1119,20 @@ static int reftable_index_record_decode(void *rec, struct strbuf key, return start.len - in.len; } -static int reftable_index_record_equal(const void *a, const void *b, int hash_size) +static int reftable_index_record_equal(const void *a, const void *b, + int hash_size UNUSED) { struct reftable_index_record *ia = (struct reftable_index_record *) a; struct reftable_index_record *ib = (struct reftable_index_record *) b; - return ia->offset == ib->offset && !strbuf_cmp(&ia->last_key, &ib->last_key); + return ia->offset == ib->offset && !reftable_buf_cmp(&ia->last_key, &ib->last_key); } static int reftable_index_record_cmp(const void *_a, const void *_b) { const struct reftable_index_record *a = _a; const struct reftable_index_record *b = _b; - return strbuf_cmp(&a->last_key, &b->last_key); -} - -static void reftable_index_record_print(const void *rec, int hash_size) -{ - const struct reftable_index_record *idx = rec; - /* TODO: escape null chars? */ - printf("\"%s\" %" PRIu64 "\n", idx->last_key.buf, idx->offset); + return reftable_buf_cmp(&a->last_key, &b->last_key); } static struct reftable_record_vtable reftable_index_record_vtable = { @@ -1155,12 +1146,11 @@ static struct reftable_record_vtable reftable_index_record_vtable = { .is_deletion = ¬_a_deletion, .equal = &reftable_index_record_equal, .cmp = &reftable_index_record_cmp, - .print = &reftable_index_record_print, }; -void reftable_record_key(struct reftable_record *rec, struct strbuf *dest) +int reftable_record_key(struct reftable_record *rec, struct reftable_buf *dest) { - reftable_record_vtable(rec)->key(reftable_record_data(rec), dest); + return reftable_record_vtable(rec)->key(reftable_record_data(rec), dest); } int reftable_record_encode(struct reftable_record *rec, struct string_view dest, @@ -1170,14 +1160,14 @@ int reftable_record_encode(struct reftable_record *rec, struct string_view dest, dest, hash_size); } -void reftable_record_copy_from(struct reftable_record *rec, +int reftable_record_copy_from(struct reftable_record *rec, struct reftable_record *src, int hash_size) { assert(src->type == rec->type); - reftable_record_vtable(rec)->copy_from(reftable_record_data(rec), - reftable_record_data(src), - hash_size); + return reftable_record_vtable(rec)->copy_from(reftable_record_data(rec), + reftable_record_data(src), + hash_size); } uint8_t reftable_record_val_type(struct reftable_record *rec) @@ -1185,9 +1175,9 @@ uint8_t reftable_record_val_type(struct reftable_record *rec) return reftable_record_vtable(rec)->val_type(reftable_record_data(rec)); } -int reftable_record_decode(struct reftable_record *rec, struct strbuf key, +int reftable_record_decode(struct reftable_record *rec, struct reftable_buf key, uint8_t extra, struct string_view src, int hash_size, - struct strbuf *scratch) + struct reftable_buf *scratch) { return reftable_record_vtable(rec)->decode(reftable_record_data(rec), key, extra, src, hash_size, @@ -1328,15 +1318,9 @@ void reftable_record_init(struct reftable_record *rec, uint8_t typ) case BLOCK_TYPE_OBJ: return; case BLOCK_TYPE_INDEX: - strbuf_init(&rec->u.idx.last_key, 0); + reftable_buf_init(&rec->u.idx.last_key); return; default: BUG("unhandled record type"); } } - -void reftable_record_print(struct reftable_record *rec, int hash_size) -{ - printf("'%c': ", rec->type); - reftable_record_vtable(rec)->print(reftable_record_data(rec), hash_size); -} diff --git a/reftable/record.h b/reftable/record.h index d778133e6e..25aa908c85 100644 --- a/reftable/record.h +++ b/reftable/record.h @@ -9,6 +9,7 @@ https://developers.google.com/open-source/licenses/bsd #ifndef RECORD_H #define RECORD_H +#include "basics.h" #include "system.h" #include <stdint.h> @@ -38,13 +39,13 @@ int put_var_int(struct string_view *dest, uint64_t val); /* Methods for records. */ struct reftable_record_vtable { - /* encode the key of to a uint8_t strbuf. */ - void (*key)(const void *rec, struct strbuf *dest); + /* encode the key of to a uint8_t reftable_buf. */ + int (*key)(const void *rec, struct reftable_buf *dest); /* The record type of ('r' for ref). */ uint8_t type; - void (*copy_from)(void *dest, const void *src, int hash_size); + int (*copy_from)(void *dest, const void *src, int hash_size); /* a value of [0..7], indicating record subvariants (eg. ref vs. symref * vs ref deletion) */ @@ -54,9 +55,9 @@ struct reftable_record_vtable { int (*encode)(const void *rec, struct string_view dest, int hash_size); /* decode data from `src` into the record. */ - int (*decode)(void *rec, struct strbuf key, uint8_t extra, + int (*decode)(void *rec, struct reftable_buf key, uint8_t extra, struct string_view src, int hash_size, - struct strbuf *scratch); + struct reftable_buf *scratch); /* deallocate and null the record. */ void (*release)(void *rec); @@ -83,7 +84,7 @@ int reftable_is_block_type(uint8_t typ); /* Encode `key` into `dest`. Sets `is_restart` to indicate a restart. Returns * number of bytes written. */ int reftable_encode_key(int *is_restart, struct string_view dest, - struct strbuf prev_key, struct strbuf key, + struct reftable_buf prev_key, struct reftable_buf key, uint8_t extra); /* Decode a record's key lengths. */ @@ -96,13 +97,13 @@ int reftable_decode_keylen(struct string_view in, * Decode into `last_key` and `extra` from `in`. `last_key` is expected to * contain the decoded key of the preceding record, if any. */ -int reftable_decode_key(struct strbuf *last_key, uint8_t *extra, +int reftable_decode_key(struct reftable_buf *last_key, uint8_t *extra, struct string_view in); /* reftable_index_record are used internally to speed up lookups. */ struct reftable_index_record { uint64_t offset; /* Offset of block */ - struct strbuf last_key; /* Last key of the block. */ + struct reftable_buf last_key; /* Last key of the block. */ }; /* reftable_obj_record stores an object ID => ref mapping. */ @@ -136,16 +137,15 @@ void reftable_record_init(struct reftable_record *rec, uint8_t typ); /* see struct record_vtable */ int reftable_record_cmp(struct reftable_record *a, struct reftable_record *b); int reftable_record_equal(struct reftable_record *a, struct reftable_record *b, int hash_size); -void reftable_record_print(struct reftable_record *rec, int hash_size); -void reftable_record_key(struct reftable_record *rec, struct strbuf *dest); -void reftable_record_copy_from(struct reftable_record *rec, - struct reftable_record *src, int hash_size); +int reftable_record_key(struct reftable_record *rec, struct reftable_buf *dest); +int reftable_record_copy_from(struct reftable_record *rec, + struct reftable_record *src, int hash_size); uint8_t reftable_record_val_type(struct reftable_record *rec); int reftable_record_encode(struct reftable_record *rec, struct string_view dest, int hash_size); -int reftable_record_decode(struct reftable_record *rec, struct strbuf key, +int reftable_record_decode(struct reftable_record *rec, struct reftable_buf key, uint8_t extra, struct string_view src, - int hash_size, struct strbuf *scratch); + int hash_size, struct reftable_buf *scratch); int reftable_record_is_deletion(struct reftable_record *rec); static inline uint8_t reftable_record_type(struct reftable_record *rec) diff --git a/reftable/reftable-basics.h b/reftable/reftable-basics.h new file mode 100644 index 0000000000..e0397ed583 --- /dev/null +++ b/reftable/reftable-basics.h @@ -0,0 +1,31 @@ +/* + * Copyright 2020 Google LLC + * + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file or at + * https://developers.google.com/open-source/licenses/bsd +*/ + +#ifndef REFTABLE_BASICS_H +#define REFTABLE_BASICS_H + +#include <stddef.h> + +/* + * Hash functions understood by the reftable library. Note that the values are + * arbitrary and somewhat random such that we can easily detect cases where the + * hash hasn't been properly set up. + */ +enum reftable_hash { + REFTABLE_HASH_SHA1 = 89, + REFTABLE_HASH_SHA256 = 247, +}; +#define REFTABLE_HASH_SIZE_SHA1 20 +#define REFTABLE_HASH_SIZE_SHA256 32 +#define REFTABLE_HASH_SIZE_MAX REFTABLE_HASH_SIZE_SHA256 + +/* Overrides the functions to use for memory management. */ +void reftable_set_alloc(void *(*malloc)(size_t), + void *(*realloc)(void *, size_t), void (*free)(void *)); + +#endif diff --git a/reftable/reftable-error.h b/reftable/reftable-error.h index 6368cd9ed9..f404826562 100644 --- a/reftable/reftable-error.h +++ b/reftable/reftable-error.h @@ -57,6 +57,9 @@ enum reftable_error { /* Trying to write out-of-date data. */ REFTABLE_OUTDATED_ERROR = -12, + + /* An allocation has failed due to an out-of-memory situation. */ + REFTABLE_OUT_OF_MEMORY_ERROR = -13, }; /* convert the numeric error code to a string. The string should not be diff --git a/reftable/reftable-generic.h b/reftable/reftable-generic.h deleted file mode 100644 index 65670ea093..0000000000 --- a/reftable/reftable-generic.h +++ /dev/null @@ -1,47 +0,0 @@ -/* -Copyright 2020 Google LLC - -Use of this source code is governed by a BSD-style -license that can be found in the LICENSE file or at -https://developers.google.com/open-source/licenses/bsd -*/ - -#ifndef REFTABLE_GENERIC_H -#define REFTABLE_GENERIC_H - -#include "reftable-iterator.h" - -struct reftable_table_vtable; - -/* - * Provides a unified API for reading tables, either merged tables, or single - * readers. */ -struct reftable_table { - struct reftable_table_vtable *ops; - void *table_arg; -}; - -void reftable_table_init_ref_iter(struct reftable_table *tab, - struct reftable_iterator *it); - -void reftable_table_init_log_iter(struct reftable_table *tab, - struct reftable_iterator *it); - -/* returns the hash ID from a generic reftable_table */ -uint32_t reftable_table_hash_id(struct reftable_table *tab); - -/* returns the max update_index covered by this table. */ -uint64_t reftable_table_max_update_index(struct reftable_table *tab); - -/* returns the min update_index covered by this table. */ -uint64_t reftable_table_min_update_index(struct reftable_table *tab); - -/* convenience function to read a single ref. Returns < 0 for error, 0 - for success, and 1 if ref not found. */ -int reftable_table_read_ref(struct reftable_table *tab, const char *name, - struct reftable_ref_record *ref); - -/* dump table contents onto stdout for debugging */ -int reftable_table_print(struct reftable_table *tab); - -#endif diff --git a/reftable/reftable-malloc.h b/reftable/reftable-malloc.h deleted file mode 100644 index 5f2185f1f3..0000000000 --- a/reftable/reftable-malloc.h +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2020 Google LLC - -Use of this source code is governed by a BSD-style -license that can be found in the LICENSE file or at -https://developers.google.com/open-source/licenses/bsd -*/ - -#ifndef REFTABLE_H -#define REFTABLE_H - -#include <stddef.h> - -/* Overrides the functions to use for memory management. */ -void reftable_set_alloc(void *(*malloc)(size_t), - void *(*realloc)(void *, size_t), void (*free)(void *)); - -#endif diff --git a/reftable/reftable-merged.h b/reftable/reftable-merged.h index 14d5fc9f05..f2d01c3ef8 100644 --- a/reftable/reftable-merged.h +++ b/reftable/reftable-merged.h @@ -26,15 +26,23 @@ https://developers.google.com/open-source/licenses/bsd /* A merged table is implements seeking/iterating over a stack of tables. */ struct reftable_merged_table; -/* A generic reftable; see below. */ -struct reftable_table; +struct reftable_reader; -/* reftable_new_merged_table creates a new merged table. It takes ownership of - the stack array. -*/ -int reftable_new_merged_table(struct reftable_merged_table **dest, - struct reftable_table *stack, size_t n, - uint32_t hash_id); +/* + * reftable_merged_table_new creates a new merged table. The readers must be + * kept alive as long as the merged table is still in use. + */ +int reftable_merged_table_new(struct reftable_merged_table **dest, + struct reftable_reader **readers, size_t n, + enum reftable_hash hash_id); + +/* Initialize a merged table iterator for reading refs. */ +int reftable_merged_table_init_ref_iterator(struct reftable_merged_table *mt, + struct reftable_iterator *it); + +/* Initialize a merged table iterator for reading logs. */ +int reftable_merged_table_init_log_iterator(struct reftable_merged_table *mt, + struct reftable_iterator *it); /* returns the max update_index covered by this merged table. */ uint64_t @@ -48,10 +56,6 @@ reftable_merged_table_min_update_index(struct reftable_merged_table *mt); void reftable_merged_table_free(struct reftable_merged_table *m); /* return the hash ID of the merged table. */ -uint32_t reftable_merged_table_hash_id(struct reftable_merged_table *m); - -/* create a generic table from reftable_merged_table */ -void reftable_table_from_merged_table(struct reftable_table *tab, - struct reftable_merged_table *table); +enum reftable_hash reftable_merged_table_hash_id(struct reftable_merged_table *m); #endif diff --git a/reftable/reftable-reader.h b/reftable/reftable-reader.h index a32f31d648..0085fbb903 100644 --- a/reftable/reftable-reader.h +++ b/reftable/reftable-reader.h @@ -23,32 +23,38 @@ /* The reader struct is a handle to an open reftable file. */ struct reftable_reader; -/* Generic table. */ -struct reftable_table; - -/* reftable_new_reader opens a reftable for reading. If successful, +/* reftable_reader_new opens a reftable for reading. If successful, * returns 0 code and sets pp. The name is used for creating a * stack. Typically, it is the basename of the file. The block source * `src` is owned by the reader, and is closed on calling * reftable_reader_destroy(). On error, the block source `src` is * closed as well. */ -int reftable_new_reader(struct reftable_reader **pp, +int reftable_reader_new(struct reftable_reader **pp, struct reftable_block_source *src, const char *name); +/* + * Manage the reference count of the reftable reader. A newly initialized + * reader starts with a refcount of 1 and will be deleted once the refcount has + * reached 0. + * + * This is required because readers may have longer lifetimes than the stack + * they belong to. The stack may for example be reloaded while the old tables + * are still being accessed by an iterator. + */ +void reftable_reader_incref(struct reftable_reader *reader); +void reftable_reader_decref(struct reftable_reader *reader); + /* Initialize a reftable iterator for reading refs. */ -void reftable_reader_init_ref_iterator(struct reftable_reader *r, - struct reftable_iterator *it); +int reftable_reader_init_ref_iterator(struct reftable_reader *r, + struct reftable_iterator *it); /* Initialize a reftable iterator for reading logs. */ -void reftable_reader_init_log_iterator(struct reftable_reader *r, - struct reftable_iterator *it); +int reftable_reader_init_log_iterator(struct reftable_reader *r, + struct reftable_iterator *it); /* returns the hash ID used in this table. */ -uint32_t reftable_reader_hash_id(struct reftable_reader *r); - -/* closes and deallocates a reader. */ -void reftable_reader_free(struct reftable_reader *); +enum reftable_hash reftable_reader_hash_id(struct reftable_reader *r); /* return an iterator for the refs pointing to `oid`. */ int reftable_reader_refs_for(struct reftable_reader *r, @@ -60,12 +66,6 @@ uint64_t reftable_reader_max_update_index(struct reftable_reader *r); /* return the min_update_index for a table */ uint64_t reftable_reader_min_update_index(struct reftable_reader *r); -/* creates a generic table from a file reader. */ -void reftable_table_from_reader(struct reftable_table *tab, - struct reftable_reader *reader); - -/* print table onto stdout for debugging. */ -int reftable_reader_print_file(const char *tablename); /* print blocks onto stdout for debugging. */ int reftable_reader_print_blocks(const char *tablename); diff --git a/reftable/reftable-record.h b/reftable/reftable-record.h index ff486eb1f7..ddd48eb579 100644 --- a/reftable/reftable-record.h +++ b/reftable/reftable-record.h @@ -9,7 +9,7 @@ https://developers.google.com/open-source/licenses/bsd #ifndef REFTABLE_RECORD_H #define REFTABLE_RECORD_H -#include "hash.h" +#include "reftable-basics.h" #include <stdint.h> /* @@ -40,10 +40,10 @@ struct reftable_ref_record { #define REFTABLE_NR_REF_VALUETYPES 4 } value_type; union { - unsigned char val1[GIT_MAX_RAWSZ]; + unsigned char val1[REFTABLE_HASH_SIZE_MAX]; struct { - unsigned char value[GIT_MAX_RAWSZ]; /* first hash */ - unsigned char target_value[GIT_MAX_RAWSZ]; /* second hash */ + unsigned char value[REFTABLE_HASH_SIZE_MAX]; /* first hash */ + unsigned char target_value[REFTABLE_HASH_SIZE_MAX]; /* second hash */ } val2; char *symref; /* referent, malloced 0-terminated string */ } value; @@ -60,10 +60,6 @@ const unsigned char *reftable_ref_record_val2(const struct reftable_ref_record * /* returns whether 'ref' represents a deletion */ int reftable_ref_record_is_deletion(const struct reftable_ref_record *ref); -/* prints a reftable_ref_record onto stdout. Useful for debugging. */ -void reftable_ref_record_print(const struct reftable_ref_record *ref, - uint32_t hash_id); - /* frees and nulls all pointer values inside `ref`. */ void reftable_ref_record_release(struct reftable_ref_record *ref); @@ -89,8 +85,8 @@ struct reftable_log_record { union { struct { - unsigned char new_hash[GIT_MAX_RAWSZ]; - unsigned char old_hash[GIT_MAX_RAWSZ]; + unsigned char new_hash[REFTABLE_HASH_SIZE_MAX]; + unsigned char old_hash[REFTABLE_HASH_SIZE_MAX]; char *name; char *email; uint64_t time; @@ -111,8 +107,4 @@ void reftable_log_record_release(struct reftable_log_record *log); int reftable_log_record_equal(const struct reftable_log_record *a, const struct reftable_log_record *b, int hash_size); -/* dumps a reftable_log_record on stdout, for debugging/testing. */ -void reftable_log_record_print(struct reftable_log_record *log, - uint32_t hash_id); - #endif diff --git a/reftable/reftable-stack.h b/reftable/reftable-stack.h index 09e97c9991..ae14270ea7 100644 --- a/reftable/reftable-stack.h +++ b/reftable/reftable-stack.h @@ -37,12 +37,21 @@ uint64_t reftable_stack_next_update_index(struct reftable_stack *st); /* holds a transaction to add tables at the top of a stack. */ struct reftable_addition; +enum { + /* + * Reload the stack when the stack is out-of-date after locking it. + */ + REFTABLE_STACK_NEW_ADDITION_RELOAD = (1 << 0), +}; + /* * returns a new transaction to add reftables to the given stack. As a side - * effect, the ref database is locked. + * effect, the ref database is locked. Accepts REFTABLE_STACK_NEW_ADDITION_* + * flags. */ int reftable_stack_new_addition(struct reftable_addition **dest, - struct reftable_stack *st); + struct reftable_stack *st, + unsigned int flags); /* Adds a reftable to transaction. */ int reftable_addition_add(struct reftable_addition *add, @@ -73,16 +82,16 @@ struct reftable_iterator; * be used to iterate through refs. The iterator is valid until the next reload * or write. */ -void reftable_stack_init_ref_iterator(struct reftable_stack *st, - struct reftable_iterator *it); +int reftable_stack_init_ref_iterator(struct reftable_stack *st, + struct reftable_iterator *it); /* * Initialize an iterator for the merged tables contained in the stack that can * be used to iterate through logs. The iterator is valid until the next reload * or write. */ -void reftable_stack_init_log_iterator(struct reftable_stack *st, - struct reftable_iterator *it); +int reftable_stack_init_log_iterator(struct reftable_stack *st, + struct reftable_iterator *it); /* returns the merged_table for seeking. This table is valid until the * next write or reload, and should not be closed or deleted. @@ -140,7 +149,7 @@ struct reftable_compaction_stats { struct reftable_compaction_stats * reftable_stack_compaction_stats(struct reftable_stack *st); -/* print the entire stack represented by the directory */ -int reftable_stack_print_directory(const char *stackdir, uint32_t hash_id); +/* Return the hash of the stack. */ +enum reftable_hash reftable_stack_hash_id(struct reftable_stack *st); #endif diff --git a/reftable/reftable-tests.h b/reftable/reftable-tests.h deleted file mode 100644 index 114cc3d053..0000000000 --- a/reftable/reftable-tests.h +++ /dev/null @@ -1,22 +0,0 @@ -/* -Copyright 2020 Google LLC - -Use of this source code is governed by a BSD-style -license that can be found in the LICENSE file or at -https://developers.google.com/open-source/licenses/bsd -*/ - -#ifndef REFTABLE_TESTS_H -#define REFTABLE_TESTS_H - -int basics_test_main(int argc, const char **argv); -int block_test_main(int argc, const char **argv); -int merged_test_main(int argc, const char **argv); -int pq_test_main(int argc, const char **argv); -int record_test_main(int argc, const char **argv); -int readwrite_test_main(int argc, const char **argv); -int stack_test_main(int argc, const char **argv); -int tree_test_main(int argc, const char **argv); -int reftable_dump_main(int argc, char *const *argv); - -#endif diff --git a/reftable/reftable-writer.h b/reftable/reftable-writer.h index 189b1f4144..5f9afa620b 100644 --- a/reftable/reftable-writer.h +++ b/reftable/reftable-writer.h @@ -33,7 +33,7 @@ struct reftable_write_options { /* 4-byte identifier ("sha1", "s256") of the hash. * Defaults to SHA1 if unset */ - uint32_t hash_id; + enum reftable_hash hash_id; /* Default mode for creating files. If unset, use 0666 (+umask) */ unsigned int default_permissions; @@ -51,6 +51,32 @@ struct reftable_write_options { * tables to compact. Defaults to 2 if unset. */ uint8_t auto_compaction_factor; + + /* + * The number of milliseconds to wait when trying to lock "tables.list". + * Note that this does not apply to locking individual tables, as these + * should only ever be locked when already holding the "tables.list" + * lock. + * + * Passing 0 will fail immediately when the file is locked, passing a + * negative value will cause us to block indefinitely. + */ + long lock_timeout_ms; + + /* + * Optional callback used to fsync files to disk. Falls back to using + * fsync(3P) when unset. + */ + int (*fsync)(int fd); + + /* + * Callback function to execute whenever the stack is being reloaded. + * This can be used e.g. to discard cached information that relies on + * the old stack's data. The payload data will be passed as argument to + * the callback. + */ + void (*on_reload)(void *payload); + void *on_reload_payload; }; /* reftable_block_stats holds statistics for a single block type */ @@ -90,11 +116,13 @@ struct reftable_stats { int object_id_len; }; -/* reftable_new_writer creates a new writer */ -struct reftable_writer * -reftable_new_writer(ssize_t (*writer_func)(void *, const void *, size_t), - int (*flush_func)(void *), - void *writer_arg, const struct reftable_write_options *opts); +struct reftable_writer; + +/* Create a new writer. */ +int reftable_writer_new(struct reftable_writer **out, + ssize_t (*writer_func)(void *, const void *, size_t), + int (*flush_func)(void *), + void *writer_arg, const struct reftable_write_options *opts); /* Set the range of update indices for the records we will add. When writing a table into a stack, the min should be at least diff --git a/reftable/stack.c b/reftable/stack.c index 737591125e..59fd695a12 100644 --- a/reftable/stack.c +++ b/reftable/stack.c @@ -8,17 +8,14 @@ https://developers.google.com/open-source/licenses/bsd #include "stack.h" -#include "../write-or-die.h" #include "system.h" #include "constants.h" #include "merged.h" #include "reader.h" #include "reftable-error.h" -#include "reftable-generic.h" #include "reftable-record.h" #include "reftable-merged.h" #include "writer.h" -#include "tempfile.h" static int stack_try_add(struct reftable_stack *st, int (*write_table)(struct reftable_writer *wr, @@ -32,58 +29,87 @@ static void reftable_addition_close(struct reftable_addition *add); static int reftable_stack_reload_maybe_reuse(struct reftable_stack *st, int reuse_open); -static void stack_filename(struct strbuf *dest, struct reftable_stack *st, - const char *name) +static int stack_filename(struct reftable_buf *dest, struct reftable_stack *st, + const char *name) { - strbuf_reset(dest); - strbuf_addstr(dest, st->reftable_dir); - strbuf_addstr(dest, "/"); - strbuf_addstr(dest, name); + int err; + reftable_buf_reset(dest); + if ((err = reftable_buf_addstr(dest, st->reftable_dir)) < 0 || + (err = reftable_buf_addstr(dest, "/")) < 0 || + (err = reftable_buf_addstr(dest, name)) < 0) + return err; + return 0; } -static ssize_t reftable_fd_write(void *arg, const void *data, size_t sz) +static int stack_fsync(const struct reftable_write_options *opts, int fd) { - int *fdp = (int *)arg; - return write_in_full(*fdp, data, sz); + if (opts->fsync) + return opts->fsync(fd); + return fsync(fd); } -static int reftable_fd_flush(void *arg) +struct fd_writer { + const struct reftable_write_options *opts; + int fd; +}; + +static ssize_t fd_writer_write(void *arg, const void *data, size_t sz) { - int *fdp = (int *)arg; + struct fd_writer *writer = arg; + return write_in_full(writer->fd, data, sz); +} - return fsync_component(FSYNC_COMPONENT_REFERENCE, *fdp); +static int fd_writer_flush(void *arg) +{ + struct fd_writer *writer = arg; + return stack_fsync(writer->opts, writer->fd); } int reftable_new_stack(struct reftable_stack **dest, const char *dir, const struct reftable_write_options *_opts) { - struct reftable_stack *p = reftable_calloc(1, sizeof(*p)); - struct strbuf list_file_name = STRBUF_INIT; - struct reftable_write_options opts = {0}; - int err = 0; + struct reftable_buf list_file_name = REFTABLE_BUF_INIT; + struct reftable_write_options opts = { 0 }; + struct reftable_stack *p; + int err; + + p = reftable_calloc(1, sizeof(*p)); + if (!p) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto out; + } if (_opts) opts = *_opts; if (opts.hash_id == 0) - opts.hash_id = GIT_SHA1_FORMAT_ID; + opts.hash_id = REFTABLE_HASH_SHA1; *dest = NULL; - strbuf_reset(&list_file_name); - strbuf_addstr(&list_file_name, dir); - strbuf_addstr(&list_file_name, "/tables.list"); + reftable_buf_reset(&list_file_name); + if ((err = reftable_buf_addstr(&list_file_name, dir)) < 0 || + (err = reftable_buf_addstr(&list_file_name, "/tables.list")) < 0) + goto out; - p->list_file = strbuf_detach(&list_file_name, NULL); + p->list_file = reftable_buf_detach(&list_file_name); p->list_fd = -1; - p->reftable_dir = xstrdup(dir); p->opts = opts; + p->reftable_dir = reftable_strdup(dir); + if (!p->reftable_dir) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto out; + } err = reftable_stack_reload_maybe_reuse(p, 1); - if (err < 0) { + if (err < 0) + goto out; + + *dest = p; + err = 0; + +out: + if (err < 0) reftable_stack_destroy(p); - } else { - *dest = p; - } return err; } @@ -103,13 +129,22 @@ static int fd_read_lines(int fd, char ***namesp) } REFTABLE_ALLOC_ARRAY(buf, size + 1); + if (!buf) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto done; + } + if (read_in_full(fd, buf, size) != size) { err = REFTABLE_IO_ERROR; goto done; } buf[size] = 0; - parse_names(buf, size, namesp); + *namesp = parse_names(buf, size); + if (!*namesp) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto done; + } done: reftable_free(buf); @@ -123,6 +158,8 @@ int read_lines(const char *filename, char ***namesp) if (fd < 0) { if (errno == ENOENT) { REFTABLE_CALLOC_ARRAY(*namesp, 1); + if (!*namesp) + return REFTABLE_OUT_OF_MEMORY_ERROR; return 0; } @@ -133,18 +170,18 @@ int read_lines(const char *filename, char ***namesp) return err; } -void reftable_stack_init_ref_iterator(struct reftable_stack *st, +int reftable_stack_init_ref_iterator(struct reftable_stack *st, struct reftable_iterator *it) { - merged_table_init_iter(reftable_stack_merged_table(st), - it, BLOCK_TYPE_REF); + return merged_table_init_iter(reftable_stack_merged_table(st), + it, BLOCK_TYPE_REF); } -void reftable_stack_init_log_iterator(struct reftable_stack *st, - struct reftable_iterator *it) +int reftable_stack_init_log_iterator(struct reftable_stack *st, + struct reftable_iterator *it) { - merged_table_init_iter(reftable_stack_merged_table(st), - it, BLOCK_TYPE_LOG); + return merged_table_init_iter(reftable_stack_merged_table(st), + it, BLOCK_TYPE_LOG); } struct reftable_merged_table * @@ -168,6 +205,10 @@ void reftable_stack_destroy(struct reftable_stack *st) { char **names = NULL; int err = 0; + + if (!st) + return; + if (st->merged) { reftable_merged_table_free(st->merged); st->merged = NULL; @@ -175,28 +216,31 @@ void reftable_stack_destroy(struct reftable_stack *st) err = read_lines(st->list_file, &names); if (err < 0) { - FREE_AND_NULL(names); + REFTABLE_FREE_AND_NULL(names); } if (st->readers) { int i = 0; - struct strbuf filename = STRBUF_INIT; + struct reftable_buf filename = REFTABLE_BUF_INIT; for (i = 0; i < st->readers_len; i++) { const char *name = reader_name(st->readers[i]); - strbuf_reset(&filename); + int try_unlinking = 1; + + reftable_buf_reset(&filename); if (names && !has_name(names, name)) { - stack_filename(&filename, st, name); + if (stack_filename(&filename, st, name) < 0) + try_unlinking = 0; } - reftable_reader_free(st->readers[i]); + reftable_reader_decref(st->readers[i]); - if (filename.len) { + if (try_unlinking && filename.len) { /* On Windows, can only unlink after closing. */ unlink(filename.buf); } } - strbuf_release(&filename); + reftable_buf_release(&filename); st->readers_len = 0; - FREE_AND_NULL(st->readers); + REFTABLE_FREE_AND_NULL(st->readers); } if (st->list_fd >= 0) { @@ -204,20 +248,20 @@ void reftable_stack_destroy(struct reftable_stack *st) st->list_fd = -1; } - FREE_AND_NULL(st->list_file); - FREE_AND_NULL(st->reftable_dir); + REFTABLE_FREE_AND_NULL(st->list_file); + REFTABLE_FREE_AND_NULL(st->reftable_dir); reftable_free(st); free_names(names); } static struct reftable_reader **stack_copy_readers(struct reftable_stack *st, - int cur_len) + size_t cur_len) { struct reftable_reader **cur = reftable_calloc(cur_len, sizeof(*cur)); - int i = 0; - for (i = 0; i < cur_len; i++) { + if (!cur) + return NULL; + for (size_t i = 0; i < cur_len; i++) cur[i] = st->readers[i]; - } return cur; } @@ -225,19 +269,31 @@ static int reftable_stack_reload_once(struct reftable_stack *st, const char **names, int reuse_open) { - size_t cur_len = !st->merged ? 0 : st->merged->stack_len; - struct reftable_reader **cur = stack_copy_readers(st, cur_len); - size_t names_len = names_length(names); - struct reftable_reader **new_readers = - reftable_calloc(names_len, sizeof(*new_readers)); - struct reftable_table *new_tables = - reftable_calloc(names_len, sizeof(*new_tables)); + size_t cur_len = !st->merged ? 0 : st->merged->readers_len; + struct reftable_reader **cur; + struct reftable_reader **reused = NULL; + struct reftable_reader **new_readers; + size_t reused_len = 0, reused_alloc = 0, names_len; size_t new_readers_len = 0; struct reftable_merged_table *new_merged = NULL; - struct strbuf table_path = STRBUF_INIT; + struct reftable_buf table_path = REFTABLE_BUF_INIT; int err = 0; size_t i; + cur = stack_copy_readers(st, cur_len); + if (!cur) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto done; + } + + names_len = names_length(names); + + new_readers = reftable_calloc(names_len, sizeof(*new_readers)); + if (!new_readers) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto done; + } + while (*names) { struct reftable_reader *rd = NULL; const char *name = *names++; @@ -248,70 +304,100 @@ static int reftable_stack_reload_once(struct reftable_stack *st, if (cur[i] && 0 == strcmp(cur[i]->name, name)) { rd = cur[i]; cur[i] = NULL; + + /* + * When reloading the stack fails, we end up + * releasing all new readers. This also + * includes the reused readers, even though + * they are still in used by the old stack. We + * thus need to keep them alive here, which we + * do by bumping their refcount. + */ + REFTABLE_ALLOC_GROW(reused, reused_len + 1, reused_alloc); + if (!reused) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto done; + } + reused[reused_len++] = rd; + reftable_reader_incref(rd); break; } } if (!rd) { struct reftable_block_source src = { NULL }; - stack_filename(&table_path, st, name); + + err = stack_filename(&table_path, st, name); + if (err < 0) + goto done; err = reftable_block_source_from_file(&src, table_path.buf); if (err < 0) goto done; - err = reftable_new_reader(&rd, &src, name); + err = reftable_reader_new(&rd, &src, name); if (err < 0) goto done; } new_readers[new_readers_len] = rd; - reftable_table_from_reader(&new_tables[new_readers_len], rd); new_readers_len++; } /* success! */ - err = reftable_new_merged_table(&new_merged, new_tables, + err = reftable_merged_table_new(&new_merged, new_readers, new_readers_len, st->opts.hash_id); if (err < 0) goto done; - new_tables = NULL; - st->readers_len = new_readers_len; - if (st->merged) - reftable_merged_table_free(st->merged); - if (st->readers) { - reftable_free(st->readers); - } - st->readers = new_readers; - new_readers = NULL; - new_readers_len = 0; - - new_merged->suppress_deletions = 1; - st->merged = new_merged; + /* + * Close the old, non-reused readers and proactively try to unlink + * them. This is done for systems like Windows, where the underlying + * file of such an open reader wouldn't have been possible to be + * unlinked by the compacting process. + */ for (i = 0; i < cur_len; i++) { if (cur[i]) { const char *name = reader_name(cur[i]); - stack_filename(&table_path, st, name); - reader_close(cur[i]); - reftable_reader_free(cur[i]); + err = stack_filename(&table_path, st, name); + if (err < 0) + goto done; - /* On Windows, can only unlink after closing. */ + reftable_reader_decref(cur[i]); unlink(table_path.buf); } } + /* Update the stack to point to the new tables. */ + if (st->merged) + reftable_merged_table_free(st->merged); + new_merged->suppress_deletions = 1; + st->merged = new_merged; + + if (st->readers) + reftable_free(st->readers); + st->readers = new_readers; + st->readers_len = new_readers_len; + new_readers = NULL; + new_readers_len = 0; + + /* + * Decrement the refcount of reused readers again. This only needs to + * happen on the successful case, because on the unsuccessful one we + * decrement their refcount via `new_readers`. + */ + for (i = 0; i < reused_len; i++) + reftable_reader_decref(reused[i]); + done: - for (i = 0; i < new_readers_len; i++) { - reader_close(new_readers[i]); - reftable_reader_free(new_readers[i]); - } + for (i = 0; i < new_readers_len; i++) + reftable_reader_decref(new_readers[i]); reftable_free(new_readers); - reftable_free(new_tables); + reftable_free(reused); reftable_free(cur); - strbuf_release(&table_path); + reftable_buf_release(&table_path); return err; } @@ -364,6 +450,10 @@ static int reftable_stack_reload_maybe_reuse(struct reftable_stack *st, } REFTABLE_CALLOC_ARRAY(names, 1); + if (!names) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto out; + } } else { err = fd_read_lines(fd, &names); if (err < 0) @@ -458,6 +548,10 @@ out: close(fd); free_names(names); free_names(names_after); + + if (st->opts.on_reload) + st->opts.on_reload(st->opts.on_reload_payload); + return err; } @@ -520,7 +614,7 @@ static int stack_uptodate(struct reftable_stack *st) } } - if (names[st->merged->stack_len]) { + if (names[st->merged->readers_len]) { err = 1; goto done; } @@ -556,18 +650,18 @@ int reftable_stack_add(struct reftable_stack *st, return 0; } -static void format_name(struct strbuf *dest, uint64_t min, uint64_t max) +static int format_name(struct reftable_buf *dest, uint64_t min, uint64_t max) { char buf[100]; uint32_t rnd = (uint32_t)git_rand(); snprintf(buf, sizeof(buf), "0x%012" PRIx64 "-0x%012" PRIx64 "-%08x", min, max, rnd); - strbuf_reset(dest); - strbuf_addstr(dest, buf); + reftable_buf_reset(dest); + return reftable_buf_addstr(dest, buf); } struct reftable_addition { - struct tempfile *lock_file; + struct reftable_flock tables_list_lock; struct reftable_stack *stack; char **new_tables; @@ -578,16 +672,17 @@ struct reftable_addition { #define REFTABLE_ADDITION_INIT {0} static int reftable_stack_init_addition(struct reftable_addition *add, - struct reftable_stack *st) + struct reftable_stack *st, + unsigned int flags) { - struct strbuf lock_file_name = STRBUF_INIT; - int err = 0; - add->stack = st; + struct reftable_buf lock_file_name = REFTABLE_BUF_INIT; + int err; - strbuf_addf(&lock_file_name, "%s.lock", st->list_file); + add->stack = st; - add->lock_file = create_tempfile(lock_file_name.buf); - if (!add->lock_file) { + err = flock_acquire(&add->tables_list_lock, st->list_file, + st->opts.lock_timeout_ms); + if (err < 0) { if (errno == EEXIST) { err = REFTABLE_LOCK_ERROR; } else { @@ -596,7 +691,8 @@ static int reftable_stack_init_addition(struct reftable_addition *add, goto done; } if (st->opts.default_permissions) { - if (chmod(add->lock_file->filename.buf, st->opts.default_permissions) < 0) { + if (chmod(add->tables_list_lock.path, + st->opts.default_permissions) < 0) { err = REFTABLE_IO_ERROR; goto done; } @@ -605,6 +701,11 @@ static int reftable_stack_init_addition(struct reftable_addition *add, err = stack_uptodate(st); if (err < 0) goto done; + if (err > 0 && flags & REFTABLE_STACK_NEW_ADDITION_RELOAD) { + err = reftable_stack_reload_maybe_reuse(add->stack, 1); + if (err) + goto done; + } if (err > 0) { err = REFTABLE_OUTDATED_ERROR; goto done; @@ -612,21 +713,20 @@ static int reftable_stack_init_addition(struct reftable_addition *add, add->next_update_index = reftable_stack_next_update_index(st); done: - if (err) { + if (err) reftable_addition_close(add); - } - strbuf_release(&lock_file_name); + reftable_buf_release(&lock_file_name); return err; } static void reftable_addition_close(struct reftable_addition *add) { - struct strbuf nm = STRBUF_INIT; + struct reftable_buf nm = REFTABLE_BUF_INIT; size_t i; for (i = 0; i < add->new_tables_len; i++) { - stack_filename(&nm, add->stack, add->new_tables[i]); - unlink(nm.buf); + if (!stack_filename(&nm, add->stack, add->new_tables[i])) + unlink(nm.buf); reftable_free(add->new_tables[i]); add->new_tables[i] = NULL; } @@ -635,8 +735,8 @@ static void reftable_addition_close(struct reftable_addition *add) add->new_tables_len = 0; add->new_tables_cap = 0; - delete_tempfile(&add->lock_file); - strbuf_release(&nm); + flock_release(&add->tables_list_lock); + reftable_buf_release(&nm); } void reftable_addition_destroy(struct reftable_addition *add) @@ -650,34 +750,38 @@ void reftable_addition_destroy(struct reftable_addition *add) int reftable_addition_commit(struct reftable_addition *add) { - struct strbuf table_list = STRBUF_INIT; - int lock_file_fd = get_tempfile_fd(add->lock_file); + struct reftable_buf table_list = REFTABLE_BUF_INIT; int err = 0; size_t i; if (add->new_tables_len == 0) goto done; - for (i = 0; i < add->stack->merged->stack_len; i++) { - strbuf_addstr(&table_list, add->stack->readers[i]->name); - strbuf_addstr(&table_list, "\n"); + for (i = 0; i < add->stack->merged->readers_len; i++) { + if ((err = reftable_buf_addstr(&table_list, add->stack->readers[i]->name)) < 0 || + (err = reftable_buf_addstr(&table_list, "\n")) < 0) + goto done; } for (i = 0; i < add->new_tables_len; i++) { - strbuf_addstr(&table_list, add->new_tables[i]); - strbuf_addstr(&table_list, "\n"); + if ((err = reftable_buf_addstr(&table_list, add->new_tables[i])) < 0 || + (err = reftable_buf_addstr(&table_list, "\n")) < 0) + goto done; } - err = write_in_full(lock_file_fd, table_list.buf, table_list.len); - strbuf_release(&table_list); + err = write_in_full(add->tables_list_lock.fd, table_list.buf, table_list.len); + reftable_buf_release(&table_list); if (err < 0) { err = REFTABLE_IO_ERROR; goto done; } - fsync_component_or_die(FSYNC_COMPONENT_REFERENCE, lock_file_fd, - get_tempfile_path(add->lock_file)); + err = stack_fsync(&add->stack->opts, add->tables_list_lock.fd); + if (err < 0) { + err = REFTABLE_IO_ERROR; + goto done; + } - err = rename_tempfile(&add->lock_file, add->stack->list_file); + err = flock_commit(&add->tables_list_lock); if (err < 0) { err = REFTABLE_IO_ERROR; goto done; @@ -715,13 +819,18 @@ done: } int reftable_stack_new_addition(struct reftable_addition **dest, - struct reftable_stack *st) + struct reftable_stack *st, + unsigned int flags) { int err = 0; struct reftable_addition empty = REFTABLE_ADDITION_INIT; + REFTABLE_CALLOC_ARRAY(*dest, 1); + if (!*dest) + return REFTABLE_OUT_OF_MEMORY_ERROR; + **dest = empty; - err = reftable_stack_init_addition(*dest, st); + err = reftable_stack_init_addition(*dest, st, flags); if (err) { reftable_free(*dest); *dest = NULL; @@ -735,7 +844,7 @@ static int stack_try_add(struct reftable_stack *st, void *arg) { struct reftable_addition add = REFTABLE_ADDITION_INIT; - int err = reftable_stack_init_addition(&add, st); + int err = reftable_stack_init_addition(&add, st, 0); if (err < 0) goto done; @@ -754,36 +863,47 @@ int reftable_addition_add(struct reftable_addition *add, void *arg), void *arg) { - struct strbuf temp_tab_file_name = STRBUF_INIT; - struct strbuf tab_file_name = STRBUF_INIT; - struct strbuf next_name = STRBUF_INIT; + struct reftable_buf temp_tab_file_name = REFTABLE_BUF_INIT; + struct reftable_buf tab_file_name = REFTABLE_BUF_INIT; + struct reftable_buf next_name = REFTABLE_BUF_INIT; struct reftable_writer *wr = NULL; - struct tempfile *tab_file = NULL; + struct reftable_tmpfile tab_file = REFTABLE_TMPFILE_INIT; + struct fd_writer writer = { + .opts = &add->stack->opts, + }; int err = 0; - int tab_fd; - strbuf_reset(&next_name); - format_name(&next_name, add->next_update_index, add->next_update_index); + reftable_buf_reset(&next_name); - stack_filename(&temp_tab_file_name, add->stack, next_name.buf); - strbuf_addstr(&temp_tab_file_name, ".temp.XXXXXX"); + err = format_name(&next_name, add->next_update_index, add->next_update_index); + if (err < 0) + goto done; - tab_file = mks_tempfile(temp_tab_file_name.buf); - if (!tab_file) { - err = REFTABLE_IO_ERROR; + err = stack_filename(&temp_tab_file_name, add->stack, next_name.buf); + if (err < 0) + goto done; + + err = reftable_buf_addstr(&temp_tab_file_name, ".temp.XXXXXX"); + if (err < 0) + goto done; + + err = tmpfile_from_pattern(&tab_file, temp_tab_file_name.buf); + if (err < 0) goto done; - } if (add->stack->opts.default_permissions) { - if (chmod(get_tempfile_path(tab_file), + if (chmod(tab_file.path, add->stack->opts.default_permissions)) { err = REFTABLE_IO_ERROR; goto done; } } - tab_fd = get_tempfile_fd(tab_file); - wr = reftable_new_writer(reftable_fd_write, reftable_fd_flush, &tab_fd, - &add->stack->opts); + writer.fd = tab_file.fd; + err = reftable_writer_new(&wr, fd_writer_write, fd_writer_flush, + &writer, &add->stack->opts); + if (err < 0) + goto done; + err = write_table(wr, arg); if (err < 0) goto done; @@ -796,46 +916,55 @@ int reftable_addition_add(struct reftable_addition *add, if (err < 0) goto done; - err = close_tempfile_gently(tab_file); - if (err < 0) { - err = REFTABLE_IO_ERROR; + err = tmpfile_close(&tab_file); + if (err < 0) goto done; - } if (wr->min_update_index < add->next_update_index) { err = REFTABLE_API_ERROR; goto done; } - format_name(&next_name, wr->min_update_index, wr->max_update_index); - strbuf_addstr(&next_name, ".ref"); - stack_filename(&tab_file_name, add->stack, next_name.buf); + err = format_name(&next_name, wr->min_update_index, wr->max_update_index); + if (err < 0) + goto done; + + err = reftable_buf_addstr(&next_name, ".ref"); + if (err < 0) + goto done; + + err = stack_filename(&tab_file_name, add->stack, next_name.buf); + if (err < 0) + goto done; /* On windows, this relies on rand() picking a unique destination name. Maybe we should do retry loop as well? */ - err = rename_tempfile(&tab_file, tab_file_name.buf); - if (err < 0) { - err = REFTABLE_IO_ERROR; + err = tmpfile_rename(&tab_file, tab_file_name.buf); + if (err < 0) goto done; - } REFTABLE_ALLOC_GROW(add->new_tables, add->new_tables_len + 1, add->new_tables_cap); - add->new_tables[add->new_tables_len++] = strbuf_detach(&next_name, NULL); + if (!add->new_tables) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto done; + } + add->new_tables[add->new_tables_len++] = reftable_buf_detach(&next_name); + done: - delete_tempfile(&tab_file); - strbuf_release(&temp_tab_file_name); - strbuf_release(&tab_file_name); - strbuf_release(&next_name); + tmpfile_delete(&tab_file); + reftable_buf_release(&temp_tab_file_name); + reftable_buf_release(&tab_file_name); + reftable_buf_release(&next_name); reftable_writer_free(wr); return err; } uint64_t reftable_stack_next_update_index(struct reftable_stack *st) { - int sz = st->merged->stack_len; + int sz = st->merged->readers_len; if (sz > 0) return reftable_reader_max_update_index(st->readers[sz - 1]) + 1; @@ -845,35 +974,46 @@ uint64_t reftable_stack_next_update_index(struct reftable_stack *st) static int stack_compact_locked(struct reftable_stack *st, size_t first, size_t last, struct reftable_log_expiry_config *config, - struct tempfile **tab_file_out) + struct reftable_tmpfile *tab_file_out) { - struct strbuf next_name = STRBUF_INIT; - struct strbuf tab_file_path = STRBUF_INIT; + struct reftable_buf next_name = REFTABLE_BUF_INIT; + struct reftable_buf tab_file_path = REFTABLE_BUF_INIT; struct reftable_writer *wr = NULL; - struct tempfile *tab_file; - int tab_fd, err = 0; + struct fd_writer writer= { + .opts = &st->opts, + }; + struct reftable_tmpfile tab_file = REFTABLE_TMPFILE_INIT; + int err = 0; + + err = format_name(&next_name, reftable_reader_min_update_index(st->readers[first]), + reftable_reader_max_update_index(st->readers[last])); + if (err < 0) + goto done; - format_name(&next_name, - reftable_reader_min_update_index(st->readers[first]), - reftable_reader_max_update_index(st->readers[last])); - stack_filename(&tab_file_path, st, next_name.buf); - strbuf_addstr(&tab_file_path, ".temp.XXXXXX"); + err = stack_filename(&tab_file_path, st, next_name.buf); + if (err < 0) + goto done; - tab_file = mks_tempfile(tab_file_path.buf); - if (!tab_file) { - err = REFTABLE_IO_ERROR; + err = reftable_buf_addstr(&tab_file_path, ".temp.XXXXXX"); + if (err < 0) + goto done; + + err = tmpfile_from_pattern(&tab_file, tab_file_path.buf); + if (err < 0) goto done; - } - tab_fd = get_tempfile_fd(tab_file); if (st->opts.default_permissions && - chmod(get_tempfile_path(tab_file), st->opts.default_permissions) < 0) { + chmod(tab_file.path, st->opts.default_permissions) < 0) { err = REFTABLE_IO_ERROR; goto done; } - wr = reftable_new_writer(reftable_fd_write, reftable_fd_flush, - &tab_fd, &st->opts); + writer.fd = tab_file.fd; + err = reftable_writer_new(&wr, fd_writer_write, fd_writer_flush, + &writer, &st->opts); + if (err < 0) + goto done; + err = stack_write_compact(st, wr, first, last, config); if (err < 0) goto done; @@ -882,18 +1022,18 @@ static int stack_compact_locked(struct reftable_stack *st, if (err < 0) goto done; - err = close_tempfile_gently(tab_file); + err = tmpfile_close(&tab_file); if (err < 0) goto done; *tab_file_out = tab_file; - tab_file = NULL; + tab_file = REFTABLE_TMPFILE_INIT; done: - delete_tempfile(&tab_file); + tmpfile_delete(&tab_file); reftable_writer_free(wr); - strbuf_release(&next_name); - strbuf_release(&tab_file_path); + reftable_buf_release(&next_name); + reftable_buf_release(&tab_file_path); return err; } @@ -902,32 +1042,28 @@ static int stack_write_compact(struct reftable_stack *st, size_t first, size_t last, struct reftable_log_expiry_config *config) { - size_t subtabs_len = last - first + 1; - struct reftable_table *subtabs = reftable_calloc( - last - first + 1, sizeof(*subtabs)); struct reftable_merged_table *mt = NULL; struct reftable_iterator it = { NULL }; struct reftable_ref_record ref = { NULL }; struct reftable_log_record log = { NULL }; + size_t subtabs_len = last - first + 1; uint64_t entries = 0; int err = 0; - for (size_t i = first, j = 0; i <= last; i++) { - struct reftable_reader *t = st->readers[i]; - reftable_table_from_reader(&subtabs[j++], t); - st->stats.bytes += t->size; - } + for (size_t i = first; i <= last; i++) + st->stats.bytes += st->readers[i]->size; reftable_writer_set_limits(wr, st->readers[first]->min_update_index, st->readers[last]->max_update_index); - err = reftable_new_merged_table(&mt, subtabs, subtabs_len, + err = reftable_merged_table_new(&mt, st->readers + first, subtabs_len, st->opts.hash_id); - if (err < 0) { - reftable_free(subtabs); + if (err < 0) + goto done; + + err = merged_table_init_iter(mt, &it, BLOCK_TYPE_REF); + if (err < 0) goto done; - } - merged_table_init_iter(mt, &it, BLOCK_TYPE_REF); err = reftable_iterator_seek_ref(&it, ""); if (err < 0) goto done; @@ -952,7 +1088,10 @@ static int stack_write_compact(struct reftable_stack *st, } reftable_iterator_destroy(&it); - merged_table_init_iter(mt, &it, BLOCK_TYPE_LOG); + err = merged_table_init_iter(mt, &it, BLOCK_TYPE_LOG); + if (err < 0) + goto done; + err = reftable_iterator_seek_log(&it, ""); if (err < 0) goto done; @@ -995,6 +1134,15 @@ done: return err; } +enum stack_compact_range_flags { + /* + * Perform a best-effort compaction. That is, even if we cannot lock + * all tables in the specified range, we will try to compact the + * remaining slice. + */ + STACK_COMPACT_RANGE_BEST_EFFORT = (1 << 0), +}; + /* * Compact all tables in the range `[first, last)` into a single new table. * @@ -1006,17 +1154,20 @@ done: */ static int stack_compact_range(struct reftable_stack *st, size_t first, size_t last, - struct reftable_log_expiry_config *expiry) + struct reftable_log_expiry_config *expiry, + unsigned int flags) { - struct strbuf tables_list_buf = STRBUF_INIT; - struct strbuf new_table_name = STRBUF_INIT; - struct strbuf new_table_path = STRBUF_INIT; - struct strbuf table_name = STRBUF_INIT; - struct lock_file tables_list_lock = LOCK_INIT; - struct lock_file *table_locks = NULL; - struct tempfile *new_table = NULL; + struct reftable_buf tables_list_buf = REFTABLE_BUF_INIT; + struct reftable_buf new_table_name = REFTABLE_BUF_INIT; + struct reftable_buf new_table_path = REFTABLE_BUF_INIT; + struct reftable_buf table_name = REFTABLE_BUF_INIT; + struct reftable_flock tables_list_lock = REFTABLE_FLOCK_INIT; + struct reftable_flock *table_locks = NULL; + struct reftable_tmpfile new_table = REFTABLE_TMPFILE_INIT; int is_empty_table = 0, err = 0; - size_t i; + size_t first_to_replace, last_to_replace; + size_t i, nlocks = 0; + char **names = NULL; if (first > last || (!expiry && first == last)) { err = 0; @@ -1029,8 +1180,7 @@ static int stack_compact_range(struct reftable_stack *st, * Hold the lock so that we can read "tables.list" and lock all tables * which are part of the user-specified range. */ - err = hold_lock_file_for_update(&tables_list_lock, st->list_file, - LOCK_NO_DEREF); + err = flock_acquire(&tables_list_lock, st->list_file, st->opts.lock_timeout_ms); if (err < 0) { if (errno == EEXIST) err = REFTABLE_LOCK_ERROR; @@ -1046,19 +1196,55 @@ static int stack_compact_range(struct reftable_stack *st, /* * Lock all tables in the user-provided range. This is the slice of our * stack which we'll compact. + * + * Note that we lock tables in reverse order from last to first. The + * intent behind this is to allow a newer process to perform best + * effort compaction of tables that it has added in the case where an + * older process is still busy compacting tables which are preexisting + * from the point of view of the newer process. */ - REFTABLE_CALLOC_ARRAY(table_locks, last - first + 1); - for (i = first; i <= last; i++) { - stack_filename(&table_name, st, reader_name(st->readers[i])); + REFTABLE_ALLOC_ARRAY(table_locks, last - first + 1); + if (!table_locks) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto done; + } + for (i = 0; i < last - first + 1; i++) + table_locks[i] = REFTABLE_FLOCK_INIT; + + for (i = last + 1; i > first; i--) { + err = stack_filename(&table_name, st, reader_name(st->readers[i - 1])); + if (err < 0) + goto done; - err = hold_lock_file_for_update(&table_locks[i - first], - table_name.buf, LOCK_NO_DEREF); + err = flock_acquire(&table_locks[nlocks], table_name.buf, 0); if (err < 0) { - if (errno == EEXIST) + /* + * When the table is locked already we may do a + * best-effort compaction and compact only the tables + * that we have managed to lock so far. This of course + * requires that we have been able to lock at least two + * tables, otherwise there would be nothing to compact. + * In that case, we return a lock error to our caller. + */ + if (errno == EEXIST && last - (i - 1) >= 2 && + flags & STACK_COMPACT_RANGE_BEST_EFFORT) { + err = 0; + /* + * The subtraction is to offset the index, the + * addition is to only compact up to the table + * of the preceding iteration. They obviously + * cancel each other out, but that may be + * non-obvious when it was omitted. + */ + first = (i - 1) + 1; + break; + } else if (errno == EEXIST) { err = REFTABLE_LOCK_ERROR; - else + goto done; + } else { err = REFTABLE_IO_ERROR; - goto done; + goto done; + } } /* @@ -1066,7 +1252,7 @@ static int stack_compact_range(struct reftable_stack *st, * run into file descriptor exhaustion when we compress a lot * of tables. */ - err = close_lock_file_gently(&table_locks[i - first]); + err = flock_close(&table_locks[nlocks++]); if (err < 0) { err = REFTABLE_IO_ERROR; goto done; @@ -1078,7 +1264,7 @@ static int stack_compact_range(struct reftable_stack *st, * "tables.list" lock while compacting the locked tables. This allows * concurrent updates to the stack to proceed. */ - err = rollback_lock_file(&tables_list_lock); + err = flock_release(&tables_list_lock); if (err < 0) { err = REFTABLE_IO_ERROR; goto done; @@ -1101,8 +1287,7 @@ static int stack_compact_range(struct reftable_stack *st, * "tables.list". We'll then replace the compacted range of tables with * the new table. */ - err = hold_lock_file_for_update(&tables_list_lock, st->list_file, - LOCK_NO_DEREF); + err = flock_acquire(&tables_list_lock, st->list_file, st->opts.lock_timeout_ms); if (err < 0) { if (errno == EEXIST) err = REFTABLE_LOCK_ERROR; @@ -1112,7 +1297,7 @@ static int stack_compact_range(struct reftable_stack *st, } if (st->opts.default_permissions) { - if (chmod(get_lock_file_path(&tables_list_lock), + if (chmod(tables_list_lock.path, st->opts.default_permissions) < 0) { err = REFTABLE_IO_ERROR; goto done; @@ -1120,20 +1305,130 @@ static int stack_compact_range(struct reftable_stack *st, } /* + * As we have unlocked the stack while compacting our slice of tables + * it may have happened that a concurrently running process has updated + * the stack while we were compacting. In that case, we need to check + * whether the tables that we have just compacted still exist in the + * stack in the exact same order as we have compacted them. + * + * If they do exist, then it is fine to continue and replace those + * tables with our compacted version. If they don't, then we need to + * abort. + */ + err = stack_uptodate(st); + if (err < 0) + goto done; + if (err > 0) { + ssize_t new_offset = -1; + int fd; + + fd = open(st->list_file, O_RDONLY); + if (fd < 0) { + err = REFTABLE_IO_ERROR; + goto done; + } + + err = fd_read_lines(fd, &names); + close(fd); + if (err < 0) + goto done; + + /* + * Search for the offset of the first table that we have + * compacted in the updated "tables.list" file. + */ + for (size_t i = 0; names[i]; i++) { + if (strcmp(names[i], st->readers[first]->name)) + continue; + + /* + * We have found the first entry. Verify that all the + * subsequent tables we have compacted still exist in + * the modified stack in the exact same order as we + * have compacted them. + */ + for (size_t j = 1; j < last - first + 1; j++) { + const char *old = first + j < st->merged->readers_len ? + st->readers[first + j]->name : NULL; + const char *new = names[i + j]; + + /* + * If some entries are missing or in case the tables + * have changed then we need to bail out. Again, this + * shouldn't ever happen because we have locked the + * tables we are compacting. + */ + if (!old || !new || strcmp(old, new)) { + err = REFTABLE_OUTDATED_ERROR; + goto done; + } + } + + new_offset = i; + break; + } + + /* + * In case we didn't find our compacted tables in the stack we + * need to bail out. In theory, this should have never happened + * because we locked the tables we are compacting. + */ + if (new_offset < 0) { + err = REFTABLE_OUTDATED_ERROR; + goto done; + } + + /* + * We have found the new range that we want to replace, so + * let's update the range of tables that we want to replace. + */ + first_to_replace = new_offset; + last_to_replace = last + (new_offset - first); + } else { + /* + * `fd_read_lines()` uses a `NULL` sentinel to indicate that + * the array is at its end. As we use `free_names()` to free + * the array, we need to include this sentinel value here and + * thus have to allocate `readers_len + 1` many entries. + */ + REFTABLE_CALLOC_ARRAY(names, st->merged->readers_len + 1); + if (!names) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto done; + } + + for (size_t i = 0; i < st->merged->readers_len; i++) { + names[i] = reftable_strdup(st->readers[i]->name); + if (!names[i]) { + err = REFTABLE_OUT_OF_MEMORY_ERROR; + goto done; + } + } + first_to_replace = first; + last_to_replace = last; + } + + /* * If the resulting compacted table is not empty, then we need to move * it into place now. */ if (!is_empty_table) { - format_name(&new_table_name, st->readers[first]->min_update_index, - st->readers[last]->max_update_index); - strbuf_addstr(&new_table_name, ".ref"); - stack_filename(&new_table_path, st, new_table_name.buf); + err = format_name(&new_table_name, st->readers[first]->min_update_index, + st->readers[last]->max_update_index); + if (err < 0) + goto done; - err = rename_tempfile(&new_table, new_table_path.buf); - if (err < 0) { - err = REFTABLE_IO_ERROR; + err = reftable_buf_addstr(&new_table_name, ".ref"); + if (err < 0) + goto done; + + err = stack_filename(&new_table_path, st, new_table_name.buf); + if (err < 0) + goto done; + + err = tmpfile_rename(&new_table, new_table_path.buf); + if (err < 0) goto done; - } } /* @@ -1141,14 +1436,23 @@ static int stack_compact_range(struct reftable_stack *st, * have just written. In case the compacted table became empty we * simply skip writing it. */ - for (i = 0; i < first; i++) - strbuf_addf(&tables_list_buf, "%s\n", st->readers[i]->name); - if (!is_empty_table) - strbuf_addf(&tables_list_buf, "%s\n", new_table_name.buf); - for (i = last + 1; i < st->merged->stack_len; i++) - strbuf_addf(&tables_list_buf, "%s\n", st->readers[i]->name); - - err = write_in_full(get_lock_file_fd(&tables_list_lock), + for (i = 0; i < first_to_replace; i++) { + if ((err = reftable_buf_addstr(&tables_list_buf, names[i])) < 0 || + (err = reftable_buf_addstr(&tables_list_buf, "\n")) < 0) + goto done; + } + if (!is_empty_table) { + if ((err = reftable_buf_addstr(&tables_list_buf, new_table_name.buf)) < 0 || + (err = reftable_buf_addstr(&tables_list_buf, "\n")) < 0) + goto done; + } + for (i = last_to_replace + 1; names[i]; i++) { + if ((err = reftable_buf_addstr(&tables_list_buf, names[i])) < 0 || + (err = reftable_buf_addstr(&tables_list_buf, "\n")) < 0) + goto done; + } + + err = write_in_full(tables_list_lock.fd, tables_list_buf.buf, tables_list_buf.len); if (err < 0) { err = REFTABLE_IO_ERROR; @@ -1156,14 +1460,14 @@ static int stack_compact_range(struct reftable_stack *st, goto done; } - err = fsync_component(FSYNC_COMPONENT_REFERENCE, get_lock_file_fd(&tables_list_lock)); + err = stack_fsync(&st->opts, tables_list_lock.fd); if (err < 0) { err = REFTABLE_IO_ERROR; unlink(new_table_path.buf); goto done; } - err = commit_lock_file(&tables_list_lock); + err = flock_commit(&tables_list_lock); if (err < 0) { err = REFTABLE_IO_ERROR; unlink(new_table_path.buf); @@ -1183,43 +1487,42 @@ static int stack_compact_range(struct reftable_stack *st, * Delete the old tables. They may still be in use by concurrent * readers, so it is expected that unlinking tables may fail. */ - for (i = first; i <= last; i++) { - struct lock_file *table_lock = &table_locks[i - first]; - char *table_path = get_locked_file_path(table_lock); - unlink(table_path); - free(table_path); + for (i = 0; i < nlocks; i++) { + struct reftable_flock *table_lock = &table_locks[i]; + + reftable_buf_reset(&table_name); + err = reftable_buf_add(&table_name, table_lock->path, + strlen(table_lock->path) - strlen(".lock")); + if (err) + continue; + + unlink(table_name.buf); } done: - rollback_lock_file(&tables_list_lock); - for (i = first; table_locks && i <= last; i++) - rollback_lock_file(&table_locks[i - first]); + flock_release(&tables_list_lock); + for (i = 0; table_locks && i < nlocks; i++) + flock_release(&table_locks[i]); reftable_free(table_locks); - delete_tempfile(&new_table); - strbuf_release(&new_table_name); - strbuf_release(&new_table_path); + tmpfile_delete(&new_table); + reftable_buf_release(&new_table_name); + reftable_buf_release(&new_table_path); + reftable_buf_release(&tables_list_buf); + reftable_buf_release(&table_name); + free_names(names); + + if (err == REFTABLE_LOCK_ERROR) + st->stats.failures++; - strbuf_release(&tables_list_buf); - strbuf_release(&table_name); return err; } int reftable_stack_compact_all(struct reftable_stack *st, struct reftable_log_expiry_config *config) { - return stack_compact_range(st, 0, st->merged->stack_len ? - st->merged->stack_len - 1 : 0, config); -} - -static int stack_compact_range_stats(struct reftable_stack *st, - size_t first, size_t last, - struct reftable_log_expiry_config *config) -{ - int err = stack_compact_range(st, first, last, config); - if (err == REFTABLE_LOCK_ERROR) - st->stats.failures++; - return err; + size_t last = st->merged->readers_len ? st->merged->readers_len - 1 : 0; + return stack_compact_range(st, 0, last, config, 0); } static int segment_size(struct segment *s) @@ -1305,27 +1608,36 @@ struct segment suggest_compaction_segment(uint64_t *sizes, size_t n, static uint64_t *stack_table_sizes_for_compaction(struct reftable_stack *st) { - uint64_t *sizes = - reftable_calloc(st->merged->stack_len, sizeof(*sizes)); - int version = (st->opts.hash_id == GIT_SHA1_FORMAT_ID) ? 1 : 2; + int version = (st->opts.hash_id == REFTABLE_HASH_SHA1) ? 1 : 2; int overhead = header_size(version) - 1; - int i = 0; - for (i = 0; i < st->merged->stack_len; i++) { + uint64_t *sizes; + + REFTABLE_CALLOC_ARRAY(sizes, st->merged->readers_len); + if (!sizes) + return NULL; + + for (size_t i = 0; i < st->merged->readers_len; i++) sizes[i] = st->readers[i]->size - overhead; - } + return sizes; } int reftable_stack_auto_compact(struct reftable_stack *st) { - uint64_t *sizes = stack_table_sizes_for_compaction(st); - struct segment seg = - suggest_compaction_segment(sizes, st->merged->stack_len, - st->opts.auto_compaction_factor); + struct segment seg; + uint64_t *sizes; + + sizes = stack_table_sizes_for_compaction(st); + if (!sizes) + return REFTABLE_OUT_OF_MEMORY_ERROR; + + seg = suggest_compaction_segment(sizes, st->merged->readers_len, + st->opts.auto_compaction_factor); reftable_free(sizes); + if (segment_size(&seg) > 0) - return stack_compact_range_stats(st, seg.start, seg.end - 1, - NULL); + return stack_compact_range(st, seg.start, seg.end - 1, + NULL, STACK_COMPACT_RANGE_BEST_EFFORT); return 0; } @@ -1339,9 +1651,31 @@ reftable_stack_compaction_stats(struct reftable_stack *st) int reftable_stack_read_ref(struct reftable_stack *st, const char *refname, struct reftable_ref_record *ref) { - struct reftable_table tab = { NULL }; - reftable_table_from_merged_table(&tab, reftable_stack_merged_table(st)); - return reftable_table_read_ref(&tab, refname, ref); + struct reftable_iterator it = { 0 }; + int ret; + + ret = reftable_merged_table_init_ref_iterator(st->merged, &it); + if (ret) + goto out; + + ret = reftable_iterator_seek_ref(&it, refname); + if (ret) + goto out; + + ret = reftable_iterator_next_ref(&it, ref); + if (ret) + goto out; + + if (strcmp(ref->refname, refname) || + reftable_ref_record_is_deletion(ref)) { + reftable_ref_record_release(ref); + ret = 1; + goto out; + } + +out: + reftable_iterator_destroy(&it); + return ret; } int reftable_stack_read_log(struct reftable_stack *st, const char *refname, @@ -1350,7 +1684,10 @@ int reftable_stack_read_log(struct reftable_stack *st, const char *refname, struct reftable_iterator it = {0}; int err; - reftable_stack_init_log_iterator(st, &it); + err = reftable_stack_init_log_iterator(st, &it); + if (err) + goto done; + err = reftable_iterator_seek_log(&it, refname); if (err) goto done; @@ -1386,25 +1723,28 @@ static void remove_maybe_stale_table(struct reftable_stack *st, uint64_t max, uint64_t update_idx = 0; struct reftable_block_source src = { NULL }; struct reftable_reader *rd = NULL; - struct strbuf table_path = STRBUF_INIT; - stack_filename(&table_path, st, name); + struct reftable_buf table_path = REFTABLE_BUF_INIT; + + err = stack_filename(&table_path, st, name); + if (err < 0) + goto done; err = reftable_block_source_from_file(&src, table_path.buf); if (err < 0) goto done; - err = reftable_new_reader(&rd, &src, name); + err = reftable_reader_new(&rd, &src, name); if (err < 0) goto done; update_idx = reftable_reader_max_update_index(rd); - reftable_reader_free(rd); + reftable_reader_decref(rd); if (update_idx <= max) { unlink(table_path.buf); } done: - strbuf_release(&table_path); + reftable_buf_release(&table_path); } static int reftable_stack_clean_locked(struct reftable_stack *st) @@ -1439,7 +1779,7 @@ static int reftable_stack_clean_locked(struct reftable_stack *st) int reftable_stack_clean(struct reftable_stack *st) { struct reftable_addition *add = NULL; - int err = reftable_stack_new_addition(&add, st); + int err = reftable_stack_new_addition(&add, st, 0); if (err < 0) { goto done; } @@ -1456,22 +1796,7 @@ done: return err; } -int reftable_stack_print_directory(const char *stackdir, uint32_t hash_id) +enum reftable_hash reftable_stack_hash_id(struct reftable_stack *st) { - struct reftable_stack *stack = NULL; - struct reftable_write_options opts = { .hash_id = hash_id }; - struct reftable_merged_table *merged = NULL; - struct reftable_table table = { NULL }; - - int err = reftable_new_stack(&stack, stackdir, &opts); - if (err < 0) - goto done; - - merged = reftable_stack_merged_table(stack); - reftable_table_from_merged_table(&table, merged); - err = reftable_table_print(&table); -done: - if (stack) - reftable_stack_destroy(stack); - return err; + return reftable_merged_table_hash_id(st->merged); } diff --git a/reftable/stack_test.c b/reftable/stack_test.c deleted file mode 100644 index e3c11e6a6e..0000000000 --- a/reftable/stack_test.c +++ /dev/null @@ -1,1034 +0,0 @@ -/* -Copyright 2020 Google LLC - -Use of this source code is governed by a BSD-style -license that can be found in the LICENSE file or at -https://developers.google.com/open-source/licenses/bsd -*/ - -#include "stack.h" - -#include "system.h" - -#include "reftable-reader.h" -#include "merged.h" -#include "basics.h" -#include "record.h" -#include "test_framework.h" -#include "reftable-tests.h" -#include "reader.h" - -#include <sys/types.h> -#include <dirent.h> - -static void clear_dir(const char *dirname) -{ - struct strbuf path = STRBUF_INIT; - strbuf_addstr(&path, dirname); - remove_dir_recursively(&path, 0); - strbuf_release(&path); -} - -static int count_dir_entries(const char *dirname) -{ - DIR *dir = opendir(dirname); - int len = 0; - struct dirent *d; - if (!dir) - return 0; - - while ((d = readdir(dir))) { - /* - * Besides skipping over "." and "..", we also need to - * skip over other files that have a leading ".". This - * is due to behaviour of NFS, which will rename files - * to ".nfs*" to emulate delete-on-last-close. - * - * In any case this should be fine as the reftable - * library will never write files with leading dots - * anyway. - */ - if (starts_with(d->d_name, ".")) - continue; - len++; - } - closedir(dir); - return len; -} - -/* - * Work linenumber into the tempdir, so we can see which tests forget to - * cleanup. - */ -static char *get_tmp_template(int linenumber) -{ - const char *tmp = getenv("TMPDIR"); - static char template[1024]; - snprintf(template, sizeof(template) - 1, "%s/stack_test-%d.XXXXXX", - tmp ? tmp : "/tmp", linenumber); - return template; -} - -static char *get_tmp_dir(int linenumber) -{ - char *dir = get_tmp_template(linenumber); - EXPECT(mkdtemp(dir)); - return dir; -} - -static void test_read_file(void) -{ - char *fn = get_tmp_template(__LINE__); - int fd = mkstemp(fn); - char out[1024] = "line1\n\nline2\nline3"; - int n, err; - char **names = NULL; - const char *want[] = { "line1", "line2", "line3" }; - int i = 0; - - EXPECT(fd > 0); - n = write_in_full(fd, out, strlen(out)); - EXPECT(n == strlen(out)); - err = close(fd); - EXPECT(err >= 0); - - err = read_lines(fn, &names); - EXPECT_ERR(err); - - for (i = 0; names[i]; i++) { - EXPECT(0 == strcmp(want[i], names[i])); - } - free_names(names); - (void) remove(fn); -} - -static int write_test_ref(struct reftable_writer *wr, void *arg) -{ - struct reftable_ref_record *ref = arg; - reftable_writer_set_limits(wr, ref->update_index, ref->update_index); - return reftable_writer_add_ref(wr, ref); -} - -struct write_log_arg { - struct reftable_log_record *log; - uint64_t update_index; -}; - -static int write_test_log(struct reftable_writer *wr, void *arg) -{ - struct write_log_arg *wla = arg; - - reftable_writer_set_limits(wr, wla->update_index, wla->update_index); - return reftable_writer_add_log(wr, wla->log); -} - -static void test_reftable_stack_add_one(void) -{ - char *dir = get_tmp_dir(__LINE__); - struct strbuf scratch = STRBUF_INIT; - int mask = umask(002); - struct reftable_write_options opts = { - .default_permissions = 0660, - }; - struct reftable_stack *st = NULL; - int err; - struct reftable_ref_record ref = { - .refname = (char *) "HEAD", - .update_index = 1, - .value_type = REFTABLE_REF_SYMREF, - .value.symref = (char *) "master", - }; - struct reftable_ref_record dest = { NULL }; - struct stat stat_result = { 0 }; - err = reftable_new_stack(&st, dir, &opts); - EXPECT_ERR(err); - - err = reftable_stack_add(st, &write_test_ref, &ref); - EXPECT_ERR(err); - - err = reftable_stack_read_ref(st, ref.refname, &dest); - EXPECT_ERR(err); - EXPECT(0 == strcmp("master", dest.value.symref)); - EXPECT(st->readers_len > 0); - - printf("testing print functionality:\n"); - err = reftable_stack_print_directory(dir, GIT_SHA1_FORMAT_ID); - EXPECT_ERR(err); - - err = reftable_stack_print_directory(dir, GIT_SHA256_FORMAT_ID); - EXPECT(err == REFTABLE_FORMAT_ERROR); - -#ifndef GIT_WINDOWS_NATIVE - strbuf_addstr(&scratch, dir); - strbuf_addstr(&scratch, "/tables.list"); - err = stat(scratch.buf, &stat_result); - EXPECT(!err); - EXPECT((stat_result.st_mode & 0777) == opts.default_permissions); - - strbuf_reset(&scratch); - strbuf_addstr(&scratch, dir); - strbuf_addstr(&scratch, "/"); - /* do not try at home; not an external API for reftable. */ - strbuf_addstr(&scratch, st->readers[0]->name); - err = stat(scratch.buf, &stat_result); - EXPECT(!err); - EXPECT((stat_result.st_mode & 0777) == opts.default_permissions); -#else - (void) stat_result; -#endif - - reftable_ref_record_release(&dest); - reftable_stack_destroy(st); - strbuf_release(&scratch); - clear_dir(dir); - umask(mask); -} - -static void test_reftable_stack_uptodate(void) -{ - struct reftable_write_options opts = { 0 }; - struct reftable_stack *st1 = NULL; - struct reftable_stack *st2 = NULL; - char *dir = get_tmp_dir(__LINE__); - - int err; - struct reftable_ref_record ref1 = { - .refname = (char *) "HEAD", - .update_index = 1, - .value_type = REFTABLE_REF_SYMREF, - .value.symref = (char *) "master", - }; - struct reftable_ref_record ref2 = { - .refname = (char *) "branch2", - .update_index = 2, - .value_type = REFTABLE_REF_SYMREF, - .value.symref = (char *) "master", - }; - - - /* simulate multi-process access to the same stack - by creating two stacks for the same directory. - */ - err = reftable_new_stack(&st1, dir, &opts); - EXPECT_ERR(err); - - err = reftable_new_stack(&st2, dir, &opts); - EXPECT_ERR(err); - - err = reftable_stack_add(st1, &write_test_ref, &ref1); - EXPECT_ERR(err); - - err = reftable_stack_add(st2, &write_test_ref, &ref2); - EXPECT(err == REFTABLE_OUTDATED_ERROR); - - err = reftable_stack_reload(st2); - EXPECT_ERR(err); - - err = reftable_stack_add(st2, &write_test_ref, &ref2); - EXPECT_ERR(err); - reftable_stack_destroy(st1); - reftable_stack_destroy(st2); - clear_dir(dir); -} - -static void test_reftable_stack_transaction_api(void) -{ - char *dir = get_tmp_dir(__LINE__); - struct reftable_write_options opts = { 0 }; - struct reftable_stack *st = NULL; - int err; - struct reftable_addition *add = NULL; - - struct reftable_ref_record ref = { - .refname = (char *) "HEAD", - .update_index = 1, - .value_type = REFTABLE_REF_SYMREF, - .value.symref = (char *) "master", - }; - struct reftable_ref_record dest = { NULL }; - - err = reftable_new_stack(&st, dir, &opts); - EXPECT_ERR(err); - - reftable_addition_destroy(add); - - err = reftable_stack_new_addition(&add, st); - EXPECT_ERR(err); - - err = reftable_addition_add(add, &write_test_ref, &ref); - EXPECT_ERR(err); - - err = reftable_addition_commit(add); - EXPECT_ERR(err); - - reftable_addition_destroy(add); - - err = reftable_stack_read_ref(st, ref.refname, &dest); - EXPECT_ERR(err); - EXPECT(REFTABLE_REF_SYMREF == dest.value_type); - EXPECT(0 == strcmp("master", dest.value.symref)); - - reftable_ref_record_release(&dest); - reftable_stack_destroy(st); - clear_dir(dir); -} - -static void test_reftable_stack_transaction_api_performs_auto_compaction(void) -{ - char *dir = get_tmp_dir(__LINE__); - struct reftable_write_options opts = {0}; - struct reftable_addition *add = NULL; - struct reftable_stack *st = NULL; - int i, n = 20, err; - - err = reftable_new_stack(&st, dir, &opts); - EXPECT_ERR(err); - - for (i = 0; i <= n; i++) { - struct reftable_ref_record ref = { - .update_index = reftable_stack_next_update_index(st), - .value_type = REFTABLE_REF_SYMREF, - .value.symref = (char *) "master", - }; - char name[100]; - - snprintf(name, sizeof(name), "branch%04d", i); - ref.refname = name; - - /* - * Disable auto-compaction for all but the last runs. Like this - * we can ensure that we indeed honor this setting and have - * better control over when exactly auto compaction runs. - */ - st->opts.disable_auto_compact = i != n; - - err = reftable_stack_new_addition(&add, st); - EXPECT_ERR(err); - - err = reftable_addition_add(add, &write_test_ref, &ref); - EXPECT_ERR(err); - - err = reftable_addition_commit(add); - EXPECT_ERR(err); - - reftable_addition_destroy(add); - - /* - * The stack length should grow continuously for all runs where - * auto compaction is disabled. When enabled, we should merge - * all tables in the stack. - */ - if (i != n) - EXPECT(st->merged->stack_len == i + 1); - else - EXPECT(st->merged->stack_len == 1); - } - - reftable_stack_destroy(st); - clear_dir(dir); -} - -static void test_reftable_stack_auto_compaction_fails_gracefully(void) -{ - struct reftable_ref_record ref = { - .refname = (char *) "refs/heads/master", - .update_index = 1, - .value_type = REFTABLE_REF_VAL1, - .value.val1 = {0x01}, - }; - struct reftable_write_options opts = {0}; - struct reftable_stack *st; - struct strbuf table_path = STRBUF_INIT; - char *dir = get_tmp_dir(__LINE__); - int err; - - err = reftable_new_stack(&st, dir, &opts); - EXPECT_ERR(err); - - err = reftable_stack_add(st, write_test_ref, &ref); - EXPECT_ERR(err); - EXPECT(st->merged->stack_len == 1); - EXPECT(st->stats.attempts == 0); - EXPECT(st->stats.failures == 0); - - /* - * Lock the newly written table such that it cannot be compacted. - * Adding a new table to the stack should not be impacted by this, even - * though auto-compaction will now fail. - */ - strbuf_addf(&table_path, "%s/%s.lock", dir, st->readers[0]->name); - write_file_buf(table_path.buf, "", 0); - - ref.update_index = 2; - err = reftable_stack_add(st, write_test_ref, &ref); - EXPECT_ERR(err); - EXPECT(st->merged->stack_len == 2); - EXPECT(st->stats.attempts == 1); - EXPECT(st->stats.failures == 1); - - reftable_stack_destroy(st); - strbuf_release(&table_path); - clear_dir(dir); -} - -static int write_error(struct reftable_writer *wr, void *arg) -{ - return *((int *)arg); -} - -static void test_reftable_stack_update_index_check(void) -{ - char *dir = get_tmp_dir(__LINE__); - struct reftable_write_options opts = { 0 }; - struct reftable_stack *st = NULL; - int err; - struct reftable_ref_record ref1 = { - .refname = (char *) "name1", - .update_index = 1, - .value_type = REFTABLE_REF_SYMREF, - .value.symref = (char *) "master", - }; - struct reftable_ref_record ref2 = { - .refname = (char *) "name2", - .update_index = 1, - .value_type = REFTABLE_REF_SYMREF, - .value.symref = (char *) "master", - }; - - err = reftable_new_stack(&st, dir, &opts); - EXPECT_ERR(err); - - err = reftable_stack_add(st, &write_test_ref, &ref1); - EXPECT_ERR(err); - - err = reftable_stack_add(st, &write_test_ref, &ref2); - EXPECT(err == REFTABLE_API_ERROR); - reftable_stack_destroy(st); - clear_dir(dir); -} - -static void test_reftable_stack_lock_failure(void) -{ - char *dir = get_tmp_dir(__LINE__); - struct reftable_write_options opts = { 0 }; - struct reftable_stack *st = NULL; - int err, i; - - err = reftable_new_stack(&st, dir, &opts); - EXPECT_ERR(err); - for (i = -1; i != REFTABLE_EMPTY_TABLE_ERROR; i--) { - err = reftable_stack_add(st, &write_error, &i); - EXPECT(err == i); - } - - reftable_stack_destroy(st); - clear_dir(dir); -} - -static void test_reftable_stack_add(void) -{ - int i = 0; - int err = 0; - struct reftable_write_options opts = { - .exact_log_message = 1, - .default_permissions = 0660, - .disable_auto_compact = 1, - }; - struct reftable_stack *st = NULL; - char *dir = get_tmp_dir(__LINE__); - struct reftable_ref_record refs[2] = { { NULL } }; - struct reftable_log_record logs[2] = { { NULL } }; - struct strbuf path = STRBUF_INIT; - struct stat stat_result; - int N = ARRAY_SIZE(refs); - - err = reftable_new_stack(&st, dir, &opts); - EXPECT_ERR(err); - - for (i = 0; i < N; i++) { - char buf[256]; - snprintf(buf, sizeof(buf), "branch%02d", i); - refs[i].refname = xstrdup(buf); - refs[i].update_index = i + 1; - refs[i].value_type = REFTABLE_REF_VAL1; - set_test_hash(refs[i].value.val1, i); - - logs[i].refname = xstrdup(buf); - logs[i].update_index = N + i + 1; - logs[i].value_type = REFTABLE_LOG_UPDATE; - logs[i].value.update.email = xstrdup("identity@invalid"); - set_test_hash(logs[i].value.update.new_hash, i); - } - - for (i = 0; i < N; i++) { - int err = reftable_stack_add(st, &write_test_ref, &refs[i]); - EXPECT_ERR(err); - } - - for (i = 0; i < N; i++) { - struct write_log_arg arg = { - .log = &logs[i], - .update_index = reftable_stack_next_update_index(st), - }; - int err = reftable_stack_add(st, &write_test_log, &arg); - EXPECT_ERR(err); - } - - err = reftable_stack_compact_all(st, NULL); - EXPECT_ERR(err); - - for (i = 0; i < N; i++) { - struct reftable_ref_record dest = { NULL }; - - int err = reftable_stack_read_ref(st, refs[i].refname, &dest); - EXPECT_ERR(err); - EXPECT(reftable_ref_record_equal(&dest, refs + i, - GIT_SHA1_RAWSZ)); - reftable_ref_record_release(&dest); - } - - for (i = 0; i < N; i++) { - struct reftable_log_record dest = { NULL }; - int err = reftable_stack_read_log(st, refs[i].refname, &dest); - EXPECT_ERR(err); - EXPECT(reftable_log_record_equal(&dest, logs + i, - GIT_SHA1_RAWSZ)); - reftable_log_record_release(&dest); - } - -#ifndef GIT_WINDOWS_NATIVE - strbuf_addstr(&path, dir); - strbuf_addstr(&path, "/tables.list"); - err = stat(path.buf, &stat_result); - EXPECT(!err); - EXPECT((stat_result.st_mode & 0777) == opts.default_permissions); - - strbuf_reset(&path); - strbuf_addstr(&path, dir); - strbuf_addstr(&path, "/"); - /* do not try at home; not an external API for reftable. */ - strbuf_addstr(&path, st->readers[0]->name); - err = stat(path.buf, &stat_result); - EXPECT(!err); - EXPECT((stat_result.st_mode & 0777) == opts.default_permissions); -#else - (void) stat_result; -#endif - - /* cleanup */ - reftable_stack_destroy(st); - for (i = 0; i < N; i++) { - reftable_ref_record_release(&refs[i]); - reftable_log_record_release(&logs[i]); - } - strbuf_release(&path); - clear_dir(dir); -} - -static void test_reftable_stack_log_normalize(void) -{ - int err = 0; - struct reftable_write_options opts = { - 0, - }; - struct reftable_stack *st = NULL; - char *dir = get_tmp_dir(__LINE__); - struct reftable_log_record input = { - .refname = (char *) "branch", - .update_index = 1, - .value_type = REFTABLE_LOG_UPDATE, - .value = { - .update = { - .new_hash = { 1 }, - .old_hash = { 2 }, - }, - }, - }; - struct reftable_log_record dest = { - .update_index = 0, - }; - struct write_log_arg arg = { - .log = &input, - .update_index = 1, - }; - - err = reftable_new_stack(&st, dir, &opts); - EXPECT_ERR(err); - - input.value.update.message = (char *) "one\ntwo"; - err = reftable_stack_add(st, &write_test_log, &arg); - EXPECT(err == REFTABLE_API_ERROR); - - input.value.update.message = (char *) "one"; - err = reftable_stack_add(st, &write_test_log, &arg); - EXPECT_ERR(err); - - err = reftable_stack_read_log(st, input.refname, &dest); - EXPECT_ERR(err); - EXPECT(0 == strcmp(dest.value.update.message, "one\n")); - - input.value.update.message = (char *) "two\n"; - arg.update_index = 2; - err = reftable_stack_add(st, &write_test_log, &arg); - EXPECT_ERR(err); - err = reftable_stack_read_log(st, input.refname, &dest); - EXPECT_ERR(err); - EXPECT(0 == strcmp(dest.value.update.message, "two\n")); - - /* cleanup */ - reftable_stack_destroy(st); - reftable_log_record_release(&dest); - clear_dir(dir); -} - -static void test_reftable_stack_tombstone(void) -{ - int i = 0; - char *dir = get_tmp_dir(__LINE__); - struct reftable_write_options opts = { 0 }; - struct reftable_stack *st = NULL; - int err; - struct reftable_ref_record refs[2] = { { NULL } }; - struct reftable_log_record logs[2] = { { NULL } }; - int N = ARRAY_SIZE(refs); - struct reftable_ref_record dest = { NULL }; - struct reftable_log_record log_dest = { NULL }; - - err = reftable_new_stack(&st, dir, &opts); - EXPECT_ERR(err); - - /* even entries add the refs, odd entries delete them. */ - for (i = 0; i < N; i++) { - const char *buf = "branch"; - refs[i].refname = xstrdup(buf); - refs[i].update_index = i + 1; - if (i % 2 == 0) { - refs[i].value_type = REFTABLE_REF_VAL1; - set_test_hash(refs[i].value.val1, i); - } - - logs[i].refname = xstrdup(buf); - /* update_index is part of the key. */ - logs[i].update_index = 42; - if (i % 2 == 0) { - logs[i].value_type = REFTABLE_LOG_UPDATE; - set_test_hash(logs[i].value.update.new_hash, i); - logs[i].value.update.email = - xstrdup("identity@invalid"); - } - } - for (i = 0; i < N; i++) { - int err = reftable_stack_add(st, &write_test_ref, &refs[i]); - EXPECT_ERR(err); - } - - for (i = 0; i < N; i++) { - struct write_log_arg arg = { - .log = &logs[i], - .update_index = reftable_stack_next_update_index(st), - }; - int err = reftable_stack_add(st, &write_test_log, &arg); - EXPECT_ERR(err); - } - - err = reftable_stack_read_ref(st, "branch", &dest); - EXPECT(err == 1); - reftable_ref_record_release(&dest); - - err = reftable_stack_read_log(st, "branch", &log_dest); - EXPECT(err == 1); - reftable_log_record_release(&log_dest); - - err = reftable_stack_compact_all(st, NULL); - EXPECT_ERR(err); - - err = reftable_stack_read_ref(st, "branch", &dest); - EXPECT(err == 1); - - err = reftable_stack_read_log(st, "branch", &log_dest); - EXPECT(err == 1); - reftable_ref_record_release(&dest); - reftable_log_record_release(&log_dest); - - /* cleanup */ - reftable_stack_destroy(st); - for (i = 0; i < N; i++) { - reftable_ref_record_release(&refs[i]); - reftable_log_record_release(&logs[i]); - } - clear_dir(dir); -} - -static void test_reftable_stack_hash_id(void) -{ - char *dir = get_tmp_dir(__LINE__); - struct reftable_write_options opts = { 0 }; - struct reftable_stack *st = NULL; - int err; - - struct reftable_ref_record ref = { - .refname = (char *) "master", - .value_type = REFTABLE_REF_SYMREF, - .value.symref = (char *) "target", - .update_index = 1, - }; - struct reftable_write_options opts32 = { .hash_id = GIT_SHA256_FORMAT_ID }; - struct reftable_stack *st32 = NULL; - struct reftable_write_options opts_default = { 0 }; - struct reftable_stack *st_default = NULL; - struct reftable_ref_record dest = { NULL }; - - err = reftable_new_stack(&st, dir, &opts); - EXPECT_ERR(err); - - err = reftable_stack_add(st, &write_test_ref, &ref); - EXPECT_ERR(err); - - /* can't read it with the wrong hash ID. */ - err = reftable_new_stack(&st32, dir, &opts32); - EXPECT(err == REFTABLE_FORMAT_ERROR); - - /* check that we can read it back with default opts too. */ - err = reftable_new_stack(&st_default, dir, &opts_default); - EXPECT_ERR(err); - - err = reftable_stack_read_ref(st_default, "master", &dest); - EXPECT_ERR(err); - - EXPECT(reftable_ref_record_equal(&ref, &dest, GIT_SHA1_RAWSZ)); - reftable_ref_record_release(&dest); - reftable_stack_destroy(st); - reftable_stack_destroy(st_default); - clear_dir(dir); -} - -static void test_suggest_compaction_segment(void) -{ - uint64_t sizes[] = { 512, 64, 17, 16, 9, 9, 9, 16, 2, 16 }; - struct segment min = - suggest_compaction_segment(sizes, ARRAY_SIZE(sizes), 2); - EXPECT(min.start == 1); - EXPECT(min.end == 10); -} - -static void test_suggest_compaction_segment_nothing(void) -{ - uint64_t sizes[] = { 64, 32, 16, 8, 4, 2 }; - struct segment result = - suggest_compaction_segment(sizes, ARRAY_SIZE(sizes), 2); - EXPECT(result.start == result.end); -} - -static void test_reflog_expire(void) -{ - char *dir = get_tmp_dir(__LINE__); - struct reftable_write_options opts = { 0 }; - struct reftable_stack *st = NULL; - struct reftable_log_record logs[20] = { { NULL } }; - int N = ARRAY_SIZE(logs) - 1; - int i = 0; - int err; - struct reftable_log_expiry_config expiry = { - .time = 10, - }; - struct reftable_log_record log = { NULL }; - - err = reftable_new_stack(&st, dir, &opts); - EXPECT_ERR(err); - - for (i = 1; i <= N; i++) { - char buf[256]; - snprintf(buf, sizeof(buf), "branch%02d", i); - - logs[i].refname = xstrdup(buf); - logs[i].update_index = i; - logs[i].value_type = REFTABLE_LOG_UPDATE; - logs[i].value.update.time = i; - logs[i].value.update.email = xstrdup("identity@invalid"); - set_test_hash(logs[i].value.update.new_hash, i); - } - - for (i = 1; i <= N; i++) { - struct write_log_arg arg = { - .log = &logs[i], - .update_index = reftable_stack_next_update_index(st), - }; - int err = reftable_stack_add(st, &write_test_log, &arg); - EXPECT_ERR(err); - } - - err = reftable_stack_compact_all(st, NULL); - EXPECT_ERR(err); - - err = reftable_stack_compact_all(st, &expiry); - EXPECT_ERR(err); - - err = reftable_stack_read_log(st, logs[9].refname, &log); - EXPECT(err == 1); - - err = reftable_stack_read_log(st, logs[11].refname, &log); - EXPECT_ERR(err); - - expiry.min_update_index = 15; - err = reftable_stack_compact_all(st, &expiry); - EXPECT_ERR(err); - - err = reftable_stack_read_log(st, logs[14].refname, &log); - EXPECT(err == 1); - - err = reftable_stack_read_log(st, logs[16].refname, &log); - EXPECT_ERR(err); - - /* cleanup */ - reftable_stack_destroy(st); - for (i = 0; i <= N; i++) { - reftable_log_record_release(&logs[i]); - } - clear_dir(dir); - reftable_log_record_release(&log); -} - -static int write_nothing(struct reftable_writer *wr, void *arg) -{ - reftable_writer_set_limits(wr, 1, 1); - return 0; -} - -static void test_empty_add(void) -{ - struct reftable_write_options opts = { 0 }; - struct reftable_stack *st = NULL; - int err; - char *dir = get_tmp_dir(__LINE__); - struct reftable_stack *st2 = NULL; - - err = reftable_new_stack(&st, dir, &opts); - EXPECT_ERR(err); - - err = reftable_stack_add(st, &write_nothing, NULL); - EXPECT_ERR(err); - - err = reftable_new_stack(&st2, dir, &opts); - EXPECT_ERR(err); - clear_dir(dir); - reftable_stack_destroy(st); - reftable_stack_destroy(st2); -} - -static int fastlog2(uint64_t sz) -{ - int l = 0; - if (sz == 0) - return 0; - for (; sz; sz /= 2) - l++; - return l - 1; -} - -static void test_reftable_stack_auto_compaction(void) -{ - struct reftable_write_options opts = { - .disable_auto_compact = 1, - }; - struct reftable_stack *st = NULL; - char *dir = get_tmp_dir(__LINE__); - int err, i; - int N = 100; - - err = reftable_new_stack(&st, dir, &opts); - EXPECT_ERR(err); - - for (i = 0; i < N; i++) { - char name[100]; - struct reftable_ref_record ref = { - .refname = name, - .update_index = reftable_stack_next_update_index(st), - .value_type = REFTABLE_REF_SYMREF, - .value.symref = (char *) "master", - }; - snprintf(name, sizeof(name), "branch%04d", i); - - err = reftable_stack_add(st, &write_test_ref, &ref); - EXPECT_ERR(err); - - err = reftable_stack_auto_compact(st); - EXPECT_ERR(err); - EXPECT(i < 3 || st->merged->stack_len < 2 * fastlog2(i)); - } - - EXPECT(reftable_stack_compaction_stats(st)->entries_written < - (uint64_t)(N * fastlog2(N))); - - reftable_stack_destroy(st); - clear_dir(dir); -} - -static void test_reftable_stack_add_performs_auto_compaction(void) -{ - struct reftable_write_options opts = { 0 }; - struct reftable_stack *st = NULL; - struct strbuf refname = STRBUF_INIT; - char *dir = get_tmp_dir(__LINE__); - int err, i, n = 20; - - err = reftable_new_stack(&st, dir, &opts); - EXPECT_ERR(err); - - for (i = 0; i <= n; i++) { - struct reftable_ref_record ref = { - .update_index = reftable_stack_next_update_index(st), - .value_type = REFTABLE_REF_SYMREF, - .value.symref = (char *) "master", - }; - - /* - * Disable auto-compaction for all but the last runs. Like this - * we can ensure that we indeed honor this setting and have - * better control over when exactly auto compaction runs. - */ - st->opts.disable_auto_compact = i != n; - - strbuf_reset(&refname); - strbuf_addf(&refname, "branch-%04d", i); - ref.refname = refname.buf; - - err = reftable_stack_add(st, &write_test_ref, &ref); - EXPECT_ERR(err); - - /* - * The stack length should grow continuously for all runs where - * auto compaction is disabled. When enabled, we should merge - * all tables in the stack. - */ - if (i != n) - EXPECT(st->merged->stack_len == i + 1); - else - EXPECT(st->merged->stack_len == 1); - } - - reftable_stack_destroy(st); - strbuf_release(&refname); - clear_dir(dir); -} - -static void test_reftable_stack_compaction_concurrent(void) -{ - struct reftable_write_options opts = { 0 }; - struct reftable_stack *st1 = NULL, *st2 = NULL; - char *dir = get_tmp_dir(__LINE__); - int err, i; - int N = 3; - - err = reftable_new_stack(&st1, dir, &opts); - EXPECT_ERR(err); - - for (i = 0; i < N; i++) { - char name[100]; - struct reftable_ref_record ref = { - .refname = name, - .update_index = reftable_stack_next_update_index(st1), - .value_type = REFTABLE_REF_SYMREF, - .value.symref = (char *) "master", - }; - snprintf(name, sizeof(name), "branch%04d", i); - - err = reftable_stack_add(st1, &write_test_ref, &ref); - EXPECT_ERR(err); - } - - err = reftable_new_stack(&st2, dir, &opts); - EXPECT_ERR(err); - - err = reftable_stack_compact_all(st1, NULL); - EXPECT_ERR(err); - - reftable_stack_destroy(st1); - reftable_stack_destroy(st2); - - EXPECT(count_dir_entries(dir) == 2); - clear_dir(dir); -} - -static void unclean_stack_close(struct reftable_stack *st) -{ - /* break abstraction boundary to simulate unclean shutdown. */ - int i = 0; - for (; i < st->readers_len; i++) { - reftable_reader_free(st->readers[i]); - } - st->readers_len = 0; - FREE_AND_NULL(st->readers); -} - -static void test_reftable_stack_compaction_concurrent_clean(void) -{ - struct reftable_write_options opts = { 0 }; - struct reftable_stack *st1 = NULL, *st2 = NULL, *st3 = NULL; - char *dir = get_tmp_dir(__LINE__); - int err, i; - int N = 3; - - err = reftable_new_stack(&st1, dir, &opts); - EXPECT_ERR(err); - - for (i = 0; i < N; i++) { - char name[100]; - struct reftable_ref_record ref = { - .refname = name, - .update_index = reftable_stack_next_update_index(st1), - .value_type = REFTABLE_REF_SYMREF, - .value.symref = (char *) "master", - }; - snprintf(name, sizeof(name), "branch%04d", i); - - err = reftable_stack_add(st1, &write_test_ref, &ref); - EXPECT_ERR(err); - } - - err = reftable_new_stack(&st2, dir, &opts); - EXPECT_ERR(err); - - err = reftable_stack_compact_all(st1, NULL); - EXPECT_ERR(err); - - unclean_stack_close(st1); - unclean_stack_close(st2); - - err = reftable_new_stack(&st3, dir, &opts); - EXPECT_ERR(err); - - err = reftable_stack_clean(st3); - EXPECT_ERR(err); - EXPECT(count_dir_entries(dir) == 2); - - reftable_stack_destroy(st1); - reftable_stack_destroy(st2); - reftable_stack_destroy(st3); - - clear_dir(dir); -} - -int stack_test_main(int argc, const char *argv[]) -{ - RUN_TEST(test_empty_add); - RUN_TEST(test_read_file); - RUN_TEST(test_reflog_expire); - RUN_TEST(test_reftable_stack_add); - RUN_TEST(test_reftable_stack_add_one); - RUN_TEST(test_reftable_stack_auto_compaction); - RUN_TEST(test_reftable_stack_add_performs_auto_compaction); - RUN_TEST(test_reftable_stack_compaction_concurrent); - RUN_TEST(test_reftable_stack_compaction_concurrent_clean); - RUN_TEST(test_reftable_stack_hash_id); - RUN_TEST(test_reftable_stack_lock_failure); - RUN_TEST(test_reftable_stack_log_normalize); - RUN_TEST(test_reftable_stack_tombstone); - RUN_TEST(test_reftable_stack_transaction_api); - RUN_TEST(test_reftable_stack_transaction_api_performs_auto_compaction); - RUN_TEST(test_reftable_stack_auto_compaction_fails_gracefully); - RUN_TEST(test_reftable_stack_update_index_check); - RUN_TEST(test_reftable_stack_uptodate); - RUN_TEST(test_suggest_compaction_segment); - RUN_TEST(test_suggest_compaction_segment_nothing); - return 0; -} diff --git a/reftable/system.c b/reftable/system.c new file mode 100644 index 0000000000..adf8e4d30b --- /dev/null +++ b/reftable/system.c @@ -0,0 +1,126 @@ +#include "system.h" +#include "basics.h" +#include "reftable-error.h" +#include "../lockfile.h" +#include "../tempfile.h" + +int tmpfile_from_pattern(struct reftable_tmpfile *out, const char *pattern) +{ + struct tempfile *tempfile; + + tempfile = mks_tempfile(pattern); + if (!tempfile) + return REFTABLE_IO_ERROR; + + out->path = tempfile->filename.buf; + out->fd = tempfile->fd; + out->priv = tempfile; + + return 0; +} + +int tmpfile_close(struct reftable_tmpfile *t) +{ + struct tempfile *tempfile = t->priv; + int ret = close_tempfile_gently(tempfile); + t->fd = -1; + if (ret < 0) + return REFTABLE_IO_ERROR; + return 0; +} + +int tmpfile_delete(struct reftable_tmpfile *t) +{ + struct tempfile *tempfile = t->priv; + int ret = delete_tempfile(&tempfile); + *t = REFTABLE_TMPFILE_INIT; + if (ret < 0) + return REFTABLE_IO_ERROR; + return 0; +} + +int tmpfile_rename(struct reftable_tmpfile *t, const char *path) +{ + struct tempfile *tempfile = t->priv; + int ret = rename_tempfile(&tempfile, path); + *t = REFTABLE_TMPFILE_INIT; + if (ret < 0) + return REFTABLE_IO_ERROR; + return 0; +} + +int flock_acquire(struct reftable_flock *l, const char *target_path, + long timeout_ms) +{ + struct lock_file *lockfile; + int err; + + lockfile = reftable_malloc(sizeof(*lockfile)); + if (!lockfile) + return REFTABLE_OUT_OF_MEMORY_ERROR; + + err = hold_lock_file_for_update_timeout(lockfile, target_path, LOCK_NO_DEREF, + timeout_ms); + if (err < 0) { + reftable_free(lockfile); + if (errno == EEXIST) + return REFTABLE_LOCK_ERROR; + return -1; + } + + l->fd = get_lock_file_fd(lockfile); + l->path = get_lock_file_path(lockfile); + l->priv = lockfile; + + return 0; +} + +int flock_close(struct reftable_flock *l) +{ + struct lock_file *lockfile = l->priv; + int ret; + + if (!lockfile) + return REFTABLE_API_ERROR; + + ret = close_lock_file_gently(lockfile); + l->fd = -1; + if (ret < 0) + return REFTABLE_IO_ERROR; + + return 0; +} + +int flock_release(struct reftable_flock *l) +{ + struct lock_file *lockfile = l->priv; + int ret; + + if (!lockfile) + return 0; + + ret = rollback_lock_file(lockfile); + reftable_free(lockfile); + *l = REFTABLE_FLOCK_INIT; + if (ret < 0) + return REFTABLE_IO_ERROR; + + return 0; +} + +int flock_commit(struct reftable_flock *l) +{ + struct lock_file *lockfile = l->priv; + int ret; + + if (!lockfile) + return REFTABLE_API_ERROR; + + ret = commit_lock_file(lockfile); + reftable_free(lockfile); + *l = REFTABLE_FLOCK_INIT; + if (ret < 0) + return REFTABLE_IO_ERROR; + + return 0; +} diff --git a/reftable/system.h b/reftable/system.h index d0cabd5d17..7d5f803eeb 100644 --- a/reftable/system.h +++ b/reftable/system.h @@ -12,12 +12,90 @@ https://developers.google.com/open-source/licenses/bsd /* This header glues the reftable library to the rest of Git */ #include "git-compat-util.h" -#include "lockfile.h" -#include "strbuf.h" -#include "tempfile.h" -#include "hash.h" /* hash ID, sizes.*/ -#include "dir.h" /* remove_dir_recursively, for tests.*/ -int hash_size(uint32_t id); +/* + * An implementation-specific temporary file. By making this specific to the + * implementation it becomes possible to tie temporary files into any kind of + * signal or atexit handlers for cleanup on abnormal situations. + */ +struct reftable_tmpfile { + const char *path; + int fd; + void *priv; +}; +#define REFTABLE_TMPFILE_INIT ((struct reftable_tmpfile) { .fd = -1, }) + +/* + * Create a temporary file from a pattern similar to how mkstemp(3p) would. + * The `pattern` shall not be modified. On success, the structure at `out` has + * been initialized such that it is ready for use. Returns 0 on success, a + * reftable error code on error. + */ +int tmpfile_from_pattern(struct reftable_tmpfile *out, const char *pattern); + +/* + * Close the temporary file's file descriptor without removing the file itself. + * This is a no-op in case the file has already been closed beforehand. Returns + * 0 on success, a reftable error code on error. + */ +int tmpfile_close(struct reftable_tmpfile *t); + +/* + * Close the temporary file and delete it. This is a no-op in case the file has + * already been deleted or renamed beforehand. Returns 0 on success, a reftable + * error code on error. + */ +int tmpfile_delete(struct reftable_tmpfile *t); + +/* + * Rename the temporary file to the provided path. The temporary file must be + * active. Return 0 on success, a reftable error code on error. Deactivates the + * temporary file. + */ +int tmpfile_rename(struct reftable_tmpfile *t, const char *path); + +/* + * An implementation-specific file lock. Same as with `reftable_tmpfile`, + * making this specific to the implementation makes it possible to tie this + * into signal or atexit handlers such that we know to clean up stale locks on + * abnormal exits. + */ +struct reftable_flock { + const char *path; + int fd; + void *priv; +}; +#define REFTABLE_FLOCK_INIT ((struct reftable_flock){ .fd = -1, }) + +/* + * Acquire the lock for the given target path by exclusively creating a file + * with ".lock" appended to it. If that lock exists, we wait up to `timeout_ms` + * to acquire the lock. If `timeout_ms` is 0 we don't wait, if it is negative + * we block indefinitely. + * + * Retrun 0 on success, a reftable error code on error. + */ +int flock_acquire(struct reftable_flock *l, const char *target_path, + long timeout_ms); + +/* + * Close the lockfile's file descriptor without removing the lock itself. This + * is a no-op in case the lockfile has already been closed beforehand. Returns + * 0 on success, a reftable error code on error. + */ +int flock_close(struct reftable_flock *l); + +/* + * Release the lock by unlinking the lockfile. This is a no-op in case the + * lockfile has already been released or committed beforehand. Returns 0 on + * success, a reftable error code on error. + */ +int flock_release(struct reftable_flock *l); + +/* + * Commit the lock by renaming the lockfile into place. Returns 0 on success, a + * reftable error code on error. + */ +int flock_commit(struct reftable_flock *l); #endif diff --git a/reftable/test_framework.c b/reftable/test_framework.c deleted file mode 100644 index 4066924eee..0000000000 --- a/reftable/test_framework.c +++ /dev/null @@ -1,27 +0,0 @@ -/* -Copyright 2020 Google LLC - -Use of this source code is governed by a BSD-style -license that can be found in the LICENSE file or at -https://developers.google.com/open-source/licenses/bsd -*/ - -#include "system.h" -#include "test_framework.h" - - -void set_test_hash(uint8_t *p, int i) -{ - memset(p, (uint8_t)i, hash_size(GIT_SHA1_FORMAT_ID)); -} - -ssize_t strbuf_add_void(void *b, const void *data, size_t sz) -{ - strbuf_add(b, data, sz); - return sz; -} - -int noop_flush(void *arg) -{ - return 0; -} diff --git a/reftable/test_framework.h b/reftable/test_framework.h deleted file mode 100644 index 687390f9c2..0000000000 --- a/reftable/test_framework.h +++ /dev/null @@ -1,61 +0,0 @@ -/* -Copyright 2020 Google LLC - -Use of this source code is governed by a BSD-style -license that can be found in the LICENSE file or at -https://developers.google.com/open-source/licenses/bsd -*/ - -#ifndef TEST_FRAMEWORK_H -#define TEST_FRAMEWORK_H - -#include "system.h" -#include "reftable-error.h" - -#define EXPECT_ERR(c) \ - do { \ - if (c != 0) { \ - fflush(stderr); \ - fflush(stdout); \ - fprintf(stderr, "%s: %d: error == %d (%s), want 0\n", \ - __FILE__, __LINE__, c, reftable_error_str(c)); \ - abort(); \ - } \ - } while (0) - -#define EXPECT_STREQ(a, b) \ - do { \ - if (strcmp(a, b)) { \ - fflush(stderr); \ - fflush(stdout); \ - fprintf(stderr, "%s:%d: %s (%s) != %s (%s)\n", __FILE__, \ - __LINE__, #a, a, #b, b); \ - abort(); \ - } \ - } while (0) - -#define EXPECT(c) \ - do { \ - if (!(c)) { \ - fflush(stderr); \ - fflush(stdout); \ - fprintf(stderr, "%s: %d: failed assertion %s\n", __FILE__, \ - __LINE__, #c); \ - abort(); \ - } \ - } while (0) - -#define RUN_TEST(f) \ - fprintf(stderr, "running %s\n", #f); \ - fflush(stderr); \ - f(); - -void set_test_hash(uint8_t *p, int i); - -/* Like strbuf_add, but suitable for passing to reftable_new_writer - */ -ssize_t strbuf_add_void(void *b, const void *data, size_t sz); - -int noop_flush(void *); - -#endif diff --git a/reftable/tree.c b/reftable/tree.c index 528f33ae38..f4dbe72090 100644 --- a/reftable/tree.c +++ b/reftable/tree.c @@ -11,53 +11,64 @@ https://developers.google.com/open-source/licenses/bsd #include "basics.h" -struct tree_node *tree_search(void *key, struct tree_node **rootp, - int (*compare)(const void *, const void *), - int insert) +struct tree_node *tree_search(struct tree_node *tree, + void *key, + int (*compare)(const void *, const void *)) { int res; + if (!tree) + return NULL; + res = compare(key, tree->key); + if (res < 0) + return tree_search(tree->left, key, compare); + else if (res > 0) + return tree_search(tree->right, key, compare); + return tree; +} + +struct tree_node *tree_insert(struct tree_node **rootp, + void *key, + int (*compare)(const void *, const void *)) +{ + int res; + if (!*rootp) { - if (!insert) { + struct tree_node *n; + + REFTABLE_CALLOC_ARRAY(n, 1); + if (!n) return NULL; - } else { - struct tree_node *n; - REFTABLE_CALLOC_ARRAY(n, 1); - n->key = key; - *rootp = n; - return *rootp; - } + + n->key = key; + *rootp = n; + return *rootp; } res = compare(key, (*rootp)->key); if (res < 0) - return tree_search(key, &(*rootp)->left, compare, insert); + return tree_insert(&(*rootp)->left, key, compare); else if (res > 0) - return tree_search(key, &(*rootp)->right, compare, insert); + return tree_insert(&(*rootp)->right, key, compare); return *rootp; } void infix_walk(struct tree_node *t, void (*action)(void *arg, void *key), void *arg) { - if (t->left) { + if (t->left) infix_walk(t->left, action, arg); - } action(arg, t->key); - if (t->right) { + if (t->right) infix_walk(t->right, action, arg); - } } void tree_free(struct tree_node *t) { - if (!t) { + if (!t) return; - } - if (t->left) { + if (t->left) tree_free(t->left); - } - if (t->right) { + if (t->right) tree_free(t->right); - } reftable_free(t); } diff --git a/reftable/tree.h b/reftable/tree.h index fbdd002e23..9604453b6d 100644 --- a/reftable/tree.h +++ b/reftable/tree.h @@ -15,12 +15,23 @@ struct tree_node { struct tree_node *left, *right; }; -/* looks for `key` in `rootp` using `compare` as comparison function. If insert - * is set, insert the key if it's not found. Else, return NULL. +/* + * Search the tree for the node matching the given key using `compare` as + * comparison function. Returns the node whose key matches or `NULL` in case + * the key does not exist in the tree. + */ +struct tree_node *tree_search(struct tree_node *tree, + void *key, + int (*compare)(const void *, const void *)); + +/* + * Insert a node into the tree. Returns the newly inserted node if the key does + * not yet exist. Otherwise it returns the preexisting node. Returns `NULL` + * when allocating the new node fails. */ -struct tree_node *tree_search(void *key, struct tree_node **rootp, - int (*compare)(const void *, const void *), - int insert); +struct tree_node *tree_insert(struct tree_node **rootp, + void *key, + int (*compare)(const void *, const void *)); /* performs an infix walk of the tree. */ void infix_walk(struct tree_node *t, void (*action)(void *arg, void *key), diff --git a/reftable/tree_test.c b/reftable/tree_test.c deleted file mode 100644 index 6961a657ad..0000000000 --- a/reftable/tree_test.c +++ /dev/null @@ -1,60 +0,0 @@ -/* -Copyright 2020 Google LLC - -Use of this source code is governed by a BSD-style -license that can be found in the LICENSE file or at -https://developers.google.com/open-source/licenses/bsd -*/ - -#include "system.h" -#include "tree.h" - -#include "test_framework.h" -#include "reftable-tests.h" - -static int test_compare(const void *a, const void *b) -{ - return (char *)a - (char *)b; -} - -struct curry { - void *last; -}; - -static void check_increasing(void *arg, void *key) -{ - struct curry *c = arg; - if (c->last) { - EXPECT(test_compare(c->last, key) < 0); - } - c->last = key; -} - -static void test_tree(void) -{ - struct tree_node *root = NULL; - - void *values[11] = { NULL }; - struct tree_node *nodes[11] = { NULL }; - int i = 1; - struct curry c = { NULL }; - do { - nodes[i] = tree_search(values + i, &root, &test_compare, 1); - i = (i * 7) % 11; - } while (i != 1); - - for (i = 1; i < ARRAY_SIZE(nodes); i++) { - EXPECT(values + i == nodes[i]->key); - EXPECT(nodes[i] == - tree_search(values + i, &root, &test_compare, 0)); - } - - infix_walk(root, check_increasing, &c); - tree_free(root); -} - -int tree_test_main(int argc, const char *argv[]) -{ - RUN_TEST(test_tree); - return 0; -} diff --git a/reftable/writer.c b/reftable/writer.c index 45b3e9ce1f..9efeab13e1 100644 --- a/reftable/writer.c +++ b/reftable/writer.c @@ -49,8 +49,14 @@ static int padded_write(struct reftable_writer *w, uint8_t *data, size_t len, { int n = 0; if (w->pending_padding > 0) { - uint8_t *zeroed = reftable_calloc(w->pending_padding, sizeof(*zeroed)); - int n = w->write(w->write_arg, zeroed, w->pending_padding); + uint8_t *zeroed; + int n; + + zeroed = reftable_calloc(w->pending_padding, sizeof(*zeroed)); + if (!zeroed) + return -1; + + n = w->write(w->write_arg, zeroed, w->pending_padding); if (n < 0) return n; @@ -73,7 +79,7 @@ static void options_set_defaults(struct reftable_write_options *opts) } if (opts->hash_id == 0) { - opts->hash_id = GIT_SHA1_FORMAT_ID; + opts->hash_id = REFTABLE_HASH_SHA1; } if (opts->block_size == 0) { opts->block_size = DEFAULT_BLOCK_SIZE; @@ -82,7 +88,7 @@ static void options_set_defaults(struct reftable_write_options *opts) static int writer_version(struct reftable_writer *w) { - return (w->opts.hash_id == 0 || w->opts.hash_id == GIT_SHA1_FORMAT_ID) ? + return (w->opts.hash_id == 0 || w->opts.hash_id == REFTABLE_HASH_SHA1) ? 1 : 2; } @@ -97,33 +103,56 @@ static int writer_write_header(struct reftable_writer *w, uint8_t *dest) put_be64(dest + 8, w->min_update_index); put_be64(dest + 16, w->max_update_index); if (writer_version(w) == 2) { - put_be32(dest + 24, w->opts.hash_id); + uint32_t hash_id; + + switch (w->opts.hash_id) { + case REFTABLE_HASH_SHA1: + hash_id = REFTABLE_FORMAT_ID_SHA1; + break; + case REFTABLE_HASH_SHA256: + hash_id = REFTABLE_FORMAT_ID_SHA256; + break; + default: + return -1; + } + + put_be32(dest + 24, hash_id); } + return header_size(writer_version(w)); } -static void writer_reinit_block_writer(struct reftable_writer *w, uint8_t typ) +static int writer_reinit_block_writer(struct reftable_writer *w, uint8_t typ) { - int block_start = 0; - if (w->next == 0) { + int block_start = 0, ret; + + if (w->next == 0) block_start = header_size(writer_version(w)); - } - strbuf_reset(&w->last_key); - block_writer_init(&w->block_writer_data, typ, w->block, - w->opts.block_size, block_start, - hash_size(w->opts.hash_id)); + reftable_buf_reset(&w->last_key); + ret = block_writer_init(&w->block_writer_data, typ, w->block, + w->opts.block_size, block_start, + hash_size(w->opts.hash_id)); + if (ret < 0) + return ret; + w->block_writer = &w->block_writer_data; w->block_writer->restart_interval = w->opts.restart_interval; + + return 0; } -struct reftable_writer * -reftable_new_writer(ssize_t (*writer_func)(void *, const void *, size_t), - int (*flush_func)(void *), - void *writer_arg, const struct reftable_write_options *_opts) +int reftable_writer_new(struct reftable_writer **out, + ssize_t (*writer_func)(void *, const void *, size_t), + int (*flush_func)(void *), + void *writer_arg, const struct reftable_write_options *_opts) { - struct reftable_writer *wp = reftable_calloc(1, sizeof(*wp)); struct reftable_write_options opts = {0}; + struct reftable_writer *wp; + + wp = reftable_calloc(1, sizeof(*wp)); + if (!wp) + return REFTABLE_OUT_OF_MEMORY_ERROR; if (_opts) opts = *_opts; @@ -131,16 +160,23 @@ reftable_new_writer(ssize_t (*writer_func)(void *, const void *, size_t), if (opts.block_size >= (1 << 24)) BUG("configured block size exceeds 16MB"); - strbuf_init(&wp->block_writer_data.last_key, 0); - strbuf_init(&wp->last_key, 0); + reftable_buf_init(&wp->block_writer_data.last_key); + reftable_buf_init(&wp->last_key); + reftable_buf_init(&wp->scratch); REFTABLE_CALLOC_ARRAY(wp->block, opts.block_size); + if (!wp->block) { + reftable_free(wp); + return REFTABLE_OUT_OF_MEMORY_ERROR; + } wp->write = writer_func; wp->write_arg = writer_arg; wp->opts = opts; wp->flush = flush_func; writer_reinit_block_writer(wp, BLOCK_TYPE_REF); - return wp; + *out = wp; + + return 0; } void reftable_writer_set_limits(struct reftable_writer *w, uint64_t min, @@ -158,7 +194,8 @@ static void writer_release(struct reftable_writer *w) block_writer_release(&w->block_writer_data); w->block_writer = NULL; writer_clear_index(w); - strbuf_release(&w->last_key); + reftable_buf_release(&w->last_key); + reftable_buf_release(&w->scratch); } } @@ -169,7 +206,7 @@ void reftable_writer_free(struct reftable_writer *w) } struct obj_index_tree_node { - struct strbuf hash; + struct reftable_buf hash; uint64_t *offsets; size_t offset_len; size_t offset_cap; @@ -177,61 +214,78 @@ struct obj_index_tree_node { #define OBJ_INDEX_TREE_NODE_INIT \ { \ - .hash = STRBUF_INIT \ + .hash = REFTABLE_BUF_INIT \ } static int obj_index_tree_node_compare(const void *a, const void *b) { - return strbuf_cmp(&((const struct obj_index_tree_node *)a)->hash, + return reftable_buf_cmp(&((const struct obj_index_tree_node *)a)->hash, &((const struct obj_index_tree_node *)b)->hash); } -static void writer_index_hash(struct reftable_writer *w, struct strbuf *hash) +static int writer_index_hash(struct reftable_writer *w, struct reftable_buf *hash) { uint64_t off = w->next; - struct obj_index_tree_node want = { .hash = *hash }; + struct obj_index_tree_node *key; + struct tree_node *node; - struct tree_node *node = tree_search(&want, &w->obj_index_tree, - &obj_index_tree_node_compare, 0); - struct obj_index_tree_node *key = NULL; + node = tree_search(w->obj_index_tree, &want, &obj_index_tree_node_compare); if (!node) { struct obj_index_tree_node empty = OBJ_INDEX_TREE_NODE_INIT; - key = reftable_malloc(sizeof(struct obj_index_tree_node)); + int err; + + key = reftable_malloc(sizeof(*key)); + if (!key) + return REFTABLE_OUT_OF_MEMORY_ERROR; + *key = empty; - strbuf_reset(&key->hash); - strbuf_addbuf(&key->hash, hash); - tree_search((void *)key, &w->obj_index_tree, - &obj_index_tree_node_compare, 1); + reftable_buf_reset(&key->hash); + err = reftable_buf_add(&key->hash, hash->buf, hash->len); + if (err < 0) + return err; + tree_insert(&w->obj_index_tree, key, + &obj_index_tree_node_compare); } else { key = node->key; } - if (key->offset_len > 0 && key->offsets[key->offset_len - 1] == off) { - return; - } + if (key->offset_len > 0 && key->offsets[key->offset_len - 1] == off) + return 0; REFTABLE_ALLOC_GROW(key->offsets, key->offset_len + 1, key->offset_cap); + if (!key->offsets) + return REFTABLE_OUT_OF_MEMORY_ERROR; key->offsets[key->offset_len++] = off; + + return 0; } static int writer_add_record(struct reftable_writer *w, struct reftable_record *rec) { - struct strbuf key = STRBUF_INIT; int err; - reftable_record_key(rec, &key); - if (strbuf_cmp(&w->last_key, &key) >= 0) { + err = reftable_record_key(rec, &w->scratch); + if (err < 0) + goto done; + + if (reftable_buf_cmp(&w->last_key, &w->scratch) >= 0) { err = REFTABLE_API_ERROR; goto done; } - strbuf_reset(&w->last_key); - strbuf_addbuf(&w->last_key, &key); - if (!w->block_writer) - writer_reinit_block_writer(w, reftable_record_type(rec)); + reftable_buf_reset(&w->last_key); + err = reftable_buf_add(&w->last_key, w->scratch.buf, w->scratch.len); + if (err < 0) + goto done; + + if (!w->block_writer) { + err = writer_reinit_block_writer(w, reftable_record_type(rec)); + if (err < 0) + goto done; + } if (block_writer_type(w->block_writer) != reftable_record_type(rec)) BUG("record of type %d added to writer of type %d", @@ -254,7 +308,9 @@ static int writer_add_record(struct reftable_writer *w, err = writer_flush_block(w); if (err < 0) goto done; - writer_reinit_block_writer(w, reftable_record_type(rec)); + err = writer_reinit_block_writer(w, reftable_record_type(rec)); + if (err < 0) + goto done; /* * Try to add the record to the writer again. If this still fails then @@ -271,7 +327,6 @@ static int writer_add_record(struct reftable_writer *w, } done: - strbuf_release(&key); return err; } @@ -284,11 +339,10 @@ int reftable_writer_add_ref(struct reftable_writer *w, .ref = *ref }, }; - int err = 0; + int err; - if (!ref->refname) - return REFTABLE_API_ERROR; - if (ref->update_index < w->min_update_index || + if (!ref->refname || + ref->update_index < w->min_update_index || ref->update_index > w->max_update_index) return REFTABLE_API_ERROR; @@ -296,24 +350,36 @@ int reftable_writer_add_ref(struct reftable_writer *w, err = writer_add_record(w, &rec); if (err < 0) - return err; + goto out; if (!w->opts.skip_index_objects && reftable_ref_record_val1(ref)) { - struct strbuf h = STRBUF_INIT; - strbuf_add(&h, (char *)reftable_ref_record_val1(ref), - hash_size(w->opts.hash_id)); - writer_index_hash(w, &h); - strbuf_release(&h); + reftable_buf_reset(&w->scratch); + err = reftable_buf_add(&w->scratch, (char *)reftable_ref_record_val1(ref), + hash_size(w->opts.hash_id)); + if (err < 0) + goto out; + + err = writer_index_hash(w, &w->scratch); + if (err < 0) + goto out; } if (!w->opts.skip_index_objects && reftable_ref_record_val2(ref)) { - struct strbuf h = STRBUF_INIT; - strbuf_add(&h, reftable_ref_record_val2(ref), - hash_size(w->opts.hash_id)); - writer_index_hash(w, &h); - strbuf_release(&h); + reftable_buf_reset(&w->scratch); + err = reftable_buf_add(&w->scratch, reftable_ref_record_val2(ref), + hash_size(w->opts.hash_id)); + if (err < 0) + goto out; + + err = writer_index_hash(w, &w->scratch); + if (err < 0) + goto out; } - return 0; + + err = 0; + +out: + return err; } int reftable_writer_add_refs(struct reftable_writer *w, @@ -353,7 +419,7 @@ int reftable_writer_add_log(struct reftable_writer *w, struct reftable_log_record *log) { char *input_log_message = NULL; - struct strbuf cleaned_message = STRBUF_INIT; + struct reftable_buf cleaned_message = REFTABLE_BUF_INIT; int err = 0; if (log->value_type == REFTABLE_LOG_DELETION) @@ -364,24 +430,34 @@ int reftable_writer_add_log(struct reftable_writer *w, input_log_message = log->value.update.message; if (!w->opts.exact_log_message && log->value.update.message) { - strbuf_addstr(&cleaned_message, log->value.update.message); + err = reftable_buf_addstr(&cleaned_message, log->value.update.message); + if (err < 0) + goto done; + while (cleaned_message.len && - cleaned_message.buf[cleaned_message.len - 1] == '\n') - strbuf_setlen(&cleaned_message, - cleaned_message.len - 1); + cleaned_message.buf[cleaned_message.len - 1] == '\n') { + err = reftable_buf_setlen(&cleaned_message, + cleaned_message.len - 1); + if (err < 0) + goto done; + } if (strchr(cleaned_message.buf, '\n')) { /* multiple lines not allowed. */ err = REFTABLE_API_ERROR; goto done; } - strbuf_addstr(&cleaned_message, "\n"); + + err = reftable_buf_addstr(&cleaned_message, "\n"); + if (err < 0) + goto done; + log->value.update.message = cleaned_message.buf; } err = reftable_writer_add_log_verbatim(w, log); log->value.update.message = input_log_message; done: - strbuf_release(&cleaned_message); + reftable_buf_release(&cleaned_message); return err; } @@ -436,7 +512,9 @@ static int writer_finish_section(struct reftable_writer *w) max_level++; index_start = w->next; - writer_reinit_block_writer(w, BLOCK_TYPE_INDEX); + err = writer_reinit_block_writer(w, BLOCK_TYPE_INDEX); + if (err < 0) + return err; idx = w->index; idx_len = w->index_len; @@ -462,7 +540,7 @@ static int writer_finish_section(struct reftable_writer *w) return err; for (i = 0; i < idx_len; i++) - strbuf_release(&idx[i].last_key); + reftable_buf_release(&idx[i].last_key); reftable_free(idx); } @@ -479,13 +557,13 @@ static int writer_finish_section(struct reftable_writer *w) bstats->max_index_level = max_level; /* Reinit lastKey, as the next section can start with any key. */ - strbuf_reset(&w->last_key); + reftable_buf_reset(&w->last_key); return 0; } struct common_prefix_arg { - struct strbuf *last; + struct reftable_buf *last; int max; }; @@ -530,7 +608,10 @@ static void write_object_record(void *void_arg, void *key) if (arg->err < 0) goto done; - writer_reinit_block_writer(arg->w, BLOCK_TYPE_OBJ); + arg->err = writer_reinit_block_writer(arg->w, BLOCK_TYPE_OBJ); + if (arg->err < 0) + goto done; + arg->err = block_writer_add(arg->w->block_writer, &rec); if (arg->err == 0) goto done; @@ -544,12 +625,12 @@ static void write_object_record(void *void_arg, void *key) done:; } -static void object_record_free(void *void_arg, void *key) +static void object_record_free(void *void_arg UNUSED, void *key) { struct obj_index_tree_node *entry = key; - FREE_AND_NULL(entry->offsets); - strbuf_release(&entry->hash); + REFTABLE_FREE_AND_NULL(entry->offsets); + reftable_buf_release(&entry->hash); reftable_free(entry); } @@ -559,16 +640,18 @@ static int writer_dump_object_index(struct reftable_writer *w) struct common_prefix_arg common = { .max = 1, /* obj_id_len should be >= 2. */ }; - if (w->obj_index_tree) { + int err; + + if (w->obj_index_tree) infix_walk(w->obj_index_tree, &update_common, &common); - } w->stats.object_id_len = common.max + 1; - writer_reinit_block_writer(w, BLOCK_TYPE_OBJ); + err = writer_reinit_block_writer(w, BLOCK_TYPE_OBJ); + if (err < 0) + return err; - if (w->obj_index_tree) { + if (w->obj_index_tree) infix_walk(w->obj_index_tree, &write_object_record, &closure); - } if (closure.err < 0) return closure.err; @@ -661,8 +744,8 @@ done: static void writer_clear_index(struct reftable_writer *w) { for (size_t i = 0; w->index && i < w->index_len; i++) - strbuf_release(&w->index[i].last_key); - FREE_AND_NULL(w->index); + reftable_buf_release(&w->index[i].last_key); + REFTABLE_FREE_AND_NULL(w->index); w->index_len = 0; w->index_cap = 0; } @@ -670,7 +753,7 @@ static void writer_clear_index(struct reftable_writer *w) static int writer_flush_nonempty_block(struct reftable_writer *w) { struct reftable_index_record index_record = { - .last_key = STRBUF_INIT, + .last_key = REFTABLE_BUF_INIT, }; uint8_t typ = block_writer_type(w->block_writer); struct reftable_block_stats *bstats; @@ -726,9 +809,15 @@ static int writer_flush_nonempty_block(struct reftable_writer *w) * case we will end up with a multi-level index. */ REFTABLE_ALLOC_GROW(w->index, w->index_len + 1, w->index_cap); + if (!w->index) + return REFTABLE_OUT_OF_MEMORY_ERROR; + index_record.offset = w->next; - strbuf_reset(&index_record.last_key); - strbuf_addbuf(&index_record.last_key, &w->block_writer->last_key); + reftable_buf_reset(&index_record.last_key); + err = reftable_buf_add(&index_record.last_key, w->block_writer->last_key.buf, + w->block_writer->last_key.len); + if (err < 0) + return err; w->index[w->index_len] = index_record; w->index_len++; diff --git a/reftable/writer.h b/reftable/writer.h index 8d0df9cc52..1f4788a430 100644 --- a/reftable/writer.h +++ b/reftable/writer.h @@ -19,7 +19,9 @@ struct reftable_writer { int (*flush)(void *); void *write_arg; int pending_padding; - struct strbuf last_key; + struct reftable_buf last_key; + /* Scratch buffer used to avoid allocations. */ + struct reftable_buf scratch; /* offset of next block to write. */ uint64_t next; |