From e987df5fe62b8b29be4cdcdeb3704681ada2b29e Mon Sep 17 00:00:00 2001 From: Matthew DeVore Date: Thu, 27 Jun 2019 15:54:08 -0700 Subject: list-objects-filter: implement composite filters Allow combining filters such that only objects accepted by all filters are shown. The motivation for this is to allow getting directory listings without also fetching blobs. This can be done by combining blob:none with tree:. There are massive repositories that have larger-than-expected trees - even if you include only a single commit. A combined filter supports any number of subfilters, and is written in the following form: combine:++ Certain non-alphanumeric characters in each filter must be URL-encoded. For now, combined filters must be specified in this form. In a subsequent commit, rev-list will support multiple --filter arguments which will have the same effect as specifying one filter argument starting with "combine:". The documentation will be updated in that commit, as the URL-encoding scheme is in general not meant to be used directly by the user, and it is better to describe the URL-encoding feature in terms of the repeated flag. Helped-by: Emily Shaffer Helped-by: Jeff Hostetler Helped-by: Johannes Schindelin Helped-by: Jonathan Tan Helped-by: Junio C Hamano Signed-off-by: Matthew DeVore Signed-off-by: Junio C Hamano --- list-objects-filter-options.h | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) (limited to 'list-objects-filter-options.h') diff --git a/list-objects-filter-options.h b/list-objects-filter-options.h index c54f0000fb..789faef1e5 100644 --- a/list-objects-filter-options.h +++ b/list-objects-filter-options.h @@ -13,6 +13,7 @@ enum list_objects_filter_choice { LOFC_BLOB_LIMIT, LOFC_TREE_DEPTH, LOFC_SPARSE_OID, + LOFC_COMBINE, LOFC__COUNT /* must be last */ }; @@ -38,13 +39,23 @@ struct list_objects_filter_options { unsigned int no_filter : 1; /* - * Parsed values (fields) from within the filter-spec. These are - * choice-specific; not all values will be defined for any given - * choice. + * BEGIN choice-specific parsed values from within the filter-spec. Only + * some values will be defined for any given choice. */ + struct object_id *sparse_oid_value; unsigned long blob_limit_value; unsigned long tree_exclude_depth; + + /* LOFC_COMBINE values */ + + /* This array contains all the subfilters which this filter combines. */ + size_t sub_nr, sub_alloc; + struct list_objects_filter_options *sub; + + /* + * END choice-specific parsed values. + */ }; /* Normalized command line arguments */ -- cgit v1.2.3 From cf9ceb5a12cad9c9153d227a0f497d1b522ce085 Mon Sep 17 00:00:00 2001 From: Matthew DeVore Date: Thu, 27 Jun 2019 15:54:10 -0700 Subject: list-objects-filter-options: make filter_spec a string_list Make the filter_spec string a string_list rather than a raw C string. The list of strings must be concatted together to make a complete filter_spec. A future patch will use this capability to build "combine:" filter specs gradually. A strbuf would seem to be a more natural choice for this object, but it unfortunately requires initialization besides just zero'ing out the memory. This results in all container structs, and all containers of those structs, etc., to also require initialization. Initializing them all would be more cumbersome that simply using a string_list, which behaves properly when its contents are zero'd. For the purposes of code simplification, change behavior in how filter specs are conveyed over the protocol: do not normalize the tree: filter specs since there should be no server in existence that supports tree:# but not tree:#k etc. Helped-by: Junio C Hamano Signed-off-by: Matthew DeVore Signed-off-by: Junio C Hamano --- builtin/clone.c | 8 +++--- builtin/fetch.c | 9 +++---- builtin/rev-list.c | 6 +++-- fetch-pack.c | 20 ++++++--------- list-objects-filter-options.c | 50 +++++++++++++++++++++++++------------ list-objects-filter-options.h | 27 ++++++++++++++------ t/t6112-rev-list-filters-objects.sh | 7 ------ transport-helper.c | 10 +++----- upload-pack.c | 11 ++++---- 9 files changed, 78 insertions(+), 70 deletions(-) (limited to 'list-objects-filter-options.h') diff --git a/builtin/clone.c b/builtin/clone.c index 5b9ebe9947..a693e6ca44 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -1149,13 +1149,11 @@ int cmd_clone(int argc, const char **argv, const char *prefix) transport->server_options = &server_options; if (filter_options.choice) { - struct strbuf expanded_filter_spec = STRBUF_INIT; - expand_list_objects_filter_spec(&filter_options, - &expanded_filter_spec); + const char *spec = + expand_list_objects_filter_spec(&filter_options); transport_set_option(transport, TRANS_OPT_LIST_OBJECTS_FILTER, - expanded_filter_spec.buf); + spec); transport_set_option(transport, TRANS_OPT_FROM_PROMISOR, "1"); - strbuf_release(&expanded_filter_spec); } if (transport->smart_options && !deepen && !filter_options.choice) diff --git a/builtin/fetch.c b/builtin/fetch.c index 4ba63d5ac6..dee89e1a19 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -1188,13 +1188,10 @@ static struct transport *prepare_transport(struct remote *remote, int deepen) if (update_shallow) set_option(transport, TRANS_OPT_UPDATE_SHALLOW, "yes"); if (filter_options.choice) { - struct strbuf expanded_filter_spec = STRBUF_INIT; - expand_list_objects_filter_spec(&filter_options, - &expanded_filter_spec); - set_option(transport, TRANS_OPT_LIST_OBJECTS_FILTER, - expanded_filter_spec.buf); + const char *spec = + expand_list_objects_filter_spec(&filter_options); + set_option(transport, TRANS_OPT_LIST_OBJECTS_FILTER, spec); set_option(transport, TRANS_OPT_FROM_PROMISOR, "1"); - strbuf_release(&expanded_filter_spec); } if (negotiation_tip.nr) { if (transport->smart_options) diff --git a/builtin/rev-list.c b/builtin/rev-list.c index 660172b014..68acbe8fd2 100644 --- a/builtin/rev-list.c +++ b/builtin/rev-list.c @@ -466,8 +466,10 @@ int cmd_rev_list(int argc, const char **argv, const char *prefix) die(_("object filtering requires --objects")); if (filter_options.choice == LOFC_SPARSE_OID && !filter_options.sparse_oid_value) - die(_("invalid sparse value '%s'"), - filter_options.filter_spec); + die( + _("invalid sparse value '%s'"), + list_objects_filter_spec( + &filter_options)); continue; } if (!strcmp(arg, ("--no-" CL_ARG__FILTER))) { diff --git a/fetch-pack.c b/fetch-pack.c index 1c10f54e78..72e13b0a1d 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -339,12 +339,9 @@ static int find_common(struct fetch_negotiator *negotiator, } } if (server_supports_filtering && args->filter_options.choice) { - struct strbuf expanded_filter_spec = STRBUF_INIT; - expand_list_objects_filter_spec(&args->filter_options, - &expanded_filter_spec); - packet_buf_write(&req_buf, "filter %s", - expanded_filter_spec.buf); - strbuf_release(&expanded_filter_spec); + const char *spec = + expand_list_objects_filter_spec(&args->filter_options); + packet_buf_write(&req_buf, "filter %s", spec); } packet_buf_flush(&req_buf); state_len = req_buf.len; @@ -1099,7 +1096,7 @@ static int add_haves(struct fetch_negotiator *negotiator, } static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out, - const struct fetch_pack_args *args, + struct fetch_pack_args *args, const struct ref *wants, struct oidset *common, int *haves_to_send, int *in_vain, int sideband_all) @@ -1140,13 +1137,10 @@ static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out, /* Add filter */ if (server_supports_feature("fetch", "filter", 0) && args->filter_options.choice) { - struct strbuf expanded_filter_spec = STRBUF_INIT; + const char *spec = + expand_list_objects_filter_spec(&args->filter_options); print_verbose(args, _("Server supports filter")); - expand_list_objects_filter_spec(&args->filter_options, - &expanded_filter_spec); - packet_buf_write(&req_buf, "filter %s", - expanded_filter_spec.buf); - strbuf_release(&expanded_filter_spec); + packet_buf_write(&req_buf, "filter %s", spec); } else if (args->filter_options.choice) { warning("filtering not recognized by server, ignoring"); } diff --git a/list-objects-filter-options.c b/list-objects-filter-options.c index 5fe2814841..01c0f13346 100644 --- a/list-objects-filter-options.c +++ b/list-objects-filter-options.c @@ -184,7 +184,7 @@ int parse_list_objects_filter(struct list_objects_filter_options *filter_options struct strbuf buf = STRBUF_INIT; if (filter_options->choice) die(_("multiple filter-specs cannot be combined")); - filter_options->filter_spec = strdup(arg); + string_list_append(&filter_options->filter_spec, xstrdup(arg)); if (gently_parse_list_objects_filter(filter_options, arg, &buf)) die("%s", buf.buf); return 0; @@ -203,19 +203,36 @@ int opt_parse_list_objects_filter(const struct option *opt, return parse_list_objects_filter(filter_options, arg); } -void expand_list_objects_filter_spec( - const struct list_objects_filter_options *filter, - struct strbuf *expanded_spec) +const char *list_objects_filter_spec(struct list_objects_filter_options *filter) { - strbuf_init(expanded_spec, strlen(filter->filter_spec)); - if (filter->choice == LOFC_BLOB_LIMIT) - strbuf_addf(expanded_spec, "blob:limit=%lu", + if (!filter->filter_spec.nr) + BUG("no filter_spec available for this filter"); + if (filter->filter_spec.nr != 1) { + struct strbuf concatted = STRBUF_INIT; + strbuf_add_separated_string_list( + &concatted, "", &filter->filter_spec); + string_list_clear(&filter->filter_spec, /*free_util=*/0); + string_list_append( + &filter->filter_spec, strbuf_detach(&concatted, NULL)); + } + + return filter->filter_spec.items[0].string; +} + +const char *expand_list_objects_filter_spec( + struct list_objects_filter_options *filter) +{ + if (filter->choice == LOFC_BLOB_LIMIT) { + struct strbuf expanded_spec = STRBUF_INIT; + strbuf_addf(&expanded_spec, "blob:limit=%lu", filter->blob_limit_value); - else if (filter->choice == LOFC_TREE_DEPTH) - strbuf_addf(expanded_spec, "tree:%lu", - filter->tree_exclude_depth); - else - strbuf_addstr(expanded_spec, filter->filter_spec); + string_list_clear(&filter->filter_spec, /*free_util=*/0); + string_list_append( + &filter->filter_spec, + strbuf_detach(&expanded_spec, NULL)); + } + + return list_objects_filter_spec(filter); } void list_objects_filter_release( @@ -225,7 +242,7 @@ void list_objects_filter_release( if (!filter_options) return; - free(filter_options->filter_spec); + string_list_clear(&filter_options->filter_spec, /*free_util=*/0); free(filter_options->sparse_oid_value); for (sub = 0; sub < filter_options->sub_nr; sub++) list_objects_filter_release(&filter_options->sub[sub]); @@ -235,7 +252,7 @@ void list_objects_filter_release( void partial_clone_register( const char *remote, - const struct list_objects_filter_options *filter_options) + struct list_objects_filter_options *filter_options) { /* * Record the name of the partial clone remote in the @@ -258,7 +275,7 @@ void partial_clone_register( * the default for subsequent fetches from this remote. */ core_partial_clone_filter_default = - xstrdup(filter_options->filter_spec); + xstrdup(expand_list_objects_filter_spec(filter_options)); git_config_set("core.partialclonefilter", core_partial_clone_filter_default); } @@ -274,7 +291,8 @@ void partial_clone_get_default_filter_spec( if (!core_partial_clone_filter_default) return; - filter_options->filter_spec = strdup(core_partial_clone_filter_default); + string_list_append(&filter_options->filter_spec, + core_partial_clone_filter_default); gently_parse_list_objects_filter(filter_options, core_partial_clone_filter_default, &errbuf); diff --git a/list-objects-filter-options.h b/list-objects-filter-options.h index 789faef1e5..bb33303f9b 100644 --- a/list-objects-filter-options.h +++ b/list-objects-filter-options.h @@ -2,7 +2,7 @@ #define LIST_OBJECTS_FILTER_OPTIONS_H #include "parse-options.h" -#include "strbuf.h" +#include "string-list.h" /* * The list of defined filters for list-objects. @@ -24,8 +24,10 @@ struct list_objects_filter_options { * commands that launch filtering sub-processes, or for communication * over the network, don't use this value; use the result of * expand_list_objects_filter_spec() instead. + * To get the raw filter spec given by the user, use the result of + * list_objects_filter_spec(). */ - char *filter_spec; + struct string_list filter_spec; /* * 'choice' is determined by parsing the filter-spec. This indicates @@ -76,13 +78,22 @@ int opt_parse_list_objects_filter(const struct option *opt, /* * Translates abbreviated numbers in the filter's filter_spec into their * fully-expanded forms (e.g., "limit:blob=1k" becomes "limit:blob=1024"). + * Returns a string owned by the list_objects_filter_options object. * - * This form should be used instead of the raw filter_spec field when - * communicating with a remote process or subprocess. + * This form should be used instead of the raw list_objects_filter_spec() + * value when communicating with a remote process or subprocess. */ -void expand_list_objects_filter_spec( - const struct list_objects_filter_options *filter, - struct strbuf *expanded_spec); +const char *expand_list_objects_filter_spec( + struct list_objects_filter_options *filter); + +/* + * Returns the filter spec string more or less in the form as the user + * entered it. This form of the filter_spec can be used in user-facing + * messages. Returns a string owned by the list_objects_filter_options + * object. + */ +const char *list_objects_filter_spec( + struct list_objects_filter_options *filter); void list_objects_filter_release( struct list_objects_filter_options *filter_options); @@ -96,7 +107,7 @@ static inline void list_objects_filter_set_no_filter( void partial_clone_register( const char *remote, - const struct list_objects_filter_options *filter_options); + struct list_objects_filter_options *filter_options); void partial_clone_get_default_filter_spec( struct list_objects_filter_options *filter_options); diff --git a/t/t6112-rev-list-filters-objects.sh b/t/t6112-rev-list-filters-objects.sh index 05d4f2e9c2..27ba15719a 100755 --- a/t/t6112-rev-list-filters-objects.sh +++ b/t/t6112-rev-list-filters-objects.sh @@ -590,11 +590,4 @@ test_expect_success 'expand blob limit in protocol' ' grep "blob:limit=1024" trace ' -test_expect_success 'expand tree depth limit in protocol' ' - GIT_TRACE_PACKET="$(pwd)/tree_trace" git -c protocol.version=2 clone \ - --filter=tree:0k "file://$(pwd)/r2" tree && - ! grep "tree:0k" tree_trace && - grep "tree:0" tree_trace -' - test_done diff --git a/transport-helper.c b/transport-helper.c index c7e17ec9cb..0a34544df0 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -682,13 +682,9 @@ static int fetch(struct transport *transport, set_helper_option(transport, "update-shallow", "true"); if (data->transport_options.filter_options.choice) { - struct strbuf expanded_filter_spec = STRBUF_INIT; - expand_list_objects_filter_spec( - &data->transport_options.filter_options, - &expanded_filter_spec); - set_helper_option(transport, "filter", - expanded_filter_spec.buf); - strbuf_release(&expanded_filter_spec); + const char *spec = expand_list_objects_filter_spec( + &data->transport_options.filter_options); + set_helper_option(transport, "filter", spec); } if (data->transport_options.negotiation_tips) diff --git a/upload-pack.c b/upload-pack.c index 4d2129e7fc..d404d88941 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -140,18 +140,17 @@ static void create_pack_file(const struct object_array *have_obj, argv_array_push(&pack_objects.args, "--delta-base-offset"); if (use_include_tag) argv_array_push(&pack_objects.args, "--include-tag"); - if (filter_options.filter_spec) { - struct strbuf expanded_filter_spec = STRBUF_INIT; - expand_list_objects_filter_spec(&filter_options, - &expanded_filter_spec); + if (filter_options.choice) { + const char *spec = + expand_list_objects_filter_spec(&filter_options); if (pack_objects.use_shell) { struct strbuf buf = STRBUF_INIT; - sq_quote_buf(&buf, expanded_filter_spec.buf); + sq_quote_buf(&buf, spec); argv_array_pushf(&pack_objects.args, "--filter=%s", buf.buf); strbuf_release(&buf); } else { argv_array_pushf(&pack_objects.args, "--filter=%s", - expanded_filter_spec.buf); + spec); } } -- cgit v1.2.3 From 489fc9ee718b7c8594f17b55f090ac5292c655e1 Mon Sep 17 00:00:00 2001 From: Matthew DeVore Date: Thu, 27 Jun 2019 15:54:12 -0700 Subject: list-objects-filter-options: allow mult. --filter Allow combining of multiple filters by simply repeating the --filter flag. Before this patch, the user had to combine them in a single flag somewhat awkwardly (e.g. --filter=combine:FOO+BAR), including URL-encoding the individual filters. To make this work, in the --filter flag parsing callback, rather than error out when we detect that the filter_options struct is already populated, we modify it in-place to contain the added sub-filter. The existing sub-filter becomes the lhs of the combined filter, and the next sub-filter becomes the rhs. We also have to URL-encode the LHS and RHS sub-filters. We can simplify the operation if the LHS is already a combine: filter. In that case, we just append the URL-encoded RHS sub-filter to the LHS spec to get the new spec. Helped-by: Emily Shaffer Helped-by: Jeff Hostetler Helped-by: Jeff King Helped-by: Junio C Hamano Signed-off-by: Matthew DeVore Signed-off-by: Junio C Hamano --- Documentation/rev-list-options.txt | 16 +++++++ list-objects-filter-options.c | 88 ++++++++++++++++++++++++++++++++++--- list-objects-filter-options.h | 11 +++++ t/t5616-partial-clone.sh | 19 ++++++++ t/t6112-rev-list-filters-objects.sh | 46 ++++++++++++++++--- transport.c | 1 + upload-pack.c | 2 + 7 files changed, 173 insertions(+), 10 deletions(-) (limited to 'list-objects-filter-options.h') diff --git a/Documentation/rev-list-options.txt b/Documentation/rev-list-options.txt index 71a1fcc093..d1f080bf6d 100644 --- a/Documentation/rev-list-options.txt +++ b/Documentation/rev-list-options.txt @@ -738,6 +738,22 @@ explicitly-given commit or tree. Note that the form '--filter=sparse:path=' that wants to read from an arbitrary path on the filesystem has been dropped for security reasons. ++ +Multiple '--filter=' flags can be specified to combine filters. Only +objects which are accepted by every filter are included. ++ +The form '--filter=combine:++...' can also be +used to combined several filters, but this is harder than just repeating +the '--filter' flag and is usually not necessary. Filters are joined by +'{plus}' and individual filters are %-encoded (i.e. URL-encoded). +Besides the '{plus}' and '%' characters, the following characters are +reserved and also must be encoded: `~!@#$^&*()[]{}\;",<>?`+'`+ +as well as all characters with ASCII code <= `0x20`, which includes +space and newline. ++ +Other arbitrary characters can also be encoded. For instance, +'combine:tree:3+blob:none' and 'combine:tree%3A3+blob%3Anone' are +equivalent. --no-filter:: Turn off any previous `--filter=` argument. diff --git a/list-objects-filter-options.c b/list-objects-filter-options.c index 01c0f13346..2506dc8327 100644 --- a/list-objects-filter-options.c +++ b/list-objects-filter-options.c @@ -6,6 +6,7 @@ #include "list-objects.h" #include "list-objects-filter.h" #include "list-objects-filter-options.h" +#include "trace.h" #include "url.h" static int parse_combine_filter( @@ -178,15 +179,92 @@ cleanup: return result; } -int parse_list_objects_filter(struct list_objects_filter_options *filter_options, - const char *arg) +static int allow_unencoded(char ch) +{ + if (ch <= ' ' || ch == '%' || ch == '+') + return 0; + return !strchr(RESERVED_NON_WS, ch); +} + +static void filter_spec_append_urlencode( + struct list_objects_filter_options *filter, const char *raw) { struct strbuf buf = STRBUF_INIT; + strbuf_addstr_urlencode(&buf, raw, allow_unencoded); + trace_printf("Add to combine filter-spec: %s\n", buf.buf); + string_list_append(&filter->filter_spec, strbuf_detach(&buf, NULL)); +} + +/* + * Changes filter_options into an equivalent LOFC_COMBINE filter options + * instance. Does not do anything if filter_options is already LOFC_COMBINE. + */ +static void transform_to_combine_type( + struct list_objects_filter_options *filter_options) +{ + assert(filter_options->choice); + if (filter_options->choice == LOFC_COMBINE) + return; + { + const int initial_sub_alloc = 2; + struct list_objects_filter_options *sub_array = + xcalloc(initial_sub_alloc, sizeof(*sub_array)); + sub_array[0] = *filter_options; + memset(filter_options, 0, sizeof(*filter_options)); + filter_options->sub = sub_array; + filter_options->sub_alloc = initial_sub_alloc; + } + filter_options->sub_nr = 1; + filter_options->choice = LOFC_COMBINE; + string_list_append(&filter_options->filter_spec, xstrdup("combine:")); + filter_spec_append_urlencode( + filter_options, + list_objects_filter_spec(&filter_options->sub[0])); + /* + * We don't need the filter_spec strings for subfilter specs, only the + * top level. + */ + string_list_clear(&filter_options->sub[0].filter_spec, /*free_util=*/0); +} + +void list_objects_filter_die_if_populated( + struct list_objects_filter_options *filter_options) +{ if (filter_options->choice) die(_("multiple filter-specs cannot be combined")); - string_list_append(&filter_options->filter_spec, xstrdup(arg)); - if (gently_parse_list_objects_filter(filter_options, arg, &buf)) - die("%s", buf.buf); +} + +int parse_list_objects_filter( + struct list_objects_filter_options *filter_options, + const char *arg) +{ + struct strbuf errbuf = STRBUF_INIT; + int parse_error; + + if (!filter_options->choice) { + string_list_append(&filter_options->filter_spec, xstrdup(arg)); + + parse_error = gently_parse_list_objects_filter( + filter_options, arg, &errbuf); + } else { + /* + * Make filter_options an LOFC_COMBINE spec so we can trivially + * add subspecs to it. + */ + transform_to_combine_type(filter_options); + + string_list_append(&filter_options->filter_spec, xstrdup("+")); + filter_spec_append_urlencode(filter_options, arg); + ALLOC_GROW(filter_options->sub, filter_options->sub_nr + 1, + filter_options->sub_alloc); + filter_options = &filter_options->sub[filter_options->sub_nr++]; + memset(filter_options, 0, sizeof(*filter_options)); + + parse_error = gently_parse_list_objects_filter( + filter_options, arg, &errbuf); + } + if (parse_error) + die("%s", errbuf.buf); return 0; } diff --git a/list-objects-filter-options.h b/list-objects-filter-options.h index bb33303f9b..d8bc7e946e 100644 --- a/list-objects-filter-options.h +++ b/list-objects-filter-options.h @@ -63,6 +63,17 @@ struct list_objects_filter_options { /* Normalized command line arguments */ #define CL_ARG__FILTER "filter" +void list_objects_filter_die_if_populated( + struct list_objects_filter_options *filter_options); + +/* + * Parses the filter spec string given by arg and either (1) simply places the + * result in filter_options if it is not yet populated or (2) combines it with + * the filter already in filter_options if it is already populated. In the case + * of (2), the filter specs are combined as if specified with 'combine:'. + * + * Dies and prints a user-facing message if an error occurs. + */ int parse_list_objects_filter( struct list_objects_filter_options *filter_options, const char *arg); diff --git a/t/t5616-partial-clone.sh b/t/t5616-partial-clone.sh index b91ef548f8..32b7d72f3c 100755 --- a/t/t5616-partial-clone.sh +++ b/t/t5616-partial-clone.sh @@ -208,6 +208,25 @@ test_expect_success 'use fsck before and after manually fetching a missing subtr test_cmp unique_types.expected unique_types.observed ' +test_expect_success 'implicitly construct combine: filter with repeated flags' ' + GIT_TRACE=$(pwd)/trace git clone --bare \ + --filter=blob:none --filter=tree:1 \ + "file://$(pwd)/srv.bare" pc2 && + grep "trace:.* git pack-objects .*--filter=combine:blob:none+tree:1" \ + trace && + git -C pc2 rev-list --objects --missing=allow-any HEAD >objects && + + # We should have gotten some root trees. + grep " $" objects && + # Should not have gotten any non-root trees or blobs. + ! grep " ." objects && + + xargs -n 1 git -C pc2 cat-file -t types && + sort -u types >unique_types.actual && + test_write_lines commit tree >unique_types.expected && + test_cmp unique_types.expected unique_types.actual +' + test_expect_success 'partial clone fetches blobs pointed to by refs even if normally filtered out' ' rm -rf src dst && git init src && diff --git a/t/t6112-rev-list-filters-objects.sh b/t/t6112-rev-list-filters-objects.sh index 27ba15719a..de0e5a5d36 100755 --- a/t/t6112-rev-list-filters-objects.sh +++ b/t/t6112-rev-list-filters-objects.sh @@ -351,7 +351,16 @@ test_expect_success 'combine:... for a simple combination' ' expect_has HEAD dir1 && # There are also 2 commit objects - test_line_count = 5 actual + test_line_count = 5 actual && + + cp actual expected && + + # Try again using repeated --filter - this is equivalent to a manual + # combine with "combine:...+..." + git -C r3 rev-list --objects --filter=combine:tree:2 \ + --filter=blob:none HEAD >actual && + + test_cmp expected actual ' test_expect_success 'combine:... with URL encoding' ' @@ -417,10 +426,12 @@ test_expect_success 'combine:... with edge-case hex digits: Ff Aa 0 9' ' test_line_count = 5 actual ' -test_expect_success 'add a sparse pattern blob whose path has reserved chars' ' +test_expect_success 'add sparse pattern blobs whose paths have reserved chars' ' cp r3/pattern r3/pattern1+renamed% && - git -C r3 add pattern1+renamed% && - git -C r3 commit -m "add sparse pattern file with reserved chars" + cp r3/pattern "r3/p;at%ter+n" && + cp r3/pattern r3/^~pattern && + git -C r3 add pattern1+renamed% "p;at%ter+n" ^~pattern && + git -C r3 commit -m "add sparse pattern files with reserved chars" ' test_expect_success 'combine:... with more than two sub-filters' ' @@ -445,7 +456,32 @@ test_expect_success 'combine:... with more than two sub-filters' ' git -C r3 rev-list --objects \ --filter=combine:tree:3+blob:limit=40+sparse:oid=master:pattern1%2brenamed%25 \ HEAD >actual && - test_cmp expect actual + test_cmp expect actual && + + # Use the same composite filter again, but with a pattern file name that + # requires encoding multiple characters, and use implicit filter + # combining. + test_when_finished "rm -f trace1" && + GIT_TRACE=$(pwd)/trace1 git -C r3 rev-list --objects \ + --filter=tree:3 --filter=blob:limit=40 \ + --filter=sparse:oid="master:p;at%ter+n" \ + HEAD >actual && + + test_cmp expect actual && + grep "Add to combine filter-spec: sparse:oid=master:p%3bat%25ter%2bn" \ + trace1 && + + # Repeat the above test, but this time, the characters to encode are in + # the LHS of the combined filter. + test_when_finished "rm -f trace2" && + GIT_TRACE=$(pwd)/trace2 git -C r3 rev-list --objects \ + --filter=sparse:oid=master:^~pattern \ + --filter=tree:3 --filter=blob:limit=40 \ + HEAD >actual && + + test_cmp expect actual && + grep "Add to combine filter-spec: sparse:oid=master:%5e%7epattern" \ + trace2 ' # Test provisional omit collection logic with a repo that has objects appearing diff --git a/transport.c b/transport.c index f1fcd2c4b0..ee7dd1c062 100644 --- a/transport.c +++ b/transport.c @@ -224,6 +224,7 @@ static int set_git_option(struct git_transport_options *opts, opts->no_dependents = !!value; return 0; } else if (!strcmp(name, TRANS_OPT_LIST_OBJECTS_FILTER)) { + list_objects_filter_die_if_populated(&opts->filter_options); parse_list_objects_filter(&opts->filter_options, value); return 0; } diff --git a/upload-pack.c b/upload-pack.c index d404d88941..f8a76ebda3 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -883,6 +883,7 @@ static void receive_needs(struct packet_reader *reader, struct object_array *wan if (skip_prefix(reader->line, "filter ", &arg)) { if (!filter_capability_requested) die("git upload-pack: filtering capability not negotiated"); + list_objects_filter_die_if_populated(&filter_options); parse_list_objects_filter(&filter_options, arg); continue; } @@ -1304,6 +1305,7 @@ static void process_args(struct packet_reader *request, } if (allow_filter && skip_prefix(arg, "filter ", &p)) { + list_objects_filter_die_if_populated(&filter_options); parse_list_objects_filter(&filter_options, p); continue; } -- cgit v1.2.3 From 90d21f9ebf6906f0ebb4fb1b20ec9536072e2916 Mon Sep 17 00:00:00 2001 From: Matthew DeVore Date: Thu, 27 Jun 2019 15:54:14 -0700 Subject: list-objects-filter-options: make parser void This function always returns 0, so make it return void instead. Signed-off-by: Matthew DeVore Signed-off-by: Junio C Hamano --- list-objects-filter-options.c | 12 +++++------- list-objects-filter-options.h | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) (limited to 'list-objects-filter-options.h') diff --git a/list-objects-filter-options.c b/list-objects-filter-options.c index 44bc1153d1..ba1425cb4a 100644 --- a/list-objects-filter-options.c +++ b/list-objects-filter-options.c @@ -232,7 +232,7 @@ void list_objects_filter_die_if_populated( die(_("multiple filter-specs cannot be combined")); } -int parse_list_objects_filter( +void parse_list_objects_filter( struct list_objects_filter_options *filter_options, const char *arg) { @@ -262,7 +262,6 @@ int parse_list_objects_filter( } if (parse_error) die("%s", errbuf.buf); - return 0; } int opt_parse_list_objects_filter(const struct option *opt, @@ -270,12 +269,11 @@ int opt_parse_list_objects_filter(const struct option *opt, { struct list_objects_filter_options *filter_options = opt->value; - if (unset || !arg) { + if (unset || !arg) list_objects_filter_set_no_filter(filter_options); - return 0; - } - - return parse_list_objects_filter(filter_options, arg); + else + parse_list_objects_filter(filter_options, arg); + return 0; } const char *list_objects_filter_spec(struct list_objects_filter_options *filter) diff --git a/list-objects-filter-options.h b/list-objects-filter-options.h index d8bc7e946e..db37dfb34a 100644 --- a/list-objects-filter-options.h +++ b/list-objects-filter-options.h @@ -74,7 +74,7 @@ void list_objects_filter_die_if_populated( * * Dies and prints a user-facing message if an error occurs. */ -int parse_list_objects_filter( +void parse_list_objects_filter( struct list_objects_filter_options *filter_options, const char *arg); -- cgit v1.2.3