diff options
Diffstat (limited to 'remote-curl.c')
| -rw-r--r-- | remote-curl.c | 961 |
1 files changed, 584 insertions, 377 deletions
diff --git a/remote-curl.c b/remote-curl.c index 99b0bedc6d..ed7e3a043a 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -1,5 +1,7 @@ -#include "cache.h" +#include "git-compat-util.h" +#include "alloc.h" #include "config.h" +#include "hex.h" #include "remote.h" #include "connect.h" #include "strbuf.h" @@ -10,12 +12,13 @@ #include "pkt-line.h" #include "string-list.h" #include "sideband.h" -#include "argv-array.h" +#include "strvec.h" #include "credential.h" -#include "sha1-array.h" +#include "oid-array.h" #include "send-pack.h" #include "protocol.h" #include "quote.h" +#include "transport.h" static struct remote *remote; /* always ends with a trailing slash */ @@ -38,8 +41,15 @@ struct options { /* One of the SEND_PACK_PUSH_CERT_* constants. */ push_cert : 2, deepen_relative : 1, + + /* see documentation of corresponding flag in fetch-pack.h */ from_promisor : 1, - no_dependents : 1; + + refetch : 1, + atomic : 1, + object_format : 1, + force_if_includes : 1; + const struct git_hash_algo *hash_algo; }; static struct options options; static struct string_list cas_options = STRING_LIST_INIT_DUP; @@ -117,10 +127,22 @@ static int set_option(const char *name, const char *value) } else if (!strcmp(name, "cas")) { struct strbuf val = STRBUF_INIT; - strbuf_addf(&val, "--" CAS_OPT_NAME "=%s", value); + strbuf_addstr(&val, "--force-with-lease="); + if (*value != '"') + strbuf_addstr(&val, value); + else if (unquote_c_style(&val, value, NULL)) + return -1; string_list_append(&cas_options, val.buf); strbuf_release(&val); return 0; + } else if (!strcmp(name, TRANS_OPT_FORCE_IF_INCLUDES)) { + if (!strcmp(value, "true")) + options.force_if_includes = 1; + else if (!strcmp(value, "false")) + options.force_if_includes = 0; + else + return -1; + return 0; } else if (!strcmp(name, "cloning")) { if (!strcmp(value, "true")) options.cloning = 1; @@ -147,19 +169,25 @@ static int set_option(const char *name, const char *value) else return -1; return 0; + } else if (!strcmp(name, "atomic")) { + if (!strcmp(value, "true")) + options.atomic = 1; + else if (!strcmp(value, "false")) + options.atomic = 0; + else + return -1; + return 0; } else if (!strcmp(name, "push-option")) { if (*value != '"') string_list_append(&options.push_options, value); else { struct strbuf unquoted = STRBUF_INIT; if (unquote_c_style(&unquoted, value, NULL) < 0) - die("invalid quoting in push-option value"); + die(_("invalid quoting in push-option value: '%s'"), value); string_list_append_nodup(&options.push_options, strbuf_detach(&unquoted, NULL)); } return 0; - -#if LIBCURL_VERSION_NUM >= 0x070a08 } else if (!strcmp(name, "family")) { if (!strcmp(value, "ipv4")) git_curl_ipresolve = CURL_IPRESOLVE_V4; @@ -170,15 +198,24 @@ static int set_option(const char *name, const char *value) else return -1; return 0; -#endif /* LIBCURL_VERSION_NUM >= 0x070a08 */ } else if (!strcmp(name, "from-promisor")) { options.from_promisor = 1; return 0; - } else if (!strcmp(name, "no-dependents")) { - options.no_dependents = 1; + } else if (!strcmp(name, "refetch")) { + options.refetch = 1; return 0; } else if (!strcmp(name, "filter")) { - options.filter = xstrdup(value);; + options.filter = xstrdup(value); + return 0; + } else if (!strcmp(name, "object-format")) { + int algo; + options.object_format = 1; + if (strcmp(value, "true")) { + algo = hash_algo_by_name(value); + if (algo == GIT_HASH_UNKNOWN) + die("unknown object format '%s'", value); + options.hash_algo = &hash_algos[algo]; + } return 0; } else { return 1 /* unsupported */; @@ -204,7 +241,8 @@ static struct ref *parse_git_refs(struct discovery *heads, int for_push) packet_reader_init(&reader, -1, heads->buf, heads->len, PACKET_READ_CHOMP_NEWLINE | - PACKET_READ_GENTLE_ON_EOF); + PACKET_READ_GENTLE_ON_EOF | + PACKET_READ_DIE_ON_ERR_PACKET); heads->version = discover_version(&reader); switch (heads->version) { @@ -220,6 +258,7 @@ static struct ref *parse_git_refs(struct discovery *heads, int for_push) case protocol_v0: get_remote_heads(&reader, &list, for_push ? REF_NORMAL : 0, NULL, &heads->shallow); + options.hash_algo = reader.hash_algo; break; case protocol_unknown_version: BUG("unknown protocol version"); @@ -228,6 +267,19 @@ static struct ref *parse_git_refs(struct discovery *heads, int for_push) return list; } +static const struct git_hash_algo *detect_hash_algo(struct discovery *heads) +{ + const char *p = memchr(heads->buf, '\t', heads->len); + int algo; + if (!p) + return the_hash_algo; + + algo = hash_algo_by_length((p - heads->buf) / 2); + if (algo == GIT_HASH_UNKNOWN) + return NULL; + return &hash_algos[algo]; +} + static struct ref *parse_info_refs(struct discovery *heads) { char *data, *start, *mid; @@ -238,6 +290,12 @@ static struct ref *parse_info_refs(struct discovery *heads) struct ref *ref = NULL; struct ref *last_ref = NULL; + options.hash_algo = detect_hash_algo(heads); + if (!options.hash_algo) + die("%sinfo/refs not valid: could not determine hash algorithm; " + "is this a git repository?", + transport_anonymize_url(url.buf)); + data = heads->buf; start = NULL; mid = data; @@ -248,13 +306,13 @@ static struct ref *parse_info_refs(struct discovery *heads) if (data[i] == '\t') mid = &data[i]; if (data[i] == '\n') { - if (mid - start != 40) - die("%sinfo/refs not valid: is this a git repository?", - url.buf); + if (mid - start != options.hash_algo->hexsz) + die(_("%sinfo/refs not valid: is this a git repository?"), + transport_anonymize_url(url.buf)); data[i] = 0; ref_name = mid + 1; ref = alloc_ref(ref_name); - get_oid_hex(start, &ref->old_oid); + get_oid_hex_algop(start, &ref->old_oid, options.hash_algo); if (!refs) refs = ref; if (last_ref) @@ -330,9 +388,63 @@ static int get_protocol_http_header(enum protocol_version version, return 0; } +static void check_smart_http(struct discovery *d, const char *service, + struct strbuf *type) +{ + const char *p; + struct packet_reader reader; + + /* + * If we don't see x-$service-advertisement, then it's not smart-http. + * But once we do, we commit to it and assume any other protocol + * violations are hard errors. + */ + if (!skip_prefix(type->buf, "application/x-", &p) || + !skip_prefix(p, service, &p) || + strcmp(p, "-advertisement")) + return; + + packet_reader_init(&reader, -1, d->buf, d->len, + PACKET_READ_CHOMP_NEWLINE | + PACKET_READ_DIE_ON_ERR_PACKET); + if (packet_reader_read(&reader) != PACKET_READ_NORMAL) + die(_("invalid server response; expected service, got flush packet")); + + if (skip_prefix(reader.line, "# service=", &p) && !strcmp(p, service)) { + /* + * The header can include additional metadata lines, up + * until a packet flush marker. Ignore these now, but + * in the future we might start to scan them. + */ + for (;;) { + packet_reader_read(&reader); + if (reader.pktlen <= 0) { + break; + } + } + + /* + * v0 smart http; callers expect us to soak up the + * service and header packets + */ + d->buf = reader.src_buffer; + d->len = reader.src_len; + d->proto_git = 1; + + } else if (!strcmp(reader.line, "version 2")) { + /* + * v2 smart http; do not consume version packet, which will + * be handled elsewhere. + */ + d->proto_git = 1; + + } else { + die(_("invalid server response; got '%s'"), reader.line); + } +} + static struct discovery *discover_refs(const char *service, int for_push) { - struct strbuf exp = STRBUF_INIT; struct strbuf type = STRBUF_INIT; struct strbuf charset = STRBUF_INIT; struct strbuf buffer = STRBUF_INIT; @@ -380,7 +492,6 @@ static struct discovery *discover_refs(const char *service, int for_push) http_options.extra_headers = &extra_headers; http_options.initial_request = 1; http_options.no_cache = 1; - http_options.keep_error = 1; http_ret = http_get_strbuf(refs_url.buf, &buffer, &http_options); switch (http_ret) { @@ -388,55 +499,35 @@ static struct discovery *discover_refs(const char *service, int for_push) break; case HTTP_MISSING_TARGET: show_http_message(&type, &charset, &buffer); - die("repository '%s' not found", url.buf); + die(_("repository '%s' not found"), + transport_anonymize_url(url.buf)); case HTTP_NOAUTH: show_http_message(&type, &charset, &buffer); - die("Authentication failed for '%s'", url.buf); + die(_("Authentication failed for '%s'"), + transport_anonymize_url(url.buf)); + case HTTP_NOMATCHPUBLICKEY: + show_http_message(&type, &charset, &buffer); + die(_("unable to access '%s' with http.pinnedPubkey configuration: %s"), + transport_anonymize_url(url.buf), curl_errorstr); default: show_http_message(&type, &charset, &buffer); - die("unable to access '%s': %s", url.buf, curl_errorstr); + die(_("unable to access '%s': %s"), + transport_anonymize_url(url.buf), curl_errorstr); } - if (options.verbosity && !starts_with(refs_url.buf, url.buf)) - warning(_("redirecting to %s"), url.buf); + if (options.verbosity && !starts_with(refs_url.buf, url.buf)) { + char *u = transport_anonymize_url(url.buf); + warning(_("redirecting to %s"), u); + free(u); + } last= xcalloc(1, sizeof(*last_discovery)); last->service = xstrdup(service); last->buf_alloc = strbuf_detach(&buffer, &last->len); last->buf = last->buf_alloc; - strbuf_addf(&exp, "application/x-%s-advertisement", service); - if (maybe_smart && - (5 <= last->len && last->buf[4] == '#') && - !strbuf_cmp(&exp, &type)) { - char *line; - - /* - * smart HTTP response; validate that the service - * pkt-line matches our request. - */ - line = packet_read_line_buf(&last->buf, &last->len, NULL); - if (!line) - die("invalid server response; expected service, got flush packet"); - - strbuf_reset(&exp); - strbuf_addf(&exp, "# service=%s", service); - if (strcmp(line, exp.buf)) - die("invalid server response; got '%s'", line); - strbuf_release(&exp); - - /* The header can include additional metadata lines, up - * until a packet flush marker. Ignore these now, but - * in the future we might start to scan them. - */ - while (packet_read_line_buf(&last->buf, &last->len, NULL)) - ; - - last->proto_git = 1; - } else if (maybe_smart && - last->len > 5 && starts_with(last->buf + 4, "version 2")) { - last->proto_git = 1; - } + if (maybe_smart) + check_smart_http(last, service, &type); if (last->proto_git) last->refs = parse_git_refs(last, for_push); @@ -444,7 +535,6 @@ static struct discovery *discover_refs(const char *service, int for_push) last->refs = parse_info_refs(last); strbuf_release(&refs_url); - strbuf_release(&exp); strbuf_release(&type); strbuf_release(&charset); strbuf_release(&effective_url); @@ -470,11 +560,18 @@ static struct ref *get_refs(int for_push) static void output_refs(struct ref *refs) { struct ref *posn; + if (options.object_format && options.hash_algo) { + printf(":object-format %s\n", options.hash_algo->name); + repo_set_hash_algo(the_repository, + hash_algo_by_ptr(options.hash_algo)); + } for (posn = refs; posn; posn = posn->next) { if (posn->symref) printf("@%s %s\n", posn->symref, posn->name); else - printf("%s %s\n", oid_to_hex(&posn->old_oid), posn->name); + printf("%s %s\n", hash_to_hex_algop(posn->old_oid.hash, + options.hash_algo), + posn->name); } printf("\n"); fflush(stdout); @@ -482,11 +579,10 @@ static void output_refs(struct ref *refs) struct rpc_state { const char *service_name; - const char **argv; - struct strbuf *stdin_preamble; char *service_url; char *hdr_content_type; char *hdr_accept; + char *hdr_accept_language; char *protocol_header; char *buf; size_t alloc; @@ -495,25 +591,125 @@ struct rpc_state { int in; int out; int any_written; - struct strbuf result; unsigned gzip_request : 1; unsigned initial_buffer : 1; + + /* + * Whenever a pkt-line is read into buf, append the 4 characters + * denoting its length before appending the payload. + */ + unsigned write_line_lengths : 1; + + /* + * Used by rpc_out; initialize to 0. This is true if a flush has been + * read, but the corresponding line length (if write_line_lengths is + * true) and EOF have not been sent to libcurl. Since each flush marks + * the end of a request, each flush must be completely sent before any + * further reading occurs. + */ + unsigned flush_read_but_not_sent : 1; }; +#define RPC_STATE_INIT { 0 } + +/* + * Appends the result of reading from rpc->out to the string represented by + * rpc->buf and rpc->len if there is enough space. Returns 1 if there was + * enough space, 0 otherwise. + * + * If rpc->write_line_lengths is true, appends the line length as a 4-byte + * hexadecimal string before appending the result described above. + * + * Writes the total number of bytes appended into appended. + */ +static int rpc_read_from_out(struct rpc_state *rpc, int options, + size_t *appended, + enum packet_read_status *status) { + size_t left; + char *buf; + int pktlen_raw; + + if (rpc->write_line_lengths) { + left = rpc->alloc - rpc->len - 4; + buf = rpc->buf + rpc->len + 4; + } else { + left = rpc->alloc - rpc->len; + buf = rpc->buf + rpc->len; + } + + if (left < LARGE_PACKET_MAX) + return 0; + + *status = packet_read_with_status(rpc->out, NULL, NULL, buf, + left, &pktlen_raw, options); + if (*status != PACKET_READ_EOF) { + *appended = pktlen_raw + (rpc->write_line_lengths ? 4 : 0); + rpc->len += *appended; + } + + if (rpc->write_line_lengths) { + switch (*status) { + case PACKET_READ_EOF: + if (!(options & PACKET_READ_GENTLE_ON_EOF)) + die(_("shouldn't have EOF when not gentle on EOF")); + break; + case PACKET_READ_NORMAL: + set_packet_header(buf - 4, *appended); + break; + case PACKET_READ_DELIM: + memcpy(buf - 4, "0001", 4); + break; + case PACKET_READ_FLUSH: + memcpy(buf - 4, "0000", 4); + break; + case PACKET_READ_RESPONSE_END: + die(_("remote server sent unexpected response end packet")); + } + } + + return 1; +} + static size_t rpc_out(void *ptr, size_t eltsize, size_t nmemb, void *buffer_) { size_t max = eltsize * nmemb; struct rpc_state *rpc = buffer_; size_t avail = rpc->len - rpc->pos; + enum packet_read_status status; if (!avail) { rpc->initial_buffer = 0; - avail = packet_read(rpc->out, NULL, NULL, rpc->buf, rpc->alloc, 0); - if (!avail) - return 0; + rpc->len = 0; rpc->pos = 0; - rpc->len = avail; + if (!rpc->flush_read_but_not_sent) { + if (!rpc_read_from_out(rpc, 0, &avail, &status)) + BUG("The entire rpc->buf should be larger than LARGE_PACKET_MAX"); + if (status == PACKET_READ_FLUSH) + rpc->flush_read_but_not_sent = 1; + } + /* + * If flush_read_but_not_sent is true, we have already read one + * full request but have not fully sent it + EOF, which is why + * we need to refrain from reading. + */ + } + if (rpc->flush_read_but_not_sent) { + if (!avail) { + /* + * The line length either does not need to be sent at + * all or has already been completely sent. Now we can + * return 0, indicating EOF, meaning that the flush has + * been fully sent. + */ + rpc->flush_read_but_not_sent = 0; + return 0; + } + /* + * If avail is non-zero, the line length for the flush still + * hasn't been fully sent. Proceed with sending the line + * length. + */ } if (max < avail) @@ -523,37 +719,97 @@ static size_t rpc_out(void *ptr, size_t eltsize, return avail; } -#ifndef NO_CURL_IOCTL -static curlioerr rpc_ioctl(CURL *handle, int cmd, void *clientp) +static int rpc_seek(void *clientp, curl_off_t offset, int origin) { struct rpc_state *rpc = clientp; - switch (cmd) { - case CURLIOCMD_NOP: - return CURLIOE_OK; + if (origin != SEEK_SET) + BUG("rpc_seek only handles SEEK_SET, not %d", origin); - case CURLIOCMD_RESTARTREAD: - if (rpc->initial_buffer) { - rpc->pos = 0; - return CURLIOE_OK; + if (rpc->initial_buffer) { + if (offset < 0 || offset > rpc->len) { + error("curl seek would be outside of rpc buffer"); + return CURL_SEEKFUNC_FAIL; } - error("unable to rewind rpc post data - try increasing http.postBuffer"); - return CURLIOE_FAILRESTART; + rpc->pos = offset; + return CURL_SEEKFUNC_OK; + } + error(_("unable to rewind rpc post data - try increasing http.postBuffer")); + return CURL_SEEKFUNC_FAIL; +} - default: - return CURLIOE_UNKNOWNCMD; +struct check_pktline_state { + char len_buf[4]; + int len_filled; + int remaining; +}; + +static void check_pktline(struct check_pktline_state *state, const char *ptr, size_t size) +{ + while (size) { + if (!state->remaining) { + int digits_remaining = 4 - state->len_filled; + if (digits_remaining > size) + digits_remaining = size; + memcpy(&state->len_buf[state->len_filled], ptr, digits_remaining); + state->len_filled += digits_remaining; + ptr += digits_remaining; + size -= digits_remaining; + + if (state->len_filled == 4) { + state->remaining = packet_length(state->len_buf); + if (state->remaining < 0) { + die(_("remote-curl: bad line length character: %.4s"), state->len_buf); + } else if (state->remaining == 2) { + die(_("remote-curl: unexpected response end packet")); + } else if (state->remaining < 4) { + state->remaining = 0; + } else { + state->remaining -= 4; + } + state->len_filled = 0; + } + } + + if (state->remaining) { + int remaining = state->remaining; + if (remaining > size) + remaining = size; + ptr += remaining; + size -= remaining; + state->remaining -= remaining; + } } } -#endif +struct rpc_in_data { + struct rpc_state *rpc; + struct active_request_slot *slot; + int check_pktline; + struct check_pktline_state pktline_state; +}; + +/* + * A callback for CURLOPT_WRITEFUNCTION. The return value is the bytes consumed + * from ptr. + */ static size_t rpc_in(char *ptr, size_t eltsize, size_t nmemb, void *buffer_) { size_t size = eltsize * nmemb; - struct rpc_state *rpc = buffer_; + struct rpc_in_data *data = buffer_; + long response_code; + + if (curl_easy_getinfo(data->slot->curl, CURLINFO_RESPONSE_CODE, + &response_code) != CURLE_OK) + return size; + if (response_code >= 300) + return size; if (size) - rpc->any_written = 1; - write_or_die(rpc->in, ptr, size); + data->rpc->any_written = 1; + if (data->check_pktline) + check_pktline(&data->pktline_state, ptr, size); + write_or_die(data->rpc->in, ptr, size); return size; } @@ -581,7 +837,7 @@ static int run_slot(struct active_request_slot *slot, strbuf_addstr(&msg, curl_errorstr); } } - error("RPC failed; %s", msg.buf); + error(_("RPC failed; %s"), msg.buf); strbuf_release(&msg); } @@ -608,7 +864,7 @@ static int probe_rpc(struct rpc_state *rpc, struct slot_results *results) curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, 4); curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, headers); curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); - curl_easy_setopt(slot->curl, CURLOPT_FILE, &buf); + curl_easy_setopt(slot->curl, CURLOPT_WRITEDATA, &buf); err = run_slot(slot, results); @@ -617,13 +873,19 @@ static int probe_rpc(struct rpc_state *rpc, struct slot_results *results) return err; } -static curl_off_t xcurl_off_t(ssize_t len) { - if (len > maximum_signed_value_of_type(curl_off_t)) - die("cannot handle pushes this big"); - return (curl_off_t) len; +static curl_off_t xcurl_off_t(size_t len) +{ + uintmax_t size = len; + if (size > maximum_signed_value_of_type(curl_off_t)) + die(_("cannot handle pushes this big")); + return (curl_off_t)size; } -static int post_rpc(struct rpc_state *rpc) +/* + * If flush_received is true, do not attempt to read any more; just use what's + * in rpc->buf. + */ +static int post_rpc(struct rpc_state *rpc, int stateless_connect, int flush_received) { struct active_request_slot *slot; struct curl_slist *headers = http_copy_default_headers(); @@ -632,26 +894,25 @@ static int post_rpc(struct rpc_state *rpc) size_t gzip_size = 0; int err, large_request = 0; int needs_100_continue = 0; + struct rpc_in_data rpc_in_data; /* Try to load the entire request, if we can fit it into the * allocated buffer space we can use HTTP/1.0 and avoid the * chunked encoding mess. */ - while (1) { - size_t left = rpc->alloc - rpc->len; - char *buf = rpc->buf + rpc->len; - int n; - - if (left < LARGE_PACKET_MAX) { - large_request = 1; - use_gzip = 0; - break; + if (!flush_received) { + while (1) { + size_t n; + enum packet_read_status status; + + if (!rpc_read_from_out(rpc, 0, &n, &status)) { + large_request = 1; + use_gzip = 0; + break; + } + if (status == PACKET_READ_FLUSH) + break; } - - n = packet_read(rpc->out, NULL, NULL, buf, left, 0); - if (!n) - break; - rpc->len += n; } if (large_request) { @@ -674,6 +935,10 @@ static int post_rpc(struct rpc_state *rpc) headers = curl_slist_append(headers, needs_100_continue ? "Expect: 100-continue" : "Expect:"); + /* Add Accept-Language header */ + if (rpc->hdr_accept_language) + headers = curl_slist_append(headers, rpc->hdr_accept_language); + /* Add the extra Git-Protocol header */ if (rpc->protocol_header) headers = curl_slist_append(headers, rpc->protocol_header); @@ -694,10 +959,8 @@ retry: rpc->initial_buffer = 1; curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, rpc_out); curl_easy_setopt(slot->curl, CURLOPT_INFILE, rpc); -#ifndef NO_CURL_IOCTL - curl_easy_setopt(slot->curl, CURLOPT_IOCTLFUNCTION, rpc_ioctl); - curl_easy_setopt(slot->curl, CURLOPT_IOCTLDATA, rpc); -#endif + curl_easy_setopt(slot->curl, CURLOPT_SEEKFUNCTION, rpc_seek); + curl_easy_setopt(slot->curl, CURLOPT_SEEKDATA, rpc); if (options.verbosity > 1) { fprintf(stderr, "POST %s (chunked)\n", rpc->service_name); fflush(stderr); @@ -714,7 +977,7 @@ retry: } else if (use_gzip && 1024 < rpc->len) { /* The client backend isn't giving us compressed data so - * we can try to deflate it ourselves, this may save on. + * we can try to deflate it ourselves, this may save on * the transfer time. */ git_zstream stream; @@ -731,11 +994,11 @@ retry: ret = git_deflate(&stream, Z_FINISH); if (ret != Z_STREAM_END) - die("cannot deflate request; zlib deflate error %d", ret); + die(_("cannot deflate request; zlib deflate error %d"), ret); ret = git_deflate_end_gently(&stream); if (ret != Z_OK) - die("cannot deflate request; zlib end error %d", ret); + die(_("cannot deflate request; zlib end error %d"), ret); gzip_size = stream.total_out; @@ -764,7 +1027,12 @@ retry: curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, headers); curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, rpc_in); - curl_easy_setopt(slot->curl, CURLOPT_FILE, rpc); + rpc_in_data.rpc = rpc; + rpc_in_data.slot = slot; + rpc_in_data.check_pktline = stateless_connect; + memset(&rpc_in_data.pktline_state, 0, sizeof(rpc_in_data.pktline_state)); + curl_easy_setopt(slot->curl, CURLOPT_WRITEDATA, &rpc_in_data); + curl_easy_setopt(slot->curl, CURLOPT_FAILONERROR, 0); rpc->any_written = 0; @@ -779,27 +1047,35 @@ retry: if (!rpc->any_written) err = -1; + if (rpc_in_data.pktline_state.len_filled) + err = error(_("%d bytes of length header were received"), rpc_in_data.pktline_state.len_filled); + if (rpc_in_data.pktline_state.remaining) + err = error(_("%d bytes of body are still expected"), rpc_in_data.pktline_state.remaining); + + if (stateless_connect) + packet_response_end(rpc->in); + curl_slist_free_all(headers); free(gzip_body); return err; } -static int rpc_service(struct rpc_state *rpc, struct discovery *heads) +static int rpc_service(struct rpc_state *rpc, struct discovery *heads, + const char **client_argv, const struct strbuf *preamble, + struct strbuf *rpc_result) { const char *svc = rpc->service_name; struct strbuf buf = STRBUF_INIT; - struct strbuf *preamble = rpc->stdin_preamble; struct child_process client = CHILD_PROCESS_INIT; int err = 0; client.in = -1; client.out = -1; client.git_cmd = 1; - client.argv = rpc->argv; + strvec_pushv(&client.args, client_argv); if (start_command(&client)) exit(1); - if (preamble) - write_or_die(client.in, preamble->buf, preamble->len); + write_or_die(client.in, preamble->buf, preamble->len); if (heads) write_or_die(client.in, heads->buf, heads->len); @@ -807,11 +1083,12 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads) rpc->buf = xmalloc(rpc->alloc); rpc->in = client.in; rpc->out = client.out; - strbuf_init(&rpc->result, 0); strbuf_addf(&buf, "%s%s", url.buf, svc); rpc->service_url = strbuf_detach(&buf, NULL); + rpc->hdr_accept_language = xstrdup_or_null(http_get_accept_language_header()); + strbuf_addf(&buf, "Content-Type: application/x-%s-request", svc); rpc->hdr_content_type = strbuf_detach(&buf, NULL); @@ -824,18 +1101,18 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads) rpc->protocol_header = NULL; while (!err) { - int n = packet_read(rpc->out, NULL, NULL, rpc->buf, rpc->alloc, 0); + int n = packet_read(rpc->out, rpc->buf, rpc->alloc, 0); if (!n) break; rpc->pos = 0; rpc->len = n; - err |= post_rpc(rpc); + err |= post_rpc(rpc, 0, 0); } close(client.in); client.in = -1; if (!err) { - strbuf_read(&rpc->result, client.out, 0); + strbuf_read(rpc_result, client.out, 0); } else { char buf[4096]; for (;;) @@ -850,6 +1127,7 @@ static int rpc_service(struct rpc_state *rpc, struct discovery *heads) free(rpc->service_url); free(rpc->hdr_content_type); free(rpc->hdr_accept); + free(rpc->hdr_accept_language); free(rpc->protocol_header); free(rpc->buf); strbuf_release(&buf); @@ -864,12 +1142,13 @@ static int fetch_dumb(int nr_heads, struct ref **to_fetch) ALLOC_ARRAY(targets, nr_heads); if (options.depth || options.deepen_since) - die("dumb http transport does not support shallow capabilities"); + die(_("dumb http transport does not support shallow capabilities")); for (i = 0; i < nr_heads; i++) targets[i] = xstrdup(oid_to_hex(&to_fetch[i]->old_oid)); walker = get_http_walker(url.buf); walker->get_verbosely = options.verbosity >= 3; + walker->get_progress = options.progress; walker->get_recover = 0; ret = walker_fetch(walker, nr_heads, targets, NULL, NULL); walker_free(walker); @@ -878,54 +1157,55 @@ static int fetch_dumb(int nr_heads, struct ref **to_fetch) free(targets[i]); free(targets); - return ret ? error("fetch failed.") : 0; + return ret ? error(_("fetch failed.")) : 0; } static int fetch_git(struct discovery *heads, int nr_heads, struct ref **to_fetch) { - struct rpc_state rpc; + struct rpc_state rpc = RPC_STATE_INIT; struct strbuf preamble = STRBUF_INIT; int i, err; - struct argv_array args = ARGV_ARRAY_INIT; + struct strvec args = STRVEC_INIT; + struct strbuf rpc_result = STRBUF_INIT; - argv_array_pushl(&args, "fetch-pack", "--stateless-rpc", - "--stdin", "--lock-pack", NULL); + strvec_pushl(&args, "fetch-pack", "--stateless-rpc", + "--stdin", "--lock-pack", NULL); if (options.followtags) - argv_array_push(&args, "--include-tag"); + strvec_push(&args, "--include-tag"); if (options.thin) - argv_array_push(&args, "--thin"); + strvec_push(&args, "--thin"); if (options.verbosity >= 3) - argv_array_pushl(&args, "-v", "-v", NULL); + strvec_pushl(&args, "-v", "-v", NULL); if (options.check_self_contained_and_connected) - argv_array_push(&args, "--check-self-contained-and-connected"); + strvec_push(&args, "--check-self-contained-and-connected"); if (options.cloning) - argv_array_push(&args, "--cloning"); + strvec_push(&args, "--cloning"); if (options.update_shallow) - argv_array_push(&args, "--update-shallow"); + strvec_push(&args, "--update-shallow"); if (!options.progress) - argv_array_push(&args, "--no-progress"); + strvec_push(&args, "--no-progress"); if (options.depth) - argv_array_pushf(&args, "--depth=%lu", options.depth); + strvec_pushf(&args, "--depth=%lu", options.depth); if (options.deepen_since) - argv_array_pushf(&args, "--shallow-since=%s", options.deepen_since); + strvec_pushf(&args, "--shallow-since=%s", options.deepen_since); for (i = 0; i < options.deepen_not.nr; i++) - argv_array_pushf(&args, "--shallow-exclude=%s", - options.deepen_not.items[i].string); + strvec_pushf(&args, "--shallow-exclude=%s", + options.deepen_not.items[i].string); if (options.deepen_relative && options.depth) - argv_array_push(&args, "--deepen-relative"); + strvec_push(&args, "--deepen-relative"); if (options.from_promisor) - argv_array_push(&args, "--from-promisor"); - if (options.no_dependents) - argv_array_push(&args, "--no-dependents"); + strvec_push(&args, "--from-promisor"); + if (options.refetch) + strvec_push(&args, "--refetch"); if (options.filter) - argv_array_pushf(&args, "--filter=%s", options.filter); - argv_array_push(&args, url.buf); + strvec_pushf(&args, "--filter=%s", options.filter); + strvec_push(&args, url.buf); for (i = 0; i < nr_heads; i++) { struct ref *ref = to_fetch[i]; if (!*ref->name) - die("cannot fetch by sha1 over smart http"); + die(_("cannot fetch by sha1 over smart http")); packet_buf_write(&preamble, "%s %s\n", oid_to_hex(&ref->old_oid), ref->name); } @@ -933,16 +1213,14 @@ static int fetch_git(struct discovery *heads, memset(&rpc, 0, sizeof(rpc)); rpc.service_name = "git-upload-pack", - rpc.argv = args.argv; - rpc.stdin_preamble = &preamble; rpc.gzip_request = 1; - err = rpc_service(&rpc, heads); - if (rpc.result.len) - write_or_die(1, rpc.result.buf, rpc.result.len); - strbuf_release(&rpc.result); + err = rpc_service(&rpc, heads, args.v, &preamble, &rpc_result); + if (rpc_result.len) + write_or_die(1, rpc_result.buf, rpc_result.len); + strbuf_release(&rpc_result); strbuf_release(&preamble); - argv_array_clear(&args); + strvec_clear(&args); return err; } @@ -968,15 +1246,16 @@ static void parse_fetch(struct strbuf *buf) const char *name; struct ref *ref; struct object_id old_oid; + const char *q; - if (get_oid_hex(p, &old_oid)) - die("protocol error: expected sha/ref, got %s'", p); - if (p[GIT_SHA1_HEXSZ] == ' ') - name = p + GIT_SHA1_HEXSZ + 1; - else if (!p[GIT_SHA1_HEXSZ]) + if (parse_oid_hex(p, &old_oid, &q)) + die(_("protocol error: expected sha/ref, got '%s'"), p); + if (*q == ' ') + name = q + 1; + else if (!*q) name = ""; else - die("protocol error: expected sha/ref, got %s'", p); + die(_("protocol error: expected sha/ref, got '%s'"), p); ref = alloc_ref(name); oidcpy(&ref->old_oid, &old_oid); @@ -988,7 +1267,7 @@ static void parse_fetch(struct strbuf *buf) to_fetch[nr_heads++] = ref; } else - die("http transport does not support %s", buf->buf); + die(_("http transport does not support %s"), buf->buf); strbuf_reset(buf); if (strbuf_getline_lf(buf, stdin) == EOF) @@ -1007,79 +1286,106 @@ static void parse_fetch(struct strbuf *buf) strbuf_reset(buf); } -static int push_dav(int nr_spec, char **specs) +static void parse_get(const char *arg) +{ + struct strbuf url = STRBUF_INIT; + struct strbuf path = STRBUF_INIT; + const char *space; + + space = strchr(arg, ' '); + + if (!space) + die(_("protocol error: expected '<url> <path>', missing space")); + + strbuf_add(&url, arg, space - arg); + strbuf_addstr(&path, space + 1); + + if (http_get_file(url.buf, path.buf, NULL)) + die(_("failed to download file at URL '%s'"), url.buf); + + strbuf_release(&url); + strbuf_release(&path); + printf("\n"); + fflush(stdout); +} + +static int push_dav(int nr_spec, const char **specs) { struct child_process child = CHILD_PROCESS_INIT; size_t i; child.git_cmd = 1; - argv_array_push(&child.args, "http-push"); - argv_array_push(&child.args, "--helper-status"); + strvec_push(&child.args, "http-push"); + strvec_push(&child.args, "--helper-status"); if (options.dry_run) - argv_array_push(&child.args, "--dry-run"); + strvec_push(&child.args, "--dry-run"); if (options.verbosity > 1) - argv_array_push(&child.args, "--verbose"); - argv_array_push(&child.args, url.buf); + strvec_push(&child.args, "--verbose"); + strvec_push(&child.args, url.buf); for (i = 0; i < nr_spec; i++) - argv_array_push(&child.args, specs[i]); + strvec_push(&child.args, specs[i]); if (run_command(&child)) - die("git-http-push failed"); + die(_("git-http-push failed")); return 0; } -static int push_git(struct discovery *heads, int nr_spec, char **specs) +static int push_git(struct discovery *heads, int nr_spec, const char **specs) { - struct rpc_state rpc; + struct rpc_state rpc = RPC_STATE_INIT; int i, err; - struct argv_array args; + struct strvec args; struct string_list_item *cas_option; struct strbuf preamble = STRBUF_INIT; + struct strbuf rpc_result = STRBUF_INIT; - argv_array_init(&args); - argv_array_pushl(&args, "send-pack", "--stateless-rpc", "--helper-status", - NULL); + strvec_init(&args); + strvec_pushl(&args, "send-pack", "--stateless-rpc", "--helper-status", + NULL); if (options.thin) - argv_array_push(&args, "--thin"); + strvec_push(&args, "--thin"); if (options.dry_run) - argv_array_push(&args, "--dry-run"); + strvec_push(&args, "--dry-run"); if (options.push_cert == SEND_PACK_PUSH_CERT_ALWAYS) - argv_array_push(&args, "--signed=yes"); + strvec_push(&args, "--signed=yes"); else if (options.push_cert == SEND_PACK_PUSH_CERT_IF_ASKED) - argv_array_push(&args, "--signed=if-asked"); + strvec_push(&args, "--signed=if-asked"); + if (options.atomic) + strvec_push(&args, "--atomic"); if (options.verbosity == 0) - argv_array_push(&args, "--quiet"); + strvec_push(&args, "--quiet"); else if (options.verbosity > 1) - argv_array_push(&args, "--verbose"); + strvec_push(&args, "--verbose"); for (i = 0; i < options.push_options.nr; i++) - argv_array_pushf(&args, "--push-option=%s", - options.push_options.items[i].string); - argv_array_push(&args, options.progress ? "--progress" : "--no-progress"); + strvec_pushf(&args, "--push-option=%s", + options.push_options.items[i].string); + strvec_push(&args, options.progress ? "--progress" : "--no-progress"); for_each_string_list_item(cas_option, &cas_options) - argv_array_push(&args, cas_option->string); - argv_array_push(&args, url.buf); + strvec_push(&args, cas_option->string); + strvec_push(&args, url.buf); - argv_array_push(&args, "--stdin"); + if (options.force_if_includes) + strvec_push(&args, "--force-if-includes"); + + strvec_push(&args, "--stdin"); for (i = 0; i < nr_spec; i++) packet_buf_write(&preamble, "%s\n", specs[i]); packet_buf_flush(&preamble); memset(&rpc, 0, sizeof(rpc)); rpc.service_name = "git-receive-pack", - rpc.argv = args.argv; - rpc.stdin_preamble = &preamble; - err = rpc_service(&rpc, heads); - if (rpc.result.len) - write_or_die(1, rpc.result.buf, rpc.result.len); - strbuf_release(&rpc.result); + err = rpc_service(&rpc, heads, args.v, &preamble, &rpc_result); + if (rpc_result.len) + write_or_die(1, rpc_result.buf, rpc_result.len); + strbuf_release(&rpc_result); strbuf_release(&preamble); - argv_array_clear(&args); + strvec_clear(&args); return err; } -static int push(int nr_spec, char **specs) +static int push(int nr_spec, const char **specs) { struct discovery *heads = discover_refs("git-receive-pack", 1); int ret; @@ -1094,16 +1400,15 @@ static int push(int nr_spec, char **specs) static void parse_push(struct strbuf *buf) { - char **specs = NULL; - int alloc_spec = 0, nr_spec = 0, i, ret; + struct strvec specs = STRVEC_INIT; + int ret; do { - if (starts_with(buf->buf, "push ")) { - ALLOC_GROW(specs, nr_spec + 1, alloc_spec); - specs[nr_spec++] = xstrdup(buf->buf + 5); - } + const char *arg; + if (skip_prefix(buf->buf, "push ", &arg)) + strvec_push(&specs, arg); else - die("http transport does not support %s", buf->buf); + die(_("http transport does not support %s"), buf->buf); strbuf_reset(buf); if (strbuf_getline_lf(buf, stdin) == EOF) @@ -1112,177 +1417,23 @@ static void parse_push(struct strbuf *buf) break; } while (1); - ret = push(nr_spec, specs); + ret = push(specs.nr, specs.v); printf("\n"); fflush(stdout); if (ret) exit(128); /* error already reported */ - free_specs: - for (i = 0; i < nr_spec; i++) - free(specs[i]); - free(specs); -} - -/* - * Used to represent the state of a connection to an HTTP server when - * communicating using git's wire-protocol version 2. - */ -struct proxy_state { - char *service_name; - char *service_url; - struct curl_slist *headers; - struct strbuf request_buffer; - int in; - int out; - struct packet_reader reader; - size_t pos; - int seen_flush; -}; - -static void proxy_state_init(struct proxy_state *p, const char *service_name, - enum protocol_version version) -{ - struct strbuf buf = STRBUF_INIT; - - memset(p, 0, sizeof(*p)); - p->service_name = xstrdup(service_name); - - p->in = 0; - p->out = 1; - strbuf_init(&p->request_buffer, 0); - - strbuf_addf(&buf, "%s%s", url.buf, p->service_name); - p->service_url = strbuf_detach(&buf, NULL); - - p->headers = http_copy_default_headers(); - - strbuf_addf(&buf, "Content-Type: application/x-%s-request", p->service_name); - p->headers = curl_slist_append(p->headers, buf.buf); - strbuf_reset(&buf); - - strbuf_addf(&buf, "Accept: application/x-%s-result", p->service_name); - p->headers = curl_slist_append(p->headers, buf.buf); - strbuf_reset(&buf); - - p->headers = curl_slist_append(p->headers, "Transfer-Encoding: chunked"); - - /* Add the Git-Protocol header */ - if (get_protocol_http_header(version, &buf)) - p->headers = curl_slist_append(p->headers, buf.buf); - - packet_reader_init(&p->reader, p->in, NULL, 0, - PACKET_READ_GENTLE_ON_EOF); - - strbuf_release(&buf); -} - -static void proxy_state_clear(struct proxy_state *p) -{ - free(p->service_name); - free(p->service_url); - curl_slist_free_all(p->headers); - strbuf_release(&p->request_buffer); -} - -/* - * CURLOPT_READFUNCTION callback function. - * Attempts to copy over a single packet-line at a time into the - * curl provided buffer. - */ -static size_t proxy_in(char *buffer, size_t eltsize, - size_t nmemb, void *userdata) -{ - size_t max; - struct proxy_state *p = userdata; - size_t avail = p->request_buffer.len - p->pos; - - - if (eltsize != 1) - BUG("curl read callback called with size = %"PRIuMAX" != 1", - (uintmax_t)eltsize); - max = nmemb; - - if (!avail) { - if (p->seen_flush) { - p->seen_flush = 0; - return 0; - } - - strbuf_reset(&p->request_buffer); - switch (packet_reader_read(&p->reader)) { - case PACKET_READ_EOF: - die("unexpected EOF when reading from parent process"); - case PACKET_READ_NORMAL: - packet_buf_write_len(&p->request_buffer, p->reader.line, - p->reader.pktlen); - break; - case PACKET_READ_DELIM: - packet_buf_delim(&p->request_buffer); - break; - case PACKET_READ_FLUSH: - packet_buf_flush(&p->request_buffer); - p->seen_flush = 1; - break; - } - p->pos = 0; - avail = p->request_buffer.len; - } - - if (max < avail) - avail = max; - memcpy(buffer, p->request_buffer.buf + p->pos, avail); - p->pos += avail; - return avail; -} - -static size_t proxy_out(char *buffer, size_t eltsize, - size_t nmemb, void *userdata) -{ - size_t size; - struct proxy_state *p = userdata; - - if (eltsize != 1) - BUG("curl read callback called with size = %"PRIuMAX" != 1", - (uintmax_t)eltsize); - size = nmemb; - - write_or_die(p->out, buffer, size); - return size; -} - -/* Issues a request to the HTTP server configured in `p` */ -static int proxy_request(struct proxy_state *p) -{ - struct active_request_slot *slot; - - slot = get_active_slot(); - - curl_easy_setopt(slot->curl, CURLOPT_ENCODING, ""); - curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0); - curl_easy_setopt(slot->curl, CURLOPT_POST, 1); - curl_easy_setopt(slot->curl, CURLOPT_URL, p->service_url); - curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, p->headers); - - /* Setup function to read request from client */ - curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, proxy_in); - curl_easy_setopt(slot->curl, CURLOPT_READDATA, p); - - /* Setup function to write server response to client */ - curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, proxy_out); - curl_easy_setopt(slot->curl, CURLOPT_WRITEDATA, p); - - if (run_slot(slot, NULL) != HTTP_OK) - return -1; - - return 0; +free_specs: + strvec_clear(&specs); } static int stateless_connect(const char *service_name) { struct discovery *discover; - struct proxy_state p; + struct rpc_state rpc = RPC_STATE_INIT; + struct strbuf buf = STRBUF_INIT; + const char *accept_language; /* * Run the info/refs request and see if the server supports protocol @@ -1301,24 +1452,63 @@ static int stateless_connect(const char *service_name) printf("\n"); fflush(stdout); } - - proxy_state_init(&p, service_name, discover->version); + accept_language = http_get_accept_language_header(); + if (accept_language) + rpc.hdr_accept_language = xstrfmt("%s", accept_language); + + rpc.service_name = service_name; + rpc.service_url = xstrfmt("%s%s", url.buf, rpc.service_name); + rpc.hdr_content_type = xstrfmt("Content-Type: application/x-%s-request", rpc.service_name); + rpc.hdr_accept = xstrfmt("Accept: application/x-%s-result", rpc.service_name); + if (get_protocol_http_header(discover->version, &buf)) { + rpc.protocol_header = strbuf_detach(&buf, NULL); + } else { + rpc.protocol_header = NULL; + strbuf_release(&buf); + } + rpc.buf = xmalloc(http_post_buffer); + rpc.alloc = http_post_buffer; + rpc.len = 0; + rpc.pos = 0; + rpc.in = 1; + rpc.out = 0; + rpc.any_written = 0; + rpc.gzip_request = 1; + rpc.initial_buffer = 0; + rpc.write_line_lengths = 1; + rpc.flush_read_but_not_sent = 0; /* * Dump the capability listing that we got from the server earlier * during the info/refs request. */ - write_or_die(p.out, discover->buf, discover->len); + write_or_die(rpc.in, discover->buf, discover->len); - /* Peek the next packet line. Until we see EOF keep sending POSTs */ - while (packet_reader_peek(&p.reader) != PACKET_READ_EOF) { - if (proxy_request(&p)) { + /* Until we see EOF keep sending POSTs */ + while (1) { + size_t avail; + enum packet_read_status status; + + if (!rpc_read_from_out(&rpc, PACKET_READ_GENTLE_ON_EOF, &avail, + &status)) + BUG("The entire rpc->buf should be larger than LARGE_PACKET_MAX"); + if (status == PACKET_READ_EOF) + break; + if (post_rpc(&rpc, 1, status == PACKET_READ_FLUSH)) /* We would have an err here */ break; - } + /* Reset the buffer for next request */ + rpc.len = 0; } - proxy_state_clear(&p); + free(rpc.service_url); + free(rpc.hdr_content_type); + free(rpc.hdr_accept); + free(rpc.hdr_accept_language); + free(rpc.protocol_header); + free(rpc.buf); + strbuf_release(&buf); + return 0; } @@ -1326,18 +1516,26 @@ int cmd_main(int argc, const char **argv) { struct strbuf buf = STRBUF_INIT; int nongit; + int ret = 1; setup_git_directory_gently(&nongit); if (argc < 2) { - error("remote-curl: usage: git remote-curl <remote> [<url>]"); - return 1; + error(_("remote-curl: usage: git remote-curl <remote> [<url>]")); + goto cleanup; } options.verbosity = 1; options.progress = !!isatty(2); options.thin = 1; - string_list_init(&options.deepen_not, 1); - string_list_init(&options.push_options, 1); + string_list_init_dup(&options.deepen_not); + string_list_init_dup(&options.push_options); + + /* + * Just report "remote-curl" here (folding all the various aliases + * ("git-remote-http", "git-remote-https", and etc.) here since they + * are all just copies of the same actual executable. + */ + trace2_cmd_name("remote-curl"); remote = remote_get(argv[1]); @@ -1354,14 +1552,14 @@ int cmd_main(int argc, const char **argv) if (strbuf_getline_lf(&buf, stdin) == EOF) { if (ferror(stdin)) - error("remote-curl: error reading command stream from git"); - return 1; + error(_("remote-curl: error reading command stream from git")); + goto cleanup; } if (buf.len == 0) break; if (starts_with(buf.buf, "fetch ")) { if (nongit) - die("remote-curl: fetch attempted without a local repo"); + die(_("remote-curl: fetch attempted without a local repo")); parse_fetch(&buf); } else if (!strcmp(buf.buf, "list") || starts_with(buf.buf, "list ")) { @@ -1389,25 +1587,34 @@ int cmd_main(int argc, const char **argv) printf("unsupported\n"); fflush(stdout); + } else if (skip_prefix(buf.buf, "get ", &arg)) { + parse_get(arg); + fflush(stdout); + } else if (!strcmp(buf.buf, "capabilities")) { printf("stateless-connect\n"); printf("fetch\n"); + printf("get\n"); printf("option\n"); printf("push\n"); printf("check-connectivity\n"); + printf("object-format\n"); printf("\n"); fflush(stdout); } else if (skip_prefix(buf.buf, "stateless-connect ", &arg)) { if (!stateless_connect(arg)) break; } else { - error("remote-curl: unknown command '%s' from git", buf.buf); - return 1; + error(_("remote-curl: unknown command '%s' from git"), buf.buf); + goto cleanup; } strbuf_reset(&buf); } while (1); http_cleanup(); + ret = 0; +cleanup: + strbuf_release(&buf); - return 0; + return ret; } |
