From 3944ba0cb0ef5119dc9d1708c572855fca88fc43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Wed, 1 Apr 2009 19:48:24 +0300 Subject: Allow curl to rewind the read buffers When using multi-pass authentication methods, the curl library may need to rewind the read buffers (depending on how much already has been fed to the server) used for providing data to HTTP PUT, POST or PROPFIND, and in order to allow the library to do so, we need to tell it how by providing either an ioctl callback or a seek callback. This patch adds an ioctl callback, which should be usable on older curl versions (since 7.12.3) than the seek callback (introduced in curl 7.18.0). Some HTTP servers (such as Apache) give an 401 error reply immediately after receiving the headers (so no data has been read from the read buffers, and thus no rewinding is needed), but other servers (such as Lighttpd) only replies after the whole request has been sent and all data has been read from the read buffers, making rewinding necessary. Signed-off-by: Martin Storsjo Signed-off-by: Junio C Hamano --- http.h | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'http.h') diff --git a/http.h b/http.h index 905b4629a4..26abebed1f 100644 --- a/http.h +++ b/http.h @@ -37,6 +37,10 @@ #define CURLE_HTTP_RETURNED_ERROR CURLE_HTTP_NOT_FOUND #endif +#if LIBCURL_VERSION_NUM < 0x070c03 +#define NO_CURL_IOCTL +#endif + struct slot_results { CURLcode curl_result; @@ -67,6 +71,9 @@ struct buffer extern size_t fread_buffer(void *ptr, size_t eltsize, size_t nmemb, void *strbuf); extern size_t fwrite_buffer(const void *ptr, size_t eltsize, size_t nmemb, void *strbuf); extern size_t fwrite_null(const void *ptr, size_t eltsize, size_t nmemb, void *strbuf); +#ifndef NO_CURL_IOCTL +extern curlioerr ioctl_buffer(CURL *handle, int cmd, void *clientp); +#endif /* Slot lifecycle functions */ extern struct active_request_slot *get_active_slot(void); -- cgit v1.2.3 From e917674597cc0b345ad2d6e29fd1a03e1039615a Mon Sep 17 00:00:00 2001 From: Tay Ray Chuan Date: Sat, 6 Jun 2009 16:43:41 +0800 Subject: http*: move common variables and macros to http.[ch] Move RANGE_HEADER_SIZE to http.h. Create no_pragma_header, the curl header list containing the header "Pragma:" in http.[ch]. It is allocated in http_init, and freed in http_cleanup. This replaces the no_pragma_header in http-push.c, and the no_pragma_header member in walker_data in http-walker.c. Create http_is_verbose. It is to be used by methods in http.c, and is modified at the entry points of http.c's users, namely http-push.c (when parsing options) and http-walker.c (in get_http_walker). Signed-off-by: Tay Ray Chuan Signed-off-by: Junio C Hamano --- http.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'http.h') diff --git a/http.h b/http.h index 26abebed1f..1ef7dc1583 100644 --- a/http.h +++ b/http.h @@ -88,11 +88,16 @@ extern void add_fill_function(void *data, int (*fill)(void *)); extern void step_active_slots(void); #endif +extern struct curl_slist *no_pragma_header; + +#define RANGE_HEADER_SIZE 30 + extern void http_init(struct remote *remote); extern void http_cleanup(void); extern int data_received; extern int active_requests; +extern int http_is_verbose; extern char curl_errorstr[CURL_ERROR_SIZE]; -- cgit v1.2.3 From e929cd20bb3557833765f95fe0e9143a47f806e7 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Sat, 6 Jun 2009 16:43:53 +0800 Subject: http.c: new functions for the http API The new functions added are: - http_request() (internal function) - http_get_strbuf() - http_get_file() - http_error() http_get_strbuf and http_get_file allow respectively to retrieve contents of an URL to a strbuf or an opened file handle. http_error prints out an error message containing the URL and the curl error (in curl_errorstr). Signed-off-by: Mike Hommey Signed-off-by: Tay Ray Chuan Signed-off-by: Junio C Hamano --- http.h | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) (limited to 'http.h') diff --git a/http.h b/http.h index 1ef7dc1583..3d878d5126 100644 --- a/http.h +++ b/http.h @@ -114,6 +114,36 @@ static inline int missing__target(int code, int result) #define missing_target(a) missing__target((a)->http_code, (a)->curl_result) +/* Options for http_request_*() */ +#define HTTP_NO_CACHE 1 + +/* Return values for http_request_*() */ +#define HTTP_OK 0 +#define HTTP_MISSING_TARGET 1 +#define HTTP_ERROR 2 +#define HTTP_START_FAILED 3 + +/* + * Requests an url and stores the result in a strbuf. + * + * If the result pointer is NULL, a HTTP HEAD request is made instead of GET. + */ +int http_get_strbuf(const char *url, struct strbuf *result, int options); + +/* + * Downloads an url and stores the result in the given file. + * + * If a previous interrupted download is detected (i.e. a previous temporary + * file is still around) the download is resumed. + */ +int http_get_file(const char *url, const char *filename, int options); + +/* + * Prints an error message using error() containing url and curl_errorstr, + * and returns ret. + */ +int http_error(const char *url, int ret); + extern int http_fetch_ref(const char *base, struct ref *ref); #endif /* HTTP_H */ -- cgit v1.2.3 From b8caac2b8ab6482e7ab59c8ec18f1c3d90e7387d Mon Sep 17 00:00:00 2001 From: Tay Ray Chuan Date: Sat, 6 Jun 2009 16:43:59 +0800 Subject: http*: add http_get_info_packs http-push.c and http-walker.c no longer have to use fetch_index or setup_index; they simply need to use http_get_info_packs, a new http method, in their fetch_indices implementations. Move fetch_index() and rename to fetch_pack_index() in http.c; this method is not meant to be used outside of http.c. It invokes end_url_with_slash with base_url; apart from that change, the code is identical. Move setup_index() and rename to fetch_and_setup_pack_index() in http.c; this method is not meant to be used outside of http.c. Do not immediately set ret to 0 in http-walker.c::fetch_indices(); instead do it in the HTTP_MISSING_TARGET case, to make it clear that the HTTP_OK and HTTP_MISSING_TARGET cases both return 0. Signed-off-by: Tay Ray Chuan Signed-off-by: Junio C Hamano --- http.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'http.h') diff --git a/http.h b/http.h index 3d878d5126..180b1c2ae2 100644 --- a/http.h +++ b/http.h @@ -146,4 +146,8 @@ int http_error(const char *url, int ret); extern int http_fetch_ref(const char *base, struct ref *ref); +/* Helpers for fetching packs */ +extern int http_get_info_packs(const char *base_url, + struct packed_git **packs_head); + #endif /* HTTP_H */ -- cgit v1.2.3 From 2264dfa5c4f11e2b0e2740072208186bee361afd Mon Sep 17 00:00:00 2001 From: Tay Ray Chuan Date: Sat, 6 Jun 2009 16:44:01 +0800 Subject: http*: add helper methods for fetching packs The code handling the fetching of packs in http-push.c and http-walker.c have been refactored into new methods and a new struct (http_pack_request) in http.c. They are not meant to be invoked elsewhere. The new methods in http.c are - new_http_pack_request - finish_http_pack_request - release_http_pack_request and the new struct is http_pack_request. Add a function, new_http_pack_request(), that deals with the details of coming up with the filename to store the retrieved packfile, resuming a previously aborted request, and making a new curl request. Update http-push.c::start_fetch_packed() and http-walker.c::fetch_pack() to use this. Add a function, finish_http_pack_request(), that deals with renaming the pack, advancing the pack list, and installing the pack. Update http-push.c::finish_request() and http-walker.c::fetch_pack to use this. Update release_request() in http-push.c and http-walker.c to invoke release_http_pack_request() to clean up pack request helper data. The local_stream member of the transfer_request struct in http-push.c has been removed, as the packfile pointer will be managed in the struct http_pack_request. Signed-off-by: Tay Ray Chuan Signed-off-by: Junio C Hamano --- http.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'http.h') diff --git a/http.h b/http.h index 180b1c2ae2..511c0c4417 100644 --- a/http.h +++ b/http.h @@ -150,4 +150,21 @@ extern int http_fetch_ref(const char *base, struct ref *ref); extern int http_get_info_packs(const char *base_url, struct packed_git **packs_head); +struct http_pack_request +{ + char *url; + struct packed_git *target; + struct packed_git **lst; + FILE *packfile; + char filename[PATH_MAX]; + char tmpfile[PATH_MAX]; + struct curl_slist *range_header; + struct active_request_slot *slot; +}; + +extern struct http_pack_request *new_http_pack_request( + struct packed_git *target, const char *base_url); +extern int finish_http_pack_request(struct http_pack_request *preq); +extern void release_http_pack_request(struct http_pack_request *preq); + #endif /* HTTP_H */ -- cgit v1.2.3 From 5424bc557fc6414660830b470dd45774b8f5f281 Mon Sep 17 00:00:00 2001 From: Tay Ray Chuan Date: Sat, 6 Jun 2009 16:44:02 +0800 Subject: http*: add helper methods for fetching objects (loose) The code handling the fetching of loose objects in http-push.c and http-walker.c have been refactored into new methods and a new struct (object_http_request) in http.c. They are not meant to be invoked elsewhere. The new methods in http.c are - new_http_object_request - process_http_object_request - finish_http_object_request - abort_http_object_request - release_http_object_request and the new struct is http_object_request. RANGER_HEADER_SIZE and no_pragma_header is no longer made available outside of http.c, since after the above changes, there are no other instances of usage outside of http.c. Remove members of the transfer_request struct in http-push.c and http-walker.c, including filename, real_sha1 and zret, as they are used no longer used. Move the methods append_remote_object_url() and get_remote_object_url() from http-push.c to http.c. Additionally, get_remote_object_url() is no longer defined only when USE_CURL_MULTI is defined, since non-USE_CURL_MULTI code in http.c uses it (namely, in new_http_object_request()). Refactor code from http-push.c::start_fetch_loose() and http-walker.c::start_object_fetch_request() that deals with the details of coming up with the filename to store the retrieved object, resuming a previously aborted request, and making a new curl request, into a new function, new_http_object_request(). Refactor code from http-walker.c::process_object_request() into the function, process_http_object_request(). Refactor code from http-push.c::finish_request() and http-walker.c::finish_object_request() into a new function, finish_http_object_request(). It returns the result of the move_temp_to_file() invocation. Add a function, release_http_object_request(), which cleans up object request data. http-push.c and http-walker.c invoke this function separately; http-push.c::release_request() and http-walker.c::release_object_request() do not invoke this function. Add a function, abort_http_object_request(), which unlink()s the object file and invokes release_http_object_request(). Update http-walker.c::abort_object_request() to use this. Signed-off-by: Tay Ray Chuan Signed-off-by: Junio C Hamano --- http.h | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) (limited to 'http.h') diff --git a/http.h b/http.h index 511c0c4417..4c4e99c2f6 100644 --- a/http.h +++ b/http.h @@ -88,10 +88,6 @@ extern void add_fill_function(void *data, int (*fill)(void *)); extern void step_active_slots(void); #endif -extern struct curl_slist *no_pragma_header; - -#define RANGE_HEADER_SIZE 30 - extern void http_init(struct remote *remote); extern void http_cleanup(void); @@ -114,6 +110,13 @@ static inline int missing__target(int code, int result) #define missing_target(a) missing__target((a)->http_code, (a)->curl_result) +/* Helpers for modifying and creating URLs */ +extern void append_remote_object_url(struct strbuf *buf, const char *url, + const char *hex, + int only_two_digit_prefix); +extern char *get_remote_object_url(const char *url, const char *hex, + int only_two_digit_prefix); + /* Options for http_request_*() */ #define HTTP_NO_CACHE 1 @@ -167,4 +170,30 @@ extern struct http_pack_request *new_http_pack_request( extern int finish_http_pack_request(struct http_pack_request *preq); extern void release_http_pack_request(struct http_pack_request *preq); +/* Helpers for fetching object */ +struct http_object_request +{ + char *url; + char filename[PATH_MAX]; + char tmpfile[PATH_MAX]; + int localfile; + CURLcode curl_result; + char errorstr[CURL_ERROR_SIZE]; + long http_code; + unsigned char sha1[20]; + unsigned char real_sha1[20]; + git_SHA_CTX c; + z_stream stream; + int zret; + int rename; + struct active_request_slot *slot; +}; + +extern struct http_object_request *new_http_object_request( + const char *base_url, unsigned char *sha1); +extern void process_http_object_request(struct http_object_request *freq); +extern int finish_http_object_request(struct http_object_request *freq); +extern void abort_http_object_request(struct http_object_request *freq); +extern void release_http_object_request(struct http_object_request *freq); + #endif /* HTTP_H */ -- cgit v1.2.3 From de1a2fdd38b138c4e4fed6412783dcb74d63d2da Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Fri, 30 Oct 2009 17:47:41 -0700 Subject: Smart push over HTTP: client side The git-remote-curl backend detects if the remote server supports the git-receive-pack service, and if so, runs git-send-pack in a pipe to dump the command and pack data as a single POST request. The advertisements from the server that were obtained during the discovery are passed into git-send-pack before the POST request starts. This permits git-send-pack to operate largely unmodified. For smaller packs (those under 1 MiB) a HTTP/1.0 POST with a Content-Length is used, permitting interaction with any server. The 1 MiB limit is arbitrary, but is sufficent to fit most deltas created by human authors against text sources with the occasional small binary file (e.g. few KiB icon image). The configuration option http.postBuffer can be used to increase (or shink) this buffer if the default is not sufficient. For larger packs which cannot be spooled entirely into the helper's memory space (due to http.postBuffer being too small), the POST request requires HTTP/1.1 and sets "Transfer-Encoding: chunked". This permits the client to upload an unknown amount of data in one HTTP transaction without needing to pregenerate the entire pack file locally. Signed-off-by: Shawn O. Pearce CC: Daniel Barkalow Signed-off-by: Junio C Hamano --- http.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'http.h') diff --git a/http.h b/http.h index 4c4e99c2f6..f828e1d806 100644 --- a/http.h +++ b/http.h @@ -79,6 +79,7 @@ extern curlioerr ioctl_buffer(CURL *handle, int cmd, void *clientp); extern struct active_request_slot *get_active_slot(void); extern int start_active_slot(struct active_request_slot *slot); extern void run_active_slot(struct active_request_slot *slot); +extern void finish_active_slot(struct active_request_slot *slot); extern void finish_all_active_slots(void); extern void release_active_slot(struct active_request_slot *slot); @@ -94,6 +95,7 @@ extern void http_cleanup(void); extern int data_received; extern int active_requests; extern int http_is_verbose; +extern size_t http_post_buffer; extern char curl_errorstr[CURL_ERROR_SIZE]; -- cgit v1.2.3 From 83e41e2e61f535d5b432437d4e8ac89c710fc482 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Mon, 11 Jan 2010 22:26:08 -0800 Subject: http.c: mark file-local functions static Signed-off-by: Junio C Hamano --- http.h | 9 --------- 1 file changed, 9 deletions(-) (limited to 'http.h') diff --git a/http.h b/http.h index f828e1d806..5c9441c10c 100644 --- a/http.h +++ b/http.h @@ -81,7 +81,6 @@ extern int start_active_slot(struct active_request_slot *slot); extern void run_active_slot(struct active_request_slot *slot); extern void finish_active_slot(struct active_request_slot *slot); extern void finish_all_active_slots(void); -extern void release_active_slot(struct active_request_slot *slot); #ifdef USE_CURL_MULTI extern void fill_active_slots(void); @@ -135,14 +134,6 @@ extern char *get_remote_object_url(const char *url, const char *hex, */ int http_get_strbuf(const char *url, struct strbuf *result, int options); -/* - * Downloads an url and stores the result in the given file. - * - * If a previous interrupted download is detected (i.e. a previous temporary - * file is still around) the download is resumed. - */ -int http_get_file(const char *url, const char *filename, int options); - /* * Prints an error message using error() containing url and curl_errorstr, * and returns ret. -- cgit v1.2.3