From 95bad7d48c4320bae152c904d95939c364083d3d Mon Sep 17 00:00:00 2001 From: Markham Date: Wed, 1 Feb 2017 17:27:51 +0100 Subject: [PATCH] use correct patch for broken wget (bb-1.26.2) --- .../busybox-1.26.2-use-wget-1.24.1.patch | 398 ------------------ ...t-fix-for-brain-damaged-HTTP-servers.patch | 87 ++++ make/applications.mk | 2 +- 3 files changed, 88 insertions(+), 399 deletions(-) delete mode 100644 archive-patches/busybox-1.26.2-use-wget-1.24.1.patch create mode 100644 archive-patches/busybox-1.26.2-wget-fix-for-brain-damaged-HTTP-servers.patch diff --git a/archive-patches/busybox-1.26.2-use-wget-1.24.1.patch b/archive-patches/busybox-1.26.2-use-wget-1.24.1.patch deleted file mode 100644 index 68d0c98..0000000 --- a/archive-patches/busybox-1.26.2-use-wget-1.24.1.patch +++ /dev/null @@ -1,398 +0,0 @@ -diff --git a/networking/wget.c b/networking/wget.c -index b082a0f..d4a9c0c 100644 ---- a/networking/wget.c -+++ b/networking/wget.c -@@ -62,10 +62,9 @@ - //config: a helper program to talk over HTTPS. - //config: - //config: OpenSSL has a simple SSL client for debug purposes. --//config: If you select "openssl" helper, wget will effectively run: --//config: "openssl s_client -quiet -connect hostname:443 --//config: -servername hostname 2>/dev/null" and pipe its data --//config: through it. -servername is not used if hostname is numeric. -+//config: If you select "openssl" helper, wget will effectively call -+//config: "openssl s_client -quiet -connect IP:443 2>/dev/null" -+//config: and pipe its data through it. - //config: Note inconvenient API: host resolution is done twice, - //config: and there is no guarantee openssl's idea of IPv6 address - //config: format is the same as ours. -@@ -100,31 +99,28 @@ - - //usage:#define wget_trivial_usage - //usage: IF_FEATURE_WGET_LONG_OPTIONS( --//usage: "[-c|--continue] [--spider] [-q|--quiet] [-O|--output-document FILE]\n" -+//usage: "[-c|--continue] [-s|--spider] [-q|--quiet] [-O|--output-document FILE]\n" - //usage: " [--header 'header: value'] [-Y|--proxy on/off] [-P DIR]\n" - /* Since we ignore these opts, we don't show them in --help */ --/* //usage: " [--no-check-certificate] [--no-cache] [--passive-ftp] [-t TRIES]" */ --/* //usage: " [-nv] [-nc] [-nH] [-np]" */ -+/* //usage: " [--no-check-certificate] [--no-cache]" */ - //usage: " [-U|--user-agent AGENT]" IF_FEATURE_WGET_TIMEOUT(" [-T SEC]") " URL..." - //usage: ) - //usage: IF_NOT_FEATURE_WGET_LONG_OPTIONS( --//usage: "[-cq] [-O FILE] [-Y on/off] [-P DIR] [-U AGENT]" -+//usage: "[-csq] [-O FILE] [-Y on/off] [-P DIR] [-U AGENT]" - //usage: IF_FEATURE_WGET_TIMEOUT(" [-T SEC]") " URL..." - //usage: ) - //usage:#define wget_full_usage "\n\n" - //usage: "Retrieve files via HTTP or FTP\n" --//usage: IF_FEATURE_WGET_LONG_OPTIONS( --//usage: "\n --spider Spider mode - only check file existence" --//usage: ) --//usage: "\n -c Continue retrieval of aborted transfer" --//usage: "\n -q Quiet" --//usage: "\n -P DIR Save to DIR (default .)" -+//usage: "\n -s Spider mode - only check file existence" -+//usage: "\n -c Continue retrieval of aborted transfer" -+//usage: "\n -q Quiet" -+//usage: "\n -P DIR Save to DIR (default .)" - //usage: IF_FEATURE_WGET_TIMEOUT( --//usage: "\n -T SEC Network read timeout is SEC seconds" -+//usage: "\n -T SEC Network read timeout is SEC seconds" - //usage: ) --//usage: "\n -O FILE Save to FILE ('-' for stdout)" --//usage: "\n -U STR Use STR for User-Agent header" --//usage: "\n -Y on/off Use proxy" -+//usage: "\n -O FILE Save to FILE ('-' for stdout)" -+//usage: "\n -U STR Use STR for User-Agent header" -+//usage: "\n -Y Use proxy ('on' or 'off')" - - #include "libbb.h" - -@@ -149,11 +145,9 @@ struct host_info { - char *host; - int port; - }; --static const char P_FTP[] ALIGN1 = "ftp"; --static const char P_HTTP[] ALIGN1 = "http"; --#if ENABLE_FEATURE_WGET_OPENSSL || ENABLE_FEATURE_WGET_SSL_HELPER --static const char P_HTTPS[] ALIGN1 = "https"; --#endif -+static const char P_FTP[] = "ftp"; -+static const char P_HTTP[] = "http"; -+static const char P_HTTPS[] = "https"; - - #if ENABLE_FEATURE_WGET_LONG_OPTIONS - /* User-specified headers prevent using our corresponding built-in headers. */ -@@ -206,7 +200,7 @@ struct globals { - const char *user_agent; /* "User-Agent" header field */ - #if ENABLE_FEATURE_WGET_TIMEOUT - unsigned timeout_seconds; -- bool die_if_timed_out; -+ bool connecting; - #endif - int output_fd; - int o_flags; -@@ -231,17 +225,17 @@ struct globals { - /* Must match option string! */ - enum { - WGET_OPT_CONTINUE = (1 << 0), -- WGET_OPT_QUIET = (1 << 1), -- WGET_OPT_OUTNAME = (1 << 2), -- WGET_OPT_PREFIX = (1 << 3), -- WGET_OPT_PROXY = (1 << 4), -- WGET_OPT_USER_AGENT = (1 << 5), -- WGET_OPT_NETWORK_READ_TIMEOUT = (1 << 6), -- WGET_OPT_RETRIES = (1 << 7), -- WGET_OPT_nsomething = (1 << 8), -- WGET_OPT_HEADER = (1 << 9) * ENABLE_FEATURE_WGET_LONG_OPTIONS, -- WGET_OPT_POST_DATA = (1 << 10) * ENABLE_FEATURE_WGET_LONG_OPTIONS, -- WGET_OPT_SPIDER = (1 << 11) * ENABLE_FEATURE_WGET_LONG_OPTIONS, -+ WGET_OPT_SPIDER = (1 << 1), -+ WGET_OPT_QUIET = (1 << 2), -+ WGET_OPT_OUTNAME = (1 << 3), -+ WGET_OPT_PREFIX = (1 << 4), -+ WGET_OPT_PROXY = (1 << 5), -+ WGET_OPT_USER_AGENT = (1 << 6), -+ WGET_OPT_NETWORK_READ_TIMEOUT = (1 << 7), -+ WGET_OPT_RETRIES = (1 << 8), -+ WGET_OPT_PASSIVE = (1 << 9), -+ WGET_OPT_HEADER = (1 << 10) * ENABLE_FEATURE_WGET_LONG_OPTIONS, -+ WGET_OPT_POST_DATA = (1 << 11) * ENABLE_FEATURE_WGET_LONG_OPTIONS, - }; - - enum { -@@ -336,44 +330,9 @@ static char* sanitize_string(char *s) - static void alarm_handler(int sig UNUSED_PARAM) - { - /* This is theoretically unsafe (uses stdio and malloc in signal handler) */ -- if (G.die_if_timed_out) -+ if (G.connecting) - bb_error_msg_and_die("download timed out"); - } --static void set_alarm(void) --{ -- if (G.timeout_seconds) { -- alarm(G.timeout_seconds); -- G.die_if_timed_out = 1; -- } --} --# define clear_alarm() ((void)(G.die_if_timed_out = 0)) --#else --# define set_alarm() ((void)0) --# define clear_alarm() ((void)0) --#endif -- --#if ENABLE_FEATURE_WGET_OPENSSL --/* -- * is_ip_address() attempts to verify whether or not a string -- * contains an IPv4 or IPv6 address (vs. an FQDN). The result -- * of inet_pton() can be used to determine this. -- * -- * TODO add proper error checking when inet_pton() returns -1 -- * (some form of system error has occurred, and errno is set) -- */ --static int is_ip_address(const char *string) --{ -- struct sockaddr_in sa; -- -- int result = inet_pton(AF_INET, string, &(sa.sin_addr)); --# if ENABLE_FEATURE_IPV6 -- if (result == 0) { -- struct sockaddr_in6 sa6; -- result = inet_pton(AF_INET6, string, &(sa6.sin6_addr)); -- } --# endif -- return (result == 1); --} - #endif - - static FILE *open_socket(len_and_sockaddr *lsa) -@@ -381,9 +340,9 @@ static FILE *open_socket(len_and_sockaddr *lsa) - int fd; - FILE *fp; - -- set_alarm(); -+ IF_FEATURE_WGET_TIMEOUT(alarm(G.timeout_seconds); G.connecting = 1;) - fd = xconnect_stream(lsa); -- clear_alarm(); -+ IF_FEATURE_WGET_TIMEOUT(G.connecting = 0;) - - /* glibc 2.4 seems to try seeking on it - ??! */ - /* hopefully it understands what ESPIPE means... */ -@@ -395,15 +354,14 @@ static FILE *open_socket(len_and_sockaddr *lsa) - } - - /* Returns '\n' if it was seen, else '\0'. Trims at first '\r' or '\n' */ -+/* FIXME: does not respect FEATURE_WGET_TIMEOUT and -T N: */ - static char fgets_and_trim(FILE *fp) - { - char c; - char *buf_ptr; - -- set_alarm(); - if (fgets(G.wget_buf, sizeof(G.wget_buf) - 1, fp) == NULL) - bb_perror_msg_and_die("error getting response"); -- clear_alarm(); - - buf_ptr = strchrnul(G.wget_buf, '\n'); - c = *buf_ptr; -@@ -452,12 +410,10 @@ static void parse_url(const char *src_url, struct host_info *h) - if (strcmp(url, P_FTP) == 0) { - h->port = bb_lookup_port(P_FTP, "tcp", 21); - } else --#if ENABLE_FEATURE_WGET_OPENSSL || ENABLE_FEATURE_WGET_SSL_HELPER - if (strcmp(url, P_HTTPS) == 0) { - h->port = bb_lookup_port(P_HTTPS, "tcp", 443); - h->protocol = P_HTTPS; - } else --#endif - if (strcmp(url, P_HTTP) == 0) { - http: - h->port = bb_lookup_port(P_HTTP, "tcp", 80); -@@ -656,7 +612,6 @@ static FILE* prepare_ftp_session(FILE **dfpp, struct host_info *target, len_and_ - static int spawn_https_helper_openssl(const char *host, unsigned port) - { - char *allocated = NULL; -- char *servername; - int sp[2]; - int pid; - IF_FEATURE_WGET_SSL_HELPER(volatile int child_failed = 0;) -@@ -667,14 +622,12 @@ static int spawn_https_helper_openssl(const char *host, unsigned port) - - if (!strchr(host, ':')) - host = allocated = xasprintf("%s:%u", host, port); -- servername = xstrdup(host); -- strrchr(servername, ':')[0] = '\0'; - - fflush_all(); - pid = xvfork(); - if (pid == 0) { - /* Child */ -- char *argv[8]; -+ char *argv[6]; - - close(sp[0]); - xmove_fd(sp[1], 0); -@@ -686,22 +639,12 @@ static int spawn_https_helper_openssl(const char *host, unsigned port) - */ - xmove_fd(2, 3); - xopen("/dev/null", O_RDWR); -- memset(&argv, 0, sizeof(argv)); - argv[0] = (char*)"openssl"; - argv[1] = (char*)"s_client"; - argv[2] = (char*)"-quiet"; - argv[3] = (char*)"-connect"; - argv[4] = (char*)host; -- /* -- * Per RFC 6066 Section 3, the only permitted values in the -- * TLS server_name (SNI) field are FQDNs (DNS hostnames). -- * IPv4 and IPv6 addresses, port numbers are not allowed. -- */ -- if (!is_ip_address(servername)) { -- argv[5] = (char*)"-servername"; -- argv[6] = (char*)servername; -- } -- -+ argv[5] = NULL; - BB_EXECVP(argv[0], argv); - xmove_fd(3, 2); - # if ENABLE_FEATURE_WGET_SSL_HELPER -@@ -714,7 +657,6 @@ static int spawn_https_helper_openssl(const char *host, unsigned port) - } - - /* Parent */ -- free(servername); - free(allocated); - close(sp[1]); - # if ENABLE_FEATURE_WGET_SSL_HELPER -@@ -1093,12 +1035,6 @@ static void download_one_url(const char *url) - } - - fflush(sfp); -- /* If we use SSL helper, keeping our end of the socket open for writing -- * makes our end (i.e. the same fd!) readable (EAGAIN instead of EOF) -- * even after child closes its copy of the fd. -- * This helps: -- */ -- shutdown(fileno(sfp), SHUT_WR); - - /* - * Retrieve HTTP response line and check for "200" status code. -@@ -1118,21 +1054,7 @@ static void download_one_url(const char *url) - while (gethdr(sfp) != NULL) - /* eat all remaining headers */; - goto read_response; -- -- /* Success responses */ - case 200: -- /* fall through */ -- case 201: /* 201 Created */ --/* "The request has been fulfilled and resulted in a new resource being created" */ -- /* Standard wget is reported to treat this as success */ -- /* fall through */ -- case 202: /* 202 Accepted */ --/* "The request has been accepted for processing, but the processing has not been completed" */ -- /* Treat as success: fall through */ -- case 203: /* 203 Non-Authoritative Information */ --/* "Use of this response code is not required and is only appropriate when the response would otherwise be 200 (OK)" */ -- /* fall through */ -- case 204: /* 204 No Content */ - /* - Response 204 doesn't say "null file", it says "metadata - has changed but data didn't": -@@ -1157,6 +1079,7 @@ is always terminated by the first empty line after the header fields." - However, in real world it was observed that some web servers - (e.g. Boa/0.94.14rc21) simply use code 204 when file size is zero. - */ -+ case 204: - if (G.beg_range != 0) { - /* "Range:..." was not honored by the server. - * Restart download from the beginning. -@@ -1164,14 +1087,11 @@ However, in real world it was observed that some web servers - reset_beg_range_to_zero(); - } - break; -- /* 205 Reset Content ?? what to do on this ?? */ -- - case 300: /* redirection */ - case 301: - case 302: - case 303: - break; -- - case 206: /* Partial Content */ - if (G.beg_range != 0) - /* "Range:..." worked. Good. */ -@@ -1288,28 +1208,26 @@ int wget_main(int argc UNUSED_PARAM, char **argv) - static const char wget_longopts[] ALIGN1 = - /* name, has_arg, val */ - "continue\0" No_argument "c" -+//FIXME: -s isn't --spider, it's --save-headers! -+ "spider\0" No_argument "s" - "quiet\0" No_argument "q" - "output-document\0" Required_argument "O" - "directory-prefix\0" Required_argument "P" - "proxy\0" Required_argument "Y" - "user-agent\0" Required_argument "U" --IF_FEATURE_WGET_TIMEOUT( -- "timeout\0" Required_argument "T") -+#if ENABLE_FEATURE_WGET_TIMEOUT -+ "timeout\0" Required_argument "T" -+#endif - /* Ignored: */ --IF_DESKTOP( "tries\0" Required_argument "t") -- "header\0" Required_argument "\xff" -- "post-data\0" Required_argument "\xfe" -- "spider\0" No_argument "\xfd" -+ // "tries\0" Required_argument "t" - /* Ignored (we always use PASV): */ --IF_DESKTOP( "passive-ftp\0" No_argument "\xf0") -+ "passive-ftp\0" No_argument "\xff" -+ "header\0" Required_argument "\xfe" -+ "post-data\0" Required_argument "\xfd" - /* Ignored (we don't do ssl) */ --IF_DESKTOP( "no-check-certificate\0" No_argument "\xf0") -+ "no-check-certificate\0" No_argument "\xfc" - /* Ignored (we don't support caching) */ --IF_DESKTOP( "no-cache\0" No_argument "\xf0") --IF_DESKTOP( "no-verbose\0" No_argument "\xf0") --IF_DESKTOP( "no-clobber\0" No_argument "\xf0") --IF_DESKTOP( "no-host-directories\0" No_argument "\xf0") --IF_DESKTOP( "no-parent\0" No_argument "\xf0") -+ "no-cache\0" No_argument "\xfb" - ; - #endif - -@@ -1329,35 +1247,17 @@ IF_DESKTOP( "no-parent\0" No_argument "\xf0") - #if ENABLE_FEATURE_WGET_LONG_OPTIONS - applet_long_options = wget_longopts; - #endif -- opt_complementary = "-1" /* at least one URL */ -- IF_FEATURE_WGET_LONG_OPTIONS(":\xff::"); /* --header is a list */ -- getopt32(argv, "cqO:P:Y:U:T:+" -- /*ignored:*/ "t:" -- /*ignored:*/ "n::" -- /* wget has exactly four -n opts, all of which we can ignore: -- * -nv --no-verbose: be moderately quiet (-q is full quiet) -- * -nc --no-clobber: abort if exists, neither download to FILE.n nor overwrite FILE -- * -nH --no-host-directories: wget -r http://host/ won't create host/ -- * -np --no-parent -- * "n::" above says that we accept -n[ARG]. -- * Specifying "n:" would be a bug: "-n ARG" would eat ARG! -- */ -- , &G.fname_out, &G.dir_prefix, -+ opt_complementary = "-1" -+ IF_FEATURE_WGET_TIMEOUT(":T+") -+ IF_FEATURE_WGET_LONG_OPTIONS(":\xfe::"); -+ getopt32(argv, "csqO:P:Y:U:T:" /*ignored:*/ "t:", -+ &G.fname_out, &G.dir_prefix, - &G.proxy_flag, &G.user_agent, - IF_FEATURE_WGET_TIMEOUT(&G.timeout_seconds) IF_NOT_FEATURE_WGET_TIMEOUT(NULL), -- NULL, /* -t RETRIES */ -- NULL /* -n[ARG] */ -+ NULL /* -t RETRIES */ - IF_FEATURE_WGET_LONG_OPTIONS(, &headers_llist) - IF_FEATURE_WGET_LONG_OPTIONS(, &G.post_data) - ); --#if 0 /* option bits debug */ -- if (option_mask32 & WGET_OPT_RETRIES) bb_error_msg("-t NUM"); -- if (option_mask32 & WGET_OPT_nsomething) bb_error_msg("-nsomething"); -- if (option_mask32 & WGET_OPT_HEADER) bb_error_msg("--header"); -- if (option_mask32 & WGET_OPT_POST_DATA) bb_error_msg("--post-data"); -- if (option_mask32 & WGET_OPT_SPIDER) bb_error_msg("--spider"); -- exit(0); --#endif - argv += optind; - - #if ENABLE_FEATURE_WGET_LONG_OPTIONS diff --git a/archive-patches/busybox-1.26.2-wget-fix-for-brain-damaged-HTTP-servers.patch b/archive-patches/busybox-1.26.2-wget-fix-for-brain-damaged-HTTP-servers.patch new file mode 100644 index 0000000..b5669d5 --- /dev/null +++ b/archive-patches/busybox-1.26.2-wget-fix-for-brain-damaged-HTTP-servers.patch @@ -0,0 +1,87 @@ +From dac762a702d01c8c2d42135795cc9bf23ff324a2 Mon Sep 17 00:00:00 2001 +From: Denys Vlasenko +Date: Wed, 11 Jan 2017 20:16:45 +0100 +Subject: wget: fix for brain-damaged HTTP servers. Closes 9471 + +write(3, "GET / HTTP/1.1\r\nUser-Agent: Wget\r\nConnection: close\r\n\r\n", 74) = 74 +shutdown(3, SHUT_WR) = 0 +alarm(900) = 900 +read(3, "", 1024) = 0 +write(2, "wget: error getting response\n", 29) = 29 +exit(1) + +The peer simply does not return anything. It closes its connection. + +Probably it detects wget closing its writing end: shutdown(3, SHUT_WR). + +The point it, closing write side of the socket is _valid_ for HTTP. +wget sent the full request, it won't be sending anything more: +it will only receive the response, and that's it. + +Signed-off-by: Denys Vlasenko +--- + networking/wget.c | 26 ++++++++++++++++++-------- + 1 file changed, 18 insertions(+), 8 deletions(-) + +diff --git a/networking/wget.c b/networking/wget.c +index b082a0f..afb09f5 100644 +--- a/networking/wget.c ++++ b/networking/wget.c +@@ -141,6 +141,8 @@ + #endif + + ++#define SSL_SUPPORTED (ENABLE_FEATURE_WGET_OPENSSL || ENABLE_FEATURE_WGET_SSL_HELPER) ++ + struct host_info { + char *allocated; + const char *path; +@@ -151,7 +153,7 @@ struct host_info { + }; + static const char P_FTP[] ALIGN1 = "ftp"; + static const char P_HTTP[] ALIGN1 = "http"; +-#if ENABLE_FEATURE_WGET_OPENSSL || ENABLE_FEATURE_WGET_SSL_HELPER ++#if SSL_SUPPORTED + static const char P_HTTPS[] ALIGN1 = "https"; + #endif + +@@ -452,7 +454,7 @@ static void parse_url(const char *src_url, struct host_info *h) + if (strcmp(url, P_FTP) == 0) { + h->port = bb_lookup_port(P_FTP, "tcp", 21); + } else +-#if ENABLE_FEATURE_WGET_OPENSSL || ENABLE_FEATURE_WGET_SSL_HELPER ++#if SSL_SUPPORTED + if (strcmp(url, P_HTTPS) == 0) { + h->port = bb_lookup_port(P_HTTPS, "tcp", 443); + h->protocol = P_HTTPS; +@@ -1093,12 +1095,20 @@ static void download_one_url(const char *url) + } + + fflush(sfp); +- /* If we use SSL helper, keeping our end of the socket open for writing +- * makes our end (i.e. the same fd!) readable (EAGAIN instead of EOF) +- * even after child closes its copy of the fd. +- * This helps: +- */ +- shutdown(fileno(sfp), SHUT_WR); ++ ++/* Tried doing this unconditionally. ++ * Cloudflare and nginx/1.11.5 are shocked to see SHUT_WR on non-HTTPS. ++ */ ++#if SSL_SUPPORTED ++ if (target.protocol == P_HTTPS) { ++ /* If we use SSL helper, keeping our end of the socket open for writing ++ * makes our end (i.e. the same fd!) readable (EAGAIN instead of EOF) ++ * even after child closes its copy of the fd. ++ * This helps: ++ */ ++ shutdown(fileno(sfp), SHUT_WR); ++ } ++#endif + + /* + * Retrieve HTTP response line and check for "200" status code. +-- +cgit v0.12 + + diff --git a/make/applications.mk b/make/applications.mk index b234e2b..6675607 100755 --- a/make/applications.mk +++ b/make/applications.mk @@ -28,7 +28,7 @@ $(D)/busybox: $(ARCHIVE)/busybox-$(BUSYBOX_VER).tar.bz2 | $(TARGETPREFIX) $(PATCH)/busybox-1.18-hack-init-s-console.patch; \ $(PATCH)/busybox-1.21.1-changing-the-order-sys-tree-is-scanned-with-mdev.diff; \ $(PATCH)/busybox-1.21.1-mdev.diff; \ - $(PATCH)/busybox-1.26.2-use-wget-1.24.1.patch; \ + $(PATCH)/busybox-1.26.2-wget-fix-for-brain-damaged-HTTP-servers.patch; \ cp $(PATCHES)/busybox-1.22.1-hd1.config .config && \ sed -i -e 's#^CONFIG_PREFIX.*#CONFIG_PREFIX="$(TARGETPREFIX)"#' .config && \ $(MAKE) all CROSS_COMPILE=$(TARGET)- CFLAGS_EXTRA="$(TARGET_CFLAGS)" && \ -- 2.39.5