Replace die handlers with error returning so download_one_url can retry from the beginning. When retries is 1 (default) the behaviour should be the same as before.
Signed-off-by: Martin Lewis <martin.lewis....@gmail.com> --- networking/wget.c | 120 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 95 insertions(+), 25 deletions(-) diff --git a/networking/wget.c b/networking/wget.c index e9fdd9f..4b1961c 100644 --- a/networking/wget.c +++ b/networking/wget.c @@ -123,14 +123,14 @@ //usage:#define wget_trivial_usage //usage: IF_FEATURE_WGET_LONG_OPTIONS( //usage: "[-c|--continue] [--spider] [-q|--quiet] [-O|--output-document FILE]\n" -//usage: " [--header 'header: value'] [-Y|--proxy on/off] [-P DIR]\n" +//usage: " [--header 'header: value'] [-Y|--proxy on/off] [-t|--tries TRIES] [-P DIR]\n" /* Since we ignore these opts, we don't show them in --help */ /* //usage: " [--no-check-certificate] [--no-cache] [--passive-ftp] [-t TRIES]" */ /* //usage: " [-nv] [-nc] [-nH] [-np]" */ //usage: " [-S|--server-response] [-U|--user-agent AGENT]" IF_FEATURE_WGET_TIMEOUT(" [-T SEC]") " URL..." //usage: ) //usage: IF_NOT_FEATURE_WGET_LONG_OPTIONS( -//usage: "[-cq] [-O FILE] [-Y on/off] [-P DIR] [-S] [-U AGENT]" +//usage: "[-cq] [-O FILE] [-Y on/off] [-t TRIES] [-P DIR] [-S] [-U AGENT]" //usage: IF_FEATURE_WGET_TIMEOUT(" [-T SEC]") " URL..." //usage: ) //usage:#define wget_full_usage "\n\n" @@ -149,6 +149,7 @@ //usage: "\n -O FILE Save to FILE ('-' for stdout)" //usage: "\n -U STR Use STR for User-Agent header" //usage: "\n -Y on/off Use proxy" +//usage: "\n -t TRIES Set number of retries to TRIES (0 unlimits)" #include "libbb.h" @@ -233,6 +234,7 @@ struct globals { char *fname_out; /* where to direct output (-O) */ const char *proxy_flag; /* Use proxies if env vars are set */ const char *user_agent; /* "User-Agent" header field */ + unsigned tries; /* For -t option */ int output_fd; int o_flags; #if ENABLE_FEATURE_WGET_TIMEOUT @@ -402,6 +404,7 @@ static int is_ip_address(const char *string) } #endif +/* Return NULL if connect() fails */ static FILE *open_socket(len_and_sockaddr *lsa) { int fd; @@ -409,13 +412,19 @@ static FILE *open_socket(len_and_sockaddr *lsa) #if ENABLE_FEATURE_WGET_TIMEOUT struct timeval timeout = {G.timeout_seconds, 0}; #endif + fd = xsocket(lsa->u.sa.sa_family, SOCK_STREAM, 0); #if ENABLE_FEATURE_WGET_TIMEOUT if (setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof (timeout)) < 0) { bb_perror_msg_and_die("setsockopt failed\n"); } #endif - xconnect(fd, &lsa->u.sa, lsa->len); + if (connect(fd, &lsa->u.sa, lsa->len) < 0) { + /* Failure */ + bb_perror_msg("connect failed"); + close(fd); + return NULL; + } /* glibc 2.4 seems to try seeking on it - ??! */ /* hopefully it understands what ESPIPE means... */ @@ -584,14 +593,16 @@ static int fread_buffered(char *buffer, size_t len, FILE *fp) return fread(buffer, 1, len, fp); } -/* Returns '\n' if it was seen, else '\0'. Trims at first '\r' or '\n' */ -static char fgets_trim_sanitize(FILE *fp, const char *fmt) +/* Returns '\n' if it was seen, -1 if timeout occured, else '\0'. Trims at first '\r' or '\n' */ +static signed char fgets_trim_sanitize(FILE *fp, const char *fmt) { char c; char *buf_ptr; - if (fgets_read_to_newline(G.wget_buf, sizeof(G.wget_buf), fp) == NULL) - bb_perror_msg_and_die("error getting response"); + if (fgets_read_to_newline(G.wget_buf, sizeof(G.wget_buf), fp) == NULL) { + bb_perror_msg("error getting response"); + return -1; + } buf_ptr = strchrnul(G.wget_buf, '\n'); c = *buf_ptr; @@ -629,7 +640,9 @@ static int ftpcmd(const char *s1, const char *s2, FILE *fp) /* Read until "Nxx something" is received */ G.wget_buf[3] = 0; do { - fgets_trim_sanitize(fp, "%s\n"); + if (fgets_trim_sanitize(fp, "%s\n") == -1) { + xfunc_die(); + } } while (!isdigit(G.wget_buf[0]) || G.wget_buf[3] != ' '); G.wget_buf[3] = '\0'; @@ -734,6 +747,11 @@ static char *get_sanitized_hdr(FILE *fp) /* retrieve header line */ c = fgets_trim_sanitize(fp, " %s\n"); + if (c == -1) { + /* Timed out */ + return NULL; + } + /* end of the headers? */ if (G.wget_buf[0] == '\0') return NULL; @@ -916,7 +934,11 @@ static FILE* prepare_ftp_session(FILE **dfpp, struct host_info *target, len_and_ char *pass; int port; + /* TODO: Add retry support for ftp */ sfp = open_socket(lsa); + if (!sfp) { + xfunc_die(); + } #if ENABLE_FEATURE_WGET_HTTPS if (target->protocol == P_FTPS) spawn_ssl_client(target->host, fileno(sfp), TLSLOOP_EXIT_ON_LOCAL_EOF); @@ -971,7 +993,11 @@ static FILE* prepare_ftp_session(FILE **dfpp, struct host_info *target, len_and_ set_nport(&lsa->u.sa, htons(port)); + /* TODO: Add retry support for ftp */ *dfpp = open_socket(lsa); + if (!*dfpp) { + xfunc_die(); + } #if ENABLE_FEATURE_WGET_HTTPS if (target->protocol == P_FTPS) { @@ -1000,7 +1026,8 @@ static FILE* prepare_ftp_session(FILE **dfpp, struct host_info *target, len_and_ return sfp; } -static void NOINLINE retrieve_file_data(FILE *dfp) +/* Return -1 if times out so we can retry */ +static int NOINLINE retrieve_file_data(FILE *dfp) { #if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT # if ENABLE_FEATURE_WGET_TIMEOUT @@ -1053,6 +1080,8 @@ static void NOINLINE retrieve_file_data(FILE *dfp) rdsz = (unsigned)G.content_len; } } + /* We probably have some data in fgets_buffer, so we need to + flush it first */ n = fread_buffered(G.wget_buf, rdsz, dfp); if (n > 0) { @@ -1093,7 +1122,7 @@ static void NOINLINE retrieve_file_data(FILE *dfp) # if ENABLE_FEATURE_WGET_TIMEOUT if (second_cnt != 0 && --second_cnt == 0) { progress_meter(PROGRESS_END); - bb_error_msg_and_die("download timed out"); + return -1; } # endif /* We used to loop back to poll here, @@ -1117,7 +1146,9 @@ static void NOINLINE retrieve_file_data(FILE *dfp) break; /* Each chunk ends with "\r\n" - eat it */ - fgets_trim_sanitize(dfp, NULL); + if (fgets_trim_sanitize(dfp, NULL) == -1 ) { + return -1; + } get_clen: /* chunk size format is "HEXNUM[;name[=val]]\r\n" */ fgets_trim_sanitize(dfp, NULL); @@ -1156,6 +1187,8 @@ static void NOINLINE retrieve_file_data(FILE *dfp) G.chunked = 0; /* makes it show 100% even for chunked download */ G.got_clen = 1; /* makes it show 100% even for download of (formerly) unknown size */ progress_meter(PROGRESS_END); + + return 0; } static void download_one_url(const char *url) @@ -1165,6 +1198,7 @@ static void download_one_url(const char *url) len_and_sockaddr *lsa; FILE *sfp; /* socket to web/ftp server */ FILE *dfp; /* socket to ftp server (data) */ + unsigned cur_tries = 0; /* number of tries so far */ char *fname_out_alloc; char *redirected_path = NULL; struct host_info server; @@ -1230,9 +1264,21 @@ static void download_one_url(const char *url) * We are not sure it exists on remote side */ } + retry: + cur_tries++; + if (G.tries != 0 && cur_tries > G.tries) { + if (G.tries != 1) /* Show message about the tries only if was set to more than one */ + bb_error_msg_and_die("Gave up after %u tries", G.tries); + else + xfunc_die(); + } redir_limit = 5; resolve_lsa: - lsa = xhost2sockaddr(server.host, server.port); + /* If DNS resolution fails, retry, don't die */ + lsa = host2sockaddr(server.host, server.port); + if (!lsa) + goto retry; + if (!(option_mask32 & WGET_OPT_QUIET)) { char *s = xmalloc_sockaddr2dotted(&lsa->u.sa); fprintf(stderr, "Connecting to %s (%s)\n", server.host, s); @@ -1262,6 +1308,8 @@ static void download_one_url(const char *url) # if ENABLE_FEATURE_WGET_HTTPS if (fd < 0) { /* no openssl? try internal */ sfp = open_socket(lsa); + if (!sfp) + goto retry; spawn_ssl_client(server.host, fileno(sfp), /*flags*/ 0); goto socket_opened; } @@ -1274,15 +1322,22 @@ static void download_one_url(const char *url) goto socket_opened; } sfp = open_socket(lsa); + if (!sfp) { + goto retry; + } socket_opened: #elif ENABLE_FEATURE_WGET_HTTPS /* Only internal TLS support is configured */ sfp = open_socket(lsa); - if (server.protocol == P_HTTPS) + if (!sfp) + goto retry; + if (server.protocol == P_HTTPS) { spawn_ssl_client(server.host, fileno(sfp), /*flags*/ 0); #else /* ssl (https) support is not configured */ sfp = open_socket(lsa); + if (!sfp) + goto retry; #endif /* Send HTTP request */ if (use_proxy) { @@ -1358,7 +1413,11 @@ static void download_one_url(const char *url) * Retrieve HTTP response line and check for "200" status code. */ read_response: - fgets_trim_sanitize(sfp, " %s\n"); + if (fgets_trim_sanitize(sfp, " %s\n") == -1) { + /* Timed out */ + bb_error_msg("timed out"); + goto retry; + } str = G.wget_buf; str = skip_non_whitespace(str); @@ -1433,7 +1492,8 @@ However, in real world it was observed that some web servers /* Partial Content even though we did not ask for it??? */ /* fall through */ default: - bb_error_msg_and_die("server returned error: %s", G.wget_buf); + bb_error_msg("server returned error: %s", G.wget_buf); + goto retry; } /* @@ -1459,19 +1519,24 @@ However, in real world it was observed that some web servers if (key == KEY_content_length) { G.content_len = BB_STRTOOFF(str, NULL, 10); if (G.content_len < 0 || errno) { - bb_error_msg_and_die("content-length %s is garbage", str); + bb_error_msg("content-length %s is garbage", str); + goto retry; } G.got_clen = 1; continue; } if (key == KEY_transfer_encoding) { - if (strcmp(str_tolower(str), "chunked") != 0) - bb_error_msg_and_die("transfer encoding '%s' is not supported", str); + if (strcmp(str_tolower(str), "chunked") != 0) { + bb_error_msg("transfer encoding '%s' is not supported", str); + goto retry; + } G.chunked = 1; } if (key == KEY_location && status >= 300) { - if (--redir_limit == 0) - bb_error_msg_and_die("too many redirections"); + if (--redir_limit == 0) { + bb_error_msg("too many redirections"); + goto retry; + } fclose(sfp); if (str[0] == '/') { free(redirected_path); @@ -1510,13 +1575,18 @@ However, in real world it was observed that some web servers free(lsa); if (!(option_mask32 & WGET_OPT_SPIDER)) { + int retrieve_retval; if (G.output_fd < 0) G.output_fd = xopen(G.fname_out, G.o_flags); - retrieve_file_data(dfp); + retrieve_retval = retrieve_file_data(dfp); if (!(option_mask32 & WGET_OPT_OUTNAME)) { xclose(G.output_fd); G.output_fd = -1; } + if (retrieve_retval < 0) { /* We timed out, if we retry so the fd is closed */ + bb_error_msg("download timed out"); + goto retry; + } } if (dfp != sfp) { @@ -1551,8 +1621,8 @@ int wget_main(int argc UNUSED_PARAM, char **argv) "user-agent\0" Required_argument "U" IF_FEATURE_WGET_TIMEOUT( "timeout\0" Required_argument "T") + "tries\0" Required_argument "t" /* Ignored: */ -IF_DESKTOP( "tries\0" Required_argument "t") "header\0" Required_argument "\xff" "post-data\0" Required_argument "\xfe" "spider\0" No_argument "\xfd" @@ -1582,14 +1652,14 @@ IF_DESKTOP( "no-parent\0" No_argument "\xf0") #if ENABLE_FEATURE_WGET_TIMEOUT G.timeout_seconds = 900; #endif + G.tries = 1; G.proxy_flag = "on"; /* use proxies if env vars are set */ G.user_agent = "Wget"; /* "User-Agent" header field */ #if ENABLE_FEATURE_WGET_LONG_OPTIONS #endif GETOPT32(argv, "^" - "cqSO:P:Y:U:T:+" - /*ignored:*/ "t:" + "cqSO:P:Y:U:T:+t:+" /*ignored:*/ "n::" /* wget has exactly four -n<letter> opts, all of which we can ignore: * -nv --no-verbose: be moderately quiet (-q is full quiet) @@ -1606,7 +1676,7 @@ IF_DESKTOP( "no-parent\0" No_argument "\xf0") , &G.fname_out, &G.dir_prefix, &G.proxy_flag, &G.user_agent, IF_FEATURE_WGET_TIMEOUT(&G.timeout_seconds) IF_NOT_FEATURE_WGET_TIMEOUT(NULL), - NULL, /* -t RETRIES */ + &G.tries, /* -t RETRIES */ NULL /* -n[ARG] */ IF_FEATURE_WGET_LONG_OPTIONS(, &headers_llist) IF_FEATURE_WGET_LONG_OPTIONS(, &G.post_data) -- 1.9.1 _______________________________________________ busybox mailing list busybox@busybox.net http://lists.busybox.net/mailman/listinfo/busybox