Replace die handlers with error returning so download_one_url can retry from 
the beginning.
When retries is 1 (default) the behaviour should be the same as before.

Signed-off-by: Martin Lewis <martin.lewis....@gmail.com>
---
 networking/wget.c | 120 ++++++++++++++++++++++++++++++++++++++++++------------
 1 file changed, 95 insertions(+), 25 deletions(-)

diff --git a/networking/wget.c b/networking/wget.c
index e9fdd9f..4b1961c 100644
--- a/networking/wget.c
+++ b/networking/wget.c
@@ -123,14 +123,14 @@
 //usage:#define wget_trivial_usage
 //usage:       IF_FEATURE_WGET_LONG_OPTIONS(
 //usage:       "[-c|--continue] [--spider] [-q|--quiet] [-O|--output-document 
FILE]\n"
-//usage:       "       [--header 'header: value'] [-Y|--proxy on/off] [-P 
DIR]\n"
+//usage:       "       [--header 'header: value'] [-Y|--proxy on/off] 
[-t|--tries TRIES] [-P DIR]\n"
 /* Since we ignore these opts, we don't show them in --help */
 /* //usage:    "       [--no-check-certificate] [--no-cache] [--passive-ftp] 
[-t TRIES]" */
 /* //usage:    "       [-nv] [-nc] [-nH] [-np]" */
 //usage:       "       [-S|--server-response] [-U|--user-agent AGENT]" 
IF_FEATURE_WGET_TIMEOUT(" [-T SEC]") " URL..."
 //usage:       )
 //usage:       IF_NOT_FEATURE_WGET_LONG_OPTIONS(
-//usage:       "[-cq] [-O FILE] [-Y on/off] [-P DIR] [-S] [-U AGENT]"
+//usage:       "[-cq] [-O FILE] [-Y on/off] [-t TRIES] [-P DIR] [-S] [-U 
AGENT]"
 //usage:                       IF_FEATURE_WGET_TIMEOUT(" [-T SEC]") " URL..."
 //usage:       )
 //usage:#define wget_full_usage "\n\n"
@@ -149,6 +149,7 @@
 //usage:     "\n       -O FILE         Save to FILE ('-' for stdout)"
 //usage:     "\n       -U STR          Use STR for User-Agent header"
 //usage:     "\n       -Y on/off       Use proxy"
+//usage:     "\n       -t TRIES        Set number of retries to TRIES (0 
unlimits)"
 
 #include "libbb.h"
 
@@ -233,6 +234,7 @@ struct globals {
        char *fname_out;        /* where to direct output (-O) */
        const char *proxy_flag; /* Use proxies if env vars are set */
        const char *user_agent; /* "User-Agent" header field */
+       unsigned tries; /* For -t option */
        int output_fd;
        int o_flags;
 #if ENABLE_FEATURE_WGET_TIMEOUT
@@ -402,6 +404,7 @@ static int is_ip_address(const char *string)
 }
 #endif
 
+/* Return NULL if connect() fails */
 static FILE *open_socket(len_and_sockaddr *lsa)
 {
        int fd;
@@ -409,13 +412,19 @@ static FILE *open_socket(len_and_sockaddr *lsa)
 #if ENABLE_FEATURE_WGET_TIMEOUT
        struct timeval timeout = {G.timeout_seconds, 0};
 #endif
+
        fd = xsocket(lsa->u.sa.sa_family, SOCK_STREAM, 0);
 #if ENABLE_FEATURE_WGET_TIMEOUT
        if (setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof (timeout)) 
< 0) {
                bb_perror_msg_and_die("setsockopt failed\n");
        }
 #endif
-       xconnect(fd, &lsa->u.sa, lsa->len);
+       if (connect(fd, &lsa->u.sa, lsa->len) < 0) {
+               /* Failure */
+               bb_perror_msg("connect failed");
+               close(fd);
+               return NULL;
+       }
 
        /* glibc 2.4 seems to try seeking on it - ??! */
        /* hopefully it understands what ESPIPE means... */
@@ -584,14 +593,16 @@ static int fread_buffered(char *buffer, size_t len, FILE 
*fp)
        return fread(buffer, 1, len, fp);
 }
 
-/* Returns '\n' if it was seen, else '\0'. Trims at first '\r' or '\n' */
-static char fgets_trim_sanitize(FILE *fp, const char *fmt)
+/* Returns '\n' if it was seen, -1 if timeout occured, else '\0'. Trims at 
first '\r' or '\n' */
+static signed char fgets_trim_sanitize(FILE *fp, const char *fmt)
 {
        char c;
        char *buf_ptr;
 
-       if (fgets_read_to_newline(G.wget_buf, sizeof(G.wget_buf), fp) == NULL)
-               bb_perror_msg_and_die("error getting response");
+       if (fgets_read_to_newline(G.wget_buf, sizeof(G.wget_buf), fp) == NULL) {
+               bb_perror_msg("error getting response");
+               return -1;
+       }
 
        buf_ptr = strchrnul(G.wget_buf, '\n');
        c = *buf_ptr;
@@ -629,7 +640,9 @@ static int ftpcmd(const char *s1, const char *s2, FILE *fp)
        /* Read until "Nxx something" is received */
        G.wget_buf[3] = 0;
        do {
-               fgets_trim_sanitize(fp, "%s\n");
+               if (fgets_trim_sanitize(fp, "%s\n") == -1) {
+                       xfunc_die();
+               }
        } while (!isdigit(G.wget_buf[0]) || G.wget_buf[3] != ' ');
 
        G.wget_buf[3] = '\0';
@@ -734,6 +747,11 @@ static char *get_sanitized_hdr(FILE *fp)
        /* retrieve header line */
        c = fgets_trim_sanitize(fp, "  %s\n");
 
+       if (c == -1) {
+               /* Timed out */
+               return NULL;
+       }
+
        /* end of the headers? */
        if (G.wget_buf[0] == '\0')
                return NULL;
@@ -916,7 +934,11 @@ static FILE* prepare_ftp_session(FILE **dfpp, struct 
host_info *target, len_and_
        char *pass;
        int port;
 
+       /* TODO: Add retry support for ftp */
        sfp = open_socket(lsa);
+       if (!sfp) {
+               xfunc_die();
+       }
 #if ENABLE_FEATURE_WGET_HTTPS
        if (target->protocol == P_FTPS)
                spawn_ssl_client(target->host, fileno(sfp), 
TLSLOOP_EXIT_ON_LOCAL_EOF);
@@ -971,7 +993,11 @@ static FILE* prepare_ftp_session(FILE **dfpp, struct 
host_info *target, len_and_
 
        set_nport(&lsa->u.sa, htons(port));
 
+       /* TODO: Add retry support for ftp */
        *dfpp = open_socket(lsa);
+       if (!*dfpp) {
+               xfunc_die();
+       }
 
 #if ENABLE_FEATURE_WGET_HTTPS
        if (target->protocol == P_FTPS) {
@@ -1000,7 +1026,8 @@ static FILE* prepare_ftp_session(FILE **dfpp, struct 
host_info *target, len_and_
        return sfp;
 }
 
-static void NOINLINE retrieve_file_data(FILE *dfp)
+/* Return -1 if times out so we can retry */
+static int NOINLINE retrieve_file_data(FILE *dfp)
 {
 #if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
 # if ENABLE_FEATURE_WGET_TIMEOUT
@@ -1053,6 +1080,8 @@ static void NOINLINE retrieve_file_data(FILE *dfp)
                                        rdsz = (unsigned)G.content_len;
                                }
                        }
+                       /* We probably have some data in fgets_buffer, so we 
need to
+                          flush it first */
                        n = fread_buffered(G.wget_buf, rdsz, dfp);
 
                        if (n > 0) {
@@ -1093,7 +1122,7 @@ static void NOINLINE retrieve_file_data(FILE *dfp)
 # if ENABLE_FEATURE_WGET_TIMEOUT
                                if (second_cnt != 0 && --second_cnt == 0) {
                                        progress_meter(PROGRESS_END);
-                                       bb_error_msg_and_die("download timed 
out");
+                                       return -1;
                                }
 # endif
                                /* We used to loop back to poll here,
@@ -1117,7 +1146,9 @@ static void NOINLINE retrieve_file_data(FILE *dfp)
                        break;
 
                /* Each chunk ends with "\r\n" - eat it */
-               fgets_trim_sanitize(dfp, NULL);
+               if (fgets_trim_sanitize(dfp, NULL) == -1 ) {
+                       return -1;
+               }
  get_clen:
                /* chunk size format is "HEXNUM[;name[=val]]\r\n" */
                fgets_trim_sanitize(dfp, NULL);
@@ -1156,6 +1187,8 @@ static void NOINLINE retrieve_file_data(FILE *dfp)
        G.chunked = 0;  /* makes it show 100% even for chunked download */
        G.got_clen = 1; /* makes it show 100% even for download of (formerly) 
unknown size */
        progress_meter(PROGRESS_END);
+
+       return 0;
 }
 
 static void download_one_url(const char *url)
@@ -1165,6 +1198,7 @@ static void download_one_url(const char *url)
        len_and_sockaddr *lsa;
        FILE *sfp;                      /* socket to web/ftp server         */
        FILE *dfp;                      /* socket to ftp server (data)      */
+       unsigned cur_tries = 0;         /* number of tries so far           */
        char *fname_out_alloc;
        char *redirected_path = NULL;
        struct host_info server;
@@ -1230,9 +1264,21 @@ static void download_one_url(const char *url)
                 * We are not sure it exists on remote side */
        }
 
+ retry:
+       cur_tries++;
+       if (G.tries != 0 && cur_tries > G.tries) {
+               if (G.tries != 1) /* Show message about the tries only if was 
set to more than one */
+                       bb_error_msg_and_die("Gave up after %u tries", G.tries);
+               else
+                       xfunc_die();
+       }
        redir_limit = 5;
  resolve_lsa:
-       lsa = xhost2sockaddr(server.host, server.port);
+       /* If DNS resolution fails, retry, don't die */
+       lsa = host2sockaddr(server.host, server.port);
+       if (!lsa)
+               goto retry;
+
        if (!(option_mask32 & WGET_OPT_QUIET)) {
                char *s = xmalloc_sockaddr2dotted(&lsa->u.sa);
                fprintf(stderr, "Connecting to %s (%s)\n", server.host, s);
@@ -1262,6 +1308,8 @@ static void download_one_url(const char *url)
 # if ENABLE_FEATURE_WGET_HTTPS
                        if (fd < 0) { /* no openssl? try internal */
                                sfp = open_socket(lsa);
+                               if (!sfp)
+                                       goto retry;
                                spawn_ssl_client(server.host, fileno(sfp), 
/*flags*/ 0);
                                goto socket_opened;
                        }
@@ -1274,15 +1322,22 @@ static void download_one_url(const char *url)
                        goto socket_opened;
                }
                sfp = open_socket(lsa);
+               if (!sfp) {
+                       goto retry;
+               }
  socket_opened:
 #elif ENABLE_FEATURE_WGET_HTTPS
                /* Only internal TLS support is configured */
                sfp = open_socket(lsa);
-               if (server.protocol == P_HTTPS)
+               if (!sfp)
+                       goto retry;
+               if (server.protocol == P_HTTPS) {
                        spawn_ssl_client(server.host, fileno(sfp), /*flags*/ 0);
 #else
                /* ssl (https) support is not configured */
                sfp = open_socket(lsa);
+               if (!sfp)
+                       goto retry;
 #endif
                /* Send HTTP request */
                if (use_proxy) {
@@ -1358,7 +1413,11 @@ static void download_one_url(const char *url)
                 * Retrieve HTTP response line and check for "200" status code.
                 */
  read_response:
-               fgets_trim_sanitize(sfp, "  %s\n");
+               if (fgets_trim_sanitize(sfp, "  %s\n") == -1) {
+                       /* Timed out */
+                       bb_error_msg("timed out");
+                       goto retry;
+               }
 
                str = G.wget_buf;
                str = skip_non_whitespace(str);
@@ -1433,7 +1492,8 @@ However, in real world it was observed that some web 
servers
                        /* Partial Content even though we did not ask for it??? 
*/
                        /* fall through */
                default:
-                       bb_error_msg_and_die("server returned error: %s", 
G.wget_buf);
+                       bb_error_msg("server returned error: %s", G.wget_buf);
+                       goto retry;
                }
 
                /*
@@ -1459,19 +1519,24 @@ However, in real world it was observed that some web 
servers
                        if (key == KEY_content_length) {
                                G.content_len = BB_STRTOOFF(str, NULL, 10);
                                if (G.content_len < 0 || errno) {
-                                       bb_error_msg_and_die("content-length %s 
is garbage", str);
+                                       bb_error_msg("content-length %s is 
garbage", str);
+                                       goto retry;
                                }
                                G.got_clen = 1;
                                continue;
                        }
                        if (key == KEY_transfer_encoding) {
-                               if (strcmp(str_tolower(str), "chunked") != 0)
-                                       bb_error_msg_and_die("transfer encoding 
'%s' is not supported", str);
+                               if (strcmp(str_tolower(str), "chunked") != 0) {
+                                       bb_error_msg("transfer encoding '%s' is 
not supported", str);
+                                       goto retry;
+                               }
                                G.chunked = 1;
                        }
                        if (key == KEY_location && status >= 300) {
-                               if (--redir_limit == 0)
-                                       bb_error_msg_and_die("too many 
redirections");
+                               if (--redir_limit == 0) {
+                                       bb_error_msg("too many redirections");
+                                       goto retry;
+                               }
                                fclose(sfp);
                                if (str[0] == '/') {
                                        free(redirected_path);
@@ -1510,13 +1575,18 @@ However, in real world it was observed that some web 
servers
        free(lsa);
 
        if (!(option_mask32 & WGET_OPT_SPIDER)) {
+               int retrieve_retval;
                if (G.output_fd < 0)
                        G.output_fd = xopen(G.fname_out, G.o_flags);
-               retrieve_file_data(dfp);
+               retrieve_retval = retrieve_file_data(dfp);
                if (!(option_mask32 & WGET_OPT_OUTNAME)) {
                        xclose(G.output_fd);
                        G.output_fd = -1;
                }
+               if (retrieve_retval < 0) { /* We timed out, if we retry so the 
fd is closed */
+                       bb_error_msg("download timed out");
+                       goto retry;
+               }
        }
 
        if (dfp != sfp) {
@@ -1551,8 +1621,8 @@ int wget_main(int argc UNUSED_PARAM, char **argv)
                "user-agent\0"       Required_argument "U"
 IF_FEATURE_WGET_TIMEOUT(
                "timeout\0"          Required_argument "T")
+               "tries\0"            Required_argument "t"
                /* Ignored: */
-IF_DESKTOP(    "tries\0"            Required_argument "t")
                "header\0"           Required_argument "\xff"
                "post-data\0"        Required_argument "\xfe"
                "spider\0"           No_argument       "\xfd"
@@ -1582,14 +1652,14 @@ IF_DESKTOP(     "no-parent\0"        No_argument       
"\xf0")
 #if ENABLE_FEATURE_WGET_TIMEOUT
        G.timeout_seconds = 900;
 #endif
+       G.tries = 1;
        G.proxy_flag = "on";   /* use proxies if env vars are set */
        G.user_agent = "Wget"; /* "User-Agent" header field */
 
 #if ENABLE_FEATURE_WGET_LONG_OPTIONS
 #endif
        GETOPT32(argv, "^"
-               "cqSO:P:Y:U:T:+"
-               /*ignored:*/ "t:"
+               "cqSO:P:Y:U:T:+t:+"
                /*ignored:*/ "n::"
                /* wget has exactly four -n<letter> opts, all of which we can 
ignore:
                 * -nv --no-verbose: be moderately quiet (-q is full quiet)
@@ -1606,7 +1676,7 @@ IF_DESKTOP(       "no-parent\0"        No_argument       
"\xf0")
                , &G.fname_out, &G.dir_prefix,
                &G.proxy_flag, &G.user_agent,
                IF_FEATURE_WGET_TIMEOUT(&G.timeout_seconds) 
IF_NOT_FEATURE_WGET_TIMEOUT(NULL),
-               NULL, /* -t RETRIES */
+               &G.tries, /* -t RETRIES */
                NULL  /* -n[ARG] */
                IF_FEATURE_WGET_LONG_OPTIONS(, &headers_llist)
                IF_FEATURE_WGET_LONG_OPTIONS(, &G.post_data)
-- 
1.9.1

_______________________________________________
busybox mailing list
busybox@busybox.net
http://lists.busybox.net/mailman/listinfo/busybox

Reply via email to