Hello.
Preambula: we have server in our local network, which is intended to share
some set of files wia http. There are a lot of files, and a lot of users,
so most of time server returns 503 http error, because it
overloaded. Sometimes, if server returns not 503 error, it sends
zero-lenght file (really don't know why, but this is not bug in wget, as
users, who try to do this with anohter downloaders, like FlashGet on
Windows or so on, report same problem), but this is invalid response, as
valid response from server is a rather big file. This is very convinient
for *nix users in our network to make wget retry downloading in theese
booth cases, 503 error and zero-lenght file returned.
Ambula: this is patch, which adds two options: --http-503-retry and
--http-zero-len-retry, which intended to do this. It should apply ok to
1.11 branch.
I'm not sure this will be useful for anyone but users of our network, or
that changes I made, done in right way, but i personally prefer to share
this patch.
Btw, i googled for "ignore 503 error with wget" and in results I found that
503 error is not fatal. Is there some special reason, why this is not
retry-case for wget?
--
E.L.K.
Saturday, 16 of May, 2009, 16:38
ps. I'm not subscribed, so, please, CC me.
diff --git a/src/http.c b/src/http.c
index be5bbe9..1510ef7 100644
--- a/src/http.c
+++ b/src/http.c
@@ -1612,6 +1612,14 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy)
contlen = parsed;
}
+ /* Check if we should retry because of zero-length content*/
+
+ if (opt.http_retry_if_zero_lenght && (contlen <= 0))
+ {
+ CLOSE_INVALIDATE(sock);
+ return ZEROLEN;
+ }
+
/* Check for keep-alive related responses. */
if (!inhibit_keep_alive && contlen != -1)
{
@@ -1702,6 +1710,13 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy)
hs->error = xstrdup (message);
xfree_null (message);
+ if (opt.http_retry_if_503 && statcode == HTTP_STATUS_UNAVAILABLE)
+ {
+ /* if statcode == 503 and we have option to retry on this error*/
+ CLOSE_INVALIDATE (sock);
+ return TEMP_UNAVAIL;
+ }
+
type = resp_header_strdup (resp, "Content-Type");
if (type)
{
@@ -2205,7 +2220,8 @@ File `%s' already there; not retrieving.\n\n"), *hstat.local_file);
{
case HERR: case HEOF: case CONSOCKERR: case CONCLOSED:
case CONERROR: case READERR: case WRITEFAILED:
- case RANGEERR: case FOPEN_EXCL_ERR:
+ case RANGEERR: case FOPEN_EXCL_ERR: case TEMP_UNAVAIL:
+ case ZEROLEN:
/* Non-fatal errors continue executing the loop, which will
bring them to "while" statement at the end, to judge
whether the number of tries was exceeded. */
diff --git a/src/init.c b/src/init.c
index e6c87a8..d23fb7e 100644
--- a/src/init.c
+++ b/src/init.c
@@ -165,9 +165,11 @@ static struct {
{ "header", NULL, cmd_spec_header },
{ "htmlextension", &opt.html_extension, cmd_boolean },
{ "htmlify", NULL, cmd_spec_htmlify },
+ { "http503retry", &opt.http_retry_if_503, cmd_boolean },
{ "httpkeepalive", &opt.http_keep_alive, cmd_boolean },
{ "httppasswd", &opt.http_passwd, cmd_string }, /* deprecated */
{ "httppassword", &opt.http_passwd, cmd_string },
{ "httpproxy", &opt.http_proxy, cmd_string },
+ { "httpretryzerolen", &opt.http_retry_if_zero_lenght, cmd_boolean },
{ "httpsproxy", &opt.https_proxy, cmd_string },
{ "httpuser", &opt.http_user, cmd_string },
diff --git a/src/main.c b/src/main.c
index f0864b8..1a74091 100644
--- a/src/main.c
+++ b/src/main.c
@@ -197,6 +197,8 @@ struct cmdline_option option_data[] =
{ "html-extension", 'E', OPT_BOOLEAN, "htmlextension", -1 },
{ "htmlify", 0, OPT_BOOLEAN, "htmlify", -1 },
{ "http-keep-alive", 0, OPT_BOOLEAN, "httpkeepalive", -1 },
+ { "http-503-retry", 0, OPT_BOOLEAN, "http503retry", -1 },
+ { "http-zero-len-retry", 0, OPT_BOOLEAN, "httpretryzerolen", -1 },
{ "http-passwd", 0, OPT_VALUE, "httppassword", -1 }, /* deprecated */
{ "http-password", 0, OPT_VALUE, "httppassword", -1 },
{ "http-user", 0, OPT_VALUE, "httpuser", -1 },
@@ -514,6 +516,10 @@ HTTP options:\n"),
N_("\
-E, --html-extension save HTML documents with `.html' extension.\n"),
N_("\
+ --http-503-retry retry if HTTP return code is 503.\n"),
+ N_("\
+ --http-zero-len-retry retry if no `Content-Length' header, or if lenght is 0.\n"),
+ N_("\
--ignore-length ignore `Content-Length' header field.\n"),
N_("\
--header=STRING insert STRING among the headers.\n"),
diff --git a/src/options.h b/src/options.h
index 714fd24..aa7d3de 100644
--- a/src/options.h
+++ b/src/options.h
@@ -91,6 +91,10 @@ struct options
char *http_passwd; /* HTTP password. */
char **user_headers; /* User-defined header(s). */
bool http_keep_alive; /* whether we use keep-alive */
+ int http_retry_if_503; /* whether we retry if http code is 503 */
+
+ int http_retry_if_zero_lenght; /* whether we should retry if no
+ content-lenght or it is 0 */
bool use_proxy; /* Do we use proxy? */
bool allow_cache; /* Do we allow server-side caching? */
diff --git a/src/wget.h b/src/wget.h
index d32ef3f..ea2fae6 100644
--- a/src/wget.h
+++ b/src/wget.h
@@ -283,5 +283,6 @@ enum
READERR, TRYLIMEXC, URLBADPATTERN, FILEBADFILE, RANGEERR,
RETRBADPATTERN, RETNOTSUP, ROBOTSOK, NOROBOTS, PROXERR,
/* 50 */
- AUTHFAILED, QUOTEXC, WRITEFAILED, SSLINITFAILED
+ AUTHFAILED, QUOTEXC, WRITEFAILED, SSLINITFAILED, ZEROLEN,
+ TEMP_UNAVAIL
} uerr_t;
