Hello Everyone, I have prepared conditional GET requests support for Wget (specifiaclly If-Modified-Since header).
Following steps were taken (each is one patch): 1) local file timestamp support added to testenv 2) 304 return code added to testenv (ie. do not send response body when 304 was requested for given file) 3) Test-condget.py - unit test for conditional requests 4-5) "--condget" option handled in src/http.c 6) Short description in user manual I would be very pleased if you could review my code and maybe point out some mistakes or any other comments/suggestions. Maybe you have ideas about some improvements to it? In case you prefer to view it on Github, here is a link: https://github.com/jy987321/Wget/commits/master-hubert -- Hubert Tarasiuk
From f4599cae4056cab7b34968b5d76700bd9ea2e547 Mon Sep 17 00:00:00 2001 From: Hubert Tarasiuk <[email protected]> Date: Thu, 7 May 2015 18:34:17 +0200 Subject: [PATCH 1/6] Implement timestamp support for local files in testenv * testenv/README: Change timestamp format definition * testenv/conf/local_files.py: Set proper timestamps --- testenv/README | 2 +- testenv/conf/local_files.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/testenv/README b/testenv/README index 081a957..cf22f89 100644 --- a/testenv/README +++ b/testenv/README @@ -124,7 +124,7 @@ WgetFile (str name, str contents, str timestamp, dict rules) None except name is a mandatory paramter, one may pass only those parameters that are required by the File object. -The timestamp string should be a valid Unix Timestamp as defined in RFC xxxx. +The timestamp string should be in a format: "YYYY-MM-DD HH:MM:SS" in UTC zone. The rules object is a dictionary element, with the key as the Rule Name and value as the Rule Data. In most cases, the Rule Data is another dictionary. diff --git a/testenv/conf/local_files.py b/testenv/conf/local_files.py index 5f9c8fa..908ced1 100644 --- a/testenv/conf/local_files.py +++ b/testenv/conf/local_files.py @@ -1,3 +1,6 @@ +from os import utime +from time import strptime +from calendar import timegm from conf import hook """ Pre-Test Hook: LocalFiles @@ -16,3 +19,8 @@ class LocalFiles: for f in self.local_files: with open(f.name, 'w') as fp: fp.write(f.content) + if f.timestamp is not None: + tstamp = timegm(strptime(f.timestamp, '%Y-%m-%d %H:%M:%S')) + atime = tstamp + mtime = tstamp + utime(f.name, (atime, mtime)) -- 2.4.0
From bcbf875fe9292ce122748b41397a29c50def2149 Mon Sep 17 00:00:00 2001 From: Hubert Tarasiuk <[email protected]> Date: Mon, 11 May 2015 12:34:05 +0200 Subject: [PATCH 2/6] Support conditional GET in testenv server. * src/exc/server_error.py: Add exception for GET to HEAD fallback. * src/server/http/http_server.py: Do not send body if 304 return code requested for a file. --- testenv/exc/server_error.py | 6 ++++++ testenv/server/http/http_server.py | 7 ++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/testenv/exc/server_error.py b/testenv/exc/server_error.py index dc676a8..fe359f5 100644 --- a/testenv/exc/server_error.py +++ b/testenv/exc/server_error.py @@ -6,6 +6,12 @@ class ServerError (Exception): def __init__(self, err_message): self.err_message = err_message +class NoBodyServerError (Exception): + """ A custom exception which is raised by the test servers. + Used if no body should be sent in response. """ + + def __init__(self, err_message): + self.err_message = err_message class AuthError (ServerError): """ A custom exception raised byt he servers when authentication of the diff --git a/testenv/server/http/http_server.py b/testenv/server/http/http_server.py index 7c0b472..2356f1c 100644 --- a/testenv/server/http/http_server.py +++ b/testenv/server/http/http_server.py @@ -1,5 +1,5 @@ from http.server import HTTPServer, BaseHTTPRequestHandler -from exc.server_error import ServerError, AuthError +from exc.server_error import ServerError, AuthError, NoBodyServerError from socketserver import BaseServer from posixpath import basename, splitext from base64 import b64encode @@ -201,6 +201,8 @@ class _Handler(BaseHTTPRequestHandler): def Response(self, resp_obj): self.send_response(resp_obj.response_code) self.finish_headers() + if resp_obj.response_code == 304: + raise NoBodyServerError("Conditional get falling to head") raise ServerError("Custom Response code sent.") def custom_response(self): @@ -401,6 +403,9 @@ class _Handler(BaseHTTPRequestHandler): except AuthError as ae: print(ae.__str__()) return(None, None) + except NoBodyServerError as nbse: + print(nbse.__str__()) + return(None, None) except ServerError as se: print(se.__str__()) return(content, None) -- 2.4.0
From 58081549e6fa3e12182ddefe5b95383dadf4fa2c Mon Sep 17 00:00:00 2001 From: Hubert Tarasiuk <[email protected]> Date: Thu, 7 May 2015 18:45:10 +0200 Subject: [PATCH 3/6] Add test for condget requests. * testenv/Test-condget.py: the test * testenv/Makefile.am: add to tests list --- testenv/Makefile.am | 3 +- testenv/Test-condget.py | 84 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) create mode 100755 testenv/Test-condget.py diff --git a/testenv/Makefile.am b/testenv/Makefile.am index 9acf0f3..1058421 100644 --- a/testenv/Makefile.am +++ b/testenv/Makefile.am @@ -54,7 +54,8 @@ if HAVE_PYTHON3 Test-504.py \ Test--spider-r.py \ Test-redirect-crash.py \ - Test-reserved-chars.py + Test-reserved-chars.py \ + Test-condget.py # added test cases expected to fail here and under TESTS XFAIL_TESTS = diff --git a/testenv/Test-condget.py b/testenv/Test-condget.py new file mode 100755 index 0000000..3670c4d --- /dev/null +++ b/testenv/Test-condget.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +from sys import exit +from test.http_test import HTTPTest +from misc.wget_file import WgetFile + +""" + Simple test for HTTP Conditional-GET Requests usiong the --condget command +""" +TEST_NAME = "HTTP Conditional-GET Requests" +############# File Definitions ############################################### +# Keep same length ! +Cont1 = """THIS IS 1 FILE""" +Cont2 = """THIS IS 2 FILE""" +Cont3 = """THIS IS 3 FILE""" +Cont4 = """THIS IS 4 FILE""" + +# Local Wget files +UpToDate_Local_File = WgetFile ("UpToDateFile", Cont1, timestamp="1995-01-01 00:00:00") +Outdated_Local_File = WgetFile ("UpdatedFile", Cont2, timestamp="1990-01-01 00:00:00") + +UpToDate_Rules = { + "SendHeader" : { + "Last-Modified" : "Sun, 01 Jan 1995 00:00:00 GMT", + }, + "Response": 304, + "ExpectHeader" : { + "If-Modified-Since" : "Sun, 01 Jan 1995 00:00:00 GMT" + }, +} + +Outdated_Rules = { + "SendHeader" : { + "Last-Modified" : "Thu, 01 Jan 2015 00:00:00 GMT", + }, + "ExpectHeader" : { + "If-Modified-Since" : "Mon, 01 Jan 1990 00:00:00 GMT", + }, +} + +UpToDate_Server_File = WgetFile ("UpToDateFile", Cont3, rules=UpToDate_Rules) +Updated_Server_File = WgetFile ("UpdatedFile", Cont4, rules=Outdated_Rules) + +WGET_OPTIONS = "--condget" +WGET_URLS = [["UpToDateFile", "UpdatedFile", ]] + +Files = [[UpToDate_Server_File, Updated_Server_File, ]] +Existing_Files = [UpToDate_Local_File, Outdated_Local_File] + +ExpectedReturnCode = 0 + +# The uptodate file should not be downloaded +ExpectedDownloadedFiles = [UpToDate_Local_File, Updated_Server_File] + +# Kind of hack to ensure proper request types +Request_List = [ + [ + "GET /UpToDateFile", + "GET /UpdatedFile", + ] +] + +################ Pre and Post Test Hooks ##################################### +pre_test = { + "ServerFiles" : Files, + "LocalFiles" : Existing_Files +} +test_options = { + "WgetCommands" : WGET_OPTIONS, + "Urls" : WGET_URLS +} +post_test = { + "ExpectedFiles" : ExpectedDownloadedFiles, + "ExpectedRetcode" : ExpectedReturnCode, + "FilesCrawled" : Request_List, +} + +err = HTTPTest ( + name=TEST_NAME, + pre_hook=pre_test, + test_params=test_options, + post_hook=post_test +).begin () + +exit (err) -- 2.4.0
From 7ecf65c2618758e011af52b63434f105564e34da Mon Sep 17 00:00:00 2001 From: Hubert Tarasiuk <[email protected]> Date: Tue, 5 May 2015 19:33:02 +0200 Subject: [PATCH 4/6] Add --condget option * src/init.c: Add to commands array. * src/main.c: Add to cmdline_option. Add to help message. * src/options.h: Add to options struct. --- src/init.c | 1 + src/main.c | 3 +++ src/options.h | 1 + 3 files changed, 5 insertions(+) diff --git a/src/init.c b/src/init.c index 33888de..f3a1c4a 100644 --- a/src/init.c +++ b/src/init.c @@ -155,6 +155,7 @@ static const struct { { "checkcertificate", &opt.check_cert, cmd_boolean }, #endif { "chooseconfig", &opt.choose_config, cmd_file }, + { "condget", &opt.cond_get, cmd_boolean }, { "connecttimeout", &opt.connect_timeout, cmd_time }, { "contentdisposition", &opt.content_disposition, cmd_boolean }, { "contentonerror", &opt.content_on_error, cmd_boolean }, diff --git a/src/main.c b/src/main.c index b59fcab..68927e2 100644 --- a/src/main.c +++ b/src/main.c @@ -301,6 +301,7 @@ static struct cmdline_option option_data[] = { "strict-comments", 0, OPT_BOOLEAN, "strictcomments", -1 }, { "timeout", 'T', OPT_VALUE, "timeout", -1 }, { "timestamping", 'N', OPT_BOOLEAN, "timestamping", -1 }, + { "condget", 0, OPT_BOOLEAN, "condget", -1 }, { "tries", 't', OPT_VALUE, "tries", -1 }, { "unlink", 0, OPT_BOOLEAN, "unlink", -1 }, { "trust-server-names", 0, OPT_BOOLEAN, "trustservernames", -1 }, @@ -516,6 +517,8 @@ Download:\n"), -N, --timestamping don't re-retrieve files unless newer than\n\ local\n"), N_("\ + --condget use conditional get requests in timestamping mode\n"), + N_("\ --no-use-server-timestamps don't set the local file's timestamp by\n\ the one on the server\n"), N_("\ diff --git a/src/options.h b/src/options.h index bded0c4..6cd3640 100644 --- a/src/options.h +++ b/src/options.h @@ -165,6 +165,7 @@ struct options #endif bool timestamping; /* Whether to use time-stamping. */ + bool cond_get; /* Whether to use conditional get requests */ bool backup_converted; /* Do we save pre-converted files as *.orig? */ int backups; /* Are numeric backups made? */ -- 2.4.0
From 4fbe0ee58f41e9d6bceb665739cdab5210ed5602 Mon Sep 17 00:00:00 2001 From: Hubert Tarasiuk <[email protected]> Date: Sat, 9 May 2015 22:47:24 +0200 Subject: [PATCH 5/6] Prototype of If-Modified-Since. * src/wget.h: Add COND_GET enum for dt. * src/http.c (time_to_rfc1123): Convert time_t do http time. * src/http.c (initialize_request): Include If-Modified-Since header if appropriate. * src/http.c (set_file_timestamp): Separate this code from check_file_output. * src/http.c (check_file_output): Use set_file_timestamp. * src/http.c (gethttp): Treat 304 code as if entire file was downloaded (to be improved). * src/http.c (http_loop): Load filename to hstat if condget was requested, use COND_GET if requested and current timestamp can be obtained. --- src/http.c | 213 ++++++++++++++++++++++++++++++++++++++++++++----------------- src/url.h | 3 +- src/wget.h | 3 +- 3 files changed, 158 insertions(+), 61 deletions(-) diff --git a/src/http.c b/src/http.c index 54eb106..5b95c23 100644 --- a/src/http.c +++ b/src/http.c @@ -1681,6 +1681,59 @@ read_response_body (struct http_stat *hs, int sock, FILE *fp, wgint contlen, } while (0) #endif /* def __VMS [else] */ +/* + Convert time_t to one of valid HTTP date formats + ie. rfc1123-date. + + HTTP-date = rfc1123-date | rfc850-date | asctime-date + rfc1123-date = wkday "," SP date1 SP time SP "GMT" + rfc850-date = weekday "," SP date2 SP time SP "GMT" + asctime-date = wkday SP date3 SP time SP 4DIGIT + date1 = 2DIGIT SP month SP 4DIGIT + ; day month year (e.g., 02 Jun 1982) + date2 = 2DIGIT "-" month "-" 2DIGIT + ; day-month-year (e.g., 02-Jun-82) + date3 = month SP ( 2DIGIT | ( SP 1DIGIT )) + ; month day (e.g., Jun 2) + time = 2DIGIT ":" 2DIGIT ":" 2DIGIT + ; 00:00:00 - 23:59:59 + wkday = "Mon" | "Tue" | "Wed" + | "Thu" | "Fri" | "Sat" | "Sun" + weekday = "Monday" | "Tuesday" | "Wednesday" + | "Thursday" | "Friday" | "Saturday" | "Sunday" + month = "Jan" | "Feb" | "Mar" | "Apr" + | "May" | "Jun" | "Jul" | "Aug" + | "Sep" | "Oct" | "Nov" | "Dec" + + source: RFC2616 */ +static uerr_t +time_to_rfc1123 (time_t time, char *buf, size_t bufsize) +{ + struct tm *gtm; + static const char *wkday[] = { "Sun", "Mon", "Tue", "Wed", + "Thu", "Fri", "Sat" }; + static const char *month[] = { "Jan", "Feb", "Mar", "Apr", + "May", "Jun", "Jul", "Aug", + "Sep", "Oct", "Nov", "Dec" }; + /* rfc1123 example: Thu, 01 Jan 1998 22:12:57 GMT */ + static const char *time_format = "%s, %02d %s %04d %02d:%02d:%02d GMT"; + + gtm = gmtime (&time); + if (!gtm) + { + logprintf (LOG_NOTQUIET, + "gmtime failed. This is probably a bug.\n"); + abort (); + } + + snprintf (buf, bufsize, time_format, wkday[gtm->tm_wday], + gtm->tm_mday, month[gtm->tm_mon], + gtm->tm_year + 1900, gtm->tm_hour, + gtm->tm_min, gtm->tm_sec); + + return RETROK; +} + static struct request * initialize_request (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, bool inhibit_keep_alive, bool *basic_auth_finished, @@ -1723,6 +1776,12 @@ initialize_request (struct url *u, struct http_stat *hs, int *dt, struct url *pr /* ... but some HTTP/1.0 caches doesn't implement Cache-Control. */ request_set_header (req, "Pragma", "no-cache", rel_none); } + if (*dt & COND_GET) + { + char strtime[32]; + time_to_rfc1123 (hs->orig_file_tstamp, strtime, countof (strtime)); + request_set_header (req, "If-Modified-Since", xstrdup(strtime), rel_value); + } if (hs->restval) request_set_header (req, "Range", aprintf ("bytes=%s-", @@ -2025,6 +2084,69 @@ establish_connection (struct url *u, struct url **conn_ref, } static uerr_t +set_file_timestamp (struct http_stat *hs) +{ + size_t filename_len = strlen (hs->local_file); + char *filename_plus_orig_suffix = alloca (filename_len + sizeof (ORIG_SFX)); + bool local_dot_orig_file_exists = false; + char *local_filename = NULL; + struct_stat st; + + if (opt.backup_converted) + /* If -K is specified, we'll act on the assumption that it was specified + last time these files were downloaded as well, and instead of just + comparing local file X against server file X, we'll compare local + file X.orig (if extant, else X) against server file X. If -K + _wasn't_ specified last time, or the server contains files called + *.orig, -N will be back to not operating correctly with -k. */ + { + /* Would a single s[n]printf() call be faster? --dan + + Definitely not. sprintf() is horribly slow. It's a + different question whether the difference between the two + affects a program. Usually I'd say "no", but at one + point I profiled Wget, and found that a measurable and + non-negligible amount of time was lost calling sprintf() + in url.c. Replacing sprintf with inline calls to + strcpy() and number_to_string() made a difference. + --hniksic */ + memcpy (filename_plus_orig_suffix, hs->local_file, filename_len); + memcpy (filename_plus_orig_suffix + filename_len, + ORIG_SFX, sizeof (ORIG_SFX)); + + /* Try to stat() the .orig file. */ + if (stat (filename_plus_orig_suffix, &st) == 0) + { + local_dot_orig_file_exists = true; + local_filename = filename_plus_orig_suffix; + } + } + + if (!local_dot_orig_file_exists) + /* Couldn't stat() <file>.orig, so try to stat() <file>. */ + if (stat (hs->local_file, &st) == 0) + local_filename = hs->local_file; + + if (local_filename != NULL) + /* There was a local file, so we'll check later to see if the version + the server has is the same version we already have, allowing us to + skip a download. */ + { + hs->orig_file_name = xstrdup (local_filename); + hs->orig_file_size = st.st_size; + hs->orig_file_tstamp = st.st_mtime; +#ifdef WINDOWS + /* Modification time granularity is 2 seconds for Windows, so + increase local time by 1 second for later comparison. */ + ++hs->orig_file_tstamp; +#endif + hs->timestamp_checked = true; + } + + return RETROK; +} + +static uerr_t check_file_output (struct url *u, struct http_stat *hs, struct response *resp, char *hdrval, size_t hdrsize) { @@ -2077,61 +2199,9 @@ check_file_output (struct url *u, struct http_stat *hs, /* Support timestamping */ if (opt.timestamping && !hs->timestamp_checked) { - size_t filename_len = strlen (hs->local_file); - char *filename_plus_orig_suffix = alloca (filename_len + sizeof (ORIG_SFX)); - bool local_dot_orig_file_exists = false; - char *local_filename = NULL; - struct_stat st; - - if (opt.backup_converted) - /* If -K is specified, we'll act on the assumption that it was specified - last time these files were downloaded as well, and instead of just - comparing local file X against server file X, we'll compare local - file X.orig (if extant, else X) against server file X. If -K - _wasn't_ specified last time, or the server contains files called - *.orig, -N will be back to not operating correctly with -k. */ - { - /* Would a single s[n]printf() call be faster? --dan - - Definitely not. sprintf() is horribly slow. It's a - different question whether the difference between the two - affects a program. Usually I'd say "no", but at one - point I profiled Wget, and found that a measurable and - non-negligible amount of time was lost calling sprintf() - in url.c. Replacing sprintf with inline calls to - strcpy() and number_to_string() made a difference. - --hniksic */ - memcpy (filename_plus_orig_suffix, hs->local_file, filename_len); - memcpy (filename_plus_orig_suffix + filename_len, - ORIG_SFX, sizeof (ORIG_SFX)); - - /* Try to stat() the .orig file. */ - if (stat (filename_plus_orig_suffix, &st) == 0) - { - local_dot_orig_file_exists = true; - local_filename = filename_plus_orig_suffix; - } - } - - if (!local_dot_orig_file_exists) - /* Couldn't stat() <file>.orig, so try to stat() <file>. */ - if (stat (hs->local_file, &st) == 0) - local_filename = hs->local_file; - - if (local_filename != NULL) - /* There was a local file, so we'll check later to see if the version - the server has is the same version we already have, allowing us to - skip a download. */ - { - hs->orig_file_name = xstrdup (local_filename); - hs->orig_file_size = st.st_size; - hs->orig_file_tstamp = st.st_mtime; -#ifdef WINDOWS - /* Modification time granularity is 2 seconds for Windows, so - increase local time by 1 second for later comparison. */ - ++hs->orig_file_tstamp; -#endif - } + uerr_t timestamp_err = set_file_timestamp (hs); + if (timestamp_err != RETROK) + return timestamp_err; } return RETROK; } @@ -2421,6 +2491,9 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, POST). */ bool head_only = !!(*dt & HEAD_ONLY); + /* Whether conditional get request will be issued */ + bool cond_get = !!(*dt & COND_GET); + char *head = NULL; struct response *resp = NULL; char hdrval[512]; @@ -3020,6 +3093,16 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, } } + if (cond_get && statcode == HTTP_STATUS_NOT_MODIFIED) + { + logprintf (LOG_VERBOSE, _("File %s not modified on server. " + "Ommiting download.\n\n"), quote(hs->local_file)); + *dt |= RETROKF; + CLOSE_FINISH (sock); + retval = RETRUNNEEDED; + goto cleanup; + } + if (statcode == HTTP_STATUS_RANGE_NOT_SATISFIABLE || (!opt.timestamping && hs->restval > 0 && statcode == HTTP_STATUS_OK && contrange == 0 && contlen >= 0 && hs->restval >= contlen)) @@ -3263,14 +3346,26 @@ http_loop (struct url *u, struct url *original_url, char **newloc, if (opt.content_disposition && opt.always_rest) send_head_first = true; - /* Send preliminary HEAD request if -N is given and we have an existing - * destination file. */ if (!opt.output_document) file_name = url_file_name (opt.trustservernames ? u : original_url, NULL); else file_name = xstrdup (opt.output_document); - if (opt.timestamping && (file_exists_p (file_name) - || opt.content_disposition)) + + /* Use conditional get request if requested + * and if timestamp is known. */ + if (opt.cond_get && file_exists_p (file_name) && !send_head_first) + { + *dt |= COND_GET; + { + uerr_t timestamp_err = set_file_timestamp (&hstat); + if (timestamp_err != RETROK) + return timestamp_err; + } + } + /* Send preliminary HEAD request if -N is given and we have an existing + * destination file. */ + else if ((opt.timestamping && file_exists_p (file_name)) + || opt.content_disposition) send_head_first = true; xfree (file_name); diff --git a/src/url.h b/src/url.h index a543f3d..dbb06a1 100644 --- a/src/url.h +++ b/src/url.h @@ -51,7 +51,8 @@ as that of the covered work. */ Defined here to avoid repetition later. #### This will require rework. */ #define ALLOW_CLOBBER (opt.noclobber || opt.always_rest || opt.timestamping \ - || opt.dirstruct || opt.output_document || opt.backups > 0) + || opt.dirstruct || opt.output_document || opt.backups > 0\ + || opt.cond_get) /* Specifies how, or whether, user auth information should be included * in URLs regenerated from URL parse structures. */ diff --git a/src/wget.h b/src/wget.h index 8d2b0f1..1440123 100644 --- a/src/wget.h +++ b/src/wget.h @@ -331,7 +331,8 @@ enum SEND_NOCACHE = 0x0008, /* send Pragma: no-cache directive */ ACCEPTRANGES = 0x0010, /* Accept-ranges header was found */ ADDED_HTML_EXTENSION = 0x0020, /* added ".html" extension due to -E */ - TEXTCSS = 0x0040 /* document is of type text/css */ + TEXTCSS = 0x0040, /* document is of type text/css */ + COND_GET = 0x0080, /* conditional get request */ }; /* Universal error type -- used almost everywhere. Error reporting of -- 2.4.0
From ce95f564c3875e493f34ab8ab3b9ffc4da44ef08 Mon Sep 17 00:00:00 2001 From: Hubert Tarasiuk <[email protected]> Date: Tue, 12 May 2015 21:21:33 +0200 Subject: [PATCH 6/6] Include --condget option in user manual. * doc/wget.texi: Add --condget section. --- doc/wget.texi | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/wget.texi b/doc/wget.texi index 689fb97..6245da9 100644 --- a/doc/wget.texi +++ b/doc/wget.texi @@ -788,6 +788,10 @@ used alongside the @samp{--logfile} option. @itemx --timestamping Turn on time-stamping. @xref{Time-Stamping}, for details. +@item --condget +Similar to @samp{-N}, but send conditional GET request instead of sending +preliminary HEAD request. This option overrides @samp{-N}. + @item --no-use-server-timestamps Don't set the local file's timestamp by the one on the server. -- 2.4.0
signature.asc
Description: OpenPGP digital signature
