diff -rupbN -x configure -x autom4te.cache wget-1.14/configure.ac wget-1.14-gz/configure.ac --- wget-1.14/configure.ac 2012-05-13 07:16:52.000000000 -0700 +++ wget-1.14-gz/configure.ac 2013-09-06 12:50:50.682436000 -0700 @@ -61,6 +61,11 @@ dnl dnl Process features. dnl +AC_ARG_WITH(gzip, + [AS_HELP_STRING([--with-gzip],[allow download of content in compressed format])], + [], + [with_gzip=yes]) + AC_ARG_WITH(ssl, [[ --without-ssl disable SSL autodetection --with-ssl={gnutls,openssl} specify the SSL backend. GNU TLS is the default.]]) @@ -133,6 +138,15 @@ if test -n "$auto_cflags"; then fi fi +if test "x${with_gzip}" = xyes +then + AC_CHECK_LIB(z, gzopen, [], + [AC_MSG_FAILURE([gzip library not found, try configure --without-gzip])]) + AC_CHECK_HEADERS(zlib.h, [], [AC_MSG_FAILURE([gzip header not found, try installing dev package])]) + AC_DEFINE([HAVE_GZIP], [], "enable download in gzip-compressed format") +dnl LIBS="-lz $LIBS" +fi + dnl dnl Checks for basic compiler characteristics. dnl diff -rupbN -x configure -x autom4te.cache wget-1.14/m4/wget.m4 wget-1.14-gz/m4/wget.m4 --- wget-1.14/m4/wget.m4 2012-05-12 08:18:27.000000000 -0700 +++ wget-1.14-gz/m4/wget.m4 2013-09-06 12:52:36.326436000 -0700 @@ -51,8 +51,8 @@ dnl /usr/local/include (!), which GCC us AC_DEFUN([WGET_FNMATCH], [ AC_MSG_CHECKING([for working fnmatch.h]) - AC_COMPILE_IFELSE([#include - ], [ + AC_COMPILE_IFELSE([AC_LANG_SOURCE([#include + ])], [ AC_MSG_RESULT(yes) AC_DEFINE([HAVE_WORKING_FNMATCH_H], 1, [Define if fnmatch.h can be included.]) @@ -270,4 +270,3 @@ else fi AC_SUBST($1)dnl ]) - diff -rupbN -x configure -x autom4te.cache wget-1.14/src/config.h.in wget-1.14-gz/src/config.h.in --- wget-1.14/src/config.h.in 2012-08-05 13:17:12.000000000 -0700 +++ wget-1.14-gz/src/config.h.in 2013-09-06 12:53:12.662436000 -0700 @@ -596,6 +596,9 @@ /* Define to 1 if you have the `gnutls_priority_set_direct' function. */ #undef HAVE_GNUTLS_PRIORITY_SET_DIRECT +/* "enable download in gzip-compressed format" */ +#undef HAVE_GZIP + /* Define if you have the iconv() function and it works. */ #undef HAVE_ICONV @@ -1826,6 +1829,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_XLOCALE_H +/* Define to 1 if you have the header file. */ +#undef HAVE_ZLIB_H + /* Define to 1 if the system has the type `_Bool'. */ #undef HAVE__BOOL diff -rupbN -x configure -x autom4te.cache wget-1.14/src/http.c wget-1.14-gz/src/http.c --- wget-1.14/src/http.c 2012-07-07 01:57:20.000000000 -0700 +++ wget-1.14-gz/src/http.c 2013-09-06 13:53:53.346436000 -0700 @@ -34,6 +34,9 @@ as that of the covered work. */ #include #include #include +#ifdef HAVE_GZIP +#include +#endif #include #include #include @@ -1621,6 +1624,50 @@ read_response_body (struct http_stat *hs } } +#ifdef HAVE_GZIP +static uerr_t decompress_file(char *file) +{ + static char buf[4096]; + char tmpf[] = "fnXXXXXX"; + int nr, _nr, nw, tf, err; + gzFile zf; + + tf = mkstemp(tmpf); + if (tf < 0) + return FOPENERR; + zf = gzopen(file, "rb"); + if (zf == NULL) { + logprintf(LOG_NOTQUIET, "gzopen: %s, file: %s\n", strerror(errno), tmpf); + return GZOPENERR; + } + nr = 0; + while (1) { + _nr = gzread(zf, (void *)buf, sizeof(buf)); + if (_nr == 0) + break; + if (_nr < 0) { + logprintf(LOG_NOTQUIET, "gzread: %s\n", strerror(errno)); + return GZREADERR; + } + nr += _nr; + nw = write(tf, buf, _nr); + if (nw < _nr) { + logprintf(LOG_NOTQUIET, "fwrite: %s\n", strerror(errno)); + return FWRITEERR; + } + } + close(tf); + gzclose(zf); + err = rename(tmpf, file); + if (err != 0) { + logprintf(LOG_NOTQUIET, "rename %s -> %s: %s\n", tmpf, file, strerror(errno)); + return RENAMEERR; + } + return 0; +} +#endif + + #define BEGINS_WITH(line, string_constant) \ (!strncasecmp (line, string_constant, sizeof (string_constant) - 1) \ && (c_isspace (line[sizeof (string_constant) - 1]) \ @@ -1714,6 +1761,11 @@ gethttp (struct url *u, struct http_stat ip_address *warc_ip = NULL; off_t warc_payload_offset = -1; +#ifdef HAVE_GZIP + /* whether response is gzipped */ + bool gzipped_resp; +#endif + /* Whether this connection will be kept alive after the HTTP request is done. */ bool keep_alive; @@ -1784,6 +1836,11 @@ gethttp (struct url *u, struct http_stat request_set_method (req, meth, meth_arg); } +#ifdef HAVE_GZIP + if (!opt.no_ae_gzip) + request_set_header(req, "Accept-Encoding", "gzip", rel_none); +#endif + request_set_header (req, "Referer", (char *) hs->referer, rel_none); if (*dt & SEND_NOCACHE) { @@ -2292,6 +2349,13 @@ read_header: && 0 == strcasecmp (hdrval, "chunked")) chunked_transfer_encoding = true; +#ifdef HAVE_GZIP + gzipped_resp = false; + if (resp_header_copy (resp, "Content-Encoding", hdrval, sizeof (hdrval)) + && 0 == strcasecmp (hdrval, "gzip")) + gzipped_resp = true; +#endif + /* Handle (possibly multiple instances of) the Set-Cookie header. */ if (opt.cookies) { @@ -2903,6 +2967,13 @@ read_header: u->url, warc_timestamp_str, warc_request_uuid, warc_ip, type, statcode, head); + if (gzipped_resp && err == RETRFINISHED) { + int ret; + + ret = decompress_file(hs->local_file); + if (ret) + err = ret; + } /* Now we no longer need to store the response header. */ xfree (head); @@ -3974,4 +4045,3 @@ test_parse_content_disposition() /* * vim: et sts=2 sw=2 cino+={s */ - diff -rupbN -x configure -x autom4te.cache wget-1.14/src/init.c wget-1.14-gz/src/init.c --- wget-1.14/src/init.c 2012-07-08 02:35:36.000000000 -0700 +++ wget-1.14-gz/src/init.c 2013-09-05 17:25:22.458436000 -0700 @@ -212,6 +212,9 @@ static const struct { { "maxredirect", &opt.max_redirect, cmd_number }, { "mirror", NULL, cmd_spec_mirror }, { "netrc", &opt.netrc, cmd_boolean }, +#ifdef HAVE_GZIP + { "noaegzip", &opt.no_ae_gzip, cmd_boolean }, +#endif { "noclobber", &opt.noclobber, cmd_boolean }, { "noparent", &opt.no_parent, cmd_boolean }, { "noproxy", &opt.no_proxy, cmd_vector }, diff -rupbN -x configure -x autom4te.cache wget-1.14/src/main.c wget-1.14-gz/src/main.c --- wget-1.14/src/main.c 2013-09-05 17:47:11.426436000 -0700 +++ wget-1.14-gz/src/main.c 2013-09-05 17:47:14.714436000 -0700 @@ -233,6 +233,9 @@ static struct cmdline_option option_data { "max-redirect", 0, OPT_VALUE, "maxredirect", -1 }, { "mirror", 'm', OPT_BOOLEAN, "mirror", -1 }, { "no", 'n', OPT__NO, NULL, required_argument }, +#ifdef HAVE_GZIP + { "no-ae-gzip", 'z', OPT_BOOLEAN, "noaegzip", -1 }, +#endif { "no-clobber", 0, OPT_BOOLEAN, "noclobber", -1 }, { "no-parent", 0, OPT_BOOLEAN, "noparent", -1 }, { "output-document", 'O', OPT_VALUE, "outputdocument", -1 }, @@ -579,6 +582,10 @@ HTTP options:\n"), this is `index.html'.).\n"), N_("\ -E, --adjust-extension save HTML/CSS documents with proper extensions.\n"), +#ifdef HAVE_GZIP + N_("\ + -z, --no-ae-gzip do NOT send Accept-Encoding: gzip header.\n"), +#endif N_("\ --ignore-length ignore `Content-Length' header field.\n"), N_("\ diff -rupbN -x configure -x autom4te.cache wget-1.14/src/options.h wget-1.14-gz/src/options.h --- wget-1.14/src/options.h 2012-06-06 04:42:10.000000000 -0700 +++ wget-1.14-gz/src/options.h 2013-09-05 17:25:39.142436000 -0700 @@ -191,6 +191,10 @@ struct options bool adjust_extension; /* Use ".html" extension on all text/html? */ +#ifdef HAVE_GZIP + int no_ae_gzip; /* don't send "Accept-encoding: gzip" header */ +#endif + bool page_requisites; /* Whether we need to download all files necessary to display a page properly. */ char *bind_address; /* What local IP address to bind to. */ diff -rupbN -x configure -x autom4te.cache wget-1.14/src/wget.h wget-1.14-gz/src/wget.h --- wget-1.14/src/wget.h 2012-06-17 13:13:44.000000000 -0700 +++ wget-1.14-gz/src/wget.h 2013-09-06 13:12:06.122436000 -0700 @@ -355,7 +355,11 @@ typedef enum AUTHFAILED, QUOTEXC, WRITEFAILED, SSLINITFAILED, VERIFCERTERR, UNLINKERR, NEWLOCATION_KEEP_POST, CLOSEFAILED, - WARC_ERR, WARC_TMP_FOPENERR, WARC_TMP_FWRITEERR + WARC_ERR, WARC_TMP_FOPENERR, WARC_TMP_FWRITEERR, +#ifdef HAVE_GZIP + /* 62 */ + GZOPENERR, GZREADERR, RENAMEERR, +#endif } uerr_t; /* 2005-02-19 SMS.