Here comes an updated version of the patches. I have applied some minor changes based on Giuseppe's suggestions posted on Github (thanks for the review!).
Another patch is added (9/9) to move some helper functions used for Metalink-over-HTTP from http.c to metalink.c. They are also made non-static so that they can be used in "metalink_from_http" function in http.c. Hubert Tarasiuk <[email protected]> writes: > > I have prepared a Metalink patch for Wget. The two main features are: > > - support Metalink v3 and Metalink v4 XML files: > https://tools.ietf.org/html/rfc5854 > using libmetalink parser: https://launchpad.net/libmetalink > > - support Metalink in HTTP headers: http://tools.ietf.org/html/rfc6249 > > Please let me know what do you think about the patches. Some test cases > are included. If you would like to test it on actual servers, here is > what I found: > - Metalink files with PGP signatures: http://curl.haxx.se/download.html > - Metalink in HTTP headers: > https://download.gnome.org/apps/3.0/3.0.0/sources/ > > The commits are also available via Github interface: > https://github.com/jy987321/Wget/commits/metalink
From cf1476f3d3be9b7cfe3cff89561f35ca79f323ea Mon Sep 17 00:00:00 2001 From: Hubert Tarasiuk <[email protected]> Date: Sat, 30 May 2015 23:51:55 +0200 Subject: [PATCH 1/9] Metalink support. * bootstrap.conf: Add crypto/sha256 * configure.ac: Look for libmetalink and GPGME * doc/wget.texi: Add --input-metalink and --metalink-over-http options description. * po/POTFILES.in: Add metalink.c * src/Makefile.am: Add new translation unit (metalink.c) * src/http.c (http_stat): Add metalink field. (free_stat): Free metalink field. (find_key_value): Find value of given key in header string. (has_key): Check if token exists in header string. (find_key_values): Find all key=value pairs in header string. (metalink_from_http): Obtain Metalink metadata from HTTP response. (gethttp): Call metalink_from_http if requested. (http_loop): Request Metalink metadata from HTTP response if should be. Fall back to regular download if no Metalink metadata found. * src/init.c: Add --input-metalink and --metalink-over-http options * src/main.c (option_data): Handle --input-metalink and --metalink-over-http cmd arguments. (print_help): Print --input-metalink option description. (main): Retrieve files from Metalink file * src/metalink.c (retrieve_from_metalink): Download files described by metalink. (metalink_res_cmp): Comparator for resources priority-sorting. * src/metalink.h: Create header for metalink.c (RES_TYPE_SUPPORTED): Define supported resources media. (DEFAULT_PRI): Default mirror priority for Metalink over HTTP. (VALID_PRI_RANGE): Valid priority range. * src/options.h (options): Add input_metalink option and metalink_over_http options. * src/utils.c (hex_to_string): Convert binary data to ASCII-hex. * src/utils.h (hex_to_string): Add prototype. * src/wget.h: Add metalink-related error enums Add METALINK_METADATA flag for document type. --- bootstrap.conf | 1 + configure.ac | 26 ++- doc/wget.texi | 12 ++ po/POTFILES.in | 1 + src/Makefile.am | 8 +- src/http.c | 607 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/init.c | 9 + src/main.c | 56 +++++- src/metalink.c | 448 +++++++++++++++++++++++++++++++++++++++++ src/metalink.h | 50 +++++ src/options.h | 4 + src/utils.c | 15 ++ src/utils.h | 2 + src/wget.h | 8 +- 14 files changed, 1241 insertions(+), 6 deletions(-) create mode 100644 src/metalink.c create mode 100644 src/metalink.h diff --git a/bootstrap.conf b/bootstrap.conf index 4fff711..ce52d99 100644 --- a/bootstrap.conf +++ b/bootstrap.conf @@ -64,6 +64,7 @@ mkstemp mkostemp crypto/md5 crypto/sha1 +crypto/sha256 quote quotearg recv diff --git a/configure.ac b/configure.ac index 01ea237..7b86605 100644 --- a/configure.ac +++ b/configure.ac @@ -475,6 +475,29 @@ else fi fi +dnl +dnl Check for libmetalink +dnl +AS_IF([test x"$with_metalink" != xno], [ + PKG_CHECK_MODULES([METALINK], libmetalink, [ + LIBS="$METALINK_LIBS $LIBS" + CFLAGS="$METALINK_CFLAGS $CFLAGS" + AC_DEFINE([HAVE_METALINK], [1], [Define if using metalink.]) + have_metalink=yes + ], [ + have_metalink=no + ]) +]) + +dnl +dnl Check for GPGME +dnl +AM_PATH_GPGME([], [ + LIBS="$GPGME_LIBS $LIBS" + CFLAGS="$GPGME_CFLAGS $CFLAGS" + AC_DEFINE([HAVE_GPGME], [1], [Define if GPGME is available.]) + have_gpg=yes + ], [have_gpg=no]) dnl ********************************************************************** dnl Checks for IPv6 @@ -714,7 +737,7 @@ AS_IF([test "X$enable_pcre" != "Xno"],[ dnl Needed by src/Makefile.am AM_CONDITIONAL([IRI_IS_ENABLED], [test "X$iri" != "Xno"]) - +AM_CONDITIONAL([METALINK_IS_ENABLED], [test "X$have_metalink" != "Xno"]) dnl dnl Create output @@ -743,4 +766,5 @@ AC_MSG_NOTICE([Summary of build options: Debugging: $ENABLE_DEBUG Assertions: $ENABLE_ASSERTION Valgrind: $VALGRIND_INFO + GPGME: $have_gpg ]) diff --git a/doc/wget.texi b/doc/wget.texi index 16cc5db..a9a0f6b 100644 --- a/doc/wget.texi +++ b/doc/wget.texi @@ -507,6 +507,18 @@ treated as @samp{html} if the Content-Type matches @samp{text/html}. Furthermore, the @var{file}'s location will be implicitly used as base href if none was specified. +@cindex input-metalink +@item --input-metalink=@var{file} +Downloads files covered in local Metalink @var{file}. Metalink version 3 +and 4 are supported. + +@cindex metalink-over-http +@item --metalink-over-http +Issues HTTP HEAD request instead of GET and extracts Metalink metadata +from response headers. Then it switches to Metalink download. +If no valid Metalink metadata is found, it falls back to ordinary HTTP download. + + @cindex force html @item -F @itemx --force-html diff --git a/po/POTFILES.in b/po/POTFILES.in index 5406e0f..0af89dc 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -26,6 +26,7 @@ src/init.c src/iri.c src/log.c src/main.c +src/metalink.c src/mswindows.c src/netrc.c src/openssl.c diff --git a/src/Makefile.am b/src/Makefile.am index e8e9373..449a27f 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -35,6 +35,10 @@ if IRI_IS_ENABLED IRI_OBJ = iri.c endif +if METALINK_IS_ENABLED +METALINK_OBJ = metalink.c +endif + # The following line is losing on some versions of make! DEFS = @DEFS@ -DSYSTEM_WGETRC=\"$(sysconfdir)/wgetrc\" -DLOCALEDIR=\"$(localedir)\" LIBS = @LIBICONV@ @LIBINTL@ @LIBS@ $(LIB_CLOCK_GETTIME) @@ -47,13 +51,13 @@ wget_SOURCES = connect.c convert.c cookies.c ftp.c \ ftp-basic.c ftp-ls.c hash.c host.c html-parse.c html-url.c \ http.c init.c log.c main.c netrc.c progress.c ptimer.c \ recur.c res.c retr.c spider.c url.c warc.c \ - utils.c exits.c build_info.c $(IRI_OBJ) \ + utils.c exits.c build_info.c $(IRI_OBJ) $(METALINK_OBJ) \ css-url.h css-tokens.h connect.h convert.h cookies.h \ ftp.h hash.h host.h html-parse.h html-url.h \ http.h http-ntlm.h init.h log.h mswindows.h netrc.h \ options.h progress.h ptimer.h recur.h res.h retr.h \ spider.h ssl.h sysdep.h url.h warc.h utils.h wget.h iri.h \ - exits.h version.h + exits.h version.h metalink.h nodist_wget_SOURCES = version.c EXTRA_wget_SOURCES = iri.c LDADD = $(LIBOBJS) ../lib/libgnu.a diff --git a/src/http.c b/src/http.c index 777903b..afbc254 100644 --- a/src/http.c +++ b/src/http.c @@ -61,6 +61,10 @@ as that of the covered work. */ #include "warc.h" #include "c-strcase.h" #include "version.h" +#ifdef HAVE_METALINK +# include "metalink.h" +# include "xstrndup.h" +#endif #ifdef TESTING #include "test.h" @@ -1497,6 +1501,9 @@ struct http_stat wgint orig_file_size; /* size of file to compare for time-stamping */ time_t orig_file_tstamp; /* time-stamp of file to compare for * time-stamping */ +#ifdef HAVE_METALINK + metalink_t *metalink; +#endif }; static void @@ -1509,6 +1516,10 @@ free_hstat (struct http_stat *hs) xfree (hs->local_file); xfree (hs->orig_file_name); xfree (hs->message); +#ifdef HAVE_METALINK + metalink_delete (hs->metalink); + hs->metalink = NULL; +#endif } static void @@ -2450,6 +2461,553 @@ set_content_type (int *dt, const char *type) *dt &= ~TEXTCSS; } +#ifdef HAVE_METALINK + +/* + Find value of given key. This is intended for Link header, but will + work with any header that uses ';' as field separator and '=' as key-value + separator. + + Link = "Link" ":" #link-value + link-value = "<" URI-Reference ">" *( ";" link-param ) + link-param = ( ( "rel" "=" relation-types ) + | ( "anchor" "=" <"> URI-Reference <"> ) + | ( "rev" "=" relation-types ) + | ( "hreflang" "=" Language-Tag ) + | ( "media" "=" ( MediaDesc | ( <"> MediaDesc <"> ) ) ) + | ( "title" "=" quoted-string ) + | ( "title*" "=" ext-value ) + | ( "type" "=" ( media-type | quoted-mt ) ) + | ( link-extension ) ) + link-extension = ( parmname [ "=" ( ptoken | quoted-string ) ] ) + | ( ext-name-star "=" ext-value ) + ext-name-star = parmname "*" ; reserved for RFC2231-profiled + ; extensions. Whitespace NOT + ; allowed in between. + ptoken = 1*ptokenchar + ptokenchar = "!" | "#" | "$" | "%" | "&" | "'" | "(" + | ")" | "*" | "+" | "-" | "." | "/" | DIGIT + | ":" | "<" | "=" | ">" | "?" | "@" | ALPHA + | "[" | "]" | "^" | "_" | "`" | "{" | "|" + | "}" | "~" + media-type = type-name "/" subtype-name + quoted-mt = <"> media-type <"> + relation-types = relation-type + | <"> relation-type *( 1*SP relation-type ) <"> + relation-type = reg-rel-type | ext-rel-type + reg-rel-type = LOALPHA *( LOALPHA | DIGIT | "." | "-" ) + ext-rel-type = URI + + See more: rfc5988 +*/ +static bool +find_key_value (const char *start, const char *end, const char *key, char **value) +{ + const char *eq; + size_t key_len = strlen (key); + const char *val_beg, *val_end; + const char *key_beg; + + key_beg = start; + + while (key_beg + key_len + 1 < end) + { + /* Skip whitespaces. */ + while (key_beg + key_len + 1 < end && c_isspace (*key_beg)) + key_beg++; + if (strncmp (key_beg, key, key_len)) + { + /* Find next token. */ + while (key_beg + key_len + 1 < end && *key_beg != ';') + key_beg++; + key_beg++; + continue; + } + else + { + /* Find equals sign. */ + eq = key_beg + key_len; + while (eq < end && c_isspace (*eq)) + eq++; + if (eq == end) + return false; + if (*eq != '=') + { + key_beg++; + continue; + } + + val_beg = eq + 1; + while (val_beg < end && c_isspace (*val_beg)) + val_beg++; + if (val_beg == end) + return false; + val_end = val_beg + 1; + while (val_end < end && *val_end != ';' && !c_isspace (*val_end)) + val_end++; + *value = xstrndup (val_beg, val_end - val_beg); + return true; + } + } + *value = NULL; + return false; +} + +/* This is to check if given token exists in HTTP header. Tokens are + separated by ';'. */ +static bool +has_key (const char *start, const char *end, const char *key) +{ + const char *pos; /* Here would the token start. */ + size_t key_len = strlen (key); + + pos = start; + while (pos + key_len <= end) + { + /* Skip whitespaces at beginning. */ + while (pos + key_len <= end && c_isspace (*pos)) + pos++; + + /* Does the prefix of pos match our key? */ + if (strncmp (key, pos, key_len)) + { + /* This was not a match. + Skip all characters until beginning of next token. */ + while (pos + key_len <= end && *pos != ';') + pos++; + pos++; + continue; + } + + /* key is prefix of pos. Is it the exact token or just a prefix? */ + pos += key_len; + while (pos < end && c_isspace (*pos)) + pos++; + if (pos == end || *pos == ';') + return true; + + /* This was not a match (just a prefix). + Skip all characters until beginning of next token. */ + while (pos + key_len <= end && *pos != ';') + pos++; + pos++; + } + return false; +} + +/* Find all key=value pairs delimited with ';' or ','. This is intended for + Digest header parsing. + The usage is: + + const char *pos; + for (pos = header_beg; pos = find_key_values (pos, header_end, &key, &val); pos++) + { + ... + } + + */ +static const char * +find_key_values (const char *start, const char *end, char **key, char **value) +{ + const char *key_start, *key_end; + const char *eq; + const char *val_start, *val_end; + + eq = start; + while (eq < end && *eq != '=') + { + /* Skip tokens without =value part. */ + if (*eq == ';' || *eq == ',') + start = eq + 1; + eq++; + } + + if (eq >= end) + return NULL; + + key_start = start; + while (key_start < eq && c_isspace (*key_start)) + key_start++; + + key_end = eq - 1; + while (key_end > key_start && c_isspace (*key_end)) + key_end--; + key_end++; + + val_start = eq + 1; + while (val_start < end && c_isspace (*val_start)) + val_start++; + + val_end = val_start; + + while (val_end < end && *val_end != ';' && + *val_end != ',' && !c_isspace (*val_end)) + val_end++; + + *key = xstrndup (key_start, key_end - key_start); + *value = xstrndup (val_start, val_end - val_start); + + /* Skip trailing whitespaces. */ + while (val_end < end && c_isspace (*val_end)) + val_end++; + + return val_end; +} + +/* Will return proper metalink_t structure if enough data was found in + http response resp. Otherwise returns NULL. + Two exit points: one for success and one for failure. */ +static metalink_t * +metalink_from_http (const struct response *resp, const struct http_stat *hs, + const struct url *u) +{ + metalink_t *metalink = NULL; + metalink_file_t *mfile = xnew0 (metalink_file_t); + const char *val_beg, *val_end; + int res_count = 0, hash_count = 0, sig_count = 0, i; + + DEBUGP (("Checking for Metalink in HTTP response\n")); + + /* Initialize metalink file for our simple use case. */ + if (hs->local_file) + mfile->name = xstrdup (hs->local_file); + else + mfile->name = url_file_name (u, NULL); + + /* Begin with 1-element array (for 0-termination). */ + mfile->checksums = xnew0 (metalink_checksum_t *); + mfile->resources = xnew0 (metalink_resource_t *); + + /* Find all Link headers. */ + for (i = 0; + (i = resp_header_locate (resp, "Link", i, &val_beg, &val_end)) != -1; + i++) + { + char *rel = NULL, *reltype = NULL; + char *urlstr = NULL; + const char *url_beg, *url_end, *attrs_beg; + size_t url_len; + + /* Sample Metalink Link headers: + + Link: <http://www2.example.com/dir1/dir2/dir3/dir4/dir5/example.ext>; + rel=duplicate; pri=1; pref; geo=gb; depth=4 + + Link: <http://example.com/example.ext.asc>; rel=describedby; + type="application/pgp-signature" + */ + + /* Find beginning of URL. */ + url_beg = val_beg; + while (url_beg < val_end - 1 && c_isspace (*url_beg)) + url_beg++; + + /* Find end of URL. */ + /* The convention here is that end ptr points to one element after + end of string. In this case, it should be pointing to the '>', which + is one element after end of actual URL. Therefore, it should never point + to val_end, which is one element after entire header value string. */ + url_end = url_beg + 1; + while (url_end < val_end - 1 && *url_end != '>') + url_end++; + + if (url_beg >= val_end || url_end >= val_end || + *url_beg != '<' || *url_end != '>') + { + DEBUGP (("This is not a valid Link header. Ignoring.\n")); + continue; + } + + /* Skip <. */ + url_beg++; + url_len = url_end - url_beg; + + /* URL found. Now handle the attributes. */ + attrs_beg = url_end + 1; + + /* First we need to find out what type of link it is. Currently, we + support rel=duplicate and rel=describedby. */ + if (!find_key_value (attrs_beg, val_end, "rel", &rel)) + { + DEBUGP (("No rel value in Link header, skipping.\n")); + continue; + } + + urlstr = xstrndup (url_beg, url_len); + DEBUGP (("URL=%s\n", urlstr)); + DEBUGP (("rel=%s\n", rel)); + + /* Handle signatures. + Libmetalink only supports one signature per file. Therefore we stop + as soon as we successfully get first supported signature. */ + if (sig_count == 0 && + !strcmp (rel, "describedby") && + find_key_value (attrs_beg, val_end, "type", &reltype) && + !strcmp (reltype, "application/pgp-signature") + ) + { + /* Download the signature to a temporary file. */ + FILE *_output_stream = output_stream; + bool _output_stream_regular = output_stream_regular; + + output_stream = tmpfile (); + if (output_stream) + { + struct iri *iri = iri_new (); + struct url *url; + int url_err; + + set_uri_encoding (iri, opt.locale, true); + url = url_parse (urlstr, &url_err, iri, false); + + if (!url) + { + char *error = url_error (urlstr, url_err); + logprintf (LOG_NOTQUIET, _("When downloading signature:\n" + "%s: %s.\n"), urlstr, error); + xfree (error); + } + else + { + /* Avoid recursive Metalink from HTTP headers. */ + bool _metalink_http = opt.metalink_over_http; + uerr_t retr_err; + + opt.metalink_over_http = false; + retr_err = retrieve_url (url, urlstr, NULL, NULL, + NULL, NULL, false, iri, false); + opt.metalink_over_http = _metalink_http; + + url_free (url); + iri_free (iri); + + if (retr_err == RETROK) + { + /* Signature is in the temporary file. Read it into + metalink resource structure. */ + metalink_signature_t msig; + size_t siglen; + + fseek (output_stream, 0, SEEK_END); + siglen = ftell (output_stream); + fseek (output_stream, 0, SEEK_SET); + + DEBUGP (("siglen=%lu\n", siglen)); + + msig.signature = xmalloc (siglen + 1); + if (fread (msig.signature, siglen, 1, output_stream) != 1) + { + logputs (LOG_NOTQUIET, + _("Unable to read signature content from " + "temporary file. Skipping.\n")); + xfree (msig.signature); + } + else + { + msig.signature[siglen] = '\0'; /* Just in case. */ + msig.mediatype = xstrdup ("application/pgp-signature"); + + DEBUGP (("Signature (%s):\n%s\n", + msig.mediatype, msig.signature)); + + mfile->signature = xnew (metalink_signature_t); + *mfile->signature = msig; + + sig_count++; + } + } + } + fclose (output_stream); + } + else + { + logputs (LOG_NOTQUIET, _("Could not create temporary file. " + "Skipping signature download.\n")); + } + output_stream_regular = _output_stream_regular; + output_stream = _output_stream; + } /* Iterate over signatures. */ + + /* Handle Metalink resources. */ + else if (!strcmp (rel, "duplicate")) + { + metalink_resource_t mres = {0}; + char *pristr; + + /* + Valid ranges for the "pri" attribute are from + 1 to 999999. Mirror servers with a lower value of the "pri" + attribute have a higher priority, while mirrors with an undefined + "pri" attribute are considered to have a value of 999999, which is + the lowest priority. + + rfc6249 section 3.1 + */ + mres.priority = DEFAULT_PRI; + if (find_key_value (url_end, val_end, "pri", &pristr)) + { + long pri; + char *end_pristr; + /* Do not care for errno since 0 is error in this case. */ + pri = strtol (pristr, &end_pristr, 10); + if (end_pristr != pristr + strlen (pristr) || + !VALID_PRI_RANGE (pri)) + { + /* This is against the specification, so let's inform the user. */ + logprintf (LOG_NOTQUIET, + _("Invalid pri value. Assuming %d.\n"), + DEFAULT_PRI); + } + else + mres.priority = pri; + xfree (pristr); + } + + switch (url_scheme (urlstr)) + { + case SCHEME_HTTP: + mres.type = xstrdup ("http"); + break; +#ifdef HAVE_SSL + case SCHEME_HTTPS: + mres.type = xstrdup ("https"); + break; +#endif + case SCHEME_FTP: + mres.type = xstrdup ("ftp"); + break; + default: + DEBUGP (("Unsupported url scheme in %s. Skipping resource.\n", urlstr)); + } + + if (mres.type) + { + DEBUGP (("TYPE=%s\n", mres.type)); + + /* At this point we have validated the new resource. */ + + find_key_value (url_end, val_end, "geo", &mres.location); + + mres.url = urlstr; + urlstr = NULL; + + mres.preference = 0; + if (has_key (url_end, val_end, "pref")) + { + DEBUGP (("This resource has preference\n")); + mres.preference = 1; + } + + /* 1 slot from new resource, 1 slot for null-termination. */ + mfile->resources = xrealloc (mfile->resources, + sizeof (metalink_resource_t *) * (res_count + 2)); + mfile->resources[res_count] = xnew0 (metalink_resource_t); + *mfile->resources[res_count] = mres; + res_count++; + } + } /* Handle resource link (rel=duplicate). */ + else + DEBUGP (("This link header was not used for Metalink\n")); + + xfree (urlstr); + xfree (reltype); + xfree (rel); + } /* Iterate over link headers. */ + + /* Null-terminate resources array. */ + mfile->resources[res_count] = 0; + + if (res_count == 0) + { + DEBUGP (("No valid metalink references found.\n")); + goto fail; + } + + /* Find all Digest headers. */ + for (i = 0; + (i = resp_header_locate (resp, "Digest", i, &val_beg, &val_end)) != -1; + i++) + { + const char *dig_pos; + char *dig_type, *dig_hash; + + /* Each Digest header can include multiple hashes. Example: + Digest: SHA=thvDyvhfIqlvFe+A9MYgxAfm1q5=,unixsum=30637 + Digest: md5=HUXZLQLMuI/KZ5KDcJPcOA== + */ + for (dig_pos = val_beg; + (dig_pos = find_key_values (dig_pos, val_end, &dig_type, &dig_hash)); + dig_pos++) + { + /* The hash here is assumed to be base64. We need the hash in hex. + Therefore we convert: base64 -> binary -> hex. */ + const size_t dig_hash_str_len = strlen (dig_hash); + char *bin_hash = alloca (dig_hash_str_len * 3 / 4 + 1); + size_t hash_bin_len; + + hash_bin_len = base64_decode (dig_hash, bin_hash); + + /* One slot for me, one for zero-termination. */ + mfile->checksums = + xrealloc (mfile->checksums, + sizeof (metalink_checksum_t *) * (hash_count + 2)); + mfile->checksums[hash_count] = xnew (metalink_checksum_t); + mfile->checksums[hash_count]->type = dig_type; + + mfile->checksums[hash_count]->hash = xmalloc (hash_bin_len * 2 + 1); + hex_to_string (mfile->checksums[hash_count]->hash, bin_hash, hash_bin_len); + + xfree (dig_hash); + + hash_count++; + } + } + + /* Zero-terminate checksums array. */ + mfile->checksums[hash_count] = 0; + + /* + If Instance Digests are not provided by the Metalink servers, the + Link header fields pertaining to this specification MUST be ignored. + + rfc6249 section 6 + */ + if (hash_count == 0) + { + logputs (LOG_VERBOSE, + _("Could not find acceptable digest for Metalink resources.\n" + "Ignoring them.\n")); + goto fail; + } + + /* Metalink data is OK. Now we just need to sort the resources based + on their priorities, preference, and perhaps location. */ + stable_sort (mfile->resources, res_count, sizeof (metalink_resource_t *), metalink_res_cmp); + + /* Restore sensible preference values (in case someone cares to look). */ + for (i = 0; i < res_count; ++i) + mfile->resources[i]->preference = 1000000 - mfile->resources[i]->priority; + + metalink = xnew0 (metalink_t); + metalink->files = xmalloc (sizeof (metalink_file_t *) * 2); + metalink->files[0] = mfile; + metalink->files[1] = 0; + metalink->origin = xstrdup (u->url); + metalink->version = METALINK_VERSION_4; + /* Leave other fields set to 0. */ + + return metalink; + +fail: + /* Free all allocated memory. */ + if (metalink) + metalink_delete (metalink); + else + metalink_file_delete (mfile); + return NULL; +} +#endif /* HAVE_METALINK */ + /* Retrieve a document through HTTP protocol. It recognizes status code, and correctly handles redirections. It closes the network socket. If it receives an error from the functions below it, it @@ -2501,6 +3059,11 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, /* Whether conditional get request will be issued. */ bool cond_get = !!(*dt & IF_MODIFIED_SINCE); +#ifdef HAVE_METALINK + /* Are we looking for metalink info in HTTP headers? */ + bool metalink = !!(*dt & METALINK_METADATA); +#endif + char *head = NULL; struct response *resp = NULL; char hdrval[512]; @@ -2838,6 +3401,19 @@ gethttp (struct url *u, struct http_stat *hs, int *dt, struct url *proxy, when we're done. This means that we can register it. */ register_persistent (conn->host, conn->port, sock, using_ssl); +#ifdef HAVE_METALINK + /* We need to check for the Metalink data in the very first response + we get from the server (before redirectionrs, authorization, etc.). */ + if (metalink) + { + hs->metalink = metalink_from_http (resp, hs, u); + xfree (hs->message); + retval = RETR_WITH_METALINK; + CLOSE_FINISH (sock); + goto cleanup; + } +#endif + if (statcode == HTTP_STATUS_UNAUTHORIZED) { /* Authorization is required. */ @@ -3383,6 +3959,14 @@ http_loop (struct url *u, struct url *original_url, char **newloc, else file_name = xstrdup (opt.output_document); +#ifdef HAVE_METALINK + if (opt.metalink_over_http) + { + *dt |= METALINK_METADATA; + send_head_first = true; + } +#endif + if (opt.timestamping) { /* Use conditional get request if requested @@ -3569,6 +4153,29 @@ Spider mode enabled. Check if remote file exists.\n")); case RETRFINISHED: /* Deal with you later. */ break; +#ifdef HAVE_METALINK + case RETR_WITH_METALINK: + { + if (hstat.metalink == NULL) + { + logputs (LOG_NOTQUIET, + _("Could not find Metalink data in HTTP response. " + "Downloading file using HTTP GET.\n")); + *dt &= ~METALINK_METADATA; + *dt &= ~HEAD_ONLY; + got_head = true; + continue; + } + + logputs (LOG_VERBOSE, + _("Metalink headers found. " + "Switching to Metalink mode.\n")); + + ret = retrieve_from_metalink (hstat.metalink); + goto exit; + } + break; +#endif default: /* All possibilities should have been exhausted. */ abort (); diff --git a/src/init.c b/src/init.c index a436ef2..47d8610 100644 --- a/src/init.c +++ b/src/init.c @@ -215,6 +215,9 @@ static const struct { { "inet6only", &opt.ipv6_only, cmd_boolean }, #endif { "input", &opt.input_filename, cmd_file }, +#ifdef HAVE_METALINK + { "input-metalink", &opt.input_metalink, cmd_file }, +#endif { "iri", &opt.enable_iri, cmd_boolean }, { "keepsessioncookies", &opt.keep_session_cookies, cmd_boolean }, { "limitrate", &opt.limit_rate, cmd_bytes }, @@ -223,6 +226,9 @@ static const struct { { "logfile", &opt.lfilename, cmd_file }, { "login", &opt.ftp_user, cmd_string },/* deprecated*/ { "maxredirect", &opt.max_redirect, cmd_number }, +#ifdef HAVE_METALINK + { "metalink-over-http", &opt.metalink_over_http, cmd_boolean }, +#endif { "method", &opt.method, cmd_string_uppercase }, { "mirror", NULL, cmd_spec_mirror }, { "netrc", &opt.netrc, cmd_boolean }, @@ -1795,6 +1801,9 @@ cleanup (void) xfree (opt.lfilename); xfree (opt.dir_prefix); xfree (opt.input_filename); +#ifdef HAVE_METALINK + xfree (opt.input_metalink); +#endif xfree (opt.output_document); free_vec (opt.accepts); free_vec (opt.rejects); diff --git a/src/main.c b/src/main.c index a0044d9..a66f7bc 100644 --- a/src/main.c +++ b/src/main.c @@ -63,6 +63,11 @@ as that of the covered work. */ #include <getpass.h> #include <quote.h> +#ifdef HAVE_METALINK +# include <metalink/metalink_parser.h> +# include "metalink.h" +#endif + #ifdef WINDOWS # include <io.h> # include <fcntl.h> @@ -241,6 +246,9 @@ static struct cmdline_option option_data[] = { "inet6-only", '6', OPT_BOOLEAN, "inet6only", -1 }, #endif { "input-file", 'i', OPT_VALUE, "input", -1 }, +#ifdef HAVE_METALINK + { "input-metalink", 0, OPT_VALUE, "input-metalink", -1 }, +#endif { "iri", 0, OPT_BOOLEAN, "iri", -1 }, { "keep-session-cookies", 0, OPT_BOOLEAN, "keepsessioncookies", -1 }, { "level", 'l', OPT_VALUE, "reclevel", -1 }, @@ -248,6 +256,9 @@ static struct cmdline_option option_data[] = { "load-cookies", 0, OPT_VALUE, "loadcookies", -1 }, { "local-encoding", 0, OPT_VALUE, "localencoding", -1 }, { "max-redirect", 0, OPT_VALUE, "maxredirect", -1 }, +#ifdef HAVE_METALINK + { "metalink-over-http", 0, OPT_BOOLEAN, "metalink-over-http", -1 }, +#endif { "method", 0, OPT_VALUE, "method", -1 }, { "mirror", 'm', OPT_BOOLEAN, "mirror", -1 }, { "no", 'n', OPT__NO, NULL, required_argument }, @@ -483,6 +494,10 @@ Logging and input file:\n"), --report-speed=TYPE output bandwidth as TYPE. TYPE can be bits\n"), N_("\ -i, --input-file=FILE download URLs found in local or external FILE\n"), +#ifdef HAVE_METALINK + N_("\ + --input-metalink=FILE download files covered in local Metalink FILE\n"), +#endif N_("\ -F, --force-html treat input file as HTML\n"), N_("\ @@ -577,6 +592,10 @@ Download:\n"), --remote-encoding=ENC use ENC as the default remote encoding\n"), N_("\ --unlink remove file before clobber\n"), +#ifdef HAVE_METALINK + N_("\ + --metalink-over-http use Metalink metadata from HTTP response headers\n"), +#endif "\n", N_("\ @@ -1405,7 +1424,11 @@ for details.\n\n")); opt.always_rest = false; } - if (!nurl && !opt.input_filename) + if (!nurl && !opt.input_filename +#ifdef HAVE_METALINK + && !opt.input_metalink +#endif + ) { /* No URL specified. */ fprintf (stderr, _("%s: missing URL\n"), exec_name); @@ -1730,6 +1753,37 @@ outputting to a regular file.\n")); opt.input_filename); } +#ifdef HAVE_METALINK + /* Finally, from metlink file, if any. */ + if (opt.input_metalink) + { + metalink_error_t meta_err; + uerr_t retr_err; + metalink_t *metalink; + + meta_err = metalink_parse_file (opt.input_metalink, &metalink); + + if (meta_err) + { + logprintf (LOG_NOTQUIET, _("Unable to parse metalink file %s.\n"), + opt.input_metalink); + retr_err = METALINK_PARSE_ERROR; + } + else + { + retr_err = retrieve_from_metalink (metalink); + if (retr_err != RETROK) + { + logprintf (LOG_NOTQUIET, + _("Could not download all resources from %s.\n"), + quote (opt.input_metalink)); + } + } + inform_exit_status (retr_err); + metalink_delete (metalink); + } +#endif /* HAVE_METALINK */ + /* Print broken links. */ if (opt.recursive && opt.spider) print_broken_links (); diff --git a/src/metalink.c b/src/metalink.c new file mode 100644 index 0000000..962dd94 --- /dev/null +++ b/src/metalink.c @@ -0,0 +1,448 @@ +/* Metalink module. + Copyright (C) 2015 Free Software Foundation, Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or (at +your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ + +#include "wget.h" +#ifdef HAVE_METALINK + +#include "metalink.h" +#include "retr.h" +#include "exits.h" +#include "utils.h" +#include "sha256.h" +#include <sys/errno.h> +#include <unistd.h> /* For unlink. */ +#include <metalink/metalink_parser.h> +#ifdef HAVE_GPGME +#include <gpgme.h> +#include <fcntl.h> /* For open and close. */ +#endif + +/* Loop through all files in metalink structure and retrieve them. + Returns RETROK if all files were downloaded. + Returns last retrieval error (from retrieve_url) if some files + could not be downloaded. */ +uerr_t +retrieve_from_metalink (const metalink_t* metalink) +{ + metalink_file_t **mfile_ptr; + uerr_t last_retr_err = RETROK; /* Store last encountered retrieve error. */ + + FILE *_output_stream = output_stream; + bool _output_stream_regular = output_stream_regular; + char *_output_document = opt.output_document; + + DEBUGP (("Retrieving from Metalink\n")); + + /* No files to download. */ + if (!metalink->files) + return RETROK; + + if (opt.output_document) + { + /* We cannot support output_document as we need to compute checksum + of downloaded file, and to remove it if the checksum is bad. */ + logputs (LOG_NOTQUIET, + _("-O not supported for metalink download. Ignoring.\n")); + } + + for (mfile_ptr = metalink->files; *mfile_ptr; mfile_ptr++) + { + metalink_file_t *mfile = *mfile_ptr; + metalink_resource_t **mres_ptr; + char *filename = NULL; + bool hash_ok = false; + + uerr_t retr_err; + + /* -1 -> file should be rejected + 0 -> could not verify + 1 -> verified successfully */ + char sig_status = 0; + + output_stream = NULL; + + DEBUGP (("Processing metalink file %s...\n", quote (mfile->name))); + + /* Resources are sorted by priority. */ + for (mres_ptr = mfile->resources; *mres_ptr; mres_ptr++) + { + metalink_resource_t *mres = *mres_ptr; + metalink_checksum_t **mchksum_ptr, *mchksum; + struct iri *iri; + struct url *url; + int url_err; + + if (!RES_TYPE_SUPPORTED (mres->type)) + { + logprintf (LOG_VERBOSE, + _("Resource type %s not supported, ignoring...\n"), + quote (mres->type)); + continue; + } + + retr_err = METALINK_RETR_ERROR; + + /* If output_stream is not NULL, then we have failed on + previous resource and are retrying. Thus, remove the file. */ + if (output_stream) + { + fclose (output_stream); + output_stream = NULL; + if (unlink (filename)) + logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno)); + xfree (filename); + } + + /* Parse our resource URL. */ + iri = iri_new (); + set_uri_encoding (iri, opt.locale, true); + url = url_parse (mres->url, &url_err, iri, false); + + if (!url) + { + char *error = url_error (mres->url, url_err); + logprintf (LOG_NOTQUIET, "%s: %s.\n", mres->url, error); + xfree (error); + inform_exit_status (URLERROR); + iri_free (iri); + continue; + } + else + { + /* Avoid recursive Metalink from HTTP headers. */ + bool _metalink_http = opt.metalink_over_http; + + /* Assure proper local file name regardless of the URL + of particular Metalink resource. + To do that we create the local file here and put + it as output_stream. We restore the original configuration + after we are finished with the file. */ + output_stream = unique_create (mfile->name, true, &filename); + output_stream_regular = true; + + /* Store the real file name for displaying in messages. */ + opt.output_document = filename; + + opt.metalink_over_http = false; + DEBUGP (("Storing to %s\n", filename)); + retr_err = retrieve_url (url, mres->url, NULL, NULL, + NULL, NULL, opt.recursive, iri, false); + opt.metalink_over_http = _metalink_http; + } + url_free (url); + iri_free (iri); + + if (retr_err == RETROK) + { + FILE *local_file; + + /* Check the digest. */ + local_file = fopen (filename, "r"); + if (!local_file) + { + logprintf (LOG_NOTQUIET, _("Could not open downloaded file.\n")); + continue; + } + + for (mchksum_ptr = mfile->checksums; *mchksum_ptr; mchksum_ptr++) + { + char sha256[SHA256_DIGEST_SIZE]; + char sha256_txt[2 * SHA256_DIGEST_SIZE + 1]; + + mchksum = *mchksum_ptr; + + /* I have seen both variants... */ + if (strcasecmp (mchksum->type, "sha256") + && strcasecmp (mchksum->type, "sha-256")) + { + DEBUGP (("Ignoring unsupported checksum type %s.\n", + quote (mchksum->type))); + continue; + } + + logprintf (LOG_VERBOSE, _("Computing checksum for %s\n"), + quote (mfile->name)); + + sha256_stream (local_file, sha256); + hex_to_string (sha256_txt, sha256, SHA256_DIGEST_SIZE); + DEBUGP (("Declared hash: %s\n", mchksum->hash)); + DEBUGP (("Computed hash: %s\n", sha256_txt)); + if (!strcmp (sha256_txt, mchksum->hash)) + { + logputs (LOG_VERBOSE, + _("Checksum matches.\n")); + hash_ok = true; + } + else + { + logprintf (LOG_NOTQUIET, + _("Checksum mismatch for file %s.\n"), + quote (mfile->name)); + hash_ok = false; + } + + /* Stop as soon as we checked the supported checksum. */ + break; + } /* Iterate over available checksums. */ + fclose (local_file); + local_file = NULL; + + if (!hash_ok) + continue; + + sig_status = 0; /* Not verified. */ + +#ifdef HAVE_GPGME + /* Check the crypto signature. */ + if (mfile->signature) + { + metalink_signature_t *msig; + gpgme_error_t gpgerr; + gpgme_ctx_t gpgctx; + gpgme_data_t gpgsigdata, gpgdata; + gpgme_verify_result_t gpgres; + int fd; + + /* Initialize the library - as name suggests. */ + gpgme_check_version (NULL); + + /* Open data file. */ + fd = open (filename, O_RDONLY); + if (fd == -1) + { + logputs (LOG_NOTQUIET, + _("Could not open downloaded file for signature " + "verification.\n")); + goto gpg_skip_verification; + } + + /* Assign file descriptor to GPG data structure. */ + gpgerr = gpgme_data_new_from_fd (&gpgdata, fd); + if (gpgerr != GPG_ERR_NO_ERROR) + { + logprintf (LOG_NOTQUIET, + "GPGME data_new_from_fd: %s\n", + gpgme_strerror (gpgerr)); + goto gpg_cleanup_fd; + } + + /* Prepare new GPGME context. */ + gpgerr = gpgme_new (&gpgctx); + if (gpgerr != GPG_ERR_NO_ERROR) + { + logprintf (LOG_NOTQUIET, + "GPGME new: %s\n", + gpgme_strerror (gpgerr)); + goto gpg_cleanup_data; + } + + /* Note that this will only work for Metalink-over-HTTP + requests (that we parse manually) due to a bug in + Libmetalink. Another problem with Libmetalink is that + it supports at most one signature per file. The below + line should be modified after Libmetalink resolves these + issues. */ + for (msig = mfile->signature; msig == mfile->signature; msig++) + { + gpgme_signature_t gpgsig; + gpgme_protocol_t gpgprot = GPGME_PROTOCOL_UNKNOWN; + + DEBUGP (("Veryfying signature %s:\n%s\n", + quote (msig->mediatype), + msig->signature)); + + /* Check signature type. */ + if (!strcmp (msig->mediatype, "application/pgp-signature")) + gpgprot = GPGME_PROTOCOL_OpenPGP; + else /* Unsupported signature type. */ + continue; + + gpgerr = gpgme_set_protocol (gpgctx, gpgprot); + if (gpgerr != GPG_ERR_NO_ERROR) + { + logprintf (LOG_NOTQUIET, + "GPGME set_protocol: %s\n", + gpgme_strerror (gpgerr)); + continue; + } + + /* Load the signature. */ + gpgerr = gpgme_data_new_from_mem (&gpgsigdata, + msig->signature, + strlen (msig->signature), + 0); + if (gpgerr != GPG_ERR_NO_ERROR) + { + logprintf (LOG_NOTQUIET, + _("GPGME data_new_from_mem: %s\n"), + gpgme_strerror (gpgerr)); + continue; + } + + /* Verify the signature. */ + gpgerr = gpgme_op_verify (gpgctx, gpgsigdata, gpgdata, NULL); + if (gpgerr != GPG_ERR_NO_ERROR) + { + logprintf (LOG_NOTQUIET, + _("GPGME op_verify: %s\n"), + gpgme_strerror (gpgerr)); + gpgme_data_release (gpgsigdata); + continue; + } + + /* Check the results. */ + gpgres = gpgme_op_verify_result (gpgctx); + if (!gpgres) + { + logputs (LOG_NOTQUIET, + _("GPGME op_verify_result: NULL\n")); + gpgme_data_release (gpgsigdata); + continue; + } + + /* The list is null-terminated. */ + for (gpgsig = gpgres->signatures; gpgsig; gpgsig = gpgsig->next) + { + DEBUGP (("Checking signature 0x%p\n", + (void *) gpgsig)); + DEBUGP (("Summary=0x%x Status=0x%x\n", + gpgsig->summary, gpgsig->status & 0xFFFF)); + + if (gpgsig->summary + & (GPGME_SIGSUM_VALID | GPGME_SIGSUM_GREEN)) + { + logputs (LOG_VERBOSE, + _("Signature validation suceeded.\n")); + sig_status = 1; + break; + } + + if (gpgsig->summary & GPGME_SIGSUM_RED) + { + logputs (LOG_NOTQUIET, + _("Invalid signature. Rejecting resource.\n")); + sig_status = -1; + break; + } + + if (gpgsig->summary == 0 + && (gpgsig->status & 0xFFFF) == GPG_ERR_NO_ERROR) + { + logputs (LOG_VERBOSE, + _("Data matches signature, but signature " + "is not trusted.\n")); + } + + if ((gpgsig->status & 0xFFFF) != GPG_ERR_NO_ERROR) + { + logprintf (LOG_NOTQUIET, + "GPGME: %s\n", + gpgme_strerror (gpgsig->status & 0xFFFF)); + } + } + + gpgme_data_release (gpgsigdata); + + if (sig_status != 0) + break; + } /* Iterate over signatures. */ + + gpgme_release (gpgctx); +gpg_cleanup_data: + gpgme_data_release (gpgdata); +gpg_cleanup_fd: + close (fd); + } /* endif (mfile->signature) */ +gpg_skip_verification: +#endif + /* Stop if file was downloaded with success. */ + if (sig_status >= 0) + break; + } /* endif RETR_OK. */ + } /* Iterate over resources. */ + + if (retr_err != RETROK) + { + logprintf (LOG_VERBOSE, _("Failed to download %s. Skipping resource.\n"), + quote (mfile->name)); + } + else if (!hash_ok) + { + retr_err = METALINK_CHKSUM_ERROR; + logprintf (LOG_NOTQUIET, + _("File %s retrieved but checksum does not match. " + "\n"), quote (mfile->name)); + } +#ifdef HAVE_GPGME + /* Signature will be only validated if hash check was successful. */ + else if (sig_status < 0) + { + retr_err = METALINK_SIG_ERROR; + logprintf (LOG_NOTQUIET, + _("File %s retrieved but signature does not match. " + "\n"), quote (mfile->name)); + } +#endif + last_retr_err = retr_err == RETROK ? last_retr_err : retr_err; + + /* Remove the file if error encountered or if option specified. + Note: the file has been downloaded using *_loop. Therefore, it + is not necessary to keep the file for continuated download. */ + if ((retr_err != RETROK || opt.delete_after) + && filename != NULL && file_exists_p (filename)) + { + logprintf (LOG_VERBOSE, _("Removing %s.\n"), quote (filename)); + if (unlink (filename)) + logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno)); + } + fclose (output_stream); + output_stream = NULL; + xfree (filename); + } /* Iterate over files. */ + + /* Restore original values. */ + opt.output_document = _output_document; + output_stream_regular = _output_stream_regular; + output_stream = _output_stream; + + return last_retr_err; +} + +int metalink_res_cmp (const void* v1, const void* v2) +{ + const metalink_resource_t *res1 = *(metalink_resource_t **) v1, + *res2 = *(metalink_resource_t **) v2; + if (res1->preference != res2->preference) + return res2->preference - res1->preference; + if (res1->priority != res2->priority) + return res1->priority - res2->priority; + return 0; +} + +#endif /* HAVE_METALINK */ diff --git a/src/metalink.h b/src/metalink.h new file mode 100644 index 0000000..202c545 --- /dev/null +++ b/src/metalink.h @@ -0,0 +1,50 @@ +/* Declarations for metalink.c. + Copyright (C) 2015 Free Software Foundation, Inc. + +This file is part of GNU Wget. + +GNU Wget is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 3 of the License, or +(at your option) any later version. + +GNU Wget is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with Wget. If not, see <http://www.gnu.org/licenses/>. + +Additional permission under GNU GPL version 3 section 7 + +If you modify this program, or any covered work, by linking or +combining it with the OpenSSL project's OpenSSL library (or a +modified version of that library), containing parts covered by the +terms of the OpenSSL or SSLeay licenses, the Free Software Foundation +grants you additional permission to convey the resulting work. +Corresponding Source for a non-source form of such a combination +shall include the source code for the parts of OpenSSL used as well +as that of the covered work. */ +#if ! defined METALINK_H && defined HAVE_METALINK +#define METALINK_H + +#include <metalink/metalink_types.h> +#include "wget.h" + +#ifdef HAVE_SSL +# define RES_TYPE_SUPPORTED(x)\ + ((!x) || !strcmp (x, "ftp") || !strcmp (x, "http") || !strcmp (x, "https")) +#else +# define RES_TYPE_SUPPORTED(x)\ + ((!x) || !strcmp (x, "ftp") || !strcmp (x, "http")) +#endif + +#define DEFAULT_PRI 999999 +#define VALID_PRI_RANGE(x) ((x) > 0 && (x) < 1000000) + +uerr_t retrieve_from_metalink (const metalink_t *metalink); + +int metalink_res_cmp (const void *res1, const void *res2); + +#endif /* METALINK_H */ diff --git a/src/options.h b/src/options.h index bef1f10..c377b50 100644 --- a/src/options.h +++ b/src/options.h @@ -58,6 +58,10 @@ struct options char *dir_prefix; /* The top of directory tree */ char *lfilename; /* Log filename */ char *input_filename; /* Input filename */ +#ifdef HAVE_METALINK + char *input_metalink; /* Input metalink file */ + bool metalink_over_http; /* Use Metalink if present in HTTP response */ +#endif char *choose_config; /* Specified config file */ bool noconfig; /* Ignore all config files? */ bool force_html; /* Is the input file an HTML file? */ diff --git a/src/utils.c b/src/utils.c index 7fccf66..4da45a1 100644 --- a/src/utils.c +++ b/src/utils.c @@ -2506,6 +2506,21 @@ get_max_length (const char *path, int length, int name) return ret; } +void +hex_to_string (char *str_buffer, const char *hex_buffer, size_t hex_len) +{ + size_t i; + + for (i = 0; i < hex_len; i++) + { + /* Each byte takes 2 characters. */ + sprintf (str_buffer + 2 * i, "%02x", hex_buffer[i] & 0xFF); + } + + /* Null-terminate result. */ + str_buffer[2 * i] = '\0'; +} + #ifdef TESTING const char * diff --git a/src/utils.h b/src/utils.h index be1888f..b265009 100644 --- a/src/utils.h +++ b/src/utils.h @@ -155,6 +155,8 @@ long get_max_length (const char *path, int length, int name); size_t strlcpy (char *dst, const char *src, size_t size); #endif +void hex_to_string (char *str_buffer, const char *hex_buffer, size_t hex_len); + extern unsigned char char_prop[]; #endif /* UTILS_H */ diff --git a/src/wget.h b/src/wget.h index 2c31713..2caa03e 100644 --- a/src/wget.h +++ b/src/wget.h @@ -332,7 +332,8 @@ enum ACCEPTRANGES = 0x0010, /* Accept-ranges header was found */ ADDED_HTML_EXTENSION = 0x0020, /* added ".html" extension due to -E */ TEXTCSS = 0x0040, /* document is of type text/css */ - IF_MODIFIED_SINCE = 0x0080 /* use if-modified-since header */ + IF_MODIFIED_SINCE = 0x0080, /* use if-modified-since header */ + METALINK_METADATA = 0x0100 /* use HTTP response for Metalink metadata */ }; /* Universal error type -- used almost everywhere. Error reporting of @@ -353,7 +354,10 @@ typedef enum AUTHFAILED, QUOTEXC, WRITEFAILED, SSLINITFAILED, VERIFCERTERR, UNLINKERR, NEWLOCATION_KEEP_POST, CLOSEFAILED, ATTRMISSING, UNKNOWNATTR, WARC_ERR, WARC_TMP_FOPENERR, WARC_TMP_FWRITEERR, - TIMECONV_ERR + TIMECONV_ERR, + METALINK_PARSE_ERROR, METALINK_RETR_ERROR, + METALINK_CHKSUM_ERROR, METALINK_SIG_ERROR, + RETR_WITH_METALINK } uerr_t; /* 2005-02-19 SMS. -- 2.4.3
From 490ea134edb8f209a331f828cef3bfb4e48fee33 Mon Sep 17 00:00:00 2001 From: Hubert Tarasiuk <[email protected]> Date: Mon, 15 Jun 2015 11:47:51 +0200 Subject: [PATCH 2/9] Start HTTP test only when calling begin(). * testenv/test/http_test.py: Move self.do_test() from __init__ to begin(). --- testenv/test/http_test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/testenv/test/http_test.py b/testenv/test/http_test.py index 230eff8..32a3335 100644 --- a/testenv/test/http_test.py +++ b/testenv/test/http_test.py @@ -23,11 +23,13 @@ class HTTPTest(BaseTest): test_params, post_hook, protocols) + self.server_setup() + + def begin(self): with self: - # if any exception occurs, self.__exit__ will be immediately called - self.server_setup() self.do_test() print_green('Test Passed.') + return super(HTTPTest, self).begin() def instantiate_server_by(self, protocol): server = {HTTP: HTTPd, -- 2.4.3
From 44b4892ed50d4f35e31727a66dbcf22b28608ecb Mon Sep 17 00:00:00 2001 From: Hubert Tarasiuk <[email protected]> Date: Mon, 15 Jun 2015 12:13:46 +0200 Subject: [PATCH 3/9] Test case for Metalink in XML. * testenv/Test-metalink-xml.py: New test. * testenv/Makefile.am: Add file for automake. --- testenv/Makefile.am | 3 +- testenv/Test-metalink-xml.py | 88 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100755 testenv/Test-metalink-xml.py diff --git a/testenv/Makefile.am b/testenv/Makefile.am index 1058421..58e116a 100644 --- a/testenv/Makefile.am +++ b/testenv/Makefile.am @@ -55,7 +55,8 @@ if HAVE_PYTHON3 Test--spider-r.py \ Test-redirect-crash.py \ Test-reserved-chars.py \ - Test-condget.py + Test-condget.py \ + Test-metalink-xml.py # added test cases expected to fail here and under TESTS XFAIL_TESTS = diff --git a/testenv/Test-metalink-xml.py b/testenv/Test-metalink-xml.py new file mode 100755 index 0000000..700418b --- /dev/null +++ b/testenv/Test-metalink-xml.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +from sys import exit +from test.http_test import HTTPTest +from misc.wget_file import WgetFile +import re +import hashlib + +""" + This is to test Metalink as XML file support in Wget. +""" +TEST_NAME = "Metalink in XML" +############# File Definitions ############################################### +File1 = "Would you like some Tea?" +File1_lowPref = "Do not take this" +File1_sha256 = hashlib.sha256 (File1.encode ('UTF-8')).hexdigest () +MetaXml = \ +"""<?xml version="1.0" encoding="utf-8"?> +<metalink version="3.0" xmlns="http://www.metalinker.org/";> + <publisher> + <name>GNU Wget</name> + </publisher> + <license> + <name>GNU GPL</name> + <url>http://www.gnu.org/licenses/gpl.html</url> + </license> + <identity>Wget Test File 1</identity> + <version>1.2.3</version> + <description>Wget Test File 1 description</description> + <files> + <file name="File1"> + <verification> + <hash type="sha256">{{FILE1_HASH}}</hash> + </verification> + <resources> + <url type="http" preference="40">http://broken.example/File1</url> + <url type="http" preference="25">http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref</url> + <url type="http" preference="30">http://{{SRV_HOST}}:{{SRV_PORT}}/File1</url> + </resources> + </file> + </files> +</metalink> +""" + +A_File = WgetFile ("File1", File1) +B_File = WgetFile ("File1_lowPref", File1_lowPref) +MetaFile = WgetFile ("test.meta4", MetaXml) + +WGET_OPTIONS = "--input-metalink test.meta4" +WGET_URLS = [[]] + +Files = [[A_File, B_File]] +Existing_Files = [MetaFile] + +ExpectedReturnCode = 0 +ExpectedDownloadedFiles = [A_File, MetaFile] + +################ Pre and Post Test Hooks ##################################### +pre_test = { + "ServerFiles" : Files, + "LocalFiles" : Existing_Files +} +test_options = { + "WgetCommands" : WGET_OPTIONS, + "Urls" : WGET_URLS +} +post_test = { + "ExpectedFiles" : ExpectedDownloadedFiles, + "ExpectedRetcode" : ExpectedReturnCode +} + +http_test = HTTPTest ( + name=TEST_NAME, + pre_hook=pre_test, + test_params=test_options, + post_hook=post_test, +) + +### Get and use dynamic server sockname +srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname () + +MetaXml = re.sub (r'{{FILE1_HASH}}', File1_sha256, MetaXml) +MetaXml = re.sub (r'{{SRV_HOST}}', srv_host, MetaXml) +MetaXml = re.sub (r'{{SRV_PORT}}', str (srv_port), MetaXml) +MetaFile.content = MetaXml + +err = http_test.begin () + +exit (err) -- 2.4.3
From f6287e13d9ae7e33adf300f48c94b4001dcf75a7 Mon Sep 17 00:00:00 2001 From: Hubert Tarasiuk <[email protected]> Date: Mon, 22 Jun 2015 20:46:01 +0200 Subject: [PATCH 4/9] Support multiple headers with same name in Python test suite. * testenv/README: Describe how to use repeated header name. * testenv/server/http/http_server.py (finish_headers): Send all values from list if the header value is a Python list. --- testenv/README | 3 ++- testenv/server/http/http_server.py | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/testenv/README b/testenv/README index cf22f89..50baf3d 100644 --- a/testenv/README +++ b/testenv/README @@ -184,7 +184,8 @@ This section lists the currently supported File Rules and their structure. * SendHeader : This list of Headers will be sent in EVERY response to a request for the respective file. It follows the same value format as - ExpectHeader. + ExpectHeader. Additionally you can specify a list of strings as <Header Data> + if you want the header repeated with multiple values. * Response : The HTTP Response Code to send to a request for this File. The value is an Integer that represents a valid HTTP Response Code. diff --git a/testenv/server/http/http_server.py b/testenv/server/http/http_server.py index 2356f1c..85769c4 100644 --- a/testenv/server/http/http_server.py +++ b/testenv/server/http/http_server.py @@ -191,7 +191,11 @@ class _Handler(BaseHTTPRequestHandler): self.send_cust_headers() try: for keyword, value in self._headers_dict.items(): - self.send_header(keyword, value) + if isinstance(value, list): + for value_el in value: + self.send_header(keyword, value_el) + else: + self.send_header(keyword, value) # Clear the dictionary of existing headers for the next request self._headers_dict.clear() except AttributeError: -- 2.4.3
From a0730ce6d3144454ba593f0d7019468cbeb93b8d Mon Sep 17 00:00:00 2001 From: Hubert Tarasiuk <[email protected]> Date: Fri, 19 Jun 2015 14:31:16 +0200 Subject: [PATCH 5/9] Test case for Metalink over HTTP. * testenv/Test-metalink-http.py: New test. * testenv/Makefile.am: Add to test list. --- testenv/Makefile.am | 3 +- testenv/Test-metalink-http.py | 126 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 1 deletion(-) create mode 100755 testenv/Test-metalink-http.py diff --git a/testenv/Makefile.am b/testenv/Makefile.am index 58e116a..ab6009d 100644 --- a/testenv/Makefile.am +++ b/testenv/Makefile.am @@ -56,7 +56,8 @@ if HAVE_PYTHON3 Test-redirect-crash.py \ Test-reserved-chars.py \ Test-condget.py \ - Test-metalink-xml.py + Test-metalink-xml.py \ + Test-metalink-http.py # added test cases expected to fail here and under TESTS XFAIL_TESTS = diff --git a/testenv/Test-metalink-http.py b/testenv/Test-metalink-http.py new file mode 100755 index 0000000..d50a370 --- /dev/null +++ b/testenv/Test-metalink-http.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +from sys import exit +from test.http_test import HTTPTest +from misc.wget_file import WgetFile +import re +import hashlib +from base64 import b64encode + +""" + This is to test Metalink as HTTP file support in Wget. +""" +TEST_NAME = "Metalink in HTTP" + +# Helper function for hostname, port and digest substitution +def SubstituteServerInfo (text, host, port, digest): + text = re.sub (r'{{FILE1_HASH}}', digest, text) + text = re.sub (r'{{SRV_HOST}}', host, text) + text = re.sub (r'{{SRV_PORT}}', str (port), text) + return text + +############# File Definitions ############################################### +File1 = "Would you like some Tea?" +File1_corrupted = "Would you like some Coffee?" +File1_lowPref = "Do not take this" +File1_sha256 = b64encode (hashlib.sha256 (File1.encode ('UTF-8')).digest ()).decode ('ascii') +Signature = '''-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.0.7 (GNU/Linux) + +This is no valid signature. But it should be downloaded. +The attempt to verify should fail but should not prevent +a successful metalink resource retrieval (the sig failure +should not be fatal). +-----END PGP SIGNATURE----- +''' +File2 = "No meta data for this file." + +LinkHeaders = [ + # This file has low priority and should not be picked. + "<http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref>; rel=duplicate; pri=9; geo=pl", + # This file should be picked second, after hash failure. + "<http://this.is.no.good.example/File1_try2_badconnection>; rel =duplicate;pref; pri=7", + # This signature download will fail. + "<http://{{SRV_HOST}}:{{SRV_PORT}}/Sig2.asc>; rel=describedby; type=application/pgp-signature", + # Two good signatures + "<http://{{SRV_HOST}}:{{SRV_PORT}}/Sig.asc>; rel=describedby; type=application/pgp-signature", + "<http://{{SRV_HOST}}:{{SRV_PORT}}/Sig.asc>; rel=describedby; type=application/pgp-signature", + # Bad URL scheme + "<invalid_url>; rel=duplicate; pri=4", + # rel missing + "<http://{{SRV_HOST}}:{{SRV_PORT}}/File1>; pri=1; pref", + # invalid rel + "<http://{{SRV_HOST}}:{{SRV_PORT}}/File1>; rel=strange; pri=4", + # This file should be picked first, because it has the lowest pri among preferred. + "<http://{{SRV_HOST}}:{{SRV_PORT}}/File1_try1_corrupted>; rel=duplicate; geo=su; pri=4; pref", + # This file should be picked as third try, and it should succeed + "<http://{{SRV_HOST}}:{{SRV_PORT}}/File1_try3_ok>; rel=duplicate; pri=5" + ] +DigestHeader = "SHA-256={{FILE1_HASH}}" + +# This will be filled as soon as we know server hostname and port +MetaFileRules = {'SendHeader' : {}} + +FileOkServer = WgetFile ("File1_try3_ok", File1) +FileBadPref = WgetFile ("File1_lowPref", File1_lowPref) +FileBadHash = WgetFile ("File1_try1_corrupted", File1_corrupted) +MetaFile = WgetFile ("test.meta", rules=MetaFileRules) +# In case of Metalink over HTTP, the local file name is +# derived from the URL suffix. +FileOkLocal = WgetFile ("test.meta", File1) +SigFile = WgetFile ("Sig.asc", Signature) +FileNoMeta = WgetFile ("File2", File2) + +WGET_OPTIONS = "--metalink-over-http " +WGET_URLS = [["test.meta", "File2"]] + +Files = [[FileOkServer, FileBadPref, FileBadHash, MetaFile, SigFile, FileNoMeta]] +Existing_Files = [] + +ExpectedReturnCode = 0 +ExpectedDownloadedFiles = [FileNoMeta, FileOkLocal] + +RequestList = [ + [ + "HEAD /test.meta", + "GET /Sig2.asc", + "GET /Sig.asc", + "GET /File1_try1_corrupted", + "GET /File1_try3_ok", + "HEAD /File2", + "GET /File2", + ] +] + +################ Pre and Post Test Hooks ##################################### +pre_test = { + "ServerFiles" : Files, + "LocalFiles" : Existing_Files +} +test_options = { + "WgetCommands" : WGET_OPTIONS, + "Urls" : WGET_URLS +} +post_test = { + "ExpectedFiles" : ExpectedDownloadedFiles, + "ExpectedRetcode" : ExpectedReturnCode, + "FilesCrawled" : RequestList, +} + +http_test = HTTPTest ( + name=TEST_NAME, + pre_hook=pre_test, + test_params=test_options, + post_hook=post_test, +) + +srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname () + +MetaFileRules["SendHeader"] = { + 'Link': [ SubstituteServerInfo (LinkHeader, srv_host, srv_port, File1_sha256) + for LinkHeader in LinkHeaders ], + 'Digest': SubstituteServerInfo (DigestHeader, srv_host, srv_port, File1_sha256), +} + +err = http_test.begin () + +exit (err) -- 2.4.3
From c02bcfaf790c2dcb73bc07f64345c80584078381 Mon Sep 17 00:00:00 2001 From: Hubert Tarasiuk <[email protected]> Date: Tue, 23 Jun 2015 16:36:21 +0200 Subject: [PATCH 6/9] Unit test for find_key_value. * src/http.c: Add test_find_key_value. * src/test.c (main): Run new test. * src/test.h: Add test_find_key_value. --- src/http.c | 49 ++++++++++++++++++++++++++++++++++++++++++++++++- src/test.c | 1 + src/test.h | 1 + 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/src/http.c b/src/http.c index afbc254..347a249 100644 --- a/src/http.c +++ b/src/http.c @@ -4998,10 +4998,57 @@ ensure_extension (struct http_stat *hs, const char *ext, int *dt) } } - #ifdef TESTING const char * +test_find_key_value (void) +{ + static const char *header_data = "key1=val1;key2=val2 ;key3=val3; key4=val4"\ + " ; key5=val5;key6 =val6;key7= val7; "\ + "key8 = val8 ; key9 = val9 "; + static const struct + { + const char *key; + const char *val; + bool result; + } test_array[] = + { + { "key1", "val1", true }, + { "key2", "val2", true }, + { "key3", "val3", true }, + { "key4", "val4", true }, + { "key5", "val5", true }, + { "key6", "val6", true }, + { "key7", "val7", true }, + { "key8", "val8", true }, + { "key9", "val9", true }, + { "key10", NULL, false }, + { "ey1", NULL, false }, + { "dey1", NULL, false } + }; + size_t i; + + for (i=0; i < countof (test_array); ++i) + { + bool result; + char *value; + + result = find_key_value (header_data, + header_data + strlen(header_data), + test_array[i].key, &value); + + mu_assert ("test_find_key_value: wrong result", + result == test_array[i].result && + ((!test_array[i].result && !value) || + !strcmp (value, test_array[i].val))); + + xfree (value); + } + + return NULL; +} + +const char * test_parse_content_disposition(void) { unsigned i; diff --git a/src/test.c b/src/test.c index e8e3669..2c7d4db 100644 --- a/src/test.c +++ b/src/test.c @@ -58,6 +58,7 @@ int tests_run; static const char * all_tests(void) { + mu_run_test (test_find_key_value); mu_run_test (test_parse_content_disposition); mu_run_test (test_subdir_p); mu_run_test (test_dir_matches_p); diff --git a/src/test.h b/src/test.h index e60c292..d427483 100644 --- a/src/test.h +++ b/src/test.h @@ -45,6 +45,7 @@ do { \ extern int tests_run; +const char *test_find_key_value (void); const char *test_parse_content_disposition(void); const char *test_commands_sorted(void); const char *test_cmd_spec_restrict_file_names(void); -- 2.4.3
From 1104bc65bd8e1145a151ebdf84d29d0213e9d062 Mon Sep 17 00:00:00 2001 From: Hubert Tarasiuk <[email protected]> Date: Tue, 23 Jun 2015 16:46:39 +0200 Subject: [PATCH 7/9] Unit test for has_key. * src/http.c: Add test_has_key. * src/test.c (main): Run new test. * src/test.h: Add test_has_key. --- src/http.c | 32 ++++++++++++++++++++++++++++++++ src/test.c | 1 + src/test.h | 1 + 3 files changed, 34 insertions(+) diff --git a/src/http.c b/src/http.c index 347a249..0d652d6 100644 --- a/src/http.c +++ b/src/http.c @@ -5049,6 +5049,38 @@ test_find_key_value (void) } const char * +test_has_key (void) +{ + static const char *header_data = "key1=val2;token1;xyz; token2;xyz;token3 ;"\ + "xyz; token4 ;xyz; token5 "; + struct + { + const char *token; + bool result; + } test_array[] = + { + { "key1=val2", true }, + { "token1", true }, + { "token2", true }, + { "token3", true }, + { "token4", true }, + { "token5", true }, + { "token6", false }, + { "oken1", false }, + { "poken1", false }, + { "key1=val2", true } + }; + size_t i; + + for (i = 0; i < countof (test_array); ++i) + mu_assert ("test_has_key: wrong result", + has_key (header_data, header_data + strlen (header_data), + test_array[i].token) == test_array[i].result); + + return NULL; +} + +const char * test_parse_content_disposition(void) { unsigned i; diff --git a/src/test.c b/src/test.c index 2c7d4db..9a76f4c 100644 --- a/src/test.c +++ b/src/test.c @@ -59,6 +59,7 @@ static const char * all_tests(void) { mu_run_test (test_find_key_value); + mu_run_test (test_has_key); mu_run_test (test_parse_content_disposition); mu_run_test (test_subdir_p); mu_run_test (test_dir_matches_p); diff --git a/src/test.h b/src/test.h index d427483..5f84184 100644 --- a/src/test.h +++ b/src/test.h @@ -45,6 +45,7 @@ do { \ extern int tests_run; +const char *test_has_key (void); const char *test_find_key_value (void); const char *test_parse_content_disposition(void); const char *test_commands_sorted(void); -- 2.4.3
From c4df1155d329bda8bcc0604ba1e747ea9838b1f5 Mon Sep 17 00:00:00 2001 From: Hubert Tarasiuk <[email protected]> Date: Tue, 23 Jun 2015 17:51:32 +0200 Subject: [PATCH 8/9] Unit test for find_key_values. * src/http.c: Add test_find_key_values. * src/test.c (main): Run new test. * src/test.h: Add test_find_key_values. --- src/http.c | 44 ++++++++++++++++++++++++++++++++++++++++++++ src/test.c | 1 + src/test.h | 1 + 3 files changed, 46 insertions(+) diff --git a/src/http.c b/src/http.c index 0d652d6..e2e98f5 100644 --- a/src/http.c +++ b/src/http.c @@ -5001,6 +5001,50 @@ ensure_extension (struct http_stat *hs, const char *ext, int *dt) #ifdef TESTING const char * +test_find_key_values (void) +{ + static const char *header_data = "key1=val1;key2=val2 ;key3=val3; key4=val4"\ + " ; key5=val5;key6 =val6;key7= val7; "\ + "key8 = val8 ; key9 = val9 "\ + " ,key10= val10,key11,key12=val12"; + static const struct + { + const char *key; + const char *val; + } test_array[] = + { + { "key1", "val1" }, + { "key2", "val2" }, + { "key3", "val3" }, + { "key4", "val4" }, + { "key5", "val5" }, + { "key6", "val6" }, + { "key7", "val7" }, + { "key8", "val8" }, + { "key9", "val9" }, + { "key10", "val10" }, + { "key12", "val12" } + }; + const char *pos; + char *key, *value; + size_t i = 0; + + for (pos = header_data; (pos = find_key_values (pos, + header_data + strlen (header_data), + &key, &value)); pos++) + { + mu_assert ("test_find_key_values: wrong result", + !strcmp (test_array[i].val, value) && + !strcmp (test_array[i].key, key)); + xfree (key); + xfree (value); + i++; + } + + return NULL; +} + +const char * test_find_key_value (void) { static const char *header_data = "key1=val1;key2=val2 ;key3=val3; key4=val4"\ diff --git a/src/test.c b/src/test.c index 9a76f4c..68f813b 100644 --- a/src/test.c +++ b/src/test.c @@ -59,6 +59,7 @@ static const char * all_tests(void) { mu_run_test (test_find_key_value); + mu_run_test (test_find_key_values); mu_run_test (test_has_key); mu_run_test (test_parse_content_disposition); mu_run_test (test_subdir_p); diff --git a/src/test.h b/src/test.h index 5f84184..58d1ce5 100644 --- a/src/test.h +++ b/src/test.h @@ -47,6 +47,7 @@ extern int tests_run; const char *test_has_key (void); const char *test_find_key_value (void); +const char *test_find_key_values (void); const char *test_parse_content_disposition(void); const char *test_commands_sorted(void); const char *test_cmd_spec_restrict_file_names(void); -- 2.4.3
From e7b58b878b287060ffb2ad19a3ff6369384a1d5e Mon Sep 17 00:00:00 2001 From: Hubert Tarasiuk <[email protected]> Date: Fri, 26 Jun 2015 02:09:51 +0200 Subject: [PATCH 9/9] Move some Metalink-related code from http.c to metalink.c. * src/http.c: Move find_key_value, has_key, find_key_values. * src/metalink.c: To here. * src/metalink.h: Make them non-static and add prototypes here. --- src/http.c | 317 -------------------------------------------------------- src/metalink.c | 322 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/metalink.h | 10 ++ 3 files changed, 332 insertions(+), 317 deletions(-) diff --git a/src/http.c b/src/http.c index e2e98f5..fe0aebd 100644 --- a/src/http.c +++ b/src/http.c @@ -2462,198 +2462,6 @@ set_content_type (int *dt, const char *type) } #ifdef HAVE_METALINK - -/* - Find value of given key. This is intended for Link header, but will - work with any header that uses ';' as field separator and '=' as key-value - separator. - - Link = "Link" ":" #link-value - link-value = "<" URI-Reference ">" *( ";" link-param ) - link-param = ( ( "rel" "=" relation-types ) - | ( "anchor" "=" <"> URI-Reference <"> ) - | ( "rev" "=" relation-types ) - | ( "hreflang" "=" Language-Tag ) - | ( "media" "=" ( MediaDesc | ( <"> MediaDesc <"> ) ) ) - | ( "title" "=" quoted-string ) - | ( "title*" "=" ext-value ) - | ( "type" "=" ( media-type | quoted-mt ) ) - | ( link-extension ) ) - link-extension = ( parmname [ "=" ( ptoken | quoted-string ) ] ) - | ( ext-name-star "=" ext-value ) - ext-name-star = parmname "*" ; reserved for RFC2231-profiled - ; extensions. Whitespace NOT - ; allowed in between. - ptoken = 1*ptokenchar - ptokenchar = "!" | "#" | "$" | "%" | "&" | "'" | "(" - | ")" | "*" | "+" | "-" | "." | "/" | DIGIT - | ":" | "<" | "=" | ">" | "?" | "@" | ALPHA - | "[" | "]" | "^" | "_" | "`" | "{" | "|" - | "}" | "~" - media-type = type-name "/" subtype-name - quoted-mt = <"> media-type <"> - relation-types = relation-type - | <"> relation-type *( 1*SP relation-type ) <"> - relation-type = reg-rel-type | ext-rel-type - reg-rel-type = LOALPHA *( LOALPHA | DIGIT | "." | "-" ) - ext-rel-type = URI - - See more: rfc5988 -*/ -static bool -find_key_value (const char *start, const char *end, const char *key, char **value) -{ - const char *eq; - size_t key_len = strlen (key); - const char *val_beg, *val_end; - const char *key_beg; - - key_beg = start; - - while (key_beg + key_len + 1 < end) - { - /* Skip whitespaces. */ - while (key_beg + key_len + 1 < end && c_isspace (*key_beg)) - key_beg++; - if (strncmp (key_beg, key, key_len)) - { - /* Find next token. */ - while (key_beg + key_len + 1 < end && *key_beg != ';') - key_beg++; - key_beg++; - continue; - } - else - { - /* Find equals sign. */ - eq = key_beg + key_len; - while (eq < end && c_isspace (*eq)) - eq++; - if (eq == end) - return false; - if (*eq != '=') - { - key_beg++; - continue; - } - - val_beg = eq + 1; - while (val_beg < end && c_isspace (*val_beg)) - val_beg++; - if (val_beg == end) - return false; - val_end = val_beg + 1; - while (val_end < end && *val_end != ';' && !c_isspace (*val_end)) - val_end++; - *value = xstrndup (val_beg, val_end - val_beg); - return true; - } - } - *value = NULL; - return false; -} - -/* This is to check if given token exists in HTTP header. Tokens are - separated by ';'. */ -static bool -has_key (const char *start, const char *end, const char *key) -{ - const char *pos; /* Here would the token start. */ - size_t key_len = strlen (key); - - pos = start; - while (pos + key_len <= end) - { - /* Skip whitespaces at beginning. */ - while (pos + key_len <= end && c_isspace (*pos)) - pos++; - - /* Does the prefix of pos match our key? */ - if (strncmp (key, pos, key_len)) - { - /* This was not a match. - Skip all characters until beginning of next token. */ - while (pos + key_len <= end && *pos != ';') - pos++; - pos++; - continue; - } - - /* key is prefix of pos. Is it the exact token or just a prefix? */ - pos += key_len; - while (pos < end && c_isspace (*pos)) - pos++; - if (pos == end || *pos == ';') - return true; - - /* This was not a match (just a prefix). - Skip all characters until beginning of next token. */ - while (pos + key_len <= end && *pos != ';') - pos++; - pos++; - } - return false; -} - -/* Find all key=value pairs delimited with ';' or ','. This is intended for - Digest header parsing. - The usage is: - - const char *pos; - for (pos = header_beg; pos = find_key_values (pos, header_end, &key, &val); pos++) - { - ... - } - - */ -static const char * -find_key_values (const char *start, const char *end, char **key, char **value) -{ - const char *key_start, *key_end; - const char *eq; - const char *val_start, *val_end; - - eq = start; - while (eq < end && *eq != '=') - { - /* Skip tokens without =value part. */ - if (*eq == ';' || *eq == ',') - start = eq + 1; - eq++; - } - - if (eq >= end) - return NULL; - - key_start = start; - while (key_start < eq && c_isspace (*key_start)) - key_start++; - - key_end = eq - 1; - while (key_end > key_start && c_isspace (*key_end)) - key_end--; - key_end++; - - val_start = eq + 1; - while (val_start < end && c_isspace (*val_start)) - val_start++; - - val_end = val_start; - - while (val_end < end && *val_end != ';' && - *val_end != ',' && !c_isspace (*val_end)) - val_end++; - - *key = xstrndup (key_start, key_end - key_start); - *value = xstrndup (val_start, val_end - val_start); - - /* Skip trailing whitespaces. */ - while (val_end < end && c_isspace (*val_end)) - val_end++; - - return val_end; -} - /* Will return proper metalink_t structure if enough data was found in http response resp. Otherwise returns NULL. Two exit points: one for success and one for failure. */ @@ -4999,131 +4807,6 @@ ensure_extension (struct http_stat *hs, const char *ext, int *dt) } #ifdef TESTING - -const char * -test_find_key_values (void) -{ - static const char *header_data = "key1=val1;key2=val2 ;key3=val3; key4=val4"\ - " ; key5=val5;key6 =val6;key7= val7; "\ - "key8 = val8 ; key9 = val9 "\ - " ,key10= val10,key11,key12=val12"; - static const struct - { - const char *key; - const char *val; - } test_array[] = - { - { "key1", "val1" }, - { "key2", "val2" }, - { "key3", "val3" }, - { "key4", "val4" }, - { "key5", "val5" }, - { "key6", "val6" }, - { "key7", "val7" }, - { "key8", "val8" }, - { "key9", "val9" }, - { "key10", "val10" }, - { "key12", "val12" } - }; - const char *pos; - char *key, *value; - size_t i = 0; - - for (pos = header_data; (pos = find_key_values (pos, - header_data + strlen (header_data), - &key, &value)); pos++) - { - mu_assert ("test_find_key_values: wrong result", - !strcmp (test_array[i].val, value) && - !strcmp (test_array[i].key, key)); - xfree (key); - xfree (value); - i++; - } - - return NULL; -} - -const char * -test_find_key_value (void) -{ - static const char *header_data = "key1=val1;key2=val2 ;key3=val3; key4=val4"\ - " ; key5=val5;key6 =val6;key7= val7; "\ - "key8 = val8 ; key9 = val9 "; - static const struct - { - const char *key; - const char *val; - bool result; - } test_array[] = - { - { "key1", "val1", true }, - { "key2", "val2", true }, - { "key3", "val3", true }, - { "key4", "val4", true }, - { "key5", "val5", true }, - { "key6", "val6", true }, - { "key7", "val7", true }, - { "key8", "val8", true }, - { "key9", "val9", true }, - { "key10", NULL, false }, - { "ey1", NULL, false }, - { "dey1", NULL, false } - }; - size_t i; - - for (i=0; i < countof (test_array); ++i) - { - bool result; - char *value; - - result = find_key_value (header_data, - header_data + strlen(header_data), - test_array[i].key, &value); - - mu_assert ("test_find_key_value: wrong result", - result == test_array[i].result && - ((!test_array[i].result && !value) || - !strcmp (value, test_array[i].val))); - - xfree (value); - } - - return NULL; -} - -const char * -test_has_key (void) -{ - static const char *header_data = "key1=val2;token1;xyz; token2;xyz;token3 ;"\ - "xyz; token4 ;xyz; token5 "; - struct - { - const char *token; - bool result; - } test_array[] = - { - { "key1=val2", true }, - { "token1", true }, - { "token2", true }, - { "token3", true }, - { "token4", true }, - { "token5", true }, - { "token6", false }, - { "oken1", false }, - { "poken1", false }, - { "key1=val2", true } - }; - size_t i; - - for (i = 0; i < countof (test_array); ++i) - mu_assert ("test_has_key: wrong result", - has_key (header_data, header_data + strlen (header_data), - test_array[i].token) == test_array[i].result); - - return NULL; -} - const char * test_parse_content_disposition(void) { diff --git a/src/metalink.c b/src/metalink.c index 962dd94..f2c4175 100644 --- a/src/metalink.c +++ b/src/metalink.c @@ -35,6 +35,7 @@ as that of the covered work. */ #include "exits.h" #include "utils.h" #include "sha256.h" +#include "xstrndup.h" #include <sys/errno.h> #include <unistd.h> /* For unlink. */ #include <metalink/metalink_parser.h> @@ -43,6 +44,10 @@ as that of the covered work. */ #include <fcntl.h> /* For open and close. */ #endif +#ifdef TESTING +#include "test.h" +#endif + /* Loop through all files in metalink structure and retrieve them. Returns RETROK if all files were downloaded. Returns last retrieval error (from retrieve_url) if some files @@ -445,4 +450,321 @@ int metalink_res_cmp (const void* v1, const void* v2) return 0; } +/* + Find value of given key. This is intended for Link header, but will + work with any header that uses ';' as field separator and '=' as key-value + separator. + + Link = "Link" ":" #link-value + link-value = "<" URI-Reference ">" *( ";" link-param ) + link-param = ( ( "rel" "=" relation-types ) + | ( "anchor" "=" <"> URI-Reference <"> ) + | ( "rev" "=" relation-types ) + | ( "hreflang" "=" Language-Tag ) + | ( "media" "=" ( MediaDesc | ( <"> MediaDesc <"> ) ) ) + | ( "title" "=" quoted-string ) + | ( "title*" "=" ext-value ) + | ( "type" "=" ( media-type | quoted-mt ) ) + | ( link-extension ) ) + link-extension = ( parmname [ "=" ( ptoken | quoted-string ) ] ) + | ( ext-name-star "=" ext-value ) + ext-name-star = parmname "*" ; reserved for RFC2231-profiled + ; extensions. Whitespace NOT + ; allowed in between. + ptoken = 1*ptokenchar + ptokenchar = "!" | "#" | "$" | "%" | "&" | "'" | "(" + | ")" | "*" | "+" | "-" | "." | "/" | DIGIT + | ":" | "<" | "=" | ">" | "?" | "@" | ALPHA + | "[" | "]" | "^" | "_" | "`" | "{" | "|" + | "}" | "~" + media-type = type-name "/" subtype-name + quoted-mt = <"> media-type <"> + relation-types = relation-type + | <"> relation-type *( 1*SP relation-type ) <"> + relation-type = reg-rel-type | ext-rel-type + reg-rel-type = LOALPHA *( LOALPHA | DIGIT | "." | "-" ) + ext-rel-type = URI + + See more: rfc5988 +*/ +bool +find_key_value (const char *start, const char *end, const char *key, char **value) +{ + const char *eq; + size_t key_len = strlen (key); + const char *val_beg, *val_end; + const char *key_beg; + + key_beg = start; + + while (key_beg + key_len + 1 < end) + { + /* Skip whitespaces. */ + while (key_beg + key_len + 1 < end && c_isspace (*key_beg)) + key_beg++; + if (strncmp (key_beg, key, key_len)) + { + /* Find next token. */ + while (key_beg + key_len + 1 < end && *key_beg != ';') + key_beg++; + key_beg++; + continue; + } + else + { + /* Find equals sign. */ + eq = key_beg + key_len; + while (eq < end && c_isspace (*eq)) + eq++; + if (eq == end) + return false; + if (*eq != '=') + { + key_beg++; + continue; + } + + val_beg = eq + 1; + while (val_beg < end && c_isspace (*val_beg)) + val_beg++; + if (val_beg == end) + return false; + val_end = val_beg + 1; + while (val_end < end && *val_end != ';' && !c_isspace (*val_end)) + val_end++; + *value = xstrndup (val_beg, val_end - val_beg); + return true; + } + } + *value = NULL; + return false; +} + +/* This is to check if given token exists in HTTP header. Tokens are + separated by ';'. */ +bool +has_key (const char *start, const char *end, const char *key) +{ + const char *pos; /* Here would the token start. */ + size_t key_len = strlen (key); + + pos = start; + while (pos + key_len <= end) + { + /* Skip whitespaces at beginning. */ + while (pos + key_len <= end && c_isspace (*pos)) + pos++; + + /* Does the prefix of pos match our key? */ + if (strncmp (key, pos, key_len)) + { + /* This was not a match. + Skip all characters until beginning of next token. */ + while (pos + key_len <= end && *pos != ';') + pos++; + pos++; + continue; + } + + /* key is prefix of pos. Is it the exact token or just a prefix? */ + pos += key_len; + while (pos < end && c_isspace (*pos)) + pos++; + if (pos == end || *pos == ';') + return true; + + /* This was not a match (just a prefix). + Skip all characters until beginning of next token. */ + while (pos + key_len <= end && *pos != ';') + pos++; + pos++; + } + return false; +} + +/* Find all key=value pairs delimited with ';' or ','. This is intended for + Digest header parsing. + The usage is: + + const char *pos; + for (pos = header_beg; pos = find_key_values (pos, header_end, &key, &val); pos++) + { + ... + } + + */ +const char * +find_key_values (const char *start, const char *end, char **key, char **value) +{ + const char *key_start, *key_end; + const char *eq; + const char *val_start, *val_end; + + eq = start; + while (eq < end && *eq != '=') + { + /* Skip tokens without =value part. */ + if (*eq == ';' || *eq == ',') + start = eq + 1; + eq++; + } + + if (eq >= end) + return NULL; + + key_start = start; + while (key_start < eq && c_isspace (*key_start)) + key_start++; + + key_end = eq - 1; + while (key_end > key_start && c_isspace (*key_end)) + key_end--; + key_end++; + + val_start = eq + 1; + while (val_start < end && c_isspace (*val_start)) + val_start++; + + val_end = val_start; + + while (val_end < end && *val_end != ';' && + *val_end != ',' && !c_isspace (*val_end)) + val_end++; + + *key = xstrndup (key_start, key_end - key_start); + *value = xstrndup (val_start, val_end - val_start); + + /* Skip trailing whitespaces. */ + while (val_end < end && c_isspace (*val_end)) + val_end++; + + return val_end; +} + +#ifdef TESTING +const char * +test_find_key_values (void) +{ + static const char *header_data = "key1=val1;key2=val2 ;key3=val3; key4=val4"\ + " ; key5=val5;key6 =val6;key7= val7; "\ + "key8 = val8 ; key9 = val9 "\ + " ,key10= val10,key11,key12=val12"; + static const struct + { + const char *key; + const char *val; + } test_array[] = + { + { "key1", "val1" }, + { "key2", "val2" }, + { "key3", "val3" }, + { "key4", "val4" }, + { "key5", "val5" }, + { "key6", "val6" }, + { "key7", "val7" }, + { "key8", "val8" }, + { "key9", "val9" }, + { "key10", "val10" }, + { "key12", "val12" } + }; + const char *pos; + char *key, *value; + size_t i = 0; + + for (pos = header_data; (pos = find_key_values (pos, + header_data + strlen (header_data), + &key, &value)); pos++) + { + mu_assert ("test_find_key_values: wrong result", + !strcmp (test_array[i].val, value) && + !strcmp (test_array[i].key, key)); + xfree (key); + xfree (value); + i++; + } + + return NULL; +} + +const char * +test_find_key_value (void) +{ + static const char *header_data = "key1=val1;key2=val2 ;key3=val3; key4=val4"\ + " ; key5=val5;key6 =val6;key7= val7; "\ + "key8 = val8 ; key9 = val9 "; + static const struct + { + const char *key; + const char *val; + bool result; + } test_array[] = + { + { "key1", "val1", true }, + { "key2", "val2", true }, + { "key3", "val3", true }, + { "key4", "val4", true }, + { "key5", "val5", true }, + { "key6", "val6", true }, + { "key7", "val7", true }, + { "key8", "val8", true }, + { "key9", "val9", true }, + { "key10", NULL, false }, + { "ey1", NULL, false }, + { "dey1", NULL, false } + }; + size_t i; + + for (i=0; i < countof (test_array); ++i) + { + bool result; + char *value; + + result = find_key_value (header_data, + header_data + strlen(header_data), + test_array[i].key, &value); + + mu_assert ("test_find_key_value: wrong result", + result == test_array[i].result && + ((!test_array[i].result && !value) || + !strcmp (value, test_array[i].val))); + + xfree (value); + } + + return NULL; +} + +const char * +test_has_key (void) +{ + static const char *header_data = "key1=val2;token1;xyz; token2;xyz;token3 ;"\ + "xyz; token4 ;xyz; token5 "; + struct + { + const char *token; + bool result; + } test_array[] = + { + { "key1=val2", true }, + { "token1", true }, + { "token2", true }, + { "token3", true }, + { "token4", true }, + { "token5", true }, + { "token6", false }, + { "oken1", false }, + { "poken1", false }, + { "key1=val2", true } + }; + size_t i; + + for (i = 0; i < countof (test_array); ++i) + mu_assert ("test_has_key: wrong result", + has_key (header_data, header_data + strlen (header_data), + test_array[i].token) == test_array[i].result); + + return NULL; +} +#endif + #endif /* HAVE_METALINK */ diff --git a/src/metalink.h b/src/metalink.h index 202c545..ec91b07 100644 --- a/src/metalink.h +++ b/src/metalink.h @@ -47,4 +47,14 @@ uerr_t retrieve_from_metalink (const metalink_t *metalink); int metalink_res_cmp (const void *res1, const void *res2); +bool find_key_value (const char *start, + const char *end, + const char *key, + char **value); +bool has_key (const char *start, const char *end, const char *key); +const char *find_key_values (const char *start, + const char *end, + char **key, + char **value); + #endif /* METALINK_H */ -- 2.4.3
signature.asc
Description: OpenPGP digital signature
