From e3f5f35e72f26d71f39ce8bd7bb3fe7a4fd66f24 Mon Sep 17 00:00:00 2001 From: Bykov Aleksey Date: Thu, 14 Mar 2013 01:07:28 +0200 Subject: [PATCH] Adding "--local-filesystem-encoding" convert.c - additional searching for original URL retr.c - original URL saved in hash every time init.c main.c options.h - added new options --- src/convert.c | 2 ++ src/init.c | 3 ++- src/iri.c | 21 +++++++++++++++++++++ src/iri.h | 5 +++++ src/main.c | 3 +++ src/options.h | 1 + src/retr.c | 4 +++- src/url.c | 21 +++++++++++++++++++++ 8 files changed, 58 insertions(+), 2 deletions(-) diff --git a/src/convert.c b/src/convert.c index f5a9cba..835862f 100644 --- a/src/convert.c +++ b/src/convert.c @@ -128,6 +128,8 @@ convert_links_in_hashtable (struct hash_table *downloaded_set, continue; local_name = hash_table_get (dl_url_file_map, u->url); + if (!local_name) + local_name = hash_table_get (dl_url_file_map, cur_url->url->url); /* Decide on the conversion type. */ if (local_name) diff --git a/src/init.c b/src/init.c index cafd456..e83448f 100644 --- a/src/init.c +++ b/src/init.c @@ -207,6 +207,7 @@ static const struct { { "limitrate", &opt.limit_rate, cmd_bytes }, { "loadcookies", &opt.cookies_input, cmd_file }, { "localencoding", &opt.locale, cmd_string }, + { "localfilesystemencoding", &opt.encoding_filesystem,cmd_string }, { "logfile", &opt.lfilename, cmd_file }, { "login", &opt.ftp_user, cmd_string },/* deprecated*/ { "maxredirect", &opt.max_redirect, cmd_number }, @@ -385,7 +386,7 @@ defaults (void) #endif opt.locale = NULL; opt.encoding_remote = NULL; - + opt.encoding_filesystem = NULL; opt.useservertimestamps = true; opt.show_all_dns_entries = false; diff --git a/src/iri.c b/src/iri.c index 9b16639..46ca3b6 100644 --- a/src/iri.c +++ b/src/iri.c @@ -298,6 +298,20 @@ remote_to_utf8 (struct iri *i, const char *str, const char **new) return ret; } +/*Tried to encode filename to given encoding*/ + +bool +utf8_to_filesystem (struct iri *i, const char *str, const char **new) +{ + iconv_t cd; + bool ret = false; + cd = iconv_open (i->filesystem_encoding,"UTF-8"); + if (do_conversion (cd, (char *) str, strlen ((char *) str), (char **) new)) + ret=true; + iconv_close(cd); + return ret; +} + /* Allocate a new iri structure and return a pointer to it. */ struct iri * iri_new (void) @@ -305,6 +319,7 @@ iri_new (void) struct iri *i = xmalloc (sizeof *i); i->uri_encoding = opt.encoding_remote ? xstrdup (opt.encoding_remote) : NULL; i->content_encoding = NULL; + i->filesystem_encoding = NULL; i->orig_url = NULL; i->utf8_encode = opt.enable_iri; return i; @@ -366,3 +381,9 @@ set_content_encoding (struct iri *i, char *charset) i->content_encoding = charset ? xstrdup (charset) : NULL; } +void +set_filesystem_encoding (struct iri *i) +{ + i->filesystem_encoding = opt.encoding_filesystem ? xstrdup (opt.encoding_filesystem) : xstrdup (opt.locale); +} + diff --git a/src/iri.h b/src/iri.h index e759e45..78dbd35 100644 --- a/src/iri.h +++ b/src/iri.h @@ -35,6 +35,7 @@ struct iri { char *content_encoding; /* Encoding of links inside the fetched file */ char *orig_url; /* */ bool utf8_encode; /* Will/Is the current url encoded in utf8 */ + char *filesystem_encoding; }; #ifdef ENABLE_IRI @@ -46,11 +47,13 @@ const char *locale_to_utf8 (const char *str); char *idn_encode (struct iri *i, char *host); char *idn_decode (char *host); bool remote_to_utf8 (struct iri *i, const char *str, const char **new); +bool utf8_to_filesystem (struct iri *i, const char *str, const char **new); struct iri *iri_new (void); struct iri *iri_dup (const struct iri *); void iri_free (struct iri *i); void set_uri_encoding (struct iri *i, char *charset, bool force); void set_content_encoding (struct iri *i, char *charset); +void set_filesystem_encoding (struct iri *i); #else /* ENABLE_IRI */ @@ -63,11 +66,13 @@ extern struct iri dummy_iri; #define idn_encode(a,b) NULL #define idn_decode(str) NULL #define remote_to_utf8(a,b,c) false +#define utf8_to_filesystem(a,b,c) false #define iri_new() (&dummy_iri) #define iri_dup(a) (&dummy_iri) #define iri_free(a) #define set_uri_encoding(a,b,c) #define set_content_encoding(a,b) +#define set_filesystem_encoding(a,b) #endif /* ENABLE_IRI */ #endif /* IRI_H */ diff --git a/src/main.c b/src/main.c index 992f60a..0baf160 100644 --- a/src/main.c +++ b/src/main.c @@ -230,6 +230,7 @@ static struct cmdline_option option_data[] = { "limit-rate", 0, OPT_VALUE, "limitrate", -1 }, { "load-cookies", 0, OPT_VALUE, "loadcookies", -1 }, { "local-encoding", 0, OPT_VALUE, "localencoding", -1 }, + { "local-filesystem-encoding", 0, OPT_VALUE, "filesystemencoding", -1 }, { "max-redirect", 0, OPT_VALUE, "maxredirect", -1 }, { "mirror", 'm', OPT_BOOLEAN, "mirror", -1 }, { "no", 'n', OPT__NO, NULL, required_argument }, @@ -547,6 +548,8 @@ Download:\n"), N_("\ --remote-encoding=ENC use ENC as the default remote encoding.\n"), N_("\ + --local-filesystem-encoding=ENC use ENC as the default remote encoding. If not specifed, use value of --local-encoding.\n"), + N_("\ --unlink remove file before clobber.\n"), "\n", diff --git a/src/options.h b/src/options.h index 44e0a70..ef705c5 100644 --- a/src/options.h +++ b/src/options.h @@ -267,6 +267,7 @@ struct options bool enable_iri; char *encoding_remote; + char *encoding_filesystem; char *locale; bool trustservernames; diff --git a/src/retr.c b/src/retr.c index 0d564ef..9ac005b 100644 --- a/src/retr.c +++ b/src/retr.c @@ -929,7 +929,9 @@ retrieve_url (struct url * orig_parsed, const char *origurl, char **file, { register_download (u->url, local_file); - if (!opt.spider && redirection_count && 0 != strcmp (origurl, u->url)) + /* if (!opt.spider && redirection_count && 0 != strcmp (origurl, u->url))*/ + if (!opt.spider && 0 != strcmp (origurl, u->url)) + register_redirection (origurl, u->url); if (*dt & TEXTHTML) diff --git a/src/url.c b/src/url.c index 87d6290..35e6cc2 100644 --- a/src/url.c +++ b/src/url.c @@ -1541,6 +1541,9 @@ url_file_name (const struct url *u, char *replaced_filename) const char *index_filename = "index.html"; /* The default index file is index.html */ size_t max_length; + char *conv_filename = NULL; + struct iri *iri = NULL; + fnres.base = NULL; fnres.size = 0; fnres.tail = 0; @@ -1592,11 +1595,21 @@ url_file_name (const struct url *u, char *replaced_filename) append_dir_structure (u, &fnres); } + if (opt.enable_iri && (opt.encoding_filesystem || opt.locale)) + { + iri= iri_new (); + set_filesystem_encoding(iri); + } + if (!replaced_filename) { /* Create the filename. */ u_file = *u->file ? u->file : index_filename; + if (iri) + if (utf8_to_filesystem (iri,u_file,(const char **)&conv_filename)) + u_file=conv_filename; + /* Append "?query" to the file name, even if empty, * and create fname_len_check. */ if (u->query) @@ -1607,9 +1620,17 @@ url_file_name (const struct url *u, char *replaced_filename) else { u_file = replaced_filename; + + if (iri) + if (utf8_to_filesystem (iri,u_file,(const char **)&conv_filename)) + u_file=conv_filename; + fname_len_check = strdupdelim (u_file, u_file + strlen (u_file)); } + if (iri) + iri_free (iri); + append_uri_pathel (fname_len_check, fname_len_check + strlen (fname_len_check), false, &temp_fnres); -- 1.8.1.msysgit.1