On 12.06.2017 11:55, Josef Moellers wrote: > On 12.06.2017 11:48, Tim Rühsen wrote: >> On 06/12/2017 10:42 AM, Josef Moellers wrote: >>> On 12.06.2017 10:00, Tim Rühsen wrote: >>>> Hi Josef, >>>> >>>> >>>> On 06/12/2017 09:23 AM, Josef Moellers wrote: >>>>> Hello Tim, >>>>> >>>>> Thanks for the reply. >>>>> >>>>> On 10.06.2017 13:36, Tim Rühsen wrote: >>>>>> On Freitag, 9. Juni 2017 17:02:15 CEST Josef Moellers wrote: >>>>>>> Hi, >>>>>> >>>>>> Hi Josef, >>>>>> >>>>>>> I'm currently trying to build test suites for openQA. >>>>>>> One of the candidates is wget and, luckily, it already provides quite an >>>>>>> extensive test suite. >>>>>>> I have successfully built an RPM which has all that is needed for the >>>>>>> tests. >>>>>>> One test, Test-ftp-iri-fallback.px, fails on SLES12-SP2 and I can't see >>>>>>> why. >>>>>> >>>>>> Look at tests/Test-ftp-iri-fallback.log, if you can't interpret the >>>>>> content >>>>>> send it here. >>>>> >>>>> I cannot find any such file, no *.log" anywhere in the vicinity of the >>>>> tests. >>>> >>>> Ok, the .log files just contain the output of each single test when >>>> tested with 'make check'. If you use run-px, copy & pasting from the >>>> console is the right thing to do. >>>> >>>>> Ah ... maybe I should have addede that I'm working on a slightly older >>>>> version of wget: 1.14, which we ship with SLES12. >>>> >>>> So I compiled 1.14 (git tag v1.14) and used run-px to run the test suite >>>> - but still can't reproduce the problem (Debian unstable here, `locale` >>>> shows all set to 'en_US.UTF-8'). >>>> >>>>> >>>>> 227 Entering Passive Mode (127,0,0,1,155,189) >>>>> --> RETR français.txt^M >>>>> >>>>> 150 Opening ASCII mode data connection. >>>>> Length: 12 (unauthoritative) >>>>> >>>>> 0K 100% >>>>> 2.95M=0s >>>>> >>>>> 226 File retrieval complete. Data connection has been closed. >>>>> 2017-06-09 16:42:53 (2.95 MB/s) - â<80><98>français.txtâ<80><99> saved >>>>> [12] >>>>> >>>>> Test failed: file français.txt not downloaded >>>> >>>> My out put looks identical except the these last lines: >>>> >>>> 227 Entering Passive Mode (127,0,0,1,175,123) >>>> --> RETR français.txt >>>> >>>> 150 Opening ASCII mode data connection. >>>> Length: 12 (unauthoritative) >>>> >>>> 0K 100% 2.05M=0s >>>> >>>> 226 File retrieval complete. Data connection has been closed. >>>> 2017-06-12 09:39:27 (2.05 MB/s) - âfran\347ais.txtâ saved [12] >>>> >>>> Test successful. >>>> >>>> >>>> I just can guess: >>>> - something with your locale (what does the 'locale' command output ?) >>> >>> LANG=POSIX >>> LC_CTYPE=en_US.UTF-8 >>> LC_NUMERIC="POSIX" >>> LC_TIME="POSIX" >>> LC_COLLATE="POSIX" >>> LC_MONETARY="POSIX" >>> LC_MESSAGES="POSIX" >>> LC_PAPER="POSIX" >>> LC_NAME="POSIX" >>> LC_ADDRESS="POSIX" >>> LC_TELEPHONE="POSIX" >>> LC_MEASUREMENT="POSIX" >>> LC_IDENTIFICATION="POSIX" >>> LC_ALL= >>> >>> Changing everything to en_US.UTF-8 doesn't help: >>> I added "export LC...=en_US.UTF-8" lines prior to running the test. >> >> I used your settings, but still the tests succeed. >> >>> I must admit, I never really understood this "locale" thingy, it bites >>> my whenever I come close. >> >> Just take your time and read up about charset conversion, it doesn't >> take too long :-) >> >>>> - something with iconv() function >>>> >>>> Does the same test fail if you use Wget 1.19.1 ? >>> >>> I can't get it to build on SLES12 as it does not have libidn2(-devel)! >>> >>> I'll keep on trying, but until then ... >> >> There is no debug output by default with wget 1.14 (I didn't think of >> it, sorry)... please add '-d' to the command line in >> Test-ftp-iri-fallback.px, and send that output. Something must go wrong >> there... > > It's attached as "out" > >> >> And then go back to v1.18 and try to build/test that (or eventually to >> v1.17.1). That should compile with libidn instead libidn2. > > I'll first try to build 1.14 without any ouf our local patches. As you > say: it works on your system (maybe I should also try on a local Ubuntu > machine), so it's strange that it doesn't work on my VM. Maybe there's > something in one of the patches ...
FYI It's the attached patch which is supposed to fix CVE-2016-4971! Without this patch, the test succeeds, with this patch, the test fails. Josef
Index: wget-1.14/src/ftp.c
===================================================================
--- wget-1.14.orig/src/ftp.c
+++ wget-1.14/src/ftp.c
@@ -234,14 +234,15 @@ print_length (wgint size, wgint start, b
logputs (LOG_VERBOSE, !authoritative ? _(" (unauthoritative)\n") : "\n");
}
-static uerr_t ftp_get_listing (struct url *, ccon *, struct fileinfo **);
+static uerr_t ftp_get_listing (struct url *, struct url *, ccon *, struct fileinfo **);
/* Retrieves a file with denoted parameters through opening an FTP
connection to the server. It always closes the data connection,
and closes the control connection in case of error. If warc_tmp
is non-NULL, the downloaded data will be written there as well. */
static uerr_t
-getftp (struct url *u, wgint passed_expected_bytes, wgint *qtyread,
+getftp (struct url *u, struct url *original_url,
+ wgint passed_expected_bytes, wgint *qtyread,
wgint restval, ccon *con, int count, FILE *warc_tmp)
{
int csock, dtsock, local_sock, res;
@@ -944,7 +945,7 @@ Error in server response, closing contro
bool exists = false;
uerr_t res;
struct fileinfo *f;
- res = ftp_get_listing (u, con, &f);
+ uerr_t _res = ftp_get_listing (u, original_url, con, &f);
/* Set the DO_RETR command flag again, because it gets unset when
calling ftp_get_listing() and would otherwise cause an assertion
failure earlier on when this function gets repeatedly called
@@ -1392,7 +1393,8 @@ Error in server response, closing contro
This loop either gets commands from con, or (if ON_YOUR_OWN is
set), makes them up to retrieve the file given by the URL. */
static uerr_t
-ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con, char **local_file)
+ftp_loop_internal (struct url *u, struct url *original_url, struct fileinfo *f,
+ ccon *con, char **local_file)
{
int count, orig_lp;
wgint restval, len = 0, qtyread = 0;
@@ -1415,7 +1417,7 @@ ftp_loop_internal (struct url *u, struct
else
{
/* URL-derived file. Consider "-O file" name. */
- con->target = url_file_name (u, NULL);
+ con->target = url_file_name (opt.trustservernames || !original_url ? u : original_url, NULL);
if (!opt.output_document)
locf = con->target;
else
@@ -1524,7 +1526,9 @@ ftp_loop_internal (struct url *u, struct
/* If we are working on a WARC record, getftp should also write
to the warc_tmp file. */
- err = getftp (u, len, &qtyread, restval, con, count, warc_tmp);
+ err = getftp (u, original_url, len, &qtyread, restval, con, count,
+ warc_tmp);
+
if (con->csock == -1)
con->st &= ~DONE_CWD;
@@ -1677,7 +1681,8 @@ Removing file due to --delete-after in f
/* Return the directory listing in a reusable format. The directory
is specifed in u->dir. */
static uerr_t
-ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
+ftp_get_listing (struct url *u, struct url *original_url, ccon *con,
+ struct fileinfo **f)
{
uerr_t err;
char *uf; /* url file name */
@@ -1698,7 +1703,7 @@ ftp_get_listing (struct url *u, ccon *co
con->target = xstrdup (lf);
xfree (lf);
- err = ftp_loop_internal (u, NULL, con, NULL);
+ err = ftp_loop_internal (u, original_url, NULL, con, NULL);
lf = xstrdup (con->target);
xfree (con->target);
con->target = old_target;
@@ -1721,8 +1726,9 @@ ftp_get_listing (struct url *u, ccon *co
return err;
}
-static uerr_t ftp_retrieve_dirs (struct url *, struct fileinfo *, ccon *);
-static uerr_t ftp_retrieve_glob (struct url *, ccon *, int);
+static uerr_t ftp_retrieve_dirs (struct url *, struct url *,
+ struct fileinfo *, ccon *);
+static uerr_t ftp_retrieve_glob (struct url *, struct url *, ccon *, int);
static struct fileinfo *delelement (struct fileinfo *, struct fileinfo **);
static void freefileinfo (struct fileinfo *f);
@@ -1734,7 +1740,8 @@ static void freefileinfo (struct fileinf
If opt.recursive is set, after all files have been retrieved,
ftp_retrieve_dirs will be called to retrieve the directories. */
static uerr_t
-ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
+ftp_retrieve_list (struct url *u, struct url *original_url,
+ struct fileinfo *f, ccon *con)
{
static int depth = 0;
uerr_t err;
@@ -1893,7 +1900,7 @@ Already have correct symlink %s -> %s\n\
else /* opt.retr_symlinks */
{
if (dlthis)
- err = ftp_loop_internal (u, f, con, NULL);
+ err = ftp_loop_internal (u, original_url, f, con, NULL);
} /* opt.retr_symlinks */
break;
case FT_DIRECTORY:
@@ -1904,7 +1911,7 @@ Already have correct symlink %s -> %s\n\
case FT_PLAINFILE:
/* Call the retrieve loop. */
if (dlthis)
- err = ftp_loop_internal (u, f, con, NULL);
+ err = ftp_loop_internal (u, original_url, f, con, NULL);
break;
case FT_UNKNOWN:
logprintf (LOG_NOTQUIET, _("%s: unknown/unsupported file type.\n"),
@@ -1969,7 +1976,7 @@ Already have correct symlink %s -> %s\n\
/* We do not want to call ftp_retrieve_dirs here */
if (opt.recursive &&
!(opt.reclevel != INFINITE_RECURSION && depth >= opt.reclevel))
- err = ftp_retrieve_dirs (u, orig, con);
+ err = ftp_retrieve_dirs (u, original_url, orig, con);
else if (opt.recursive)
DEBUGP ((_("Will not retrieve dirs since depth is %d (max %d).\n"),
depth, opt.reclevel));
@@ -1982,7 +1989,8 @@ Already have correct symlink %s -> %s\n\
ftp_retrieve_glob on each directory entry. The function knows
about excluded directories. */
static uerr_t
-ftp_retrieve_dirs (struct url *u, struct fileinfo *f, ccon *con)
+ftp_retrieve_dirs (struct url *u, struct url *original_url,
+ struct fileinfo *f, ccon *con)
{
char *container = NULL;
int container_size = 0;
@@ -2032,7 +2040,7 @@ Not descending to %s as it is excluded/n
odir = xstrdup (u->dir); /* because url_set_dir will free
u->dir. */
url_set_dir (u, newdir);
- ftp_retrieve_glob (u, con, GLOB_GETALL);
+ ftp_retrieve_glob (u, original_url, con, GLOB_GETALL);
url_set_dir (u, odir);
xfree (odir);
@@ -2068,14 +2076,15 @@ has_insecure_name_p (const char *s)
GLOB_GLOBALL, use globbing; if it's GLOB_GETALL, download the whole
directory. */
static uerr_t
-ftp_retrieve_glob (struct url *u, ccon *con, int action)
+ftp_retrieve_glob (struct url *u, struct url *original_url,
+ ccon *con, int action)
{
struct fileinfo *f, *start;
uerr_t res;
con->cmd |= LEAVE_PENDING;
- res = ftp_get_listing (u, con, &start);
+ res = ftp_get_listing (u, original_url, con, &start);
if (res != RETROK)
return res;
/* First: weed out that do not conform the global rules given in
@@ -2171,7 +2180,7 @@ ftp_retrieve_glob (struct url *u, ccon *
if (start)
{
/* Just get everything. */
- res = ftp_retrieve_list (u, start, con);
+ res = ftp_retrieve_list (u, original_url, start, con);
}
else
{
@@ -2187,7 +2196,7 @@ ftp_retrieve_glob (struct url *u, ccon *
{
/* Let's try retrieving it anyway. */
con->st |= ON_YOUR_OWN;
- res = ftp_loop_internal (u, NULL, con, NULL);
+ res = ftp_loop_internal (u, original_url, NULL, con, NULL);
return res;
}
@@ -2207,8 +2216,8 @@ ftp_retrieve_glob (struct url *u, ccon *
of URL. Inherently, its capabilities are limited on what can be
encoded into a URL. */
uerr_t
-ftp_loop (struct url *u, char **local_file, int *dt, struct url *proxy,
- bool recursive, bool glob)
+ftp_loop (struct url *u, struct url *original_url, char **local_file, int *dt,
+ struct url *proxy, bool recursive, bool glob)
{
ccon con; /* FTP connection */
uerr_t res;
@@ -2229,16 +2238,17 @@ ftp_loop (struct url *u, char **local_fi
if (!*u->file && !recursive)
{
struct fileinfo *f;
- res = ftp_get_listing (u, &con, &f);
+ res = ftp_get_listing (u, original_url, &con, &f);
if (res == RETROK)
{
if (opt.htmlify && !opt.spider)
{
+ struct url *url_file = opt.trustservernames ? u : original_url;
char *filename = (opt.output_document
? xstrdup (opt.output_document)
: (con.target ? xstrdup (con.target)
- : url_file_name (u, NULL)));
+ : url_file_name (url_file, NULL)));
res = ftp_index (filename, u, f);
if (res == FTPOK && opt.verbose)
{
@@ -2283,11 +2293,11 @@ ftp_loop (struct url *u, char **local_fi
/* ftp_retrieve_glob is a catch-all function that gets called
if we need globbing, time-stamping or recursion. Its
third argument is just what we really need. */
- res = ftp_retrieve_glob (u, &con,
+ res = ftp_retrieve_glob (u, original_url, &con,
ispattern ? GLOB_GLOBALL : GLOB_GETONE);
}
else
- res = ftp_loop_internal (u, NULL, &con, local_file);
+ res = ftp_loop_internal (u, original_url, NULL, &con, local_file);
}
if (res == FTPOK)
res = RETROK;
Index: wget-1.14/src/ftp.h
===================================================================
--- wget-1.14.orig/src/ftp.h
+++ wget-1.14/src/ftp.h
@@ -129,7 +129,8 @@ enum wget_ftp_fstatus
};
struct fileinfo *ftp_parse_ls (const char *, const enum stype);
-uerr_t ftp_loop (struct url *, char **, int *, struct url *, bool, bool);
+uerr_t ftp_loop (struct url *, struct url *, char **, int *, struct url *,
+ bool, bool);
uerr_t ftp_index (const char *, struct url *, struct fileinfo *);
Index: wget-1.14/src/retr.c
===================================================================
--- wget-1.14.orig/src/retr.c
+++ wget-1.14/src/retr.c
@@ -798,7 +798,8 @@ retrieve_url (struct url * orig_parsed,
if (redirection_count)
oldrec = glob = false;
- result = ftp_loop (u, &local_file, dt, proxy_url, recursive, glob);
+ result = ftp_loop (u, orig_parsed, &local_file, dt, proxy_url,
+ recursive, glob);
recursive = oldrec;
/* There is a possibility of having HTTP being redirected to
signature.asc
Description: OpenPGP digital signature
