This patch seems to do user-agent checks correctly (it might have been
broken previously) with a correction to a string comparison macro.
The patch also uses the value of the --user-agent option when enforcing
robots.txt rules.
this patch is against CVS, more on that here:
http://www.gnu.org/software/wget/
Index: init.c
===
RCS file: /pack/anoncvs/wget/src/init.c,v
retrieving revision 1.54
diff -u -u -r1.54 init.c
--- init.c 2002/08/03 20:34:57 1.54
+++ init.c 2003/05/29 17:51:50
@@ -271,6 +271,7 @@
opt.timeout = 900;
#endif
opt.use_robots = 1;
+ opt.useragent = xstrdup (Wget);
opt.remove_listing = 1;
Index: res.c
===
RCS file: /pack/anoncvs/wget/src/res.c,v
retrieving revision 1.7
diff -u -u -r1.7 res.c
--- res.c 2002/05/18 02:16:24 1.7
+++ res.c 2003/05/29 17:51:50
@@ -115,7 +115,7 @@
*matches = 1;
*exact_match = 0;
}
- else if (BOUNDED_EQUAL_NO_CASE (agent, agent + length, wget))
+ else if (BOUNDED_EQUAL_NO_CASE (agent, agent + length, opt.useragent))
{
*matches = 1;
*exact_match = 1;
@@ -355,7 +355,7 @@
}
else
{
- DEBUGP ((Ignoring unknown field at line %d, line_count));
+ DEBUGP ((Ignoring unknown field at line %d\n, line_count));
goto next;
}
Index: wget.h
===
RCS file: /pack/anoncvs/wget/src/wget.h,v
retrieving revision 1.34
diff -u -u -r1.34 wget.h
--- wget.h 2002/05/18 02:16:25 1.34
+++ wget.h 2003/05/29 17:51:50
@@ -189,15 +189,15 @@
/* Return non-zero if string bounded between BEG and END is equal to
STRING_LITERAL. The comparison is case-sensitive. */
#define BOUNDED_EQUAL(beg, end, string_literal)\
- ((end) - (beg) == sizeof (string_literal) - 1\
+ ((end) - (beg) == strlen (string_literal) - 1\
!memcmp ((beg), (string_literal),\
- sizeof (string_literal) - 1))
+ strlen (string_literal) - 1))
/* The same as above, except the comparison is case-insensitive. */
#define BOUNDED_EQUAL_NO_CASE(beg, end, string_literal)\
- ((end) - (beg) == sizeof (string_literal) - 1\
+ ((end) - (beg) == strlen (string_literal)\
!strncasecmp ((beg), (string_literal), \
- sizeof (string_literal) - 1))
+ strlen (string_literal)))
/* Note that this much more elegant definition cannot be used:
On Wed, 28 May 2003, Christian von Ferber wrote:
Hi,
I am mirroring a friendly site that excludes robots in general but
is supposed to allow my FriendlyMirror using wget.
For this purpose I asked the webadmin to set up his robots.txt as follows:
User-agent: FriendlyMirror
Disallow:
User-agent: *
Disallow: /
Starting Wget by
wget --user-agent FriendlyMirror -m http://Friendly.Site
Wget indeed identifies as user-agent FriendlyMirror to Friendly.Site
but considers itself to be user-agent Wget when implementing the rules
of robots.txt.
I think it would be nice if Wget could be told to interpret robots.txt
such that only my FriendlyMirror and not all other robots using wget
will continue automatic download.
Any Ideas ?
Cheers,
Christian
--
#undef MACROS? .libs
? Makefile
? config.h
? wget
Index: init.c
===
RCS file: /pack/anoncvs/wget/src/init.c,v
retrieving revision 1.54
diff -u -u -r1.54 init.c
--- init.c 2002/08/03 20:34:57 1.54
+++ init.c 2003/05/29 17:51:50
@@ -271,6 +271,7 @@
opt.timeout = 900;
#endif
opt.use_robots = 1;
+ opt.useragent = xstrdup (Wget);
opt.remove_listing = 1;
Index: res.c
===
RCS file: /pack/anoncvs/wget/src/res.c,v
retrieving revision 1.7
diff -u -u -r1.7 res.c
--- res.c 2002/05/18 02:16:24 1.7
+++ res.c 2003/05/29 17:51:50
@@ -115,7 +115,7 @@
*matches = 1;
*exact_match = 0;
}
- else if (BOUNDED_EQUAL_NO_CASE (agent, agent + length, wget))
+ else if (BOUNDED_EQUAL_NO_CASE (agent, agent + length, opt.useragent))
{
*matches = 1;
*exact_match = 1;
@@ -355,7 +355,7 @@
}
else
{
- DEBUGP ((Ignoring unknown field at line %d, line_count));
+ DEBUGP ((Ignoring unknown field at line %d\n, line_count));
goto next;
}
Index: wget.h
===
RCS file: /pack/anoncvs/wget/src/wget.h,v
retrieving revision 1.34
diff -u -u -r1.34 wget.h
--- wget.h 2002/05/18 02:16:25 1.34
+++ wget.h 2003/05/29 17:51:50
@@ -189,15 +189,15 @@
/* Return non-zero if string bounded between BEG and END is equal to
STRING_LITERAL. The comparison is
