Hello again,
WWW::Robot implements a generic webspider (robot) with all kinds of
user-supplied test functions to control the spidering process.
But it turns out to have several hardcoded filters that limit
retrieval to HTML documents found over 'ordinary' links.
For my purposes I need every document, not just HTML. The attached
patch adds three Boolean flags to turn off these hardcoded filters.
With all three turned on, traversal/retrieval is only limited by the
robot exclusion protocol.
--
Reinier Post [EMAIL PROTECTED]
--- Robot.pm.orig Sun Aug 6 19:48:34 2000
+++ Robot.pm Mon Aug 7 16:00:53 2000
@@ -174,7 +174,7 @@
#------------------------------------------------------------------------------
use vars qw( $VERSION );
-$VERSION = '0.022';
+$VERSION = '0.0221';
#------------------------------------------------------------------------------
#
@@ -193,6 +193,9 @@
'CHECK_MIME_TYPES' => 'should we check the MIME types of links?',
'ACCEPT_LANGUAGE' => 'array ref to list of languages to accept',
'DELAY' => 'delay between robot requests (minutes)',
+ 'ANY_URL' => 'whether to omit default URL filtering',
+ 'ANY_LINK' => 'whether to restrict to default link types',
+ 'ANY_CONTENT' => 'whether to offer content of non-HTML documents',
);
my %ATTRIBUTE_DEFAULT = (
@@ -201,6 +204,9 @@
'IGNORE_TEXT' => 1,
'IGNORE_UNKNOWN' => 1,
'CHECK_MIME_TYPES' => 1,
+ 'ANY_URL' => 0,
+ 'ANY_LINK' => 0,
+ 'ANY_CONTENT' => 0,
);
my %SUPPORTED_HOOKS = (
@@ -341,8 +347,20 @@
$url = new URI::URL( $base );
}
- if ( $response->content_type eq 'text/html' )
+ if ( $response->content_type ne 'text/html' )
{
+ if ( $self->{'ANY_CONTENT'} )
+ {
+ $self->invoke_hook_procedures(
+ 'invoke-on-contents',
+ $url,
+ $response,
+ undef # no $structure
+ );
+ }
+ }
+ else
+ {
my $contents = $response->content;
$self->verbose( "Parse $url into HTML::TreeBuilder ..." );
@@ -853,7 +871,7 @@
$self->verbose( "Extract links from $url (base = $base) ..." );
- my ( @links );
+ my ( @abslinks );
my $link_extor = new HTML::LinkExtor(
@@ -862,23 +880,34 @@
sub {
my ( $tag, %attr ) = @_;
- my ( $link );
-
- # grab anchor / area / frame links
+ my ( @eltlinks );
- if( lc( $tag ) =~ /^a(?:rea)?$/ )
+ if ( $self->{ 'ANY_LINK' } )
{
- return unless defined( $link = $attr{ 'href' } );
+ # anything goes
+ @eltlinks = values %attr;
+ }
+ # else, grab anchor / area / frame links
+ elsif ( lc( $tag ) =~ /^a(?:rea)?$/ )
+ {
+ @eltlinks =
+ grep { $_ eq 'href' ? $attr{$_} : undef } keys %attr
+ or return;
}
elsif ( lc( $tag ) eq 'frame' )
{
- return unless defined( $link = $attr{ 'src' } );
+ @eltlinks =
+ grep { $_ eq 'src' ? $attr{$_} : undef } keys %attr
+ or return;
}
else
{
return;
}
+ foreach my $link (@eltlinks)
+ {
+
$self->verbose( "\n\t$link ..." );
# ignore page internal links
@@ -888,10 +917,10 @@
$link =~ s!#.*!!;
- # only follow html links (.html or .htm or no extension)
-
- unless ( $link =~ /\.s?html?/ || $link =~ m{/$} )
- # lets assume .s?html or "/" type links really are text/html
+ unless ( $self->{ 'ANY_URL' } ||
+ # only follow html links (.html or .htm or no extension)
+ $link =~ /\.s?html?/ || $link =~ m{/$} )
+ # lets assume .s?html or "/" type links really are text/html
{
# put in some obvious ones here ...
return if $link =~
@@ -900,6 +929,7 @@
return if $link =~ /\.(?:gif|jpe?g)/;
if ( $self->{ 'CHECK_MIME_TYPES' } )
{
+ # grab anchor / area / frame links
$self->verbose( " check mime type ..." );
return unless
$self->check_mime_type( $link, [ 'text/html' ] )
@@ -929,8 +959,10 @@
$link_url_abs
)
);
- push( @links, $link_url_abs );
+ push( @abslinks, $link_url_abs );
$self->verbose( " OK" );
+
+ }
},
$base
);
@@ -942,7 +974,7 @@
# ... and return the links created in the callback
$self->verbose( "\n" );
- return( @links );
+ return( @abslinks );
}
#------------------------------------------------------------------------------
@@ -1386,6 +1418,29 @@
Optionally set the delay between requests for the user agent, in minutes. The
default for this is 1 (see LWP::RobotUA).
+
+=item ANY_URL
+
+Setting this to true will omit the filtering that older versions
+of WWW::Robot use to weed out all non-HTML and many non-HTTP URLs
+before even offering them to 'add-url-test'.
+
+Default: 0 (false)
+
+=item ANY_LINK
+
+Setting this to true will omit the selection of <A HREF>, <AREA>
+and <FRAME SRC> attributes to find links; instead, we use everything
+that may contain a link (as determined by HTML::LinkExtor).
+
+Default: 0 (false)
+
+=item ANY_CONTENT
+
+Setting this to true will call the invoke-on-content hook on documents
+other than those with MIME type text/html.
+
+Default: 0 (false)
=back
