#!/usr/bin/perl

require LWP::Parallel::UserAgent ;
use HTTP::Request;

my $ua = LWP::Parallel::UserAgent->new('agent1.0','mneuman@netcurrents.com');

$ua->remember_failures(1);
$ua->redirect(0) ;        
$ua->max_hosts(150) ;     
$ua->max_req(150) ;       
$ua->in_order (0) ;       
$ua->duplicates(0) ;      
$ua->timeout(2) ;        

open FH, "site_list2.txt";
@sites = <FH> ;
close FH ;


foreach $site (@sites){
  chomp $site ;
  my $req = HTTP::Request->new('GET', $site) ;
  if ( my $res = $ua->register($req)){
     print STDERR $res->error_as_html ;}
  else {  print " registering :\t $site \n";}
}
print "i am now waiting ..... \n" ;
my $entry = $ua->wait() ;
print "i am now done waiting ......... \n" ;