It seems maxPerHost could cause us not to fill each segment to topN even
when there are more than enough URLs for this job.
We should only count URLs we keep instead of all URLs considered.
There were also two variables named count which is probably bad form
(not a Java person, but it certainly looked odd).
--
Rod Taylor <[EMAIL PROTECTED]>
*** src/java/org/apache/nutch/crawl/Generator.java.orig 2005-10-31 23:35:20.000000000 -0500
--- src/java/org/apache/nutch/crawl/Generator.java 2005-11-10 12:58:25.756074384 -0500
***************
*** 76,98 ****
OutputCollector output, Reporter reporter)
throws IOException {
! while (values.hasNext() && ++count < limit) {
UTF8 url = (UTF8)values.next();
if (maxPerHost > 0) { // are we counting hosts?
String host = new URL(url.toString()).getHost();
! Integer count = (Integer)hostCounts.get(host);
! if (count != null) {
! if (count.intValue() >= maxPerHost)
continue; // too many from host
! hostCounts.put(host, new Integer(count.intValue()+1));
} else { // update host count
hostCounts.put(host, new Integer(1));
}
}
output.collect(key, url);
}
}
--- 76,102 ----
OutputCollector output, Reporter reporter)
throws IOException {
! while (values.hasNext() && count < limit) {
UTF8 url = (UTF8)values.next();
if (maxPerHost > 0) { // are we counting hosts?
String host = new URL(url.toString()).getHost();
! Integer hostCount = (Integer)hostCounts.get(host);
! if (hostCount != null) {
! if (hostCount.intValue() >= maxPerHost)
continue; // too many from host
! hostCounts.put(host, new Integer(hostCount.intValue()+1));
} else { // update host count
hostCounts.put(host, new Integer(1));
}
}
output.collect(key, url);
+
+ // Count is incremented only when we keep the URL
+ // maxPerHost may cause us to skip it.
+ count++;
}
}
