Author: ab
Date: Wed Aug 11 08:33:34 2010
New Revision: 984337
URL: http://svn.apache.org/viewvc?rev=984337&view=rev
Log:
NUTCH-876 Remove remaining robots/IP blocking code in lib-http.
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
nutch/trunk/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
Modified: nutch/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=984337&r1=984336&r2=984337&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Wed Aug 11 08:33:34 2010
@@ -2,6 +2,8 @@ Nutch Change Log
Release 2.0 - Current Development
+* NUTCH-876 Remove remaining robots/IP blocking code in lib-http (ab)
+
* NUTCH-851 Port logging to slf4j (jnioche)
* NUTCH-564 External parser supports encoding attribute (Antony Bowesman,
mattmann)
Modified:
nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java?rev=984337&r1=984336&r2=984337&view=diff
==============================================================================
---
nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
(original)
+++
nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
Wed Aug 11 08:33:34 2010
@@ -18,9 +18,7 @@ package org.apache.nutch.protocol.http.a
// JDK imports
import java.io.IOException;
-import java.net.InetAddress;
import java.net.URL;
-import java.net.UnknownHostException;
import java.util.HashMap;
import java.util.LinkedList;
@@ -68,21 +66,6 @@ public abstract class HttpBase implement
/** The length limit for downloaded content, in bytes. */
protected int maxContent = 64 * 1024;
- /** The number of times a thread will delay when trying to fetch a page. */
- protected int maxDelays = 3;
-
- /**
- * The maximum number of threads that should be allowed
- * to access a host at one time.
- */
- protected int maxThreadsPerHost = 1;
-
- /**
- * The number of seconds the fetcher will delay between
- * successive requests to the same server.
- */
- protected long serverDelay = 1000;
-
/** The Nutch 'User-Agent' request header */
protected String userAgent = getAgentString(
"NutchCVS", null, "Nutch",
@@ -92,27 +75,6 @@ public abstract class HttpBase implement
/** The "Accept-Language" request header value. */
protected String acceptLanguage = "en-us,en-gb,en;q=0.7,*;q=0.3";
-
- /**
- * Maps from host to a Long naming the time it should be unblocked.
- * The Long is zero while the host is in use, then set to now+wait when
- * a request finishes. This way only one thread at a time accesses a
- * host.
- */
- private static HashMap<String, Long> BLOCKED_ADDR_TO_TIME =
- new HashMap<String, Long>();
-
- /**
- * Maps a host to the number of threads accessing that host.
- */
- private static HashMap<String, Integer> THREADS_PER_HOST_COUNT =
- new HashMap<String, Integer>();
-
- /**
- * Queue of blocked hosts. This contains all of the non-zero entries
- * from BLOCKED_ADDR_TO_TIME, ordered by increasing time.
- */
- private static LinkedList<String> BLOCKED_ADDR_QUEUE = new
LinkedList<String>();
/** The default logger */
private final static Logger LOGGER = LoggerFactory.getLogger(HttpBase.class);
@@ -123,23 +85,11 @@ public abstract class HttpBase implement
/** The nutch configuration */
private Configuration conf = null;
- /** Do we block by IP addresses or by hostnames? */
- private boolean byIP = true;
-
private MimeUtil mimeTypes;
/** Do we use HTTP/1.1? */
protected boolean useHttp11 = false;
- /** Skip page if Crawl-Delay longer than this value. */
- protected long maxCrawlDelay = -1L;
-
- /** Plugin should handle host blocking internally. */
- protected boolean checkBlocking = true;
-
- /** Plugin should handle robot rules checking internally. */
- protected boolean checkRobots = true;
-
/** Creates a new instance of HttpBase */
public HttpBase() {
this(null);
@@ -161,20 +111,12 @@ public abstract class HttpBase implement
this.useProxy = (proxyHost != null && proxyHost.length() > 0);
this.timeout = conf.getInt("http.timeout", 10000);
this.maxContent = conf.getInt("http.content.limit", 64 * 1024);
- this.maxDelays = conf.getInt("http.max.delays", 3);
- this.maxThreadsPerHost = conf.getInt("fetcher.threads.per.host", 1);
this.userAgent = getAgentString(conf.get("http.agent.name"),
conf.get("http.agent.version"), conf
.get("http.agent.description"), conf.get("http.agent.url"),
conf.get("http.agent.email"));
this.acceptLanguage = conf.get("http.accept.language", acceptLanguage);
- this.serverDelay = (long) (conf.getFloat("fetcher.server.delay", 1.0f) *
1000);
- this.maxCrawlDelay = (conf.getInt("fetcher.max.crawl.delay", -1) * 1000);
- // backward-compatible default setting
- this.byIP = conf.getBoolean("fetcher.threads.per.host.by.ip", true);
this.mimeTypes = new MimeUtil(conf);
this.useHttp11 = conf.getBoolean("http.useHttp11", false);
this.robots.setConf(conf);
- this.checkBlocking = conf.getBoolean(Protocol.CHECK_BLOCKING, true);
- this.checkRobots = conf.getBoolean(Protocol.CHECK_ROBOTS, true);
logConf();
}
@@ -189,44 +131,8 @@ public abstract class HttpBase implement
try {
URL u = new URL(url);
- long delay = serverDelay;
- if (checkRobots) {
- try {
- if (!robots.isAllowed(this, u)) {
- return new ProtocolOutput(null,
-
ProtocolStatusUtils.makeStatus(ProtocolStatusCodes.ROBOTS_DENIED, url));
- }
- } catch (Throwable e) {
- // XXX Maybe bogus: assume this is allowed.
- if (logger.isTraceEnabled()) {
- logger.trace("Exception checking robot rules for " + url + ": " +
e);
- }
- }
-
- long crawlDelay = robots.getCrawlDelay(this, u);
- delay = crawlDelay > 0 ? crawlDelay : serverDelay;
- }
- if (checkBlocking && maxCrawlDelay >= 0 && delay > maxCrawlDelay) {
- // skip this page, otherwise the thread would block for too long.
- LOGGER.info("Skipping: " + u + " exceeds fetcher.max.crawl.delay, max="
- + (maxCrawlDelay / 1000) + ", Crawl-Delay=" + (delay / 1000));
- return new ProtocolOutput(null, ProtocolStatusUtils.STATUS_WOULDBLOCK);
- }
String host = null;
- if (checkBlocking) {
- try {
- host = blockAddr(u, delay);
- } catch (BlockedException be) {
- return new ProtocolOutput(null, ProtocolStatusUtils.STATUS_BLOCKED);
- }
- }
- Response response;
- try {
- response = getResponse(u, page, false); // make a request
- } finally {
- if (checkBlocking) unblockAddr(host, delay);
- }
-
+ Response response = getResponse(u, page, false); // make a request
int code = response.getCode();
byte[] content = response.getContent();
Content c = new Content(u.toString(), u.toString(),
@@ -319,18 +225,6 @@ public abstract class HttpBase implement
return maxContent;
}
- public int getMaxDelays() {
- return maxDelays;
- }
-
- public int getMaxThreadsPerHost() {
- return maxThreadsPerHost;
- }
-
- public long getServerDelay() {
- return serverDelay;
- }
-
public String getUserAgent() {
return userAgent;
}
@@ -346,94 +240,6 @@ public abstract class HttpBase implement
return useHttp11;
}
- private String blockAddr(URL url, long crawlDelay) throws ProtocolException {
-
- String host;
- if (byIP) {
- try {
- InetAddress addr = InetAddress.getByName(url.getHost());
- host = addr.getHostAddress();
- } catch (UnknownHostException e) {
- // unable to resolve it, so don't fall back to host name
- throw new HttpException(e);
- }
- } else {
- host = url.getHost();
- if (host == null)
- throw new HttpException("Unknown host for url: " + url);
- host = host.toLowerCase();
- }
-
- int delays = 0;
- while (true) {
- cleanExpiredServerBlocks(); // free held addresses
-
- Long time;
- synchronized (BLOCKED_ADDR_TO_TIME) {
- time = BLOCKED_ADDR_TO_TIME.get(host);
- if (time == null) { // address is free
-
- // get # of threads already accessing this addr
- Integer counter = THREADS_PER_HOST_COUNT.get(host);
- int count = (counter == null) ? 0 : counter.intValue();
-
- count++; // increment & store
- THREADS_PER_HOST_COUNT.put(host, new Integer(count));
-
- if (count >= maxThreadsPerHost) {
- BLOCKED_ADDR_TO_TIME.put(host, new Long(0)); // block it
- }
- return host;
- }
- }
-
- if (delays == maxDelays)
- throw new BlockedException("Exceeded http.max.delays: retry later.");
-
- long done = time.longValue();
- long now = System.currentTimeMillis();
- long sleep = 0;
- if (done == 0) { // address is still in use
- sleep = crawlDelay; // wait at least delay
-
- } else if (now < done) { // address is on hold
- sleep = done - now; // wait until its free
- }
-
- try {
- Thread.sleep(sleep);
- } catch (InterruptedException e) {}
- delays++;
- }
- }
-
- private void unblockAddr(String host, long crawlDelay) {
- synchronized (BLOCKED_ADDR_TO_TIME) {
- int addrCount = THREADS_PER_HOST_COUNT.get(host).intValue();
- if (addrCount == 1) {
- THREADS_PER_HOST_COUNT.remove(host);
- BLOCKED_ADDR_QUEUE.addFirst(host);
- BLOCKED_ADDR_TO_TIME.put
- (host, new Long(System.currentTimeMillis() + crawlDelay));
- } else {
- THREADS_PER_HOST_COUNT.put(host, new Integer(addrCount - 1));
- }
- }
- }
-
- private static void cleanExpiredServerBlocks() {
- synchronized (BLOCKED_ADDR_TO_TIME) {
- for (int i = BLOCKED_ADDR_QUEUE.size() - 1; i >= 0; i--) {
- String host = BLOCKED_ADDR_QUEUE.get(i);
- long time = BLOCKED_ADDR_TO_TIME.get(host).longValue();
- if (time <= System.currentTimeMillis()) {
- BLOCKED_ADDR_TO_TIME.remove(host);
- BLOCKED_ADDR_QUEUE.remove(i);
- }
- }
- }
- }
-
private static String getAgentString(String agentName,
String agentVersion,
String agentDesc,
@@ -487,12 +293,6 @@ public abstract class HttpBase implement
logger.info("http.content.limit = " + maxContent);
logger.info("http.agent = " + userAgent);
logger.info("http.accept.language = " + acceptLanguage);
- logger.info(Protocol.CHECK_BLOCKING + " = " + checkBlocking);
- logger.info(Protocol.CHECK_ROBOTS + " = " + checkRobots);
- if (checkBlocking) {
- logger.info("fetcher.server.delay = " + serverDelay);
- logger.info("http.max.delays = " + maxDelays);
- }
}
}
Modified:
nutch/trunk/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java?rev=984337&r1=984336&r2=984337&view=diff
==============================================================================
---
nutch/trunk/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
(original)
+++
nutch/trunk/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
Wed Aug 11 08:33:34 2010
@@ -190,11 +190,6 @@ public class Http extends HttpBase {
params.setSendBufferSize(BUFFER_SIZE);
params.setReceiveBufferSize(BUFFER_SIZE);
params.setMaxTotalConnections(maxThreadsTotal);
- if (maxThreadsTotal > maxThreadsPerHost) {
-
params.setDefaultMaxConnectionsPerHost(maxThreadsPerHost);
- } else {
- params.setDefaultMaxConnectionsPerHost(maxThreadsTotal);
- }
// executeMethod(HttpMethod) seems to ignore the connection
timeout on
// the connection manager.
