Repository: nutch Updated Branches: refs/heads/master d27c351f4 -> d37b7ce13
Remove obsolete properties protocol.plugin.check.blocking and protocol.plugin.check.robots Project: http://git-wip-us.apache.org/repos/asf/nutch/repo Commit: http://git-wip-us.apache.org/repos/asf/nutch/commit/070a637b Tree: http://git-wip-us.apache.org/repos/asf/nutch/tree/070a637b Diff: http://git-wip-us.apache.org/repos/asf/nutch/diff/070a637b Branch: refs/heads/master Commit: 070a637babedc324948c0c58b333668bab6b813d Parents: d27c351 Author: Sebastian Nagel <sna...@apache.org> Authored: Mon Aug 15 11:19:46 2016 +0200 Committer: Sebastian Nagel <sna...@apache.org> Committed: Mon Aug 15 11:19:46 2016 +0200 ---------------------------------------------------------------------- src/java/org/apache/nutch/fetcher/Fetcher.java | 4 ---- src/java/org/apache/nutch/protocol/Protocol.java | 18 ------------------ 2 files changed, 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nutch/blob/070a637b/src/java/org/apache/nutch/fetcher/Fetcher.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/nutch/fetcher/Fetcher.java b/src/java/org/apache/nutch/fetcher/Fetcher.java index aad9ee9..e60b10f 100644 --- a/src/java/org/apache/nutch/fetcher/Fetcher.java +++ b/src/java/org/apache/nutch/fetcher/Fetcher.java @@ -209,10 +209,6 @@ MapRunnable<Text, CrawlDatum, Text, NutchWritable> { feeder.setTimeLimit(timelimit); feeder.start(); - // set non-blocking & no-robots mode for HTTP protocol plugins. - getConf().setBoolean(Protocol.CHECK_BLOCKING, false); - getConf().setBoolean(Protocol.CHECK_ROBOTS, false); - for (int i = 0; i < threadCount; i++) { // spawn threads FetcherThread t = new FetcherThread(getConf(), getActiveThreads(), fetchQueues, feeder, spinWaiting, lastRequestStart, reporter, errors, segmentName, http://git-wip-us.apache.org/repos/asf/nutch/blob/070a637b/src/java/org/apache/nutch/protocol/Protocol.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/nutch/protocol/Protocol.java b/src/java/org/apache/nutch/protocol/Protocol.java index 0aa5d29..efd0100 100755 --- a/src/java/org/apache/nutch/protocol/Protocol.java +++ b/src/java/org/apache/nutch/protocol/Protocol.java @@ -33,24 +33,6 @@ public interface Protocol extends Pluggable, Configurable { public final static String X_POINT_ID = Protocol.class.getName(); /** - * Property name. If in the current configuration this property is set to - * true, protocol implementations should handle "politeness" limits - * internally. If this is set to false, it is assumed that these limits are - * enforced elsewhere, and protocol implementations should not enforce them - * internally. - */ - public final static String CHECK_BLOCKING = "protocol.plugin.check.blocking"; - - /** - * Property name. If in the current configuration this property is set to - * true, protocol implementations should handle robot exclusion rules - * internally. If this is set to false, it is assumed that these limits are - * enforced elsewhere, and protocol implementations should not enforce them - * internally. - */ - public final static String CHECK_ROBOTS = "protocol.plugin.check.robots"; - - /** * Returns the {@link Content} for a fetchlist entry. */ ProtocolOutput getProtocolOutput(Text url, CrawlDatum datum);
