svn commit: r1603094 - in /nutch/trunk: CHANGES.txt src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java

[jnioche]

Author: jnioche
Date: Tue Jun 17 08:41:57 2014
New Revision: 1603094

URL: http://svn.apache.org/r1603094
Log:
NUTCH-1793 HttpRobotRulesParser not configured properly (jnioche)

Modified:
    nutch/trunk/CHANGES.txt
    
nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java

Modified: nutch/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1603094&r1=1603093&r2=1603094&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Tue Jun 17 08:41:57 2014
@@ -2,6 +2,8 @@ Nutch Change Log
 
 Nutch Current Development
 
+* NUTCH-1793 HttpRobotRulesParser not configured properly (jnioche)
+
 * NUTCH-1647 protocol-http throws 'unzipBestEffort returned null' for 
redirected pages (jnioche)
 
 * NUTCH-1736 Can't fetch page if http response header contains 
Transfer-Encoding：chunked (ysc via jnioche)

Modified: 
nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java
URL: 
http://svn.apache.org/viewvc/nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java?rev=1603094&r1=1603093&r2=1603094&view=diff
==============================================================================
--- 
nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java
 (original)
+++ 
nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java
 Tue Jun 17 08:41:57 2014
@@ -44,8 +44,12 @@ public class HttpRobotRulesParser extend
   HttpRobotRulesParser() { }
 
   public HttpRobotRulesParser(Configuration conf) {
-    super(conf);
-    allowForbidden = conf.getBoolean("http.robots.403.allow", false);
+           setConf(conf);
+  }
+ 
+  public void setConf(Configuration conf) {
+           super.setConf(conf);
+           allowForbidden = conf.getBoolean("http.robots.403.allow", true);
   }
 
   /** Compose unique key to store and access robot rules in cache for given 
URL */