Author: mattmann
Date: Sat Sep 11 03:49:03 2010
New Revision: 996059
URL: http://svn.apache.org/viewvc?rev=996059&view=rev
Log:
fix for NUTCH-905 Configurable file protocol parent directory crawling
Modified:
nutch/branches/branch-1.2/CHANGES.txt
nutch/branches/branch-1.2/conf/nutch-default.xml
nutch/branches/branch-1.2/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
nutch/branches/branch-1.2/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java
Modified: nutch/branches/branch-1.2/CHANGES.txt
URL:
http://svn.apache.org/viewvc/nutch/branches/branch-1.2/CHANGES.txt?rev=996059&r1=996058&r2=996059&view=diff
==============================================================================
--- nutch/branches/branch-1.2/CHANGES.txt (original)
+++ nutch/branches/branch-1.2/CHANGES.txt Sat Sep 11 03:49:03 2010
@@ -2,6 +2,8 @@ Nutch Change Log
Release 1.2 - 08/07/2010
+* NUTCH-905 Configurable file protocol parent directory crawling (Thorsten
Scherler, mattmann, ab)
+
* NUTCH-877 Allow setting of slop values for non-quote phrase queries on
query-basic plugin (kubes via jnioche)
* NUTCH-716 Make subcollection index filed multivalued (Dmitry Lihachev via
jnioche)
Modified: nutch/branches/branch-1.2/conf/nutch-default.xml
URL:
http://svn.apache.org/viewvc/nutch/branches/branch-1.2/conf/nutch-default.xml?rev=996059&r1=996058&r2=996059&view=diff
==============================================================================
--- nutch/branches/branch-1.2/conf/nutch-default.xml (original)
+++ nutch/branches/branch-1.2/conf/nutch-default.xml Sat Sep 11 03:49:03 2010
@@ -45,6 +45,17 @@
</description>
</property>
+<property>
+ <name>file.crawl.parent</name>
+ <value>true</value>
+ <description>The crawler is not restricted to the directories that you
specified in the
+ urls file but it is jumping into the parent directories as well. For your
own crawling you can
+ change this behavior (set to false) the way that only directories beneath
the directories that you specify get
+ crawled.
+ </description>
+</property>
+
+
<!-- HTTP properties -->
<property>
Modified:
nutch/branches/branch-1.2/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
URL:
http://svn.apache.org/viewvc/nutch/branches/branch-1.2/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java?rev=996059&r1=996058&r2=996059&view=diff
==============================================================================
---
nutch/branches/branch-1.2/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
(original)
+++
nutch/branches/branch-1.2/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
Sat Sep 11 03:49:03 2010
@@ -53,6 +53,8 @@ public class File implements Protocol {
static final int MAX_REDIRECTS = 5;
int maxContentLength;
+
+ boolean crawlParents;
private Configuration conf;
@@ -156,6 +158,7 @@ public class File implements Protocol {
public void setConf(Configuration conf) {
this.conf = conf;
this.maxContentLength = conf.getInt("file.content.limit", 64 * 1024);
+ this.crawlParents = conf.getBoolean("file.crawl.parent", true);
}
public Configuration getConf() {
Modified:
nutch/branches/branch-1.2/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java
URL:
http://svn.apache.org/viewvc/nutch/branches/branch-1.2/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java?rev=996059&r1=996058&r2=996059&view=diff
==============================================================================
---
nutch/branches/branch-1.2/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java
(original)
+++
nutch/branches/branch-1.2/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java
Sat Sep 11 03:49:03 2010
@@ -222,7 +222,10 @@ public class FileResponse {
throws IOException {
String path = f.toString();
- this.content = list2html(f.listFiles(), path, "/".equals(path) ? false :
true);
+ if (this.file.crawlParents)
+ this.content = list2html(f.listFiles(), path, "/".equals(path) ? false
: true);
+ else
+ this.content = list2html(f.listFiles(), path, false);
// set headers
headers.set(Response.CONTENT_LENGTH,
