Author: mattmann
Date: Sat Sep 11 02:09:43 2010
New Revision: 996045
URL: http://svn.apache.org/viewvc?rev=996045&view=rev
Log:
- fix for NUTCH-905 Configurable file protocol parent directory crawling
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/conf/nutch-default.xml
nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java
Modified: nutch/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=996045&r1=996044&r2=996045&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Sat Sep 11 02:09:43 2010
@@ -2,6 +2,8 @@ Nutch Change Log
Release 2.0 - Current Development
+* NUTCH-905 Configurable file protocol parent directory crawling (Thorsten
Scherler, mattmann, ab)
+
* NUTCH-716 Make subcollection index filed multivalued (Dmitry Lihachev via
jnioche)
* NUTCH-884 FetcherJob should run more reduce tasks than default (ab)
Modified: nutch/trunk/conf/nutch-default.xml
URL:
http://svn.apache.org/viewvc/nutch/trunk/conf/nutch-default.xml?rev=996045&r1=996044&r2=996045&view=diff
==============================================================================
--- nutch/trunk/conf/nutch-default.xml (original)
+++ nutch/trunk/conf/nutch-default.xml Sat Sep 11 02:09:43 2010
@@ -45,6 +45,17 @@
</description>
</property>
+<property>
+ <name>file.crawl.parent</name>
+ <value>true</value>
+ <description>The crawler is not restricted to the directories that you
specified in the
+ urls file but it is jumping into the parent directories as well. For your
own crawling you can
+ change this behavior (set to false) the way that only directories beneath
the directories that you specify get
+ crawled.
+ </description>
+</property>
+
+
<!-- HTTP properties -->
<property>
Modified:
nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java?rev=996045&r1=996044&r2=996045&view=diff
==============================================================================
---
nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
(original)
+++
nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
Sat Sep 11 02:09:43 2010
@@ -59,6 +59,8 @@ public class File implements Protocol {
static final int MAX_REDIRECTS = 5;
int maxContentLength;
+
+ boolean crawlParents;
private Configuration conf;
@@ -173,6 +175,7 @@ public class File implements Protocol {
public void setConf(Configuration conf) {
this.conf = conf;
this.maxContentLength = conf.getInt("file.content.limit", 64 * 1024);
+ this.crawlParents = conf.getBoolean("file.crawl.parent", true);
}
public Configuration getConf() {
Modified:
nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java
URL:
http://svn.apache.org/viewvc/nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java?rev=996045&r1=996044&r2=996045&view=diff
==============================================================================
---
nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java
(original)
+++
nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java
Sat Sep 11 02:09:43 2010
@@ -215,7 +215,10 @@ public class FileResponse {
throws IOException {
String path = f.toString();
- this.content = list2html(f.listFiles(), path, "/".equals(path) ? false :
true);
+ if (this.file.crawlParents)
+ this.content = list2html(f.listFiles(), path, "/".equals(path) ? false
: true);
+ else
+ this.content = list2html(f.listFiles(), path, false);
// set headers
headers.set(Response.CONTENT_LENGTH,
