Gal Nitzan wrote:
I believe you are right. When I checked the file I noticed it exists.
However, I run the fetcher only once on that segment.
Please try the attached patch and tell me if it fixes this for you.
Doug
Index: src/java/org/apache/nutch/ndfs/NDFSClient.java
===================================================================
--- src/java/org/apache/nutch/ndfs/NDFSClient.java (revision 292500)
+++ src/java/org/apache/nutch/ndfs/NDFSClient.java (working copy)
@@ -71,14 +71,6 @@
return new NDFSInputStream(src.toString());
}
- /**
- * Create an output stream that writes to all the right places.
- * Basically creates instance of inner subclass of OutputStream
- * that handles datanode/namenode negotiation.
- */
- public NFSOutputStream create(UTF8 src) throws IOException {
- return create(src, false);
- }
public NFSOutputStream create(UTF8 src, boolean overwrite) throws IOException {
return new NDFSOutputStream(src, overwrite);
}
Index: src/java/org/apache/nutch/fs/LocalFileSystem.java
===================================================================
--- src/java/org/apache/nutch/fs/LocalFileSystem.java (revision 292500)
+++ src/java/org/apache/nutch/fs/LocalFileSystem.java (working copy)
@@ -95,13 +95,6 @@
return new LocalNFSFileInputStream(f);
}
- /**
- * Create the file at f.
- */
- public NFSOutputStream create(File f) throws IOException {
- return create(f, false);
- }
-
/*********************************************************
* For create()'s NFSOutputStream.
*********************************************************/
@@ -128,8 +121,6 @@
public void write(int b) throws IOException { fos.write(b); }
}
- /**
- */
public NFSOutputStream create(File f, boolean overwrite) throws IOException {
if (f.exists() && ! overwrite) {
throw new IOException("File already exists:"+f);
Index: src/java/org/apache/nutch/fs/NutchFileSystem.java
===================================================================
--- src/java/org/apache/nutch/fs/NutchFileSystem.java (revision 292500)
+++ src/java/org/apache/nutch/fs/NutchFileSystem.java (working copy)
@@ -122,10 +122,18 @@
public abstract NFSInputStream open(File f) throws IOException;
/**
- * Opens an OutputStream at the indicated File, whether local
- * or via NDFS.
+ * Opens an OutputStream at the indicated File.
+ * Files are overwritten by default.
*/
- public abstract NFSOutputStream create(File f) throws IOException;
+ public NFSOutputStream create(File f) throws IOException {
+ return create(f, true);
+ }
+
+ /** Opens an OutputStream at the indicated File.
+ * @param f the file name to open
+ * @param overwrite if a file with this name already exists, then if true,
+ * the file will be overwritten, and if false an error will be thrown.
+ */
public abstract NFSOutputStream create(File f, boolean overwrite) throws IOException;
/**
