Update of /cvsroot/nutch/nutch/src/java/net/nutch/util
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv20100/src/java/net/nutch/util
Modified Files:
LocalFileSystem.java NDFSFileSystem.java NutchFileSystem.java
Log Message:
Modify the NutchFileSystem so caller can more transparently
modify files that could be local or remote. In the local case,
they are modified directly in place. In the remote case, they
are copied to a local path, then placed back on the remote
fs.
This was the effect of previous code, but it could sometimes
result in unnecessary copying. This API I think is more clear,
and has no inefficiencies for the localfs case. (And is as
fast as possible for remotefs, given that we have no NDFS
API for Lucene-formatted files.)
Index: NutchFileSystem.java
===================================================================
RCS file: /cvsroot/nutch/nutch/src/java/net/nutch/util/NutchFileSystem.java,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** NutchFileSystem.java 8 Sep 2004 16:29:14 -0000 1.3
--- NutchFileSystem.java 4 Oct 2004 15:42:59 -0000 1.4
***************
*** 174,191 ****
public abstract void putToLocalFile(File src, File dst) throws IOException;
/**
! * Returns a local File that contains everything in the NFS "src".
! * If the NFS is local, we just return "src". If the NFS is remote,
! * we return a local file (starting with "tmpName") that contains the
! * info.
*/
! public abstract File makeLocal(File src, String tmpName) throws IOException;
/**
! * Called when we're done with local work. The "local" file has been
! * obtained via "makeLocal". We copy it back to "src" on the NFS.
! * If the NFS is local, no work has to take place.
*/
! public abstract void completeLocalWork(File local, File src) throws IOException;
/**
--- 174,209 ----
public abstract void putToLocalFile(File src, File dst) throws IOException;
+
/**
! * Returns a local File that the user can write output to. The caller
! * provides both the eventual NFS target name and the local working
! * file. If the NFS is local, we write directly into the target. If
! * the NFS is remote, we write into the tmp local area.
*/
! public abstract File startLocalOutput(File nfsOutputFile, File tmpLocalFile)
throws IOException;
/**
! * Called when we're all done writing to the target. A local NFS will
! * do nothing, because we've written to exactly the right place. A remote
! * NFS will copy the contents of tmpLocalFile to the correct target at
! * nfsOutputFile.
*/
! public abstract void completeLocalOutput(File nfsOutputFile, File tmpLocalFile)
throws IOException;
!
! /**
! * Returns a local File that the user can read from. The caller
! * provides both the eventual NFS target name and the local working
! * file. If the NFS is local, we read directly from the source. If
! * the NFS is remote, we write data into the tmp local area.
! */
! public abstract File startLocalInput(File nfsInputFile, File tmpLocalFile)
throws IOException;
!
! /**
! * Called when we're all done writing to the target. A local NFS will
! * do nothing, because we've written to exactly the right place. A remote
! * NFS will copy the contents of tmpLocalFile to the correct target at
! * nfsOutputFile.
! */
! public abstract void completeLocalInput(File localFile) throws IOException;
/**
Index: LocalFileSystem.java
===================================================================
RCS file: /cvsroot/nutch/nutch/src/java/net/nutch/util/LocalFileSystem.java,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** LocalFileSystem.java 8 Sep 2004 16:29:13 -0000 1.2
--- LocalFileSystem.java 4 Oct 2004 15:42:58 -0000 1.3
***************
*** 302,306 ****
*/
public void addLocalFile(File src, File dst) throws IOException {
! src.renameTo(dst);
}
--- 302,308 ----
*/
public void addLocalFile(File src, File dst) throws IOException {
! if (! src.equals(dst)) {
! src.renameTo(dst);
! }
}
***************
*** 309,319 ****
*/
public void putToLocalFile(File src, File dst) throws IOException {
! FileUtil.copyContents(this, src, dst, true);
}
! public File makeLocal(File src, String tmpName) throws IOException {
! return src;
}
! public void completeLocalWork(File local, File src) throws IOException {
}
--- 311,344 ----
*/
public void putToLocalFile(File src, File dst) throws IOException {
! if (! src.equals(dst)) {
! FileUtil.copyContents(this, src, dst, true);
! }
}
! /**
! * We can write output directly to the final location
! */
! public File startLocalOutput(File nfsOutputFile, File tmpLocalFile) throws
IOException {
! return nfsOutputFile;
}
!
! /**
! * It's in the right place - nothing to do.
! */
! public void completeLocalOutput(File nfsWorkingFile, File tmpLocalFile) throws
IOException {
! }
!
! /**
! * We can read directly from the real local fs.
! */
! public File startLocalInput(File nfsInputFile, File tmpLocalFile) throws
IOException {
! return nfsInputFile;
! }
!
! /**
! * We're done reading. Nothing to clean up.
! */
! public void completeLocalInput(File localFile) throws IOException {
! // Ignore the file, it's at the right destination!
}
Index: NDFSFileSystem.java
===================================================================
RCS file: /cvsroot/nutch/nutch/src/java/net/nutch/util/NDFSFileSystem.java,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** NDFSFileSystem.java 8 Sep 2004 16:29:14 -0000 1.2
--- NDFSFileSystem.java 4 Oct 2004 15:42:59 -0000 1.3
***************
*** 198,209 ****
/**
*/
! public File makeLocal(File src, String tmpName) throws IOException {
! File local = new File(tmpName + new
SimpleDateFormat("yyyyMMddHHmmss").format(new Date(System.currentTimeMillis())));
! putToLocalFile(src, local);
! return local;
}
! public void completeLocalWork(File local, File src) throws IOException {
! addLocalFile(local, src);
}
--- 198,232 ----
/**
+ * Output will go to the tmp working area. There may be some source
+ * material that we obtain first.
*/
! public File startLocalOutput(File nfsOutputFile, File tmpLocalFile) throws
IOException {
! if (exists(nfsOutputFile)) {
! putToLocalFile(nfsOutputFile, tmpLocalFile);
! }
! return tmpLocalFile;
}
!
! /**
! * Move completed local data to NDFS destination
! */
! public void completeLocalOutput(File nfsOutputFile, File tmpLocalFile) throws
IOException {
! addLocalFile(tmpLocalFile, nfsOutputFile);
! }
!
! /**
! * Fetch remote NDFS file, place at tmpLocalFile
! */
! public File startLocalInput(File nfsInputFile, File tmpLocalFile) throws
IOException {
! putToLocalFile(nfsInputFile, tmpLocalFile);
! return tmpLocalFile;
! }
!
! /**
! * We're done with the local stuff, so delete it
! */
! public void completeLocalInput(File localFile) throws IOException {
! // Get rid of the local copy - we don't need it anymore.
! FileUtil.fullyDelete(localFile);
}
-------------------------------------------------------
This SF.net email is sponsored by: IT Product Guide on ITManagersJournal
Use IT products in your business? Tell us what you think of them. Give us
Your Opinions, Get Free ThinkGeek Gift Certificates! Click to find out more
http://productguide.itmanagersjournal.com/guidepromo.tmpl
_______________________________________________
Nutch-cvs mailing list
[EMAIL PROTECTED]
https://lists.sourceforge.net/lists/listinfo/nutch-cvs
