Author: mc
Date: Wed Dec 14 17:21:52 2005
New Revision: 356964
URL: http://svn.apache.org/viewcvs?rev=356964&view=rev
Log:
Change filesystem getHints() to work for
multiple block-regions.
Fix a filesystem bug when moving files into
directories. They're now moved correctly.
Modified:
lucene/nutch/branches/mapred/src/java/org/apache/nutch/fs/LocalFileSystem.java
lucene/nutch/branches/mapred/src/java/org/apache/nutch/fs/NDFSFileSystem.java
lucene/nutch/branches/mapred/src/java/org/apache/nutch/fs/NutchFileSystem.java
lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/ClientProtocol.java
lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/FSDirectory.java
lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/FSNamesystem.java
lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/NDFSClient.java
lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/NameNode.java
Modified:
lucene/nutch/branches/mapred/src/java/org/apache/nutch/fs/LocalFileSystem.java
URL:
http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/fs/LocalFileSystem.java?rev=356964&r1=356963&r2=356964&view=diff
==============================================================================
---
lucene/nutch/branches/mapred/src/java/org/apache/nutch/fs/LocalFileSystem.java
(original)
+++
lucene/nutch/branches/mapred/src/java/org/apache/nutch/fs/LocalFileSystem.java
Wed Dec 14 17:21:52 2005
@@ -51,6 +51,20 @@
// useCopyForRename = false;
}
+ /**
+ * Return 1x1 'localhost' cell if the file exists.
+ * Return null if otherwise.
+ */
+ public String[][] getFileCacheHints(File f, long start, long len) throws
IOException {
+ if (! f.exists()) {
+ return null;
+ } else {
+ String result[][] = new String[1][];
+ result[0] = new String[1];
+ result[0][0] = "localhost";
+ return result;
+ }
+ }
public String getName() { return "local"; }
Modified:
lucene/nutch/branches/mapred/src/java/org/apache/nutch/fs/NDFSFileSystem.java
URL:
http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/fs/NDFSFileSystem.java?rev=356964&r1=356963&r2=356964&view=diff
==============================================================================
---
lucene/nutch/branches/mapred/src/java/org/apache/nutch/fs/NDFSFileSystem.java
(original)
+++
lucene/nutch/branches/mapred/src/java/org/apache/nutch/fs/NDFSFileSystem.java
Wed Dec 14 17:21:52 2005
@@ -58,6 +58,10 @@
return new UTF8(path);
}
+ public String[][] getFileCacheHints(File f, long start, long len) throws
IOException {
+ return ndfs.getHints(getPath(f), start, len);
+ }
+
public NFSInputStream openRaw(File f) throws IOException {
return ndfs.open(getPath(f));
}
Modified:
lucene/nutch/branches/mapred/src/java/org/apache/nutch/fs/NutchFileSystem.java
URL:
http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/fs/NutchFileSystem.java?rev=356964&r1=356963&r2=356964&view=diff
==============================================================================
---
lucene/nutch/branches/mapred/src/java/org/apache/nutch/fs/NutchFileSystem.java
(original)
+++
lucene/nutch/branches/mapred/src/java/org/apache/nutch/fs/NutchFileSystem.java
Wed Dec 14 17:21:52 2005
@@ -131,6 +131,18 @@
}
/**
+ * Return a 2D array of size 1x1 or greater, containing hostnames
+ * where portions of the given file can be found. For a nonexistent
+ * file or regions, null will be returned.
+ *
+ * This call is most helpful with NDFS, where it returns
+ * hostnames of machines that contain the given file.
+ *
+ * The NutchFileSystem will simply return an elt containing 'localhost'.
+ */
+ public abstract String[][] getFileCacheHints(File f, long start, long len)
throws IOException;
+
+ /**
* Opens an NFSDataInputStream for the indicated File.
*/
public NFSDataInputStream open(File f) throws IOException {
Modified:
lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/ClientProtocol.java
URL:
http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/ClientProtocol.java?rev=356964&r1=356963&r2=356964&view=diff
==============================================================================
---
lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/ClientProtocol.java
(original)
+++
lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/ClientProtocol.java
Wed Dec 14 17:21:52 2005
@@ -73,9 +73,10 @@
/**
* The client wants to read the indicated filename at a certain offset.
- * Return a list of hostnames where the data can be found.
+ * Return a list of hostnames where the data can be found. (Return
+ * a set of hostnames for every block.)
*/
- public String[] getHints(String src, long offset) throws IOException;
+ public String[][] getHints(String src, long start, long len) throws
IOException;
/**
* Rename an item in the fs namespace
Modified:
lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/FSDirectory.java
URL:
http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/FSDirectory.java?rev=356964&r1=356963&r2=356964&view=diff
==============================================================================
---
lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/FSDirectory.java
(original)
+++
lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/FSDirectory.java
Wed Dec 14 17:21:52 2005
@@ -507,7 +507,9 @@
return false;
}
removedNode.removeNode();
-
+ if (isDir(dst)) {
+ dst = new UTF8(dst.toString() + "/" + new
File(src.toString()).getName());
+ }
INode newNode = rootDir.addNode(dst.toString(),
removedNode.blocks);
if (newNode != null) {
newNode.children = removedNode.children;
@@ -517,7 +519,7 @@
}
return true;
} else {
- removedNode.parent.children.put(new
File(dst.toString()).getName(), removedNode);
+ rootDir.addNode(src.toString(), removedNode.blocks);
return false;
}
}
Modified:
lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/FSNamesystem.java
URL:
http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/FSNamesystem.java?rev=356964&r1=356963&r2=356964&view=diff
==============================================================================
---
lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/FSNamesystem.java
(original)
+++
lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/FSNamesystem.java
Wed Dec 14 17:21:52 2005
@@ -529,39 +529,58 @@
/**
* Figure out a few hosts that are likely to contain the
- * block referred to by the given filename, offset pair.
+ * block(s) referred to by the given (filename, start, len) tuple.
*/
- public UTF8[] getDatanodeHints(UTF8 src, long offset) {
- Block targetBlock = null;
+ public UTF8[][] getDatanodeHints(UTF8 src, long start, long len) {
+ if (start < 0 || len < 0) {
+ return new UTF8[0][];
+ }
+
+ int startBlock = -1;
+ int endBlock = -1;
Block blocks[] = dir.getFile(src);
//
- // First, figure out where the offset would fall in
+ // First, figure out where the range falls in
// the blocklist.
//
+ long startpos = start;
+ long endpos = start + len;
for (int i = 0; i < blocks.length; i++) {
- offset -= blocks[i].getNumBytes();
- if (offset <= 0) {
- targetBlock = blocks[i];
- break;
+ if (startpos >= 0) {
+ startpos -= blocks[i].getNumBytes();
+ if (startpos <= 0) {
+ startBlock = i;
+ }
+ }
+ if (endpos >= 0) {
+ endpos -= blocks[i].getNumBytes();
+ if (endpos <= 0) {
+ endBlock = i;
+ break;
+ }
}
}
//
- // Next, create an array of hosts where that block can
+ // Next, create an array of hosts where each block can
// be found
//
- UTF8 hosts[] = null;
- if (targetBlock != null) {
- TreeSet containingNodes = (TreeSet) blocksMap.get(targetBlock);
- Vector v = new Vector();
- for (Iterator it = containingNodes.iterator(); it.hasNext(); ) {
- DatanodeInfo cur = (DatanodeInfo) it.next();
- v.add(cur.getHost());
+ if (startBlock < 0 || endBlock < 0) {
+ return new UTF8[0][];
+ } else {
+ UTF8 hosts[][] = new UTF8[endBlock - startBlock + 1][];
+ for (int i = startBlock; i <= endBlock; i++) {
+ TreeSet containingNodes = (TreeSet) blocksMap.get(blocks[i]);
+ Vector v = new Vector();
+ for (Iterator it = containingNodes.iterator(); it.hasNext(); )
{
+ DatanodeInfo cur = (DatanodeInfo) it.next();
+ v.add(cur.getHost());
+ }
+ hosts[i] = (UTF8[]) v.toArray(new UTF8[v.size()]);
}
- hosts = (UTF8[]) v.toArray(new UTF8[v.size()]);
+ return hosts;
}
- return hosts;
}
/************************************************************
Modified:
lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/NDFSClient.java
URL:
http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/NDFSClient.java?rev=356964&r1=356963&r2=356964&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/NDFSClient.java
(original)
+++ lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/NDFSClient.java
Wed Dec 14 17:21:52 2005
@@ -60,6 +60,15 @@
}
/**
+ * Get hints about the location of the indicated block(s). The
+ * array returned is as long as there are blocks in the indicated
+ * range. Each block may have one or more locations.
+ */
+ public String[][] getHints(UTF8 src, long start, long len) throws
IOException {
+ return namenode.getHints(src.toString(), start, len);
+ }
+
+ /**
* Create an input stream that obtains a nodelist from the
* namenode, and then reads from all the right places. Creates
* inner subclass of InputStream that does the right out-of-band
Modified:
lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/NameNode.java
URL:
http://svn.apache.org/viewcvs/lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/NameNode.java?rev=356964&r1=356963&r2=356964&view=diff
==============================================================================
--- lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/NameNode.java
(original)
+++ lucene/nutch/branches/mapred/src/java/org/apache/nutch/ndfs/NameNode.java
Wed Dec 14 17:21:52 2005
@@ -188,14 +188,17 @@
}
/**
*/
- public String[] getHints(String src, long offset) throws IOException {
- UTF8 hosts[] = namesystem.getDatanodeHints(new UTF8(src), offset);
+ public String[][] getHints(String src, long start, long len) throws
IOException {
+ UTF8 hosts[][] = namesystem.getDatanodeHints(new UTF8(src), start,
len);
if (hosts == null) {
- return new String[0];
+ return new String[0][];
} else {
- String results[] = new String[hosts.length];
+ String results[][] = new String[hosts.length][];
for (int i = 0; i < hosts.length; i++) {
- results[i] = hosts[i].toString();
+ results[i] = new String[hosts[i].length];
+ for (int j = 0; j < results[i].length; j++) {
+ results[i][j] = hosts[i][j].toString();
+ }
}
return results;
}
