Author: dhruba
Date: Mon Nov 10 22:28:20 2008
New Revision: 712962
URL: http://svn.apache.org/viewvc?rev=712962&view=rev
Log:
HADOOP-4567. GetFileBlockLocations returns the NetworkTopology
information of the machines on where the blocks reside. (dhruba)
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/core/org/apache/hadoop/fs/BlockLocation.java
hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java
hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/TestReplication.java
Modified: hadoop/core/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=712962&r1=712961&r2=712962&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon Nov 10 22:28:20 2008
@@ -22,6 +22,9 @@
NameNode(bindAddress, conf) is removed.
(shv)
+ HADOOP-4567. GetFileBlockLocations returns the NetworkTopology
+ information of the machines where the blocks reside. (dhruba)
+
NEW FEATURES
HADOOP-4575. Add a proxy service for relaying HsftpFileSystem requests.
Modified: hadoop/core/trunk/src/core/org/apache/hadoop/fs/BlockLocation.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/fs/BlockLocation.java?rev=712962&r1=712961&r2=712962&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/fs/BlockLocation.java
(original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/fs/BlockLocation.java Mon Nov
10 22:28:20 2008
@@ -38,6 +38,7 @@
private String[] hosts; //hostnames of datanodes
private String[] names; //hostname:portNumber of datanodes
+ private String[] topologyPaths; // full path name in network topology
private long offset; //offset of the of the block in the file
private long length;
@@ -65,6 +66,20 @@
}
this.offset = offset;
this.length = length;
+ this.topologyPaths = new String[0];
+ }
+
+ /**
+ * Constructor with host, name, network topology, offset and length
+ */
+ public BlockLocation(String[] names, String[] hosts, String[] topologyPaths,
+ long offset, long length) {
+ this(names, hosts, offset, length);
+ if (topologyPaths == null) {
+ this.topologyPaths = new String[0];
+ } else {
+ this.topologyPaths = topologyPaths;
+ }
}
/**
@@ -88,6 +103,18 @@
return this.names;
}
}
+
+ /**
+ * Get the list of network topology paths for each of the hosts.
+ * The last component of the path is the host.
+ */
+ public String[] getTopologyPaths() throws IOException {
+ if ((topologyPaths == null) || (topologyPaths.length == 0)) {
+ return new String[0];
+ } else {
+ return this.topologyPaths;
+ }
+ }
/**
* Get the start offset of file associated with this block
@@ -140,6 +167,17 @@
}
/**
+ * Set the network topology paths of the hosts
+ */
+ public void setTopologyPaths(String[] topologyPaths) throws IOException {
+ if (topologyPaths == null) {
+ this.topologyPaths = new String[0];
+ } else {
+ this.topologyPaths = topologyPaths;
+ }
+ }
+
+ /**
* Implement write of Writable
*/
public void write(DataOutput out) throws IOException {
@@ -155,6 +193,11 @@
Text host = new Text(hosts[i]);
host.write(out);
}
+ out.writeInt(topologyPaths.length);
+ for (int i=0; i < topologyPaths.length; i++) {
+ Text host = new Text(topologyPaths[i]);
+ host.write(out);
+ }
}
/**
@@ -176,6 +219,12 @@
host.readFields(in);
hosts[i] = host.toString();
}
+ int numTops = in.readInt();
+ Text path = new Text();
+ for (int i = 0; i < numTops; i++) {
+ path.readFields(in);
+ topologyPaths[i] = path.toString();
+ }
}
public String toString() {
Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java?rev=712962&r1=712961&r2=712962&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/DFSClient.java Mon Nov 10
22:28:20 2008
@@ -25,6 +25,7 @@
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.ipc.*;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.net.NodeBase;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.hdfs.DistributedFileSystem.DiskStatus;
import org.apache.hadoop.hdfs.protocol.*;
@@ -309,11 +310,16 @@
DatanodeInfo[] locations = blk.getLocations();
String[] hosts = new String[locations.length];
String[] names = new String[locations.length];
+ String[] racks = new String[locations.length];
for (int hCnt = 0; hCnt < locations.length; hCnt++) {
hosts[hCnt] = locations[hCnt].getHostName();
names[hCnt] = locations[hCnt].getName();
+ NodeBase node = new NodeBase(names[hCnt],
+ locations[hCnt].getNetworkLocation());
+ racks[hCnt] = node.toString();
}
- blkLocations[idx] = new BlockLocation(names, hosts, blk.getStartOffset(),
+ blkLocations[idx] = new BlockLocation(names, hosts, racks,
+ blk.getStartOffset(),
blk.getBlockSize());
idx++;
}
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/TestReplication.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/TestReplication.java?rev=712962&r1=712961&r2=712962&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/TestReplication.java
(original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/hdfs/TestReplication.java Mon
Nov 10 22:28:20 2008
@@ -35,6 +35,8 @@
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.BlockLocation;
/**
* This class tests the replication of a DFS file.
@@ -74,6 +76,28 @@
LocatedBlocks locations = namenode.getBlockLocations(name.toString(),0,
Long.MAX_VALUE);
+ FileStatus stat = fileSys.getFileStatus(name);
+ BlockLocation[] blockLocations = fileSys.getFileBlockLocations(stat,0L,
+ Long.MAX_VALUE);
+ // verify that rack locations match
+ assertTrue(blockLocations.length == locations.locatedBlockCount());
+ for (int i = 0; i < blockLocations.length; i++) {
+ LocatedBlock blk = locations.get(i);
+ DatanodeInfo[] datanodes = blk.getLocations();
+ String[] topologyPaths = blockLocations[i].getTopologyPaths();
+ assertTrue(topologyPaths.length == datanodes.length);
+ for (int j = 0; j < topologyPaths.length; j++) {
+ boolean found = false;
+ for (int k = 0; k < racks.length; k++) {
+ if (topologyPaths[j].startsWith(racks[k])) {
+ found = true;
+ break;
+ }
+ }
+ assertTrue(found);
+ }
+ }
+
boolean isOnSameRack = true, isNotOnSameRack = true;
for (LocatedBlock blk : locations.getLocatedBlocks()) {
DatanodeInfo[] datanodes = blk.getLocations();
