Author: wang
Date: Thu Jun 12 20:45:52 2014
New Revision: 1602287
URL: http://svn.apache.org/r1602287
Log:
HDFS-6460. Ignore stale and decommissioned nodes in
NetworkTopology#sortByDistance. Contributed by Yongjun Zhang.
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java?rev=1602287&r1=1602286&r2=1602287&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
Thu Jun 12 20:45:52 2014
@@ -884,15 +884,16 @@ public class NetworkTopology {
* @param seed Used to seed the pseudo-random generator that randomizes the
* set of nodes at each network distance.
*/
- public void sortByDistance(Node reader, Node[] nodes, long seed) {
+ public void sortByDistance(Node reader, Node[] nodes,
+ int activeLen, long seed) {
/** Sort weights for the nodes array */
- int[] weights = new int[nodes.length];
- for (int i=0; i<nodes.length; i++) {
+ int[] weights = new int[activeLen];
+ for (int i=0; i<activeLen; i++) {
weights[i] = getWeight(reader, nodes[i]);
}
// Add weight/node pairs to a TreeMap to sort
TreeMap<Integer, List<Node>> tree = new TreeMap<Integer, List<Node>>();
- for (int i=0; i<nodes.length; i++) {
+ for (int i=0; i<activeLen; i++) {
int weight = weights[i];
Node node = nodes[i];
List<Node> list = tree.get(weight);
@@ -918,7 +919,7 @@ public class NetworkTopology {
}
}
}
- Preconditions.checkState(idx == nodes.length,
+ Preconditions.checkState(idx == activeLen,
"Sorted the wrong number of nodes!");
}
}
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java?rev=1602287&r1=1602286&r2=1602287&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
Thu Jun 12 20:45:52 2014
@@ -279,7 +279,8 @@ public class NetworkTopologyWithNodeGrou
* set of nodes at each network distance.
*/
@Override
- public void sortByDistance( Node reader, Node[] nodes, long seed) {
+ public void sortByDistance( Node reader, Node[] nodes,
+ int activeLen, long seed) {
// If reader is not a datanode (not in NetworkTopology tree), we need to
// replace this reader with a sibling leaf node in tree.
if (reader != null && !this.contains(reader)) {
@@ -292,7 +293,7 @@ public class NetworkTopologyWithNodeGrou
return;
}
}
- super.sortByDistance(reader, nodes, seed);
+ super.sortByDistance(reader, nodes, nodes.length, seed);
}
/** InnerNodeWithNodeGroup represents a switch/router of a data center, rack
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java?rev=1602287&r1=1602286&r2=1602287&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetworkTopologyWithNodeGroup.java
Thu Jun 12 20:45:52 2014
@@ -104,7 +104,8 @@ public class TestNetworkTopologyWithNode
testNodes[1] = dataNodes[2];
testNodes[2] = dataNodes[3];
testNodes[3] = dataNodes[0];
- cluster.sortByDistance(dataNodes[0], testNodes, 0xDEADBEEF);
+ cluster.sortByDistance(dataNodes[0], testNodes,
+ testNodes.length, 0xDEADBEEF);
assertTrue(testNodes[0] == dataNodes[0]);
assertTrue(testNodes[1] == dataNodes[1]);
assertTrue(testNodes[2] == dataNodes[2]);
@@ -115,7 +116,8 @@ public class TestNetworkTopologyWithNode
testNodes[1] = dataNodes[4];
testNodes[2] = dataNodes[1];
testNodes[3] = dataNodes[0];
- cluster.sortByDistance(dataNodes[0], testNodes, 0xDEADBEEF);
+ cluster.sortByDistance(dataNodes[0], testNodes,
+ testNodes.length, 0xDEADBEEF);
assertTrue(testNodes[0] == dataNodes[0]);
assertTrue(testNodes[1] == dataNodes[1]);
@@ -124,7 +126,8 @@ public class TestNetworkTopologyWithNode
testNodes[1] = dataNodes[3];
testNodes[2] = dataNodes[2];
testNodes[3] = dataNodes[0];
- cluster.sortByDistance(dataNodes[0], testNodes, 0xDEADBEEF);
+ cluster.sortByDistance(dataNodes[0], testNodes,
+ testNodes.length, 0xDEADBEEF);
assertTrue(testNodes[0] == dataNodes[0]);
assertTrue(testNodes[1] == dataNodes[2]);
@@ -133,7 +136,8 @@ public class TestNetworkTopologyWithNode
testNodes[1] = dataNodes[7];
testNodes[2] = dataNodes[2];
testNodes[3] = dataNodes[0];
- cluster.sortByDistance(computeNode, testNodes, 0xDEADBEEF);
+ cluster.sortByDistance(computeNode, testNodes,
+ testNodes.length, 0xDEADBEEF);
assertTrue(testNodes[0] == dataNodes[0]);
assertTrue(testNodes[1] == dataNodes[2]);
}
