Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java?rev=1523402&r1=1523401&r2=1523402&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java Sun Sep 15 04:32:39 2013 @@ -39,6 +39,7 @@ import org.apache.hadoop.hdfs.HdfsConfig import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.net.NetworkTopology; +import org.apache.hadoop.net.NetworkTopologyWithNodeGroup; import org.apache.hadoop.net.Node; import org.apache.hadoop.test.PathUtils; import org.junit.After; @@ -101,10 +102,10 @@ public class TestReplicationPolicyWithNo FileSystem.setDefaultUri(CONF, "hdfs://localhost:0"); CONF.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0"); // Set properties to make HDFS aware of NodeGroup. - CONF.set("dfs.block.replicator.classname", - "org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyWithNodeGroup"); + CONF.set(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY, + BlockPlacementPolicyWithNodeGroup.class.getName()); CONF.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_IMPL_KEY, - "org.apache.hadoop.net.NetworkTopologyWithNodeGroup"); + NetworkTopologyWithNodeGroup.class.getName()); File baseDir = PathUtils.getTestDir(TestReplicationPolicyWithNodeGroup.class); @@ -156,6 +157,35 @@ public class TestReplicationPolicyWithNo return true; } + private DatanodeDescriptor[] chooseTarget(int numOfReplicas) { + return chooseTarget(numOfReplicas, dataNodes[0]); + } + + private DatanodeDescriptor[] chooseTarget(int numOfReplicas, + DatanodeDescriptor writer) { + return chooseTarget(numOfReplicas, writer, + new ArrayList<DatanodeDescriptor>()); + } + + private DatanodeDescriptor[] chooseTarget(int numOfReplicas, + List<DatanodeDescriptor> chosenNodes) { + return chooseTarget(numOfReplicas, dataNodes[0], chosenNodes); + } + + private DatanodeDescriptor[] chooseTarget(int numOfReplicas, + DatanodeDescriptor writer, List<DatanodeDescriptor> chosenNodes) { + return chooseTarget(numOfReplicas, writer, chosenNodes, null); + } + + private DatanodeDescriptor[] chooseTarget( + int numOfReplicas, + DatanodeDescriptor writer, + List<DatanodeDescriptor> chosenNodes, + Map<Node, Node> excludedNodes) { + return replicator.chooseTarget(filename, numOfReplicas, writer, chosenNodes, + false, excludedNodes, BLOCK_SIZE); + } + /** * In this testcase, client is dataNodes[0]. So the 1st replica should be * placed on dataNodes[0], the 2nd replica should be placed on @@ -172,31 +202,26 @@ public class TestReplicationPolicyWithNo HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 4, 0); // overloaded DatanodeDescriptor[] targets; - targets = replicator.chooseTarget(filename, 0, dataNodes[0], - new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); + targets = chooseTarget(0); assertEquals(targets.length, 0); - targets = replicator.chooseTarget(filename, 1, dataNodes[0], - new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); + targets = chooseTarget(1); assertEquals(targets.length, 1); assertEquals(targets[0], dataNodes[0]); - targets = replicator.chooseTarget(filename, 2, dataNodes[0], - new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); + targets = chooseTarget(2); assertEquals(targets.length, 2); assertEquals(targets[0], dataNodes[0]); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); - targets = replicator.chooseTarget(filename, 3, dataNodes[0], - new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); + targets = chooseTarget(3); assertEquals(targets.length, 3); assertEquals(targets[0], dataNodes[0]); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); assertTrue(cluster.isOnSameRack(targets[1], targets[2])); assertFalse(cluster.isOnSameNodeGroup(targets[1], targets[2])); - targets = replicator.chooseTarget(filename, 4, dataNodes[0], - new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); + targets = chooseTarget(4); assertEquals(targets.length, 4); assertEquals(targets[0], dataNodes[0]); assertTrue(cluster.isOnSameRack(targets[1], targets[2]) || @@ -235,7 +260,7 @@ public class TestReplicationPolicyWithNo excludedNodes = new HashMap<Node, Node>(); excludedNodes.put(dataNodes[1], dataNodes[1]); - targets = repl.chooseTarget(4, dataNodes[0], chosenNodes, false, + targets = repl.chooseTarget(filename, 4, dataNodes[0], chosenNodes, false, excludedNodes, BLOCK_SIZE); assertEquals(targets.length, 4); assertEquals(targets[0], dataNodes[0]); @@ -252,7 +277,7 @@ public class TestReplicationPolicyWithNo chosenNodes.clear(); excludedNodes.put(dataNodes[1], dataNodes[1]); chosenNodes.add(dataNodes[2]); - targets = repl.chooseTarget(1, dataNodes[0], chosenNodes, true, + targets = repl.chooseTarget(filename, 1, dataNodes[0], chosenNodes, true, excludedNodes, BLOCK_SIZE); System.out.println("targets=" + Arrays.asList(targets)); assertEquals(2, targets.length); @@ -278,30 +303,25 @@ public class TestReplicationPolicyWithNo (HdfsConstants.MIN_BLOCKS_FOR_WRITE-1)*BLOCK_SIZE, 0L, 0, 0); // no space DatanodeDescriptor[] targets; - targets = replicator.chooseTarget(filename, 0, dataNodes[0], - new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); + targets = chooseTarget(0); assertEquals(targets.length, 0); - targets = replicator.chooseTarget(filename, 1, dataNodes[0], - new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); + targets = chooseTarget(1); assertEquals(targets.length, 1); assertEquals(targets[0], dataNodes[1]); - targets = replicator.chooseTarget(filename, 2, dataNodes[0], - new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); + targets = chooseTarget(2); assertEquals(targets.length, 2); assertEquals(targets[0], dataNodes[1]); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); - targets = replicator.chooseTarget(filename, 3, dataNodes[0], - new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); + targets = chooseTarget(3); assertEquals(targets.length, 3); assertEquals(targets[0], dataNodes[1]); assertTrue(cluster.isOnSameRack(targets[1], targets[2])); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); - targets = replicator.chooseTarget(filename, 4, dataNodes[0], - new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); + targets = chooseTarget(4); assertEquals(targets.length, 4); assertEquals(targets[0], dataNodes[1]); assertTrue(cluster.isNodeGroupAware()); @@ -333,23 +353,19 @@ public class TestReplicationPolicyWithNo } DatanodeDescriptor[] targets; - targets = replicator.chooseTarget(filename, 0, dataNodes[0], - new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); + targets = chooseTarget(0); assertEquals(targets.length, 0); - targets = replicator.chooseTarget(filename, 1, dataNodes[0], - new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); + targets = chooseTarget(1); assertEquals(targets.length, 1); assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0])); - targets = replicator.chooseTarget(filename, 2, dataNodes[0], - new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); + targets = chooseTarget(2); assertEquals(targets.length, 2); assertFalse(cluster.isOnSameRack(targets[0], dataNodes[0])); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); - targets = replicator.chooseTarget(filename, 3, dataNodes[0], - new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); + targets = chooseTarget(3); assertEquals(targets.length, 3); for(int i=0; i<3; i++) { assertFalse(cluster.isOnSameRack(targets[i], dataNodes[0])); @@ -371,21 +387,17 @@ public class TestReplicationPolicyWithNo public void testChooseTarget5() throws Exception { setupDataNodeCapacity(); DatanodeDescriptor[] targets; - targets = replicator.chooseTarget(filename, 0, NODE, - new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); + targets = chooseTarget(0, NODE); assertEquals(targets.length, 0); - targets = replicator.chooseTarget(filename, 1, NODE, - new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); + targets = chooseTarget(1, NODE); assertEquals(targets.length, 1); - targets = replicator.chooseTarget(filename, 2, NODE, - new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); + targets = chooseTarget(2, NODE); assertEquals(targets.length, 2); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); - targets = replicator.chooseTarget(filename, 3, NODE, - new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); + targets = chooseTarget(3, NODE); assertEquals(targets.length, 3); assertTrue(cluster.isOnSameRack(targets[1], targets[2])); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); @@ -406,23 +418,19 @@ public class TestReplicationPolicyWithNo chosenNodes.add(dataNodes[0]); DatanodeDescriptor[] targets; - targets = replicator.chooseTarget(filename, - 0, dataNodes[0], chosenNodes, BLOCK_SIZE); + targets = chooseTarget(0, chosenNodes); assertEquals(targets.length, 0); - targets = replicator.chooseTarget(filename, - 1, dataNodes[0], chosenNodes, BLOCK_SIZE); + targets = chooseTarget(1, chosenNodes); assertEquals(targets.length, 1); assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0])); - targets = replicator.chooseTarget(filename, - 2, dataNodes[0], chosenNodes, BLOCK_SIZE); + targets = chooseTarget(2, chosenNodes); assertEquals(targets.length, 2); assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); - targets = replicator.chooseTarget(filename, - 3, dataNodes[0], chosenNodes, BLOCK_SIZE); + targets = chooseTarget(3, chosenNodes); assertEquals(targets.length, 3); assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); assertFalse(cluster.isOnSameNodeGroup(dataNodes[0], targets[0])); @@ -444,17 +452,14 @@ public class TestReplicationPolicyWithNo chosenNodes.add(dataNodes[1]); DatanodeDescriptor[] targets; - targets = replicator.chooseTarget(filename, - 0, dataNodes[0], chosenNodes, BLOCK_SIZE); + targets = chooseTarget(0, chosenNodes); assertEquals(targets.length, 0); - targets = replicator.chooseTarget(filename, - 1, dataNodes[0], chosenNodes, BLOCK_SIZE); + targets = chooseTarget(1, chosenNodes); assertEquals(targets.length, 1); assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0])); - targets = replicator.chooseTarget(filename, - 2, dataNodes[0], chosenNodes, BLOCK_SIZE); + targets = chooseTarget(2, chosenNodes); assertEquals(targets.length, 2); assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0]) && cluster.isOnSameRack(dataNodes[0], targets[1])); @@ -475,30 +480,26 @@ public class TestReplicationPolicyWithNo chosenNodes.add(dataNodes[3]); DatanodeDescriptor[] targets; - targets = replicator.chooseTarget(filename, - 0, dataNodes[0], chosenNodes, BLOCK_SIZE); + targets = chooseTarget(0, chosenNodes); assertEquals(targets.length, 0); - targets = replicator.chooseTarget(filename, - 1, dataNodes[0], chosenNodes, BLOCK_SIZE); + targets = chooseTarget(1, chosenNodes); assertEquals(targets.length, 1); assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); assertFalse(cluster.isOnSameRack(dataNodes[3], targets[0])); - targets = replicator.chooseTarget(filename, - 1, dataNodes[3], chosenNodes, BLOCK_SIZE); + targets = chooseTarget(1, dataNodes[3], chosenNodes); assertEquals(targets.length, 1); assertTrue(cluster.isOnSameRack(dataNodes[3], targets[0])); assertFalse(cluster.isOnSameNodeGroup(dataNodes[3], targets[0])); assertFalse(cluster.isOnSameRack(dataNodes[0], targets[0])); - targets = replicator.chooseTarget(filename, - 2, dataNodes[0], chosenNodes, BLOCK_SIZE); + targets = chooseTarget(2, chosenNodes); assertEquals(targets.length, 2); assertTrue(cluster.isOnSameRack(dataNodes[0], targets[0])); assertFalse(cluster.isOnSameNodeGroup(dataNodes[0], targets[0])); - targets = replicator.chooseTarget(filename, - 2, dataNodes[3], chosenNodes, BLOCK_SIZE); + + targets = chooseTarget(2, dataNodes[3], chosenNodes); assertEquals(targets.length, 2); assertTrue(cluster.isOnSameRack(dataNodes[3], targets[0])); } @@ -584,21 +585,17 @@ public class TestReplicationPolicyWithNo } DatanodeDescriptor[] targets; - targets = replicator.chooseTarget(filename, 0, dataNodesInBoundaryCase[0], - new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); + targets = chooseTarget(0, dataNodesInBoundaryCase[0]); assertEquals(targets.length, 0); - targets = replicator.chooseTarget(filename, 1, dataNodesInBoundaryCase[0], - new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); + targets = chooseTarget(1, dataNodesInBoundaryCase[0]); assertEquals(targets.length, 1); - targets = replicator.chooseTarget(filename, 2, dataNodesInBoundaryCase[0], - new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); + targets = chooseTarget(2, dataNodesInBoundaryCase[0]); assertEquals(targets.length, 2); assertFalse(cluster.isOnSameRack(targets[0], targets[1])); - targets = replicator.chooseTarget(filename, 3, dataNodesInBoundaryCase[0], - new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); + targets = chooseTarget(3, dataNodesInBoundaryCase[0]); assertEquals(targets.length, 3); assertTrue(checkTargetsOnDifferentNodeGroup(targets)); } @@ -621,8 +618,7 @@ public class TestReplicationPolicyWithNo chosenNodes.add(dataNodesInBoundaryCase[0]); chosenNodes.add(dataNodesInBoundaryCase[5]); DatanodeDescriptor[] targets; - targets = replicator.chooseTarget(filename, 1, dataNodesInBoundaryCase[0], - chosenNodes, BLOCK_SIZE); + targets = chooseTarget(1, dataNodesInBoundaryCase[0], chosenNodes); assertFalse(cluster.isOnSameNodeGroup(targets[0], dataNodesInBoundaryCase[0])); assertFalse(cluster.isOnSameNodeGroup(targets[0], @@ -661,14 +657,12 @@ public class TestReplicationPolicyWithNo DatanodeDescriptor[] targets; // Test normal case -- 3 replicas - targets = replicator.chooseTarget(filename, 3, dataNodesInMoreTargetsCase[0], - new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); + targets = chooseTarget(3, dataNodesInMoreTargetsCase[0]); assertEquals(targets.length, 3); assertTrue(checkTargetsOnDifferentNodeGroup(targets)); // Test special case -- replica number over node groups. - targets = replicator.chooseTarget(filename, 10, dataNodesInMoreTargetsCase[0], - new ArrayList<DatanodeDescriptor>(), BLOCK_SIZE); + targets = chooseTarget(10, dataNodesInMoreTargetsCase[0]); assertTrue(checkTargetsOnDifferentNodeGroup(targets)); // Verify it only can find 6 targets for placing replicas. assertEquals(targets.length, 6);
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java?rev=1523402&r1=1523401&r2=1523402&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDNFencing.java Sun Sep 15 04:32:39 2013 @@ -96,8 +96,8 @@ public class TestDNFencing { // Increase max streams so that we re-replicate quickly. conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 1000); // See RandomDeleterPolicy javadoc. - conf.setClass("dfs.block.replicator.classname", RandomDeleterPolicy.class, - BlockPlacementPolicy.class); + conf.setClass(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY, + RandomDeleterPolicy.class, BlockPlacementPolicy.class); conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); cluster = new MiniDFSCluster.Builder(conf) .nnTopology(MiniDFSNNTopology.simpleHATopology())
