liubingxing commented on a change in pull request #3679:
URL: https://github.com/apache/hadoop/pull/3679#discussion_r763930129
##########
File path:
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
##########
@@ -1615,6 +1619,92 @@ public void testBalancerWithStripedFile() throws
Exception {
NameNodeConnector.setWrite2IdFile(false);
}
+ @Test
+ public void testBalancerWithExcludeListWithStripedFile() throws Exception {
+ Configuration conf = new Configuration();
+ initConfWithStripe(conf);
+ NameNodeConnector.setWrite2IdFile(true);
+ doTestBalancerWithExcludeListWithStripedFile(conf);
+ NameNodeConnector.setWrite2IdFile(false);
+ }
+
+ private void doTestBalancerWithExcludeListWithStripedFile(Configuration
conf) throws Exception {
+ int numOfDatanodes = dataBlocks + parityBlocks + 3;
+ int numOfRacks = dataBlocks;
+ long capacity = 20 * defaultBlockSize;
+ long[] capacities = new long[numOfDatanodes];
+ Arrays.fill(capacities, capacity);
+ String[] racks = new String[numOfDatanodes];
+ for (int i = 0; i < numOfDatanodes; i++) {
+ racks[i] = "/rack" + (i % numOfRacks);
+ }
+ cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(numOfDatanodes)
+ .racks(racks)
+ .simulatedCapacities(capacities)
+ .build();
+
+ try {
+ cluster.waitActive();
+ client = NameNodeProxies.createProxy(conf,
cluster.getFileSystem(0).getUri(),
+ ClientProtocol.class).getProxy();
+ client.enableErasureCodingPolicy(
+ StripedFileTestUtil.getDefaultECPolicy().getName());
+ client.setErasureCodingPolicy("/",
+ StripedFileTestUtil.getDefaultECPolicy().getName());
+
+ long totalCapacity = sum(capacities);
+
+ // fill up the cluster with 30% data. It'll be 45% full plus parity.
+ long fileLen = totalCapacity * 3 / 10;
+ long totalUsedSpace = fileLen * (dataBlocks + parityBlocks) / dataBlocks;
+ FileSystem fs = cluster.getFileSystem(0);
+ DFSTestUtil.createFile(fs, filePath, fileLen, (short) 3, r.nextLong());
+
+ // verify locations of striped blocks
+ LocatedBlocks locatedBlocks = client.getBlockLocations(fileName, 0,
fileLen);
+ StripedFileTestUtil.verifyLocatedStripedBlocks(locatedBlocks, groupSize);
+
+ // get datanode report
+ DatanodeInfo[] datanodeReport =
client.getDatanodeReport(DatanodeReportType.ALL);
+
+ // add datanode in new rack
+ String newRack = "/rack" + (++numOfRacks);
+ cluster.startDataNodes(conf, 2, true, null,
+ new String[]{newRack, newRack}, null,
+ new long[]{capacity, capacity});
+ totalCapacity += capacity*2;
+ cluster.triggerHeartbeats();
+
+ // add datanode to exclude list
+ Set<String> dnList = new HashSet<>();
+ dnList.add(datanodeReport[0].getHostName());
+ BalancerParameters.Builder pBuilder = new BalancerParameters.Builder();
+ pBuilder.setExcludedNodes(dnList);
+ waitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster);
+
+ // start balancer and check the failed num of moving task
+ Collection<URI> namenodes = DFSUtil.getInternalNsRpcUris(conf);
+ final int run = runBalancer(namenodes, pBuilder.build(), conf, true);
+ if (conf.getInt(
+ DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY,
+ DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_DEFAULT)
+ == 0) {
+ assertEquals(ExitStatus.NO_MOVE_PROGRESS.getExitCode(), run);
+ } else {
+ assertEquals(ExitStatus.SUCCESS.getExitCode(), run);
+ }
+ waitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster);
Review comment:
I updated the UT and calling `runBalancer` from the unit test.
And I also add a parameter `boolean checkExcludeNodesUtilization` in
`waitForBalancer` to determine whether to check the nodeUtilization of
excluded datanode
``` java
DatanodeInfo[] datanodeReport =
client.getDatanodeReport(DatanodeReportType.ALL);
assertEquals(datanodeReport.length, cluster.getDataNodes().size());
balanced = true;
int actualExcludedNodeCount = 0;
for (DatanodeInfo datanode : datanodeReport) {
double nodeUtilization = ((double)datanode.getDfsUsed())
/ datanode.getCapacity();
if (Dispatcher.Util.isExcluded(p.getExcludedNodes(), datanode)) {
if (checkExcludeNodesUtilization) {
assertTrue(nodeUtilization == 0);
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
