KevinWikant commented on a change in pull request #3675:
URL: https://github.com/apache/hadoop/pull/3675#discussion_r771734015
##########
File path:
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
##########
@@ -1654,4 +1658,204 @@ public Boolean get() {
cleanupFile(fileSys, file);
}
+
+ /**
+ * Test DatanodeAdminManager logic to re-queue unhealthy decommissioning
nodes
+ * which are blocking the decommissioning of healthy nodes.
+ * Force the tracked nodes set to be filled with nodes lost while
decommissioning,
+ * then decommission healthy nodes & validate they are decommissioned
eventually.
+ */
+ @Test(timeout = 120000)
+ public void testRequeueUnhealthyDecommissioningNodes() throws Exception {
+ // Create a MiniDFSCluster with 3 live datanode in AdminState=NORMAL and
+ // 2 dead datanodes in AdminState=DECOMMISSION_INPROGRESS and a file
+ // with replication factor of 5.
+ final int numLiveNodes = 3;
+ final int numDeadNodes = 2;
+ final int numNodes = numLiveNodes + numDeadNodes;
+ final List<DatanodeDescriptor> liveNodes = new ArrayList<>();
+ final Map<DatanodeDescriptor, MiniDFSCluster.DataNodeProperties>
deadNodeProps =
+ new HashMap<>();
+ final ArrayList<DatanodeInfo> decommissionedNodes = new ArrayList<>();
+ final Path filePath = new Path("/tmp/test");
+ createClusterWithDeadNodesDecommissionInProgress(numLiveNodes, liveNodes,
numDeadNodes,
+ deadNodeProps, decommissionedNodes, filePath);
+ final FSNamesystem namesystem = getCluster().getNamesystem();
+ final BlockManager blockManager = namesystem.getBlockManager();
+ final DatanodeManager datanodeManager = blockManager.getDatanodeManager();
+ final DatanodeAdminManager decomManager =
datanodeManager.getDatanodeAdminManager();
+
+ // Validate the 2 "dead" nodes are not removed from the tracked nodes set
+ // after several seconds of operation
+ final Duration checkDuration = Duration.ofSeconds(5);
+ Instant checkUntil = Instant.now().plus(checkDuration);
+ while (Instant.now().isBefore(checkUntil)) {
+ BlockManagerTestUtil.recheckDecommissionState(datanodeManager);
+ assertEquals(0, decomManager.getNumPendingNodes());
+ assertEquals(numDeadNodes, decomManager.getNumTrackedNodes());
+ assertTrue(deadNodeProps.keySet().stream()
+ .allMatch(node ->
node.getAdminState().equals(AdminStates.DECOMMISSION_INPROGRESS)));
+ Thread.sleep(500);
+ }
+
+ // Delete the file such that its no longer a factor blocking
decommissioning of live nodes
+ // which have block replicas for that file
+ getCluster().getFileSystem().delete(filePath, true);
+
+ // Start decommissioning 2 "live" datanodes
+ int numLiveDecommNodes = 2;
+ final List<DatanodeDescriptor> liveDecommNodes = liveNodes.subList(0,
numLiveDecommNodes);
+ for (final DatanodeDescriptor liveNode : liveDecommNodes) {
+ takeNodeOutofService(0, liveNode.getDatanodeUuid(), 0,
decommissionedNodes,
+ AdminStates.DECOMMISSION_INPROGRESS);
+ decommissionedNodes.add(liveNode);
+ }
+
+ // Write a new file such that there are under-replicated blocks preventing
decommissioning
+ // of dead nodes
+ writeFile(getCluster().getFileSystem(), filePath, numNodes, 10);
+
+ // Validate that the live datanodes are put into the pending
decommissioning queue
+ GenericTestUtils.waitFor(() -> decomManager.getNumTrackedNodes() ==
numDeadNodes
+ && decomManager.getNumPendingNodes() == numLiveDecommNodes
+ && liveDecommNodes.stream().allMatch(
+ node ->
node.getAdminState().equals(AdminStates.DECOMMISSION_INPROGRESS)),
+ 500, 30000);
+ for (final DatanodeDescriptor node : decomManager.getPendingNodes()) {
+ assertTrue(liveDecommNodes.contains(node));
+ }
Review comment:
that API seems a lot cleaner, thanks! I have made the change
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
