This is an automated email from the ASF dual-hosted git repository.
chungen0126 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 19ca844b89f HDDS-15918. Optimize TestSCMNodeManager by reducing
heartbeat configs (#10837)
19ca844b89f is described below
commit 19ca844b89f4a514cbbb44479c088751b77b8581
Author: KUAN-HAO HUANG <[email protected]>
AuthorDate: Tue Sep 22 15:08:05 2026 +0800
HDDS-15918. Optimize TestSCMNodeManager by reducing heartbeat configs
(#10837)
Co-authored-by: Doroszlai, Attila <[email protected]>
---
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 160 +++++++++++----------
1 file changed, 83 insertions(+), 77 deletions(-)
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
index 78814b53e81..868ca469d9c 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
@@ -108,6 +108,7 @@
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager;
+import org.apache.hadoop.hdds.utils.HddsServerUtil;
import org.apache.hadoop.ozone.container.upgrade.UpgradeUtils;
import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode;
@@ -147,6 +148,11 @@ public class TestSCMNodeManager {
private SCMContext scmContext;
private static final int MAX_LV =
HDDSLayoutVersionManager.maxLayoutVersion();
+ // Max time to wait for a node to reach an expected state.
+ private static final int WAIT_FOR_TIMEOUT_MS = 10 * 1000;
+ // Stale-node window used by testScmClusterIsInExpectedState1; the sleeps in
+ // that test are expressed as fractions of it.
+ private static final int STALE_INTERVAL_MS = 1000;
private static final LayoutVersionProto LARGER_SLV_LAYOUT_PROTO =
toLayoutVersionProto(MAX_LV, MAX_LV + 1);
private static final LayoutVersionProto SMALLER_MLV_LAYOUT_PROTO =
@@ -771,16 +777,20 @@ public void testSetNodeOpStateAndCommandFired()
*/
@Test
public void testScmDetectStaleAndDeadNode()
- throws IOException, InterruptedException, AuthenticationException {
+ throws IOException, InterruptedException, AuthenticationException,
+ TimeoutException {
+ final int healthCheckInterval = 50;
final int interval = 100;
final int nodeCount = 10;
OzoneConfiguration conf = getConf();
- conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, interval,
- MILLISECONDS);
- conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
- conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
- conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
+ conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL,
healthCheckInterval, MILLISECONDS);
+ conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, interval, MILLISECONDS);
+ conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 300, MILLISECONDS);
+ conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 600, MILLISECONDS);
+
+ assertThat(HddsServerUtil.getStaleNodeInterval(conf)).isEqualTo(300);
+ assertThat(HddsServerUtil.getDeadNodeInterval(conf)).isEqualTo(600);
try (SCMNodeManager nodeManager = createNodeManager(conf)) {
@@ -790,28 +800,20 @@ public void testScmDetectStaleAndDeadNode()
DatanodeDetails staleNode =
HddsTestUtils.createRandomDatanodeAndRegister(
nodeManager);
- // Heartbeat once
+ // Heartbeat the stale node once; it ages into stale then dead while the
+ // other nodes are kept alive.
nodeManager.processHeartbeat(staleNode);
- // Heartbeat all other nodes.
- for (DatanodeDetails dn : nodeList) {
- nodeManager.processHeartbeat(dn);
- }
-
- // Wait for 2 seconds .. and heartbeat good nodes again.
- Thread.sleep(2 * 1000);
-
- for (DatanodeDetails dn : nodeList) {
- nodeManager.processHeartbeat(dn);
- }
+ // Keep the good nodes alive until the stale node moves into stale state.
+ GenericTestUtils.waitFor(() -> {
+ for (DatanodeDetails dn : nodeList) {
+ nodeManager.processHeartbeat(dn);
+ }
+ return nodeManager.getNodeCount(NodeStatus.inServiceStale()) == 1;
+ }, interval, WAIT_FOR_TIMEOUT_MS);
- // Wait for 2 seconds, wait a total of 4 seconds to make sure that the
- // node moves into stale state.
- Thread.sleep(2 * 1000);
List<DatanodeDetails> staleNodeList =
nodeManager.getNodes(NodeStatus.inServiceStale());
- assertEquals(1, nodeManager.getNodeCount(NodeStatus.inServiceStale()),
- "Expected to find 1 stale node");
assertEquals(1, staleNodeList.size(),
"Expected to find 1 stale node");
assertEquals(staleNode.getID(), staleNodeList.get(0).getID(),
@@ -822,23 +824,21 @@ public void testScmDetectStaleAndDeadNode()
nodeCounts.get(HddsProtos.NodeOperationalState.IN_SERVICE.name())
.get(HddsProtos.NodeState.STALE.name()).intValue());
- Thread.sleep(1000);
- // heartbeat good nodes again.
- for (DatanodeDetails dn : nodeList) {
- nodeManager.processHeartbeat(dn);
- }
-
- // 6 seconds is the dead window for this test , so we wait a total of
- // 7 seconds to make sure that the node moves into dead state.
- Thread.sleep(2 * 1000);
+ // Keep the good nodes alive until the stale node moves into dead state.
+ GenericTestUtils.waitFor(() -> {
+ for (DatanodeDetails dn : nodeList) {
+ nodeManager.processHeartbeat(dn);
+ }
+ return nodeManager.getNodeCount(NodeStatus.inServiceDead()) == 1;
+ }, interval, WAIT_FOR_TIMEOUT_MS);
- // the stale node has been removed
+ // the stale node has moved on to dead
staleNodeList = nodeManager.getNodes(NodeStatus.inServiceStale());
nodeCounts = nodeManager.getNodeCount();
assertEquals(0, nodeManager.getNodeCount(NodeStatus.inServiceStale()),
- "Expected to find 1 stale node");
+ "Expected to find 0 stale node");
assertEquals(0, staleNodeList.size(),
- "Expected to find 1 stale node");
+ "Expected to find 0 stale node");
assertEquals(0,
nodeCounts.get(HddsProtos.NodeOperationalState.IN_SERVICE.name())
.get(HddsProtos.NodeState.STALE.name()).intValue());
@@ -866,21 +866,22 @@ public void testScmDetectStaleAndDeadNode()
*/
@Test
void testScmHandleJvmPause() throws Exception {
- final int healthCheckInterval = 200; // milliseconds
- final int heartbeatInterval = 1; // seconds
- final int staleNodeInterval = 3; // seconds
- final int deadNodeInterval = 6; // seconds
+ final int healthCheckInterval = 100; // milliseconds
+ final int heartbeatInterval = 100; // milliseconds
+ final int staleNodeInterval = 1000; // milliseconds
+ // Keep the dead window large so the stale node does not race into dead.
+ final int deadNodeInterval = 10000; // milliseconds
ScheduledFuture schedFuture;
OzoneConfiguration conf = getConf();
conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL,
healthCheckInterval, MILLISECONDS);
conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL,
- heartbeatInterval, SECONDS);
+ heartbeatInterval, MILLISECONDS);
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL,
- staleNodeInterval, SECONDS);
+ staleNodeInterval, MILLISECONDS);
conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL,
- deadNodeInterval, SECONDS);
+ deadNodeInterval, MILLISECONDS);
try (SCMNodeManager nodeManager = createNodeManager(conf)) {
DatanodeDetails node1 =
@@ -912,7 +913,7 @@ void testScmHandleJvmPause() throws Exception {
// Step 1 : stop health check process (simulate JVM pause)
nodeManager.pauseHealthCheck();
- Thread.sleep(MILLISECONDS.convert(staleNodeInterval, SECONDS));
+ Thread.sleep(staleNodeInterval);
// Step 2 : resume health check
assertEquals(0, nodeManager.getSkippedHealthChecks(),
@@ -933,8 +934,12 @@ void testScmHandleJvmPause() throws Exception {
// Step 5 : heartbeat for node1
nodeManager.processHeartbeat(node1);
- // Step 6 : wait for health check process to run
- Thread.sleep(1000);
+ // Step 6 : wait for node2 (not heartbeated) to transition to STALE while
+ // node1 (just heartbeated) stays HEALTHY.
+ GenericTestUtils.waitFor(
+ () -> nodeManager.getNodeCount(NodeStatus.inServiceHealthy()) == 1
+ && nodeManager.getNodeCount(NodeStatus.inServiceStale()) == 1,
+ healthCheckInterval, staleNodeInterval);
// Step 7 : node2 should transition to STALE
assertEquals(1, nodeManager.getNodeCount(NodeStatus.inServiceHealthy()));
@@ -1236,9 +1241,9 @@ public void testScmClusterIsInExpectedState1()
OzoneConfiguration conf = getConf();
conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100,
MILLISECONDS);
- conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
- conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
- conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
+ conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 100, MILLISECONDS);
+ conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 1, SECONDS);
+ conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 2, SECONDS);
/**
@@ -1263,10 +1268,11 @@ public void testScmClusterIsInExpectedState1()
assertEquals(3, nodeManager.getNodeCount(NodeStatus.inServiceHealthy()));
/**
- * Cluster state: Quiesced: We are going to sleep for 3 seconds. Which
- * means that no node is heartbeating. All nodes should move to Stale.
+ * Cluster state: Quiesced: We are going to sleep past the 1 second stale
+ * interval. Which means that no node is heartbeating. All nodes should
+ * move to Stale.
*/
- Thread.sleep(3 * 1000);
+ Thread.sleep(STALE_INTERVAL_MS);
assertEquals(3, nodeManager.getAllNodes().size());
assertEquals(3, nodeManager.getNodeCount(NodeStatus.inServiceStale()));
@@ -1275,21 +1281,21 @@ public void testScmClusterIsInExpectedState1()
* Cluster State : Move healthy node back to healthy state, move other 2
* nodes to Stale State.
*
- * We heartbeat healthy node after 1 second and let other 2 nodes elapse
- * the 3 second windows.
+ * We heartbeat healthy node partway through and let other 2 nodes elapse
+ * the 1 second stale window.
*/
nodeManager.processHeartbeat(healthyNode);
nodeManager.processHeartbeat(staleNode);
nodeManager.processHeartbeat(deadNode);
- Thread.sleep(1500);
+ Thread.sleep(STALE_INTERVAL_MS / 2);
nodeManager.processHeartbeat(healthyNode);
- Thread.sleep(2 * 1000);
+ Thread.sleep(2 * STALE_INTERVAL_MS / 3);
assertEquals(1, nodeManager.getNodeCount(NodeStatus.inServiceHealthy()));
- // 3.5 seconds from last heartbeat for the stale and deadNode. So those
+ // ~1.17 seconds from last heartbeat for the stale and deadNode. So those
// 2 nodes must move to Stale state and the healthy node must
// remain in the healthy State.
List<DatanodeDetails> healthyList = nodeManager.getNodes(
@@ -1307,13 +1313,13 @@ public void testScmClusterIsInExpectedState1()
nodeManager.processHeartbeat(healthyNode);
nodeManager.processHeartbeat(staleNode);
- Thread.sleep(1500);
+ Thread.sleep(STALE_INTERVAL_MS / 2);
nodeManager.processHeartbeat(healthyNode);
- Thread.sleep(2 * 1000);
+ Thread.sleep(2 * STALE_INTERVAL_MS / 3);
- // 3.5 seconds have elapsed for stale node, so it moves into Stale.
- // 7 seconds have elapsed for dead node, so it moves into dead.
- // 2 Seconds have elapsed for healthy node, so it stays in healthy state.
+ // ~1.17 seconds have elapsed for stale node, so it moves into Stale.
+ // ~2.33 seconds have elapsed for dead node, so it moves into dead.
+ // ~0.67 seconds have elapsed for healthy node, so it stays healthy.
healthyList = nodeManager.getNodes((NodeStatus.inServiceHealthy()));
List<DatanodeDetails> staleList =
nodeManager.getNodes(NodeStatus.inServiceStale());
@@ -1416,9 +1422,9 @@ public void testScmClusterIsInExpectedState2()
OzoneConfiguration conf = getConf();
conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100,
MILLISECONDS);
- conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
- conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
- conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
+ conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 100, MILLISECONDS);
+ conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 1, SECONDS);
+ conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 2, SECONDS);
try (SCMNodeManager nodeManager = createNodeManager(conf)) {
@@ -1431,17 +1437,17 @@ public void testScmClusterIsInExpectedState2()
Runnable healthyNodeTask = () -> {
try {
- // 2 second heartbeat makes these nodes stay healthy.
- heartbeatNodeSet(nodeManager, healthyNodeList, 2 * 1000);
+ // Sub-stale heartbeat keeps these nodes healthy.
+ heartbeatNodeSet(nodeManager, healthyNodeList, 600);
} catch (InterruptedException ignored) {
}
};
Runnable staleNodeTask = () -> {
try {
- // 4 second heartbeat makes these nodes go to stale and back to
- // healthy again.
- heartbeatNodeSet(nodeManager, staleNodeList, 4 * 1000);
+ // Heartbeat between the stale and dead windows makes these nodes go
+ // to stale and back to healthy again.
+ heartbeatNodeSet(nodeManager, staleNodeList, 1500);
} catch (InterruptedException ignored) {
}
};
@@ -1462,7 +1468,7 @@ public void testScmClusterIsInExpectedState2()
thread2.setDaemon(true);
thread2.start();
- Thread.sleep(10 * 1000);
+ Thread.sleep(4 * 1000);
// Assert all healthy nodes are healthy now, this has to be a greater
// than check since Stale nodes can be healthy when we check the state.
@@ -1510,11 +1516,11 @@ public void testScmCanHandleScale()
OzoneConfiguration conf = getConf();
conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100,
MILLISECONDS);
- conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1,
- SECONDS);
- conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3 * 1000,
+ conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 100,
MILLISECONDS);
- conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6 * 1000,
+ conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 1000,
+ MILLISECONDS);
+ conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 2000,
MILLISECONDS);
try (SCMNodeManager nodeManager = createNodeManager(conf)) {
@@ -1525,7 +1531,7 @@ public void testScmCanHandleScale()
Runnable healthyNodeTask = () -> {
try {
- heartbeatNodeSet(nodeManager, healthyList, 2 * 1000);
+ heartbeatNodeSet(nodeManager, healthyList, 600);
} catch (InterruptedException ignored) {
}
@@ -1533,7 +1539,7 @@ public void testScmCanHandleScale()
Runnable staleNodeTask = () -> {
try {
- heartbeatNodeSet(nodeManager, staleList, 4 * 1000);
+ heartbeatNodeSet(nodeManager, staleList, 1500);
} catch (InterruptedException ignored) {
}
};
@@ -1545,7 +1551,7 @@ public void testScmCanHandleScale()
Thread thread2 = new Thread(staleNodeTask);
thread2.setDaemon(true);
thread2.start();
- Thread.sleep(3 * 1000);
+ Thread.sleep(1000);
GenericTestUtils.waitFor(() -> findNodes(nodeManager, staleCount, STALE),
500, 20 * 1000);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
