This is an automated email from the ASF dual-hosted git repository.
smengcl pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 84fc9f52ee7 HDDS-15350. SCM crashes with ArithmeticException when
topology reports zero racks during DN decommission (#10339)
84fc9f52ee7 is described below
commit 84fc9f52ee709992e88ca745770f031c4466e13b
Author: Siyao Meng <[email protected]>
AuthorDate: Fri May 29 23:25:34 2026 -0700
HDDS-15350. SCM crashes with ArithmeticException when topology reports zero
racks during DN decommission (#10339)
Co-authored-by: Copilot Autofix powered by AI
<[email protected]>
Co-authored-by: Claude Opus 4.7 <[email protected]>
Co-authored-by: Wei-Chiu Chuang <[email protected]>
---
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 27 ++++++++++++---
.../hdds/scm/TestSCMCommonPlacementPolicy.java | 40 ++++++++++++++++++++++
2 files changed, 62 insertions(+), 5 deletions(-)
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java
index c51f6d0429f..93940b7770f 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/SCMCommonPlacementPolicy.java
@@ -411,6 +411,18 @@ protected int getRequiredRackCount(int numReplicas, int
excludedRackCount) {
* @return The max number of replicas per rack
*/
protected int getMaxReplicasPerRack(int numReplicas, int numberOfRacks) {
+ if (numberOfRacks <= 0) {
+ // No rack information means there is no per-rack constraint to
+ // enforce. Callers are expected to short-circuit before reaching
+ // here, but guard the divide site against transient empty-topology
+ // windows (HDDS-15350). The WARN makes the silent path observable;
+ // configure log4j appender-side filtering if it floods.
+ LOG.warn("Empty rack topology in placement validation: numReplicas={} "
+ + "numberOfRacks={}; returning numReplicas to avoid divide-by-zero "
+ + "(HDDS-15350).",
+ numReplicas, numberOfRacks);
+ return numReplicas;
+ }
return numReplicas / numberOfRacks
+ Math.min(numReplicas % numberOfRacks, 1);
}
@@ -436,7 +448,16 @@ public ContainerPlacementStatus validateContainerPlacement(
NetworkTopology topology = nodeManager.getClusterNetworkTopologyMap();
// We have a network topology so calculate if it is satisfied or not.
int requiredRacks = getRequiredRackCount(replicas, 0);
- if (topology == null || replicas == 1 || requiredRacks == 1) {
+ // The leaf nodes are all at max level, so the number of nodes at
+ // maxLevel - 1 is the rack count. Compute up front so we can
+ // short-circuit when the topology has no rack information, which
+ // would otherwise reach getMaxReplicasPerRack with numberOfRacks
+ // == 0 (HDDS-15350: transient empty-topology window during a DN
+ // decommission crashed the ReplicationMonitor with "/ by zero").
+ final int numRacks = topology == null ? 0
+ : topology.getNumOfNodes(topology.getMaxLevel() - 1);
+ if (topology == null || replicas == 1 || requiredRacks <= 1
+ || numRacks <= 0) {
if (!dns.isEmpty()) {
// placement is always satisfied if there is at least one DN.
return validPlacement;
@@ -471,10 +492,6 @@ public ContainerPlacementStatus validateContainerPlacement(
Function.identity(),
Collectors.reducing(0, e -> 1, Integer::sum)))
.values());
- final int maxLevel = topology.getMaxLevel();
- // The leaf nodes are all at max level, so the number of nodes at
- // leafLevel - 1 is the rack count
- int numRacks = topology.getNumOfNodes(maxLevel - 1);
if (replicas < requiredRacks) {
requiredRacks = replicas;
}
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestSCMCommonPlacementPolicy.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestSCMCommonPlacementPolicy.java
index dba2d60b98c..5ecd3fa1c74 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestSCMCommonPlacementPolicy.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestSCMCommonPlacementPolicy.java
@@ -566,6 +566,46 @@ public void testValidatePlacementWithDeadMaintenanceNode()
throws NodeNotFoundEx
assertTrue(placementStatus.isPolicySatisfied());
}
+ /**
+ * HDDS-15350: when the network topology transiently reports zero racks
+ * (due to DNS resolution problems), validateContainerPlacement must
+ * not crash with ArithmeticException ("/ by zero") in
+ * getMaxReplicasPerRack. Without the fix this test throws and SCM's
+ * ReplicationMonitor thread dies along with it.
+ */
+ @Test
+ public void testValidateContainerPlacementWithZeroRackTopology() {
+ List<DatanodeDetails> nodes = ImmutableList.of(
+ MockDatanodeDetails.randomDatanodeDetails(),
+ MockDatanodeDetails.randomDatanodeDetails(),
+ MockDatanodeDetails.randomDatanodeDetails());
+ NodeManager mockNodeManager = mock(NodeManager.class);
+ when(mockNodeManager.getAllNodes()).thenAnswer(inv -> nodes);
+
+ // Topology that reports zero racks at the rack level during the
+ // empty-topology window observed during DN decommission.
+ NetworkTopology topology = mock(NetworkTopology.class);
+ when(topology.getMaxLevel()).thenReturn(3);
+ when(topology.getNumOfNodes(anyInt())).thenReturn(0);
+ when(mockNodeManager.getClusterNetworkTopologyMap()).thenReturn(topology);
+
+ // rackCnt=2 makes DummyPlacementPolicy.getRequiredRackCount return
+ // min(replicas, 2) > 1, so the original early-return guard does NOT
+ // fire and execution proceeds to the divide site.
+ Map<Integer, Integer> rackMap = new HashMap<>();
+ rackMap.put(0, 0);
+ rackMap.put(1, 0);
+ rackMap.put(2, 0);
+ DummyPlacementPolicy policy = new DummyPlacementPolicy(
+ mockNodeManager, conf, rackMap, 2);
+
+ ContainerPlacementStatus status =
+ policy.validateContainerPlacement(nodes, 3);
+ assertTrue(status.isPolicySatisfied(),
+ "placement should not crash and should be considered satisfied "
+ + "when the topology reports no racks");
+ }
+
private static class DummyPlacementPolicy extends SCMCommonPlacementPolicy {
private Map<DatanodeDetails, Node> rackMap;
private List<Node> racks;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
