This is an automated email from the ASF dual-hosted git repository.
devmadhuu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 1b9cbd1dd60 HDDS-16320. Recon incorrectly marks healthy EC containers
as REPLICA_MISMATCH by comparing checksums across different EC replica indexes.
(#11215)
1b9cbd1dd60 is described below
commit 1b9cbd1dd6009091aeb28e3014146122f7a965df
Author: Devesh Kumar Singh <[email protected]>
AuthorDate: Wed Sep 9 14:55:21 2026 +0530
HDDS-16320. Recon incorrectly marks healthy EC containers as
REPLICA_MISMATCH by comparing checksums across different EC replica indexes.
(#11215)
---
.../ozone/recon/fsck/ReconReplicationManager.java | 36 ++++++++--------
.../recon/fsck/TestReconReplicationManager.java | 50 ++++++++++++++++++++++
2 files changed, 69 insertions(+), 17 deletions(-)
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ReconReplicationManager.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ReconReplicationManager.java
index 09005cadb18..940b91ffc64 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ReconReplicationManager.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ReconReplicationManager.java
@@ -21,12 +21,13 @@
import java.time.Clock;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
import java.util.Set;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.container.ContainerHealthState;
import org.apache.hadoop.hdds.scm.container.ContainerID;
@@ -239,28 +240,29 @@ public synchronized void start() {
* </ul>
* </p>
*
- * <p>This uses checksum mismatch logic:
- * {@code
replicas.stream().map(ContainerReplica::getDataChecksum).distinct().count() !=
1}
- * </p>
+ * <p>EC replicas are compared only within the same replica index, since
different indexes contain different data or
+ * parity fragments. Non-EC replicas are compared together.</p>
*
+ * @param container Container whose replicas are checked
* @param replicas Set of container replicas to check
- * @return true if replicas have different data checksums
+ * @return true if comparable replicas have different data checksums
*/
- private boolean hasDataChecksumMismatch(Set<ContainerReplica> replicas) {
+ private boolean hasDataChecksumMismatch(ContainerInfo container,
Set<ContainerReplica> replicas) {
if (replicas == null || replicas.isEmpty()) {
return false;
}
- // Count distinct checksums (filter out nulls)
- long distinctChecksums = replicas.stream()
- .map(ContainerReplica::getDataChecksum)
- .filter(Objects::nonNull)
- .distinct()
- .count();
-
- // More than 1 distinct checksum = data mismatch
- // 0 distinct checksums = all nulls, no mismatch
- return distinctChecksums > 1;
+ boolean isEC = container.getReplicationType() == ReplicationType.EC;
+ Map<Integer, Long> checksumsByIndex = new HashMap<>();
+ for (ContainerReplica replica : replicas) {
+ int replicaIndex = isEC ? replica.getReplicaIndex() : 0;
+ long checksum = replica.getDataChecksum();
+ Long previousChecksum = checksumsByIndex.putIfAbsent(replicaIndex,
checksum);
+ if (previousChecksum != null && previousChecksum != checksum) {
+ return true;
+ }
+ }
+ return false;
}
/**
@@ -314,7 +316,7 @@ public synchronized void processAll() {
processContainer(container, replicas, pendingOps, nullQueue, report,
true);
// ADDITIONAL CHECK: Detect REPLICA_MISMATCH (Recon-specific, not in
SCM)
- if (hasDataChecksumMismatch(replicas)) {
+ if (hasDataChecksumMismatch(container, replicas)) {
report.addReplicaMismatchContainer(cid);
LOG.debug("Container {} has data checksum mismatch across replicas",
cid);
}
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestReconReplicationManager.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestReconReplicationManager.java
index 0678caa86eb..b803642b3e5 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestReconReplicationManager.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/fsck/TestReconReplicationManager.java
@@ -58,6 +58,8 @@
import
org.apache.ozone.recon.schema.generated.tables.daos.UnhealthyContainersDao;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
/**
* Smoke tests for ReconReplicationManager Local ReplicationManager.
@@ -222,6 +224,54 @@ public void testProcessAllStoresAllPrimaryV2States()
throws Exception {
UnHealthyContainerStates.REPLICA_MISMATCH, 0, 0, 10).size());
}
+ @ParameterizedTest
+ @CsvSource({"EC, 1", "EC, 4", "RATIS, 0"})
+ public void testReplicaChecksumMismatch(HddsProtos.ReplicationType
replicationType, int duplicateIndex)
+ throws Exception {
+ long containerId = 306L;
+ boolean isEC = replicationType == HddsProtos.ReplicationType.EC;
+ int requiredNodes = isEC ? 5 : 3;
+ ContainerInfo container = mockContainerInfo(containerId, 5, 1024L,
requiredNodes);
+ when(container.getReplicationType()).thenReturn(replicationType);
+
when(containerManager.getContainers()).thenReturn(Collections.singletonList(container));
+
when(containerManager.getContainer(container.containerID())).thenReturn(container);
+
+ Set<ContainerReplica> replicas = new HashSet<>();
+ for (int i = 1; i <= requiredNodes; i++) {
+ ContainerReplica replica = mock(ContainerReplica.class);
+ int replicaIndex = isEC ? i : 0;
+ when(replica.getReplicaIndex()).thenReturn(replicaIndex);
+ when(replica.getDataChecksum()).thenReturn(1000L + replicaIndex);
+ replicas.add(replica);
+ }
+
when(containerManager.getContainerReplicas(container.containerID())).thenReturn(replicas);
+ reconRM = createStateInjectingReconRM(Collections.emptyMap());
+
+ reconRM.processAll();
+
assertTrue(schemaManagerV2.getUnhealthyContainers(UnHealthyContainerStates.REPLICA_MISMATCH,
0, 0, 10).isEmpty(),
+ "Different EC indexes may have different checksums");
+
+ ContainerReplica duplicate = mock(ContainerReplica.class);
+ when(duplicate.getReplicaIndex()).thenReturn(duplicateIndex);
+ when(duplicate.getDataChecksum()).thenReturn(1000L + duplicateIndex);
+ replicas.add(duplicate);
+ reconRM.processAll();
+
assertTrue(schemaManagerV2.getUnhealthyContainers(UnHealthyContainerStates.REPLICA_MISMATCH,
0, 0, 10).isEmpty(),
+ "Replicas of the same index with matching checksums should not report
a mismatch");
+
+ when(duplicate.getDataChecksum()).thenReturn(2000L);
+ reconRM.processAll();
+ List<ContainerHealthSchemaManager.UnhealthyContainerRecord> mismatches =
+
schemaManagerV2.getUnhealthyContainers(UnHealthyContainerStates.REPLICA_MISMATCH,
0, 0, 10);
+ assertEquals(1, mismatches.size());
+ assertEquals(containerId, mismatches.get(0).getContainerId());
+
+ when(duplicate.getDataChecksum()).thenReturn(1000L + duplicateIndex);
+ reconRM.processAll();
+
assertTrue(schemaManagerV2.getUnhealthyContainers(UnHealthyContainerStates.REPLICA_MISMATCH,
0, 0, 10).isEmpty(),
+ "Resolved checksum mismatches should be removed");
+ }
+
@Test
public void testProcessAllMapsCompositeScmStatesToBaseStates() throws
Exception {
final long unhealthyUnderContainerId = 401L;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
