This is an automated email from the ASF dual-hosted git repository.
aswinshakil pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 5e5243eca2a HDDS-14714. Support keeping a configurable number of extra
copies of quasi-closed containers. (#9829)
5e5243eca2a is described below
commit 5e5243eca2a28ba5127be5e10bba97a99adf9d52
Author: Aswin Shakil Balasubramanian <[email protected]>
AuthorDate: Tue Mar 24 11:06:32 2026 -0700
HDDS-14714. Support keeping a configurable number of extra copies of
quasi-closed containers. (#9829)
---
.../QuasiClosedStuckOverReplicationHandler.java | 4 +-
.../replication/QuasiClosedStuckReplicaCount.java | 77 +++++-
.../QuasiClosedStuckUnderReplicationHandler.java | 4 +-
.../container/replication/ReplicationManager.java | 42 ++-
.../health/QuasiClosedStuckReplicationCheck.java | 20 +-
.../container/replication/ReplicationTestUtil.java | 24 ++
...TestQuasiClosedStuckOverReplicationHandler.java | 96 +++++--
.../TestQuasiClosedStuckReplicaCount.java | 302 ++++++++++++++-------
...estQuasiClosedStuckUnderReplicationHandler.java | 61 +++--
.../TestQuasiClosedStuckReplicationCheck.java | 40 ++-
.../replicationManagerTests/quasi_closed.json | 57 ++--
11 files changed, 520 insertions(+), 207 deletions(-)
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/QuasiClosedStuckOverReplicationHandler.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/QuasiClosedStuckOverReplicationHandler.java
index 3ff23b495e8..e7eee2aeb48 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/QuasiClosedStuckOverReplicationHandler.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/QuasiClosedStuckOverReplicationHandler.java
@@ -76,8 +76,10 @@ public int processAndSendCommands(Set<ContainerReplica>
replicas, List<Container
})
.collect(Collectors.toSet());
+ ReplicationManager.ReplicationManagerConfiguration rmConf =
replicationManager.getConfig();
QuasiClosedStuckReplicaCount replicaCount =
- new QuasiClosedStuckReplicaCount(healthyReplicas,
remainingMaintenanceRedundancy);
+ new QuasiClosedStuckReplicaCount(healthyReplicas,
remainingMaintenanceRedundancy,
+ rmConf.getQuasiClosedStuckBestOriginCopies(),
rmConf.getQuasiClosedStuckOtherOriginCopies());
List<QuasiClosedStuckReplicaCount.MisReplicatedOrigin> misReplicatedOrigins
= replicaCount.getOverReplicatedOrigins();
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/QuasiClosedStuckReplicaCount.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/QuasiClosedStuckReplicaCount.java
index 97df8cf958f..0e62913287d 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/QuasiClosedStuckReplicaCount.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/QuasiClosedStuckReplicaCount.java
@@ -17,6 +17,8 @@
package org.apache.hadoop.hdds.scm.container.replication;
+import static
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.UNHEALTHY;
+
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -26,11 +28,16 @@
import java.util.Set;
import org.apache.hadoop.hdds.protocol.DatanodeID;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
import org.apache.hadoop.hdds.scm.container.ContainerReplica;
/**
* Class to count the replicas in a quasi-closed stuck container.
+ *
+ * <p>Origins are ranked by their highest healthy BCSID (sequenceId). The
origins with the
+ * highest BCSID receive {@code bestOriginCopies} replicas, while all other
origins receive
+ * {@code otherOriginCopies}. If multiple origins share the same highest BCSID
they are all treated
+ * as "best". For a single-origin container, {@code bestOriginCopies} is
always used.
+ *
*/
public class QuasiClosedStuckReplicaCount {
@@ -40,13 +47,20 @@ public class QuasiClosedStuckReplicaCount {
private final int minHealthyForMaintenance;
private final boolean hasHealthyReplicas;
private final boolean hasOutOfServiceReplicas;
+ private final int bestOriginCopies;
+ private final int otherOriginCopies;
+ private final Set<DatanodeID> bestOrigins;
- public QuasiClosedStuckReplicaCount(Set<ContainerReplica> replicas, int
minHealthyForMaintenance) {
+ public QuasiClosedStuckReplicaCount(Set<ContainerReplica> replicas, int
minHealthyForMaintenance,
+ int bestOriginCopies, int otherOriginCopies) {
this.minHealthyForMaintenance = minHealthyForMaintenance;
+ this.bestOriginCopies = bestOriginCopies;
+ this.otherOriginCopies = otherOriginCopies;
+
boolean hasHealthy = false;
boolean hasOutOfService = false;
for (ContainerReplica r : replicas) {
- if (r.getState() !=
StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.UNHEALTHY) {
+ if (r.getState() != UNHEALTHY) {
hasHealthy = true;
}
replicasByOrigin.computeIfAbsent(r.getOriginDatanodeId(), k -> new
HashSet<>()).add(r);
@@ -64,6 +78,39 @@ public QuasiClosedStuckReplicaCount(Set<ContainerReplica>
replicas, int minHealt
this.hasHealthyReplicas = hasHealthy;
this.hasOutOfServiceReplicas = hasOutOfService;
+ this.bestOrigins = computeBestOrigins();
+ }
+
+ /**
+ * Identifies the best origins whose maximum BCSID among healthy replicas
equals the cluster-wide maximum healthy
+ * BCSID. Origins with only UNHEALTHY replicas are excluded. If all replicas
are UNHEALTHY the returned set is empty.
+ */
+ private Set<DatanodeID> computeBestOrigins() {
+ long maxBcsid = Long.MIN_VALUE;
+
+ for (Map.Entry<DatanodeID, Set<ContainerReplica>> entry :
replicasByOrigin.entrySet()) {
+ for (ContainerReplica r : entry.getValue()) {
+ if (r.getState() != UNHEALTHY && r.getSequenceId() != null &&
r.getSequenceId() > maxBcsid) {
+ maxBcsid = r.getSequenceId();
+ }
+ }
+ }
+
+ if (maxBcsid == Long.MIN_VALUE) {
+ return Collections.emptySet();
+ }
+
+ final long highestBcsid = maxBcsid;
+ Set<DatanodeID> best = new HashSet<>();
+ for (Map.Entry<DatanodeID, Set<ContainerReplica>> entry :
replicasByOrigin.entrySet()) {
+ boolean hasBestBcsid = entry.getValue().stream()
+ .anyMatch(r -> r.getState() != UNHEALTHY && r.getSequenceId() != null
+ && r.getSequenceId() == highestBcsid);
+ if (hasBestBcsid) {
+ best.add(entry.getKey());
+ }
+ }
+ return best;
}
public int availableOrigins() {
@@ -92,6 +139,10 @@ private int getMaintenanceCount(DatanodeID origin) {
return maintenance == null ? 0 : maintenance.size();
}
+ private int targetCopiesForOrigin(DatanodeID origin) {
+ return bestOrigins.contains(origin) ? bestOriginCopies : otherOriginCopies;
+ }
+
public List<MisReplicatedOrigin> getUnderReplicatedReplicas() {
List<MisReplicatedOrigin> misReplicatedOrigins = new ArrayList<>();
@@ -114,20 +165,21 @@ public List<MisReplicatedOrigin>
getUnderReplicatedReplicas() {
return misReplicatedOrigins;
}
- // If there are multiple origins, we expect 2 copies of each origin
- // For maintenance, we expect 1 copy of each origin and ignore the
minHealthyForMaintenance parameter
+ // Multiple origins: the best origins target bestOriginCopies; all others
target otherOriginCopies.
+ // For maintenance, we expect 1 copy of each origin online regardless of
best/other designation.
for (Map.Entry<DatanodeID, Set<ContainerReplica>> entry :
replicasByOrigin.entrySet()) {
final Set<ContainerReplica> inService = getInService(entry.getKey());
final int maintenanceCount = getMaintenanceCount(entry.getKey());
+ final int target = targetCopiesForOrigin(entry.getKey());
- if (inService.size() < 2) {
+ if (inService.size() < target) {
if (maintenanceCount > 0) {
if (inService.isEmpty()) {
// We need 1 copy online for maintenance
misReplicatedOrigins.add(new MisReplicatedOrigin(entry.getValue(),
1));
}
} else {
- misReplicatedOrigins.add(new MisReplicatedOrigin(entry.getValue(), 2
- inService.size()));
+ misReplicatedOrigins.add(new MisReplicatedOrigin(entry.getValue(),
target - inService.size()));
}
}
}
@@ -136,7 +188,7 @@ public List<MisReplicatedOrigin>
getUnderReplicatedReplicas() {
/**
* Returns True is the container is over-replicated. This means that if we
have a single origin, there are more than
- * 3 copies. If we have multiple origins, there are more than 2 copies of
each origin.
+ * 3 copies. If we have multiple origins, there are more than target copies
of each origin.
* The over replication check ignore maintenance replicas. The container may
become over replicated when maintenance
* ends.
*
@@ -147,7 +199,7 @@ public boolean isOverReplicated() {
}
public List<MisReplicatedOrigin> getOverReplicatedOrigins() {
- // If there is only a single origin, we expect 3 copies, otherwise we
expect 2 copies of each origin
+ // If there is only a single origin, we expect 3 copies.
if (replicasByOrigin.size() == 1) {
final DatanodeID origin = replicasByOrigin.keySet().iterator().next();
final Set<ContainerReplica> inService = getInService(origin);
@@ -157,15 +209,14 @@ public List<MisReplicatedOrigin>
getOverReplicatedOrigins() {
return Collections.emptyList();
}
- // If there are multiple origins, we expect 2 copies of each origin
List<MisReplicatedOrigin> overReplicatedOrigins = new ArrayList<>();
for (DatanodeID origin : replicasByOrigin.keySet()) {
final Set<ContainerReplica> replicas = getInService(origin);
- if (replicas.size() > 2) {
- overReplicatedOrigins.add(new MisReplicatedOrigin(replicas,
replicas.size() - 2));
+ final int target = targetCopiesForOrigin(origin);
+ if (replicas.size() > target) {
+ overReplicatedOrigins.add(new MisReplicatedOrigin(replicas,
replicas.size() - target));
}
}
- // If we have 2 copies or less of each origin, we are not over-replicated
return overReplicatedOrigins;
}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/QuasiClosedStuckUnderReplicationHandler.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/QuasiClosedStuckUnderReplicationHandler.java
index 55b503822fd..91ab321859a 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/QuasiClosedStuckUnderReplicationHandler.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/QuasiClosedStuckUnderReplicationHandler.java
@@ -83,8 +83,10 @@ public int processAndSendCommands(Set<ContainerReplica>
replicas, List<Container
return 0;
}
+ ReplicationManager.ReplicationManagerConfiguration rmConf =
replicationManager.getConfig();
QuasiClosedStuckReplicaCount replicaCount =
- new QuasiClosedStuckReplicaCount(replicas,
remainingMaintenanceRedundancy);
+ new QuasiClosedStuckReplicaCount(replicas,
remainingMaintenanceRedundancy,
+ rmConf.getQuasiClosedStuckBestOriginCopies(),
rmConf.getQuasiClosedStuckOtherOriginCopies());
List<QuasiClosedStuckReplicaCount.MisReplicatedOrigin> misReplicatedOrigins
= replicaCount.getUnderReplicatedReplicas();
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java
index 32808fc85a5..7096c18e6c5 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java
@@ -269,7 +269,7 @@ public ReplicationManager(final
ReplicationManagerConfiguration rmConf,
.addNext(new MismatchedReplicasHandler(this))
.addNext(new EmptyContainerHandler(this))
.addNext(new DeletingContainerHandler(this))
- .addNext(new QuasiClosedStuckReplicationCheck())
+ .addNext(new QuasiClosedStuckReplicationCheck(rmConf))
.addNext(ecReplicationCheckHandler)
.addNext(ratisReplicationCheckHandler)
.addNext(new ClosedWithUnhealthyReplicasHandler(this))
@@ -1285,6 +1285,30 @@ public static class ReplicationManagerConfiguration
)
private int containerSampleLimit = 100;
+ @Config(key = "hdds.scm.replication.quasi.closed.stuck.best.origin.copies",
+ type = ConfigType.INT,
+ defaultValue = "3",
+ reconfigurable = true,
+ tags = { SCM },
+ description = "For quasi-closed stuck containers with multiple
diverged origins, " +
+ "the number of replicas to maintain for the origin with the
highest bcsId " +
+ "among healthy replicas. This origin is considered the 'best' copy
and receives " +
+ "extra fault-tolerance. If multiple origins share the same highest
bcsId, all of them receive this count."
+ )
+ private int quasiClosedStuckBestOriginCopies = 3;
+
+ @Config(key =
"hdds.scm.replication.quasi.closed.stuck.other.origin.copies",
+ type = ConfigType.INT,
+ defaultValue = "2",
+ reconfigurable = true,
+ tags = { SCM },
+ description = "For quasi-closed stuck containers with multiple
diverged origins, " +
+ "the number of replicas to maintain for each origin that does not
have the " +
+ "highest block commit sequence ID (BCSID). These replicas are kept
to preserve " +
+ "data integrity across diverged copies."
+ )
+ private int quasiClosedStuckOtherOriginCopies = 2;
+
public long getDatanodeTimeoutOffset() {
return datanodeTimeoutOffset;
}
@@ -1377,6 +1401,22 @@ public void setContainerSampleLimit(int sampleLimit) {
this.containerSampleLimit = sampleLimit;
}
+ public int getQuasiClosedStuckBestOriginCopies() {
+ return quasiClosedStuckBestOriginCopies;
+ }
+
+ public void setQuasiClosedStuckBestOriginCopies(int copies) {
+ this.quasiClosedStuckBestOriginCopies = copies;
+ }
+
+ public int getQuasiClosedStuckOtherOriginCopies() {
+ return quasiClosedStuckOtherOriginCopies;
+ }
+
+ public void setQuasiClosedStuckOtherOriginCopies(int copies) {
+ this.quasiClosedStuckOtherOriginCopies = copies;
+ }
+
@PostConstruct
public void validate() {
if (datanodeTimeoutOffset < 0) {
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/QuasiClosedStuckReplicationCheck.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/QuasiClosedStuckReplicationCheck.java
index d89c3bb351e..e520153ad84 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/QuasiClosedStuckReplicationCheck.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/QuasiClosedStuckReplicationCheck.java
@@ -28,17 +28,25 @@
import org.apache.hadoop.hdds.scm.container.replication.ContainerHealthResult;
import org.apache.hadoop.hdds.scm.container.replication.ContainerReplicaOp;
import
org.apache.hadoop.hdds.scm.container.replication.QuasiClosedStuckReplicaCount;
+import org.apache.hadoop.hdds.scm.container.replication.ReplicationManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Class to check for the replication of the replicas in quasi-closed stuck
containers. As we want to maintain
- * as much data and information as possible, the rule for QC stuck container
is to maintain 2 copies of each origin
- * if there is more than 1 origin. If there is only 1 origin, then we need to
maintain 3 copies.
+ * as much data and information as possible, the rule for QC stuck container
is to maintain a configurable number
+ * of copies of the origin with the highest BCSID (bestOriginCopies, default
3), and a configurable number of copies
+ * of each other origin (otherOriginCopies, default 2). If there is only 1
origin, bestOriginCopies copies are kept.
*/
public class QuasiClosedStuckReplicationCheck extends AbstractCheck {
private static final Logger LOG =
LoggerFactory.getLogger(QuasiClosedStuckReplicationCheck.class);
+ private final ReplicationManager.ReplicationManagerConfiguration rmConf;
+
+ public
QuasiClosedStuckReplicationCheck(ReplicationManager.ReplicationManagerConfiguration
rmConf) {
+ this.rmConf = rmConf;
+ }
+
public static boolean shouldHandleAsQuasiClosedStuck(ContainerInfo
containerInfo, Set<ContainerReplica> replicas) {
if (containerInfo.getState() != QUASI_CLOSED) {
return false;
@@ -46,7 +54,7 @@ public static boolean
shouldHandleAsQuasiClosedStuck(ContainerInfo containerInfo
if (!QuasiClosedContainerHandler.isQuasiClosedStuck(containerInfo,
replicas)) {
return false;
}
- QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 0);
+ QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 0, 3, 2);
if (replicaCount.availableOrigins() == 1) {
// This is the 3 copies of a single origin case, so allow it to be
handled via the normal under-replicated
// handler.
@@ -70,8 +78,10 @@ public boolean handle(ContainerCheckRequest request) {
return true;
}
- QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(
- request.getContainerReplicas(), request.getMaintenanceRedundancy());
+ QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(request.getContainerReplicas(),
+ request.getMaintenanceRedundancy(),
+ rmConf.getQuasiClosedStuckBestOriginCopies(),
+ rmConf.getQuasiClosedStuckOtherOriginCopies());
if (!replicaCount.hasHealthyReplicas()) {
// All unhealthy are handled by a different handler
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationTestUtil.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationTestUtil.java
index ad5a30e3ace..63f4c6c9f33 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationTestUtil.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationTestUtil.java
@@ -161,6 +161,30 @@ public static Set<ContainerReplica>
createReplicasWithOriginAndOpState(
return replicas;
}
+ /**
+ * Creates a single replica with a specific origin, operational state,
replica state, and BCSID.
+ */
+ public static ContainerReplica createReplicaWithOriginAndSeqId(
+ ContainerID containerID, DatanodeID originNodeId,
+ HddsProtos.NodeOperationalState opState,
+ ContainerReplicaProto.State replicaState, long seqId) {
+ return createContainerReplica(containerID, 0, opState, replicaState, 123L,
1234L,
+ MockDatanodeDetails.randomDatanodeDetails(), originNodeId, seqId);
+ }
+
+ /**
+ * Adds {@code count} replicas with the given origin, operational state,
replica state, and BCSID.
+ * to the provided set.
+ */
+ public static void addReplicasWithOriginAndSeqId(
+ Set<ContainerReplica> replicas, ContainerID containerID,
+ DatanodeID originNodeId, HddsProtos.NodeOperationalState opState,
+ ContainerReplicaProto.State replicaState, long seqId, int count) {
+ for (int i = 0; i < count; i++) {
+ replicas.add(createReplicaWithOriginAndSeqId(containerID, originNodeId,
opState, replicaState, seqId));
+ }
+ }
+
public static ContainerReplica createContainerReplica(ContainerID
containerID,
int replicaIndex, HddsProtos.NodeOperationalState opState,
ContainerReplicaProto.State replicaState) {
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestQuasiClosedStuckOverReplicationHandler.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestQuasiClosedStuckOverReplicationHandler.java
index 67d1fe09371..5a493b1981a 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestQuasiClosedStuckOverReplicationHandler.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestQuasiClosedStuckOverReplicationHandler.java
@@ -17,7 +17,9 @@
package org.apache.hadoop.hdds.scm.container.replication;
+import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE;
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE;
+import static
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.QUASI_CLOSED;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
@@ -97,11 +99,11 @@ void setup() throws NodeNotFoundException,
@Test
public void testReturnsZeroIfNotOverReplicated() throws IOException {
Set<ContainerReplica> replicas =
ReplicationTestUtil.createReplicasWithOriginAndOpState(container.containerID(),
-
StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.QUASI_CLOSED,
- Pair.of(origin1, HddsProtos.NodeOperationalState.IN_SERVICE),
- Pair.of(origin1, HddsProtos.NodeOperationalState.IN_SERVICE),
- Pair.of(origin2, HddsProtos.NodeOperationalState.IN_SERVICE),
- Pair.of(origin2, HddsProtos.NodeOperationalState.IN_SERVICE));
+ QUASI_CLOSED,
+ Pair.of(origin1, IN_SERVICE),
+ Pair.of(origin1, IN_SERVICE),
+ Pair.of(origin2, IN_SERVICE),
+ Pair.of(origin2, IN_SERVICE));
int count = handler.processAndSendCommands(replicas,
Collections.emptyList(), getOverReplicatedHealthResult(), 1);
assertEquals(0, count);
@@ -110,13 +112,13 @@ public void testReturnsZeroIfNotOverReplicated() throws
IOException {
@Test
public void testNoCommandsScheduledIfPendingOps() throws IOException {
Set<ContainerReplica> replicas =
ReplicationTestUtil.createReplicasWithOriginAndOpState(container.containerID(),
-
StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.QUASI_CLOSED,
- Pair.of(origin1, HddsProtos.NodeOperationalState.IN_SERVICE),
- Pair.of(origin1, HddsProtos.NodeOperationalState.IN_SERVICE),
- Pair.of(origin1, HddsProtos.NodeOperationalState.IN_SERVICE),
- Pair.of(origin2, HddsProtos.NodeOperationalState.IN_SERVICE),
- Pair.of(origin2, HddsProtos.NodeOperationalState.IN_SERVICE),
- Pair.of(origin2, HddsProtos.NodeOperationalState.IN_SERVICE));
+ QUASI_CLOSED,
+ Pair.of(origin1, IN_SERVICE),
+ Pair.of(origin1, IN_SERVICE),
+ Pair.of(origin1, IN_SERVICE),
+ Pair.of(origin2, IN_SERVICE),
+ Pair.of(origin2, IN_SERVICE),
+ Pair.of(origin2, IN_SERVICE));
List<ContainerReplicaOp> pendingOps = new ArrayList<>();
pendingOps.add(new ContainerReplicaOp(
ContainerReplicaOp.PendingOpType.DELETE,
@@ -132,13 +134,21 @@ public void testNoCommandsScheduledIfPendingOps() throws
IOException {
@Test
public void testCommandScheduledForOverReplicatedContainer() throws
IOException {
- Set<ContainerReplica> replicas =
ReplicationTestUtil.createReplicasWithOriginAndOpState(container.containerID(),
-
StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.QUASI_CLOSED,
- Pair.of(origin1, HddsProtos.NodeOperationalState.IN_SERVICE),
- Pair.of(origin1, HddsProtos.NodeOperationalState.IN_SERVICE),
- Pair.of(origin1, HddsProtos.NodeOperationalState.IN_SERVICE),
- Pair.of(origin2, HddsProtos.NodeOperationalState.IN_SERVICE),
- Pair.of(origin2, HddsProtos.NodeOperationalState.IN_SERVICE));
+ // origin1 is the best origin (BCSID=10, target=3): has 4 copies,
over-replicated by 1.
+ // origin2 is an other origin (BCSID=5, target=2): has 2 copies, at target.
+ Set<ContainerReplica> replicas = new HashSet<>();
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(container.containerID(),
+ origin1, IN_SERVICE, QUASI_CLOSED, 10));
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(container.containerID(),
+ origin1, IN_SERVICE, QUASI_CLOSED, 10));
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(container.containerID(),
+ origin1, IN_SERVICE, QUASI_CLOSED, 10));
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(container.containerID(),
+ origin1, IN_SERVICE, QUASI_CLOSED, 10));
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(container.containerID(),
+ origin2, IN_SERVICE, QUASI_CLOSED, 5));
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(container.containerID(),
+ origin2, IN_SERVICE, QUASI_CLOSED, 5));
int count = handler.processAndSendCommands(replicas,
Collections.emptyList(), getOverReplicatedHealthResult(), 1);
assertEquals(1, count);
@@ -148,14 +158,25 @@ public void
testCommandScheduledForOverReplicatedContainer() throws IOException
@Test
public void testOverloadedExceptionContinuesAndThrows() throws
NotLeaderException, CommandTargetOverloadedException {
- Set<ContainerReplica> replicas =
ReplicationTestUtil.createReplicasWithOriginAndOpState(container.containerID(),
-
StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.QUASI_CLOSED,
- Pair.of(origin1, HddsProtos.NodeOperationalState.IN_SERVICE),
- Pair.of(origin1, HddsProtos.NodeOperationalState.IN_SERVICE),
- Pair.of(origin1, HddsProtos.NodeOperationalState.IN_SERVICE),
- Pair.of(origin2, HddsProtos.NodeOperationalState.IN_SERVICE),
- Pair.of(origin2, HddsProtos.NodeOperationalState.IN_SERVICE),
- Pair.of(origin2, HddsProtos.NodeOperationalState.IN_SERVICE));
+ // origin1 is the best origin (BCSID=10, target=3): has 4 copies,
over-replicated by 1.
+ // origin2 is an other origin (BCSID=5, target=2): has 3 copies,
over-replicated by 1.
+ // With the first delete overloaded: the first delete command throws, the
second succeeds.
+ // Expect 1 command sent total and CommandTargetOverloadedException thrown.
+ Set<ContainerReplica> replicas = new HashSet<>();
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(container.containerID(),
+ origin1, IN_SERVICE, QUASI_CLOSED, 10));
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(container.containerID(),
+ origin1, IN_SERVICE, QUASI_CLOSED, 10));
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(container.containerID(),
+ origin1, IN_SERVICE, QUASI_CLOSED, 10));
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(container.containerID(),
+ origin1, IN_SERVICE, QUASI_CLOSED, 10));
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(container.containerID(),
+ origin2, IN_SERVICE, QUASI_CLOSED, 5));
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(container.containerID(),
+ origin2, IN_SERVICE, QUASI_CLOSED, 5));
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(container.containerID(),
+ origin2, IN_SERVICE, QUASI_CLOSED, 5));
ReplicationTestUtil.mockRMSendThrottledDeleteCommand(replicationManager,
commandsSent, new AtomicBoolean(true));
@@ -164,6 +185,27 @@ public void testOverloadedExceptionContinuesAndThrows()
throws NotLeaderExceptio
assertEquals(1, commandsSent.size());
}
+ @Test
+ public void testBestOriginNotOverReplicated() throws IOException {
+ // origin1 is the best origin (BCSID=10, target=3): has exactly 3 copies,
at target.
+ // origin2 is an other origin (BCSID=5, target=2): has exactly 2 copies,
at target.
+ // Neither origin is over-replicated, so no delete commands should be
scheduled.
+ Set<ContainerReplica> replicas = new HashSet<>();
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(container.containerID(),
+ origin1, IN_SERVICE, QUASI_CLOSED, 10));
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(container.containerID(),
+ origin1, IN_SERVICE, QUASI_CLOSED, 10));
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(container.containerID(),
+ origin1, IN_SERVICE, QUASI_CLOSED, 10));
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(container.containerID(),
+ origin2, IN_SERVICE, QUASI_CLOSED, 5));
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(container.containerID(),
+ origin2, IN_SERVICE, QUASI_CLOSED, 5));
+
+ int count = handler.processAndSendCommands(replicas,
Collections.emptyList(), getOverReplicatedHealthResult(), 1);
+ assertEquals(0, count);
+ }
+
private ContainerHealthResult.OverReplicatedHealthResult
getOverReplicatedHealthResult() {
ContainerHealthResult.OverReplicatedHealthResult
healthResult =
mock(ContainerHealthResult.OverReplicatedHealthResult.class);
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestQuasiClosedStuckReplicaCount.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestQuasiClosedStuckReplicaCount.java
index 3b74eccc72f..bfa27006d54 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestQuasiClosedStuckReplicaCount.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestQuasiClosedStuckReplicaCount.java
@@ -27,6 +27,7 @@
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.commons.lang3.tuple.Pair;
@@ -46,13 +47,17 @@ public class TestQuasiClosedStuckReplicaCount {
@Test
public void testCorrectlyReplicationWithThreeOrigins() {
- Set<ContainerReplica> replicas =
ReplicationTestUtil.createReplicasWithOriginAndOpState(
- ContainerID.valueOf(1), QUASI_CLOSED,
- Pair.of(origin1, IN_SERVICE), Pair.of(origin1, IN_SERVICE),
- Pair.of(origin2, IN_SERVICE), Pair.of(origin2, IN_SERVICE),
- Pair.of(origin3, IN_SERVICE), Pair.of(origin3, IN_SERVICE));
-
- QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1);
+ // origin1 is best (BCSID=10) with 3 copies at target; origin2 and origin3
are other (BCSID=5)
+ // with 2 copies each at target.
+ Set<ContainerReplica> replicas = new HashSet<>();
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin1,
+ IN_SERVICE, QUASI_CLOSED, 10, 3);
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin2,
+ IN_SERVICE, QUASI_CLOSED, 5, 2);
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin3,
+ IN_SERVICE, QUASI_CLOSED, 5, 2);
+
+ QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1, 3, 2);
assertFalse(replicaCount.isUnderReplicated());
assertFalse(replicaCount.isOverReplicated());
assertTrue(replicaCount.getUnderReplicatedReplicas().isEmpty());
@@ -60,12 +65,14 @@ public void testCorrectlyReplicationWithThreeOrigins() {
@Test
public void testCorrectReplicationWithTwoOrigins() {
- Set<ContainerReplica> replicas =
ReplicationTestUtil.createReplicasWithOriginAndOpState(
- ContainerID.valueOf(1), QUASI_CLOSED,
- Pair.of(origin1, IN_SERVICE), Pair.of(origin1, IN_SERVICE),
- Pair.of(origin2, IN_SERVICE), Pair.of(origin2, IN_SERVICE));
-
- QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1);
+ // origin1 is best (BCSID=10) with 3 copies at target; origin2 is other
(BCSID=5) with 2 copies at target.
+ Set<ContainerReplica> replicas = new HashSet<>();
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin1,
+ IN_SERVICE, QUASI_CLOSED, 10, 3);
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin2,
+ IN_SERVICE, QUASI_CLOSED, 5, 2);
+
+ QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1, 3, 2);
assertFalse(replicaCount.isUnderReplicated());
assertFalse(replicaCount.isOverReplicated());
assertTrue(replicaCount.getUnderReplicatedReplicas().isEmpty());
@@ -77,7 +84,7 @@ public void testCorrectReplicationWithOneOrigin() {
ContainerID.valueOf(1), QUASI_CLOSED,
Pair.of(origin1, IN_SERVICE), Pair.of(origin1, IN_SERVICE),
Pair.of(origin1, IN_SERVICE));
- QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1);
+ QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1, 3, 2);
assertFalse(replicaCount.isUnderReplicated());
assertFalse(replicaCount.isOverReplicated());
assertTrue(replicaCount.getUnderReplicatedReplicas().isEmpty());
@@ -85,13 +92,18 @@ public void testCorrectReplicationWithOneOrigin() {
@Test
public void testUnderReplicationWithThreeOrigins() {
- Set<ContainerReplica> replicas =
ReplicationTestUtil.createReplicasWithOriginAndOpState(
- ContainerID.valueOf(1), QUASI_CLOSED,
- Pair.of(origin1, IN_SERVICE), Pair.of(origin1, IN_SERVICE),
- Pair.of(origin2, IN_SERVICE), Pair.of(origin2, IN_SERVICE),
- Pair.of(origin3, IN_SERVICE));
-
- QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1);
+ // origin1 is best (BCSID=10) with 3 copies at target.
+ // origin2 is other (BCSID=5) with 2 copies at target.
+ // origin3 is other (BCSID=5) with only 1 copy, so it is under-replicated
by 1.
+ Set<ContainerReplica> replicas = new HashSet<>();
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin1,
+ IN_SERVICE, QUASI_CLOSED, 10, 3);
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin2,
+ IN_SERVICE, QUASI_CLOSED, 5, 2);
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin3,
+ IN_SERVICE, QUASI_CLOSED, 5, 1);
+
+ QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1, 3, 2);
assertTrue(replicaCount.isUnderReplicated());
assertFalse(replicaCount.isOverReplicated());
validateMisReplicatedOrigins(replicaCount.getUnderReplicatedReplicas(), 1,
1, 1, origin3);
@@ -99,19 +111,23 @@ public void testUnderReplicationWithThreeOrigins() {
@Test
public void testUnderReplicationWithThreeOriginsTwoUnderReplicated() {
- Set<ContainerReplica> replicas =
ReplicationTestUtil.createReplicasWithOriginAndOpState(
- ContainerID.valueOf(1), QUASI_CLOSED,
- Pair.of(origin1, IN_SERVICE),
- Pair.of(origin2, IN_SERVICE), Pair.of(origin2, IN_SERVICE),
- Pair.of(origin3, IN_SERVICE));
-
- QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1);
+ // origin2 is best (BCSID=10) with 3 copies at target.
+ // origin1 and origin3 are other (BCSID=5) with only 1 copy each, so both
are under-replicated.
+ Set<ContainerReplica> replicas = new HashSet<>();
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin1,
+ IN_SERVICE, QUASI_CLOSED, 5, 1);
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin2,
+ IN_SERVICE, QUASI_CLOSED, 10, 3);
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin3,
+ IN_SERVICE, QUASI_CLOSED, 5, 1);
+
+ QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1, 3, 2);
assertTrue(replicaCount.isUnderReplicated());
assertFalse(replicaCount.isOverReplicated());
List<QuasiClosedStuckReplicaCount.MisReplicatedOrigin>
misReplicatedOrigins =
replicaCount.getUnderReplicatedReplicas();
- assertTrue(misReplicatedOrigins.size() == 2);
+ assertEquals(2, misReplicatedOrigins.size());
for (QuasiClosedStuckReplicaCount.MisReplicatedOrigin misReplicatedOrigin
: misReplicatedOrigins) {
final DatanodeID source =
misReplicatedOrigin.getSources().iterator().next().getOriginDatanodeId();
@@ -121,12 +137,15 @@ public void
testUnderReplicationWithThreeOriginsTwoUnderReplicated() {
@Test
public void testUnderReplicationWithTwoOrigins() {
- Set<ContainerReplica> replicas =
ReplicationTestUtil.createReplicasWithOriginAndOpState(
- ContainerID.valueOf(1), QUASI_CLOSED,
- Pair.of(origin1, IN_SERVICE), Pair.of(origin1, IN_SERVICE),
- Pair.of(origin2, IN_SERVICE));
-
- QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1);
+ // origin1 is best (BCSID=10) with 3 copies at target.
+ // origin2 is other (BCSID=5) with only 1 copy, so it is under-replicated
by 1.
+ Set<ContainerReplica> replicas = new HashSet<>();
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin1,
+ IN_SERVICE, QUASI_CLOSED, 10, 3);
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin2,
+ IN_SERVICE, QUASI_CLOSED, 5, 1);
+
+ QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1, 3, 2);
assertTrue(replicaCount.isUnderReplicated());
assertFalse(replicaCount.isOverReplicated());
validateMisReplicatedOrigins(replicaCount.getUnderReplicatedReplicas(), 1,
1, 1, origin2);
@@ -138,7 +157,7 @@ public void testUnderReplicationWithOneOrigin() {
ContainerID.valueOf(1), QUASI_CLOSED,
Pair.of(origin1, IN_SERVICE));
- QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1);
+ QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1, 3, 2);
assertTrue(replicaCount.isUnderReplicated());
assertFalse(replicaCount.isOverReplicated());
validateMisReplicatedOrigins(replicaCount.getUnderReplicatedReplicas(), 1,
1, 2, origin1);
@@ -146,13 +165,18 @@ public void testUnderReplicationWithOneOrigin() {
@Test
public void testOverReplicationWithThreeOrigins() {
- Set<ContainerReplica> replicas =
ReplicationTestUtil.createReplicasWithOriginAndOpState(
- ContainerID.valueOf(1), QUASI_CLOSED,
- Pair.of(origin1, IN_SERVICE), Pair.of(origin1, IN_SERVICE),
- Pair.of(origin2, IN_SERVICE), Pair.of(origin2, IN_SERVICE),
- Pair.of(origin3, IN_SERVICE), Pair.of(origin3, IN_SERVICE),
Pair.of(origin3, IN_SERVICE));
-
- QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1);
+ // origin1 is best (BCSID=10) with 3 copies at target.
+ // origin2 is other (BCSID=5) with 2 copies at target.
+ // origin3 is other (BCSID=5) with 3 copies, exceeding its target of 2 by
1.
+ Set<ContainerReplica> replicas = new HashSet<>();
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin1,
+ IN_SERVICE, QUASI_CLOSED, 10, 3);
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin2,
+ IN_SERVICE, QUASI_CLOSED, 5, 2);
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin3,
+ IN_SERVICE, QUASI_CLOSED, 5, 3);
+
+ QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1, 3, 2);
assertFalse(replicaCount.isUnderReplicated());
assertTrue(replicaCount.isOverReplicated());
validateMisReplicatedOrigins(replicaCount.getOverReplicatedOrigins(), 1,
3, 1, origin3);
@@ -160,12 +184,15 @@ public void testOverReplicationWithThreeOrigins() {
@Test
public void testOverReplicationWithTwoOrigins() {
- Set<ContainerReplica> replicas =
ReplicationTestUtil.createReplicasWithOriginAndOpState(
- ContainerID.valueOf(1), QUASI_CLOSED,
- Pair.of(origin1, IN_SERVICE), Pair.of(origin1, IN_SERVICE),
- Pair.of(origin2, IN_SERVICE), Pair.of(origin2, IN_SERVICE),
Pair.of(origin2, IN_SERVICE));
-
- QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1);
+ // origin1 is best (BCSID=10) with 3 copies at target.
+ // origin2 is other (BCSID=5) with 3 copies, exceeding its target of 2 by
1.
+ Set<ContainerReplica> replicas = new HashSet<>();
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin1,
+ IN_SERVICE, QUASI_CLOSED, 10, 3);
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin2,
+ IN_SERVICE, QUASI_CLOSED, 5, 3);
+
+ QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1, 3, 2);
assertFalse(replicaCount.isUnderReplicated());
assertTrue(replicaCount.isOverReplicated());
validateMisReplicatedOrigins(replicaCount.getOverReplicatedOrigins(), 1,
3, 1, origin2);
@@ -178,7 +205,7 @@ public void testOverReplicationWithOneOrigin() {
Pair.of(origin1, IN_SERVICE), Pair.of(origin1, IN_SERVICE),
Pair.of(origin1, IN_SERVICE),
Pair.of(origin1, IN_SERVICE));
- QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1);
+ QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1, 3, 2);
assertFalse(replicaCount.isUnderReplicated());
assertTrue(replicaCount.isOverReplicated());
validateMisReplicatedOrigins(replicaCount.getOverReplicatedOrigins(), 1,
4, 1, origin1);
@@ -186,29 +213,38 @@ public void testOverReplicationWithOneOrigin() {
@Test
public void testUnderReplicationDueToDecommissionWithThreeOrigins() {
- Set<ContainerReplica> replicas =
ReplicationTestUtil.createReplicasWithOriginAndOpState(
- ContainerID.valueOf(1), QUASI_CLOSED,
- Pair.of(origin1, DECOMMISSIONING), Pair.of(origin1, DECOMMISSIONING),
- Pair.of(origin2, IN_SERVICE), Pair.of(origin2, IN_SERVICE),
- Pair.of(origin3, IN_SERVICE), Pair.of(origin3, IN_SERVICE));
-
- QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1);
+ // origin1 is best (BCSID=10) with 2 decommissioning copies and 0
in-service, so it is under-replicated by 3.
+ // origin2 and origin3 are other (BCSID=5) with 2 in-service copies each
at target 2.
+ Set<ContainerReplica> replicas = new HashSet<>();
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin1,
+ DECOMMISSIONING, QUASI_CLOSED, 10, 2);
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin2,
+ IN_SERVICE, QUASI_CLOSED, 5, 2);
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin3,
+ IN_SERVICE, QUASI_CLOSED, 5, 2);
+
+ QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1, 3, 2);
assertTrue(replicaCount.isUnderReplicated());
assertFalse(replicaCount.isOverReplicated());
- validateMisReplicatedOrigins(replicaCount.getUnderReplicatedReplicas(), 1,
2, 2, origin1);
+ validateMisReplicatedOrigins(replicaCount.getUnderReplicatedReplicas(), 1,
2, 3, origin1);
}
@Test
public void testUnderReplicationDueToDecommissionWithTwoOrigins() {
- Set<ContainerReplica> replicas =
ReplicationTestUtil.createReplicasWithOriginAndOpState(
- ContainerID.valueOf(1), QUASI_CLOSED,
- Pair.of(origin1, IN_SERVICE), Pair.of(origin1, DECOMMISSIONING),
- Pair.of(origin2, IN_SERVICE), Pair.of(origin2, IN_SERVICE));
-
- QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1);
+ // origin1 is best (BCSID=10) with 1 in-service and 1 decommissioning
copy; it needs 2 more to reach target 3.
+ // origin2 is other (BCSID=5) with 2 in-service copies at target 2.
+ Set<ContainerReplica> replicas = new HashSet<>();
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(ContainerID.valueOf(1),
origin1,
+ IN_SERVICE, QUASI_CLOSED, 10));
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(ContainerID.valueOf(1),
origin1,
+ DECOMMISSIONING, QUASI_CLOSED, 10));
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin2,
+ IN_SERVICE, QUASI_CLOSED, 5, 2);
+
+ QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1, 3, 2);
assertTrue(replicaCount.isUnderReplicated());
assertFalse(replicaCount.isOverReplicated());
- validateMisReplicatedOrigins(replicaCount.getUnderReplicatedReplicas(), 1,
2, 1, origin1);
+ validateMisReplicatedOrigins(replicaCount.getUnderReplicatedReplicas(), 1,
2, 2, origin1);
}
@Test
@@ -217,7 +253,7 @@ public void
testUnderReplicationDueToDecommissionWithOneOrigin() {
ContainerID.valueOf(1), QUASI_CLOSED,
Pair.of(origin1, IN_SERVICE), Pair.of(origin1, DECOMMISSIONING),
Pair.of(origin1, DECOMMISSIONING));
- QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1);
+ QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1, 3, 2);
assertTrue(replicaCount.isUnderReplicated());
assertFalse(replicaCount.isOverReplicated());
validateMisReplicatedOrigins(replicaCount.getUnderReplicatedReplicas(), 1,
3, 2, origin1);
@@ -225,25 +261,37 @@ public void
testUnderReplicationDueToDecommissionWithOneOrigin() {
@Test
public void testNoOverReplicationWithOutOfServiceReplicasWithThreeOrigins() {
- Set<ContainerReplica> replicas =
ReplicationTestUtil.createReplicasWithOriginAndOpState(
- ContainerID.valueOf(1), QUASI_CLOSED,
- Pair.of(origin1, IN_SERVICE), Pair.of(origin1, IN_SERVICE),
Pair.of(origin1, DECOMMISSIONED),
- Pair.of(origin2, IN_SERVICE), Pair.of(origin2, IN_SERVICE),
- Pair.of(origin3, IN_SERVICE), Pair.of(origin3, IN_SERVICE));
-
- QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1);
+ // origin1 is best (BCSID=10) with 3 in-service copies at target plus 1
decommissioned.
+ // The decommissioned replica is out-of-service and must not be counted
toward over-replication.
+ // origin2 and origin3 are other (BCSID=5) with 2 in-service copies each
at target 2.
+ Set<ContainerReplica> replicas = new HashSet<>();
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin1,
+ IN_SERVICE, QUASI_CLOSED, 10, 3);
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(ContainerID.valueOf(1),
origin1,
+ DECOMMISSIONED, QUASI_CLOSED, 10));
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin2,
+ IN_SERVICE, QUASI_CLOSED, 5, 2);
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin3,
+ IN_SERVICE, QUASI_CLOSED, 5, 2);
+
+ QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1, 3, 2);
assertFalse(replicaCount.isUnderReplicated());
assertFalse(replicaCount.isOverReplicated());
}
@Test
public void testNoOverReplicationWithOutOfServiceReplicasWithTwoOrigins() {
- Set<ContainerReplica> replicas =
ReplicationTestUtil.createReplicasWithOriginAndOpState(
- ContainerID.valueOf(1), QUASI_CLOSED,
- Pair.of(origin1, IN_SERVICE), Pair.of(origin1, IN_SERVICE),
Pair.of(origin1, DECOMMISSIONED),
- Pair.of(origin2, IN_SERVICE), Pair.of(origin2, IN_SERVICE));
-
- QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1);
+ // origin1 is best (BCSID=10) with 3 in-service copies at target plus 1
decommissioned.
+ // origin2 is other (BCSID=5) with 2 in-service copies at target 2.
+ Set<ContainerReplica> replicas = new HashSet<>();
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin1,
+ IN_SERVICE, QUASI_CLOSED, 10, 3);
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(ContainerID.valueOf(1),
origin1,
+ DECOMMISSIONED, QUASI_CLOSED, 10));
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin2,
+ IN_SERVICE, QUASI_CLOSED, 5, 2);
+
+ QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1, 3, 2);
assertFalse(replicaCount.isUnderReplicated());
assertFalse(replicaCount.isOverReplicated());
}
@@ -255,7 +303,7 @@ public void
testNoOverReplicationWithOutOfServiceReplicasWithOneOrigin() {
Pair.of(origin1, IN_SERVICE), Pair.of(origin1, IN_SERVICE),
Pair.of(origin1, IN_SERVICE),
Pair.of(origin1, DECOMMISSIONED));
- QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1);
+ QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1, 3, 2);
assertFalse(replicaCount.isUnderReplicated());
assertFalse(replicaCount.isOverReplicated());
}
@@ -266,7 +314,7 @@ public void
testUnderReplicationWithMaintenanceWithOneOrigin() {
ContainerID.valueOf(1), QUASI_CLOSED,
Pair.of(origin1, IN_SERVICE), Pair.of(origin1, IN_SERVICE),
Pair.of(origin1, ENTERING_MAINTENANCE));
- QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1);
+ QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1, 3, 2);
assertFalse(replicaCount.isUnderReplicated());
assertFalse(replicaCount.isOverReplicated());
@@ -274,7 +322,7 @@ public void
testUnderReplicationWithMaintenanceWithOneOrigin() {
ContainerID.valueOf(1), QUASI_CLOSED,
Pair.of(origin1, IN_SERVICE), Pair.of(origin1, ENTERING_MAINTENANCE),
Pair.of(origin1, ENTERING_MAINTENANCE));
- replicaCount = new QuasiClosedStuckReplicaCount(replicas, 2);
+ replicaCount = new QuasiClosedStuckReplicaCount(replicas, 2, 3, 2);
assertTrue(replicaCount.isUnderReplicated());
assertFalse(replicaCount.isOverReplicated());
validateMisReplicatedOrigins(replicaCount.getUnderReplicatedReplicas(), 1,
3, 1, origin1);
@@ -282,21 +330,30 @@ public void
testUnderReplicationWithMaintenanceWithOneOrigin() {
@Test
public void testUnderReplicationWithMaintenanceWithTwoOrigins() {
- Set<ContainerReplica> replicas =
ReplicationTestUtil.createReplicasWithOriginAndOpState(
- ContainerID.valueOf(1), QUASI_CLOSED,
- Pair.of(origin1, IN_SERVICE), Pair.of(origin1, ENTERING_MAINTENANCE),
- Pair.of(origin2, IN_SERVICE), Pair.of(origin2, IN_SERVICE));
-
- QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1);
+ // origin1 is best (BCSID=10) with 1 in-service and 1 entering-maintenance
copy.
+ // Because origin1 has at least 1 in-service replica while under
maintenance, it is not under-replicated.
+ // origin2 is other (BCSID=5) with 2 in-service copies at target 2.
+ Set<ContainerReplica> replicas = new HashSet<>();
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(ContainerID.valueOf(1),
origin1,
+ IN_SERVICE, QUASI_CLOSED, 10));
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(ContainerID.valueOf(1),
origin1,
+ ENTERING_MAINTENANCE, QUASI_CLOSED, 10));
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin2,
+ IN_SERVICE, QUASI_CLOSED, 5, 2);
+
+ QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1, 3, 2);
assertFalse(replicaCount.isUnderReplicated());
assertFalse(replicaCount.isOverReplicated());
- replicas = ReplicationTestUtil.createReplicasWithOriginAndOpState(
- ContainerID.valueOf(1), QUASI_CLOSED,
- Pair.of(origin1, ENTERING_MAINTENANCE), Pair.of(origin1,
ENTERING_MAINTENANCE),
- Pair.of(origin2, IN_SERVICE), Pair.of(origin2, IN_SERVICE));
+ // When all of origin1's replicas are entering-maintenance it has no
in-service copies,
+ // so it is under-replicated by 1 (the maintenance path requires at least
1 online copy).
+ replicas = new HashSet<>();
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin1,
+ ENTERING_MAINTENANCE, QUASI_CLOSED, 10, 2);
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin2,
+ IN_SERVICE, QUASI_CLOSED, 5, 2);
- replicaCount = new QuasiClosedStuckReplicaCount(replicas, 1);
+ replicaCount = new QuasiClosedStuckReplicaCount(replicas, 1, 3, 2);
assertTrue(replicaCount.isUnderReplicated());
assertFalse(replicaCount.isOverReplicated());
validateMisReplicatedOrigins(replicaCount.getUnderReplicatedReplicas(), 1,
2, 1, origin1);
@@ -304,12 +361,18 @@ public void
testUnderReplicationWithMaintenanceWithTwoOrigins() {
@Test
public void testNoOverReplicationWithExcessMaintenanceReplicasTwoOrigins() {
- Set<ContainerReplica> replicas =
ReplicationTestUtil.createReplicasWithOriginAndOpState(
- ContainerID.valueOf(1), QUASI_CLOSED,
- Pair.of(origin1, IN_SERVICE), Pair.of(origin1, IN_SERVICE),
Pair.of(origin1, IN_MAINTENANCE),
- Pair.of(origin2, IN_SERVICE), Pair.of(origin2, IN_SERVICE));
-
- QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1);
+ // origin1 is best (BCSID=10) with 2 in-service copies and 1
in-maintenance copy.
+ // The maintenance replica is excluded from the over-replication check.
+ // origin2 is other (BCSID=5) with 2 in-service copies at target 2.
+ Set<ContainerReplica> replicas = new HashSet<>();
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin1,
+ IN_SERVICE, QUASI_CLOSED, 10, 2);
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(ContainerID.valueOf(1),
origin1,
+ IN_MAINTENANCE, QUASI_CLOSED, 10));
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin2,
+ IN_SERVICE, QUASI_CLOSED, 5, 2);
+
+ QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1, 3, 2);
assertFalse(replicaCount.isUnderReplicated());
assertFalse(replicaCount.isOverReplicated());
}
@@ -321,11 +384,50 @@ public void
testNoOverReplicationWithExcessMaintenanceReplicasOneOrigin() {
Pair.of(origin1, IN_SERVICE), Pair.of(origin1, IN_SERVICE),
Pair.of(origin1, IN_SERVICE),
Pair.of(origin1, IN_MAINTENANCE));
- QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1);
+ QuasiClosedStuckReplicaCount replicaCount = new
QuasiClosedStuckReplicaCount(replicas, 1, 3, 2);
assertFalse(replicaCount.isUnderReplicated());
assertFalse(replicaCount.isOverReplicated());
}
+ @Test
+ public void testShiftInBestOriginOnNewHigherBcsid() {
+ // Initial state: origin1 is best (BCSID=10, target=3) with 3 copies;
+ // origin2 is other (BCSID=5, target=2) with 2 copies. Both are at target.
+ Set<ContainerReplica> replicas = new HashSet<>();
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin1,
+ IN_SERVICE, QUASI_CLOSED, 10, 3);
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(replicas,
ContainerID.valueOf(1), origin2,
+ IN_SERVICE, QUASI_CLOSED, 5, 2);
+
+ QuasiClosedStuckReplicaCount initialCount = new
QuasiClosedStuckReplicaCount(replicas, 1, 3, 2);
+ assertFalse(initialCount.isUnderReplicated());
+ assertFalse(initialCount.isOverReplicated());
+
+ // A new replica arrives from origin3 with BCSID=20, higher than the
previous best of 10.
+ // origin3 is now the best origin (BCSID=20, target=3) but has only 1
copy: under-replicated by 2.
+ // origin1 is now an other origin (BCSID=10, target=2) but still has 3
copies: over-replicated by 1.
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(ContainerID.valueOf(1),
origin3,
+ IN_SERVICE, QUASI_CLOSED, 20));
+
+ QuasiClosedStuckReplicaCount updatedCount = new
QuasiClosedStuckReplicaCount(replicas, 1, 3, 2);
+
+ // The new best origin (origin3, BCSID=20) is under-replicated and must
receive more copies.
+ assertTrue(updatedCount.isUnderReplicated());
+ List<QuasiClosedStuckReplicaCount.MisReplicatedOrigin> underRep =
updatedCount.getUnderReplicatedReplicas();
+ assertEquals(1, underRep.size());
+ assertEquals(2, underRep.get(0).getReplicaDelta());
+ assertTrue(underRep.get(0).getSources().stream()
+ .allMatch(r -> r.getOriginDatanodeId().equals(origin3)));
+
+ // The old best origin (origin1, BCSID=10) is now over-replicated and one
copy must be deleted.
+ assertTrue(updatedCount.isOverReplicated());
+ List<QuasiClosedStuckReplicaCount.MisReplicatedOrigin> overRep =
updatedCount.getOverReplicatedOrigins();
+ assertEquals(1, overRep.size());
+ assertEquals(1, overRep.get(0).getReplicaDelta());
+ assertTrue(overRep.get(0).getSources().stream()
+ .allMatch(r -> r.getOriginDatanodeId().equals(origin1)));
+ }
+
private void validateMisReplicatedOrigins(
List<QuasiClosedStuckReplicaCount.MisReplicatedOrigin>
misReplicatedOrigins,
int expectedUnderRepOrigins, int expectedSources, int expectedDelta,
DatanodeID expectedOrigin) {
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestQuasiClosedStuckUnderReplicationHandler.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestQuasiClosedStuckUnderReplicationHandler.java
index ff7853f8c8d..0ae73131856 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestQuasiClosedStuckUnderReplicationHandler.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestQuasiClosedStuckUnderReplicationHandler.java
@@ -17,7 +17,9 @@
package org.apache.hadoop.hdds.scm.container.replication;
+import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE;
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE;
+import static
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.QUASI_CLOSED;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
@@ -39,7 +41,6 @@
import org.apache.hadoop.hdds.protocol.DatanodeID;
import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
import org.apache.hadoop.hdds.scm.container.ContainerReplica;
@@ -115,10 +116,7 @@ void setup(@TempDir File testDir) throws
NodeNotFoundException,
public void testReturnsZeroIfNotUnderReplicated() throws IOException {
final DatanodeID origin = DatanodeID.randomID();
Set<ContainerReplica> replicas =
ReplicationTestUtil.createReplicasWithOriginAndOpState(container.containerID(),
-
StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.QUASI_CLOSED,
- Pair.of(origin, HddsProtos.NodeOperationalState.IN_SERVICE),
- Pair.of(origin, HddsProtos.NodeOperationalState.IN_SERVICE),
- Pair.of(origin, HddsProtos.NodeOperationalState.IN_SERVICE));
+ QUASI_CLOSED, Pair.of(origin, IN_SERVICE), Pair.of(origin,
IN_SERVICE), Pair.of(origin, IN_SERVICE));
int count = handler.processAndSendCommands(replicas,
Collections.emptyList(), getUnderReplicatedHealthResult(), 1);
assertEquals(0, count);
@@ -128,9 +126,7 @@ public void testReturnsZeroIfNotUnderReplicated() throws
IOException {
public void testNoCommandsScheduledIfPendingOps() throws IOException {
final DatanodeID origin = DatanodeID.randomID();
Set<ContainerReplica> replicas =
ReplicationTestUtil.createReplicasWithOriginAndOpState(container.containerID(),
-
StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.QUASI_CLOSED,
- Pair.of(origin, HddsProtos.NodeOperationalState.IN_SERVICE),
- Pair.of(origin, HddsProtos.NodeOperationalState.IN_SERVICE));
+ QUASI_CLOSED, Pair.of(origin, IN_SERVICE), Pair.of(origin,
IN_SERVICE));
List<ContainerReplicaOp> pendingOps = new ArrayList<>();
pendingOps.add(new ContainerReplicaOp(
ContainerReplicaOp.PendingOpType.ADD,
MockDatanodeDetails.randomDatanodeDetails(), 0, null, Long.MAX_VALUE, 0));
@@ -143,8 +139,7 @@ public void testNoCommandsScheduledIfPendingOps() throws
IOException {
public void testCommandScheduledForUnderReplicatedContainer() throws
IOException {
final DatanodeID origin = DatanodeID.randomID();
Set<ContainerReplica> replicas =
ReplicationTestUtil.createReplicasWithOriginAndOpState(container.containerID(),
-
StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.QUASI_CLOSED,
- Pair.of(origin, HddsProtos.NodeOperationalState.IN_SERVICE));
+ QUASI_CLOSED, Pair.of(origin, IN_SERVICE));
int count = handler.processAndSendCommands(replicas,
Collections.emptyList(), getUnderReplicatedHealthResult(), 1);
assertEquals(2, count);
@@ -155,10 +150,13 @@ public void
testCommandScheduledForUnderReplicatedContainer() throws IOException
public void testOverloadedExceptionContinuesAndThrows() throws
NotLeaderException, CommandTargetOverloadedException {
final DatanodeID origin1 = DatanodeID.randomID();
final DatanodeID origin2 = DatanodeID.randomID();
- Set<ContainerReplica> replicas =
ReplicationTestUtil.createReplicasWithOriginAndOpState(container.containerID(),
-
StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.QUASI_CLOSED,
- Pair.of(origin1, HddsProtos.NodeOperationalState.IN_SERVICE),
- Pair.of(origin2, HddsProtos.NodeOperationalState.IN_SERVICE));
+ Set<ContainerReplica> replicas = new HashSet<>();
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(container.containerID(),
+ origin1, IN_SERVICE, QUASI_CLOSED, 10));
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(container.containerID(),
+ origin1, IN_SERVICE, QUASI_CLOSED, 10));
+
replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(container.containerID(),
+ origin2, IN_SERVICE, QUASI_CLOSED, 5));
ReplicationTestUtil.mockRMSendThrottleReplicateCommand(replicationManager,
commandsSent, new AtomicBoolean(true));
@@ -172,9 +170,7 @@ public void testInsufficientNodesExceptionThrown() {
final DatanodeID origin1 = DatanodeID.randomID();
final DatanodeID origin2 = DatanodeID.randomID();
Set<ContainerReplica> replicas =
ReplicationTestUtil.createReplicasWithOriginAndOpState(container.containerID(),
-
StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.QUASI_CLOSED,
- Pair.of(origin1, HddsProtos.NodeOperationalState.IN_SERVICE),
- Pair.of(origin2, HddsProtos.NodeOperationalState.IN_SERVICE));
+ QUASI_CLOSED, Pair.of(origin1, IN_SERVICE), Pair.of(origin2,
IN_SERVICE));
PlacementPolicy policy =
ReplicationTestUtil.getNoNodesTestPlacementPolicy(nodeManager, conf);
handler = new QuasiClosedStuckUnderReplicationHandler(policy, conf,
replicationManager);
@@ -188,8 +184,7 @@ public void testInsufficientNodesExceptionThrown() {
public void testPartialReplicationExceptionThrown() {
final DatanodeID origin1 = DatanodeID.randomID();
Set<ContainerReplica> replicas =
ReplicationTestUtil.createReplicasWithOriginAndOpState(container.containerID(),
-
StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.QUASI_CLOSED,
- Pair.of(origin1, HddsProtos.NodeOperationalState.IN_SERVICE));
+ QUASI_CLOSED, Pair.of(origin1, IN_SERVICE));
PlacementPolicy policy =
ReplicationTestUtil.getInsufficientNodesTestPlacementPolicy(nodeManager, conf,
2);
handler = new QuasiClosedStuckUnderReplicationHandler(policy, conf,
replicationManager);
@@ -199,6 +194,34 @@ public void testPartialReplicationExceptionThrown() {
assertEquals(1, commandsSent.size());
}
+ @Test
+ public void testOriginCopies() throws IOException {
+ final DatanodeID origin1 = DatanodeID.randomID();
+ final DatanodeID origin2 = DatanodeID.randomID();
+
+ // Configure default copy counts: best origin gets 3 copies, other origins
get 2 copies.
+ ReplicationManager.ReplicationManagerConfiguration rmConf =
+ new
OzoneConfiguration().getObject(ReplicationManager.ReplicationManagerConfiguration.class);
+ rmConf.setQuasiClosedStuckBestOriginCopies(3);
+ rmConf.setQuasiClosedStuckOtherOriginCopies(2);
+ when(replicationManager.getConfig()).thenReturn(rmConf);
+
+ // origin1 is the best origin (BCSID=10, target=3): has 2 copies, needs 1
more.
+ // origin2 is an other origin (BCSID=5, target=2): has 2 copies, already
at target.
+ Set<ContainerReplica> replicas = new HashSet<>();
+ replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(
+ container.containerID(), origin1, IN_SERVICE, QUASI_CLOSED, 10));
+ replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(
+ container.containerID(), origin1, IN_SERVICE, QUASI_CLOSED, 10));
+ replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(
+ container.containerID(), origin2, IN_SERVICE, QUASI_CLOSED, 5));
+ replicas.add(ReplicationTestUtil.createReplicaWithOriginAndSeqId(
+ container.containerID(), origin2, IN_SERVICE, QUASI_CLOSED, 5));
+
+ int count = handler.processAndSendCommands(replicas,
Collections.emptyList(), getUnderReplicatedHealthResult(), 1);
+ assertEquals(1, count);
+ }
+
private ContainerHealthResult.UnderReplicatedHealthResult
getUnderReplicatedHealthResult() {
ContainerHealthResult.UnderReplicatedHealthResult
healthResult =
mock(ContainerHealthResult.UnderReplicatedHealthResult.class);
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestQuasiClosedStuckReplicationCheck.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestQuasiClosedStuckReplicationCheck.java
index c11450a79c3..1d8738787c4 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestQuasiClosedStuckReplicationCheck.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestQuasiClosedStuckReplicationCheck.java
@@ -24,7 +24,6 @@
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.mockito.Mockito.mock;
import java.util.ArrayList;
import java.util.Collections;
@@ -33,6 +32,7 @@
import java.util.Set;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.client.RatisReplicationConfig;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeID;
import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State;
@@ -64,8 +64,8 @@ public class TestQuasiClosedStuckReplicationCheck {
@BeforeEach
public void setup() {
- rmConf = mock(ReplicationManager.ReplicationManagerConfiguration.class);
- handler = new QuasiClosedStuckReplicationCheck();
+ rmConf = new
OzoneConfiguration().getObject(ReplicationManager.ReplicationManagerConfiguration.class);
+ handler = new QuasiClosedStuckReplicationCheck(rmConf);
report = new ReplicationManagerReport(rmConf.getContainerSampleLimit());
report.resetContainerHealthState(); // Reset before each test
queue = new ReplicationQueue();
@@ -148,10 +148,14 @@ public void testCorrectlyReplicated() {
ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo(
RatisReplicationConfig.getInstance(THREE), 1, QUASI_CLOSED);
- Set<ContainerReplica> containerReplicas = ReplicationTestUtil
- .createReplicasWithOriginAndOpState(containerInfo.containerID(),
State.QUASI_CLOSED,
- Pair.of(origin1, IN_SERVICE), Pair.of(origin1, IN_SERVICE),
- Pair.of(origin2, IN_SERVICE), Pair.of(origin2, IN_SERVICE));
+ // origin1 is the best origin (seqId=10, target=3): has 3 copies, at
target.
+ // origin2 is an other origin (seqId=5, target=2): has 2 copies, at target.
+ Set<ContainerReplica> containerReplicas = new HashSet<>();
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(containerReplicas,
containerInfo.containerID(),
+ origin1, IN_SERVICE, State.QUASI_CLOSED, 10, 3);
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(containerReplicas,
containerInfo.containerID(),
+ origin2, IN_SERVICE, State.QUASI_CLOSED, 5, 2);
+
ContainerCheckRequest request = new ContainerCheckRequest.Builder()
.setPendingOps(Collections.emptyList())
.setReport(report)
@@ -246,10 +250,13 @@ public void testOverReplicatedIsQueued() {
ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo(
RatisReplicationConfig.getInstance(THREE), 1, QUASI_CLOSED);
- Set<ContainerReplica> containerReplicas = ReplicationTestUtil
- .createReplicasWithOriginAndOpState(containerInfo.containerID(),
State.QUASI_CLOSED,
- Pair.of(origin1, IN_SERVICE), Pair.of(origin1, IN_SERVICE),
Pair.of(origin1, IN_SERVICE),
- Pair.of(origin2, IN_SERVICE), Pair.of(origin2, IN_SERVICE));
+ // origin1 is the best origin (seqId=10, target=3): has 3 copies, at
target.
+ // origin2 is an other origin (seqId=5, target=2): has 3 copies,
over-replicated by 1.
+ Set<ContainerReplica> containerReplicas = new HashSet<>();
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(containerReplicas,
containerInfo.containerID(),
+ origin1, IN_SERVICE, State.QUASI_CLOSED, 10, 3);
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(containerReplicas,
containerInfo.containerID(),
+ origin2, IN_SERVICE, State.QUASI_CLOSED, 5, 3);
ContainerCheckRequest request = new ContainerCheckRequest.Builder()
.setPendingOps(Collections.emptyList())
@@ -273,10 +280,13 @@ public void
testOverReplicatedWithPendingDeleteIsNotQueued() {
ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo(
RatisReplicationConfig.getInstance(THREE), 1, QUASI_CLOSED);
- Set<ContainerReplica> containerReplicas = ReplicationTestUtil
- .createReplicasWithOriginAndOpState(containerInfo.containerID(),
State.QUASI_CLOSED,
- Pair.of(origin1, IN_SERVICE), Pair.of(origin1, IN_SERVICE),
- Pair.of(origin2, IN_SERVICE), Pair.of(origin2, IN_SERVICE),
Pair.of(origin2, IN_SERVICE));
+ // origin1 is the best origin (seqId=10, target=3): has 3 copies, at
target.
+ // origin2 is an other origin (seqId=5, target=2): has 3 copies,
over-replicated by 1.
+ Set<ContainerReplica> containerReplicas = new HashSet<>();
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(containerReplicas,
containerInfo.containerID(),
+ origin1, IN_SERVICE, State.QUASI_CLOSED, 10, 3);
+ ReplicationTestUtil.addReplicasWithOriginAndSeqId(containerReplicas,
containerInfo.containerID(),
+ origin2, IN_SERVICE, State.QUASI_CLOSED, 5, 3);
List<ContainerReplicaOp> pendingOps = new ArrayList<>();
pendingOps.add(new ContainerReplicaOp(
diff --git
a/hadoop-hdds/server-scm/src/test/resources/replicationManagerTests/quasi_closed.json
b/hadoop-hdds/server-scm/src/test/resources/replicationManagerTests/quasi_closed.json
index 6d229d652c8..2137b523754 100644
---
a/hadoop-hdds/server-scm/src/test/resources/replicationManagerTests/quasi_closed.json
+++
b/hadoop-hdds/server-scm/src/test/resources/replicationManagerTests/quasi_closed.json
@@ -14,11 +14,12 @@
{ "description": "Quasi-closed with 2 replicas", "containerState":
"QUASI_CLOSED", "replicationConfig": "RATIS:THREE", "sequenceId": 12,
"replicas": [
{ "state": "QUASI_CLOSED", "index": 0, "datanode": "d1", "sequenceId":
12, "isEmpty": false, "origin": "o1"},
- { "state": "QUASI_CLOSED", "index": 0, "datanode": "d2", "sequenceId":
12, "isEmpty": false, "origin": "o2"}
+ { "state": "QUASI_CLOSED", "index": 0, "datanode": "d2", "sequenceId":
10, "isEmpty": false, "origin": "o2"}
],
"expectation": { "quasiClosedStuckUnderReplicated": 1,
"underReplicatedQueue": 1, "overReplicated": 0, "overReplicatedQueue": 0,
"quasiClosedStuck": 1},
"checkCommands": [],
"commands": [
+ { "type": "replicateContainerCommand", "datanode": "d1" },
{ "type": "replicateContainerCommand", "datanode": "d1" },
{ "type": "replicateContainerCommand", "datanode": "d2" }
]
@@ -26,12 +27,13 @@
{ "description": "Quasi-closed with 3 replicas 2 origins", "containerState":
"QUASI_CLOSED", "replicationConfig": "RATIS:THREE", "sequenceId": 12,
"replicas": [
{ "state": "QUASI_CLOSED", "index": 0, "datanode": "d1", "sequenceId":
12, "isEmpty": false, "origin": "o1"},
- { "state": "QUASI_CLOSED", "index": 0, "datanode": "d2", "sequenceId":
12, "isEmpty": false, "origin": "o2"},
- { "state": "QUASI_CLOSED", "index": 0, "datanode": "d3", "sequenceId":
12, "isEmpty": false, "origin": "o2"}
+ { "state": "QUASI_CLOSED", "index": 0, "datanode": "d2", "sequenceId":
10, "isEmpty": false, "origin": "o2"},
+ { "state": "QUASI_CLOSED", "index": 0, "datanode": "d3", "sequenceId":
10, "isEmpty": false, "origin": "o2"}
],
"expectation": { "quasiClosedStuckUnderReplicated": 1,
"underReplicatedQueue": 1, "overReplicated": 0, "overReplicatedQueue": 0,
"quasiClosedStuck": 1, "unhealthy": 0 },
"checkCommands": [],
"commands": [
+ { "type": "replicateContainerCommand", "datanode": "d1" },
{ "type": "replicateContainerCommand", "datanode": "d1" }
]
},
@@ -95,12 +97,13 @@
{ "description": "Quasi-Closed with 2 replicas and unhealthy where unhealthy
is highest BCSID", "containerState": "QUASI_CLOSED", "replicationConfig":
"RATIS:THREE", "sequenceId": 11,
"replicas": [
{ "state": "QUASI_CLOSED", "index": 0, "datanode": "d1", "sequenceId":
10, "isEmpty": false, "origin": "o1"},
- { "state": "QUASI_CLOSED", "index": 0, "datanode": "d2", "sequenceId":
10, "isEmpty": false, "origin": "o2"},
+ { "state": "QUASI_CLOSED", "index": 0, "datanode": "d2", "sequenceId":
8, "isEmpty": false, "origin": "o2"},
{ "state": "UNHEALTHY", "index": 0, "datanode": "d3", "sequenceId":
11, "isEmpty": false, "origin": "o3"}
],
"expectation": { "quasiClosedStuckUnderReplicated": 1,
"underReplicatedQueue": 1, "overReplicated": 0, "overReplicatedQueue": 0,
"quasiClosedStuck": 1, "unhealthy": 0 },
"checkCommands": [],
"commands": [
+ { "type": "replicateContainerCommand", "datanode": "d1" },
{ "type": "replicateContainerCommand", "datanode": "d1" },
{ "type": "replicateContainerCommand", "datanode": "d2" },
{ "type": "replicateContainerCommand", "datanode": "d3" }
@@ -120,8 +123,9 @@
"replicas": [
{ "state": "QUASI_CLOSED", "index": 0, "datanode": "d1", "sequenceId":
10, "isEmpty": false, "origin": "o1"},
{ "state": "QUASI_CLOSED", "index": 0, "datanode": "d2", "sequenceId":
10, "isEmpty": false, "origin": "o1"},
- { "state": "QUASI_CLOSED", "index": 0, "datanode": "d3", "sequenceId":
10, "isEmpty": false, "origin": "o2"},
- { "state": "QUASI_CLOSED", "index": 0, "datanode": "d4", "sequenceId":
10, "isEmpty": false, "origin": "o2"}
+ { "state": "QUASI_CLOSED", "index": 0, "datanode": "d5", "sequenceId":
10, "isEmpty": false, "origin": "o1"},
+ { "state": "QUASI_CLOSED", "index": 0, "datanode": "d3", "sequenceId":
8, "isEmpty": false, "origin": "o2"},
+ { "state": "QUASI_CLOSED", "index": 0, "datanode": "d4", "sequenceId":
8, "isEmpty": false, "origin": "o2"}
],
"expectation": { "underReplicated": 0, "underReplicatedQueue": 0,
"overReplicated": 0, "overReplicatedQueue": 0, "quasiClosedStuck": 1,
"unhealthy": 0 },
"checkCommands": [],
@@ -131,8 +135,9 @@
"replicas": [
{ "state": "QUASI_CLOSED", "index": 0, "datanode": "d1", "sequenceId":
10, "isEmpty": false, "origin": "o1"},
{ "state": "QUASI_CLOSED", "index": 0, "datanode": "d2", "sequenceId":
10, "isEmpty": false, "origin": "o1"},
- { "state": "QUASI_CLOSED", "index": 0, "datanode": "d3", "sequenceId":
10, "isEmpty": false, "origin": "o2"},
- { "state": "QUASI_CLOSED", "index": 0, "datanode": "d4", "sequenceId":
10, "isEmpty": false, "origin": "o2"},
+ { "state": "QUASI_CLOSED", "index": 0, "datanode": "d7", "sequenceId":
10, "isEmpty": false, "origin": "o1"},
+ { "state": "QUASI_CLOSED", "index": 0, "datanode": "d3", "sequenceId":
8, "isEmpty": false, "origin": "o2"},
+ { "state": "QUASI_CLOSED", "index": 0, "datanode": "d4", "sequenceId":
8, "isEmpty": false, "origin": "o2"},
{ "state": "UNHEALTHY", "index": 0, "datanode": "d5", "sequenceId":
11, "isEmpty": false, "origin": "o3"},
{ "state": "UNHEALTHY", "index": 0, "datanode": "d6", "sequenceId":
11, "isEmpty": false, "origin": "o3"}
],
@@ -143,13 +148,14 @@
{ "description": "Quasi-Closed with 3 QC and one unhealthy",
"containerState": "QUASI_CLOSED", "replicationConfig": "RATIS:THREE",
"sequenceId": 11,
"replicas": [
{ "state": "QUASI_CLOSED", "index": 0, "datanode": "d1", "sequenceId":
10, "isEmpty": false, "origin": "o1"},
- { "state": "QUASI_CLOSED", "index": 0, "datanode": "d2", "sequenceId":
10, "isEmpty": false, "origin": "o2"},
- { "state": "QUASI_CLOSED", "index": 0, "datanode": "d3", "sequenceId":
10, "isEmpty": false, "origin": "o2"},
+ { "state": "QUASI_CLOSED", "index": 0, "datanode": "d2", "sequenceId":
8, "isEmpty": false, "origin": "o2"},
+ { "state": "QUASI_CLOSED", "index": 0, "datanode": "d3", "sequenceId":
8, "isEmpty": false, "origin": "o2"},
{ "state": "UNHEALTHY", "index": 0, "datanode": "d4", "sequenceId":
11, "isEmpty": false, "origin": "o3"}
],
"expectation": { "quasiClosedStuckUnderReplicated": 1,
"underReplicatedQueue": 1, "overReplicated": 0, "overReplicatedQueue": 0,
"quasiClosedStuck": 1, "unhealthy": 0 },
"checkCommands": [],
"commands": [
+ { "type": "replicateContainerCommand", "datanode": "d1" },
{ "type": "replicateContainerCommand", "datanode": "d1" },
{ "type": "replicateContainerCommand", "datanode": "d4" }
]
@@ -199,14 +205,13 @@
{ "state": "QUASI_CLOSED", "index": 0, "datanode": "d1", "sequenceId":
10, "isEmpty": false, "origin": "o1"},
{ "state": "QUASI_CLOSED", "index": 0, "datanode": "d2", "sequenceId":
10, "isEmpty": false, "origin": "o1"},
{ "state": "QUASI_CLOSED", "index": 0, "datanode": "d3", "sequenceId":
10, "isEmpty": false, "origin": "o1"},
- { "state": "QUASI_CLOSED", "index": 0, "datanode": "d4", "sequenceId":
10, "isEmpty": false, "origin": "o2"},
- { "state": "QUASI_CLOSED", "index": 0, "datanode": "d5", "sequenceId":
10, "isEmpty": false, "origin": "o2"},
- { "state": "QUASI_CLOSED", "index": 0, "datanode": "d6", "sequenceId":
10, "isEmpty": false, "origin": "o2"}
+ { "state": "QUASI_CLOSED", "index": 0, "datanode": "d4", "sequenceId":
8, "isEmpty": false, "origin": "o2"},
+ { "state": "QUASI_CLOSED", "index": 0, "datanode": "d5", "sequenceId":
8, "isEmpty": false, "origin": "o2"},
+ { "state": "QUASI_CLOSED", "index": 0, "datanode": "d6", "sequenceId":
8, "isEmpty": false, "origin": "o2"}
],
"expectation": { "underReplicated": 0, "underReplicatedQueue": 0,
"overReplicated": 0, "quasiClosedStuckOverReplicated": 1,
"overReplicatedQueue": 1, "quasiClosedStuck": 1, "unhealthy": 0 },
"checkCommands": [],
"commands": [
- { "type": "deleteContainerCommand", "datanode": "d1|d2|d3" },
{ "type": "deleteContainerCommand", "datanode": "d4|d5|d6" }
]
},
@@ -227,10 +232,11 @@
"replicas": [
{ "state": "QUASI_CLOSED", "index": 0, "datanode": "d1", "sequenceId":
10, "isEmpty": false, "origin": "o1"},
{ "state": "QUASI_CLOSED", "index": 0, "datanode": "d2", "sequenceId":
10, "isEmpty": false, "origin": "o1"},
+ { "state": "QUASI_CLOSED", "index": 0, "datanode": "d7", "sequenceId":
10, "isEmpty": false, "origin": "o1"},
{ "state": "QUASI_CLOSED", "index": 0, "datanode": "d3", "sequenceId":
10, "isEmpty": false, "origin": "o1", "operationalState": "DECOMMISSIONED" },
- { "state": "QUASI_CLOSED", "index": 0, "datanode": "d4", "sequenceId":
10, "isEmpty": false, "origin": "o2"},
- { "state": "QUASI_CLOSED", "index": 0, "datanode": "d5", "sequenceId":
10, "isEmpty": false, "origin": "o2"},
- { "state": "QUASI_CLOSED", "index": 0, "datanode": "d6", "sequenceId":
10, "isEmpty": false, "origin": "o2", "operationalState": "DECOMMISSIONED"}
+ { "state": "QUASI_CLOSED", "index": 0, "datanode": "d4", "sequenceId":
8, "isEmpty": false, "origin": "o2"},
+ { "state": "QUASI_CLOSED", "index": 0, "datanode": "d5", "sequenceId":
8, "isEmpty": false, "origin": "o2"},
+ { "state": "QUASI_CLOSED", "index": 0, "datanode": "d6", "sequenceId":
8, "isEmpty": false, "origin": "o2", "operationalState": "DECOMMISSIONED"}
],
"expectation": { "underReplicated": 0, "underReplicatedQueue": 0,
"overReplicated": 0, "overReplicatedQueue": 0, "quasiClosedStuck": 1,
"unhealthy": 0 },
"checkCommands": [],
@@ -241,15 +247,16 @@
{ "state": "QUASI_CLOSED", "index": 0, "datanode": "d1", "sequenceId":
10, "isEmpty": false, "origin": "o1"},
{ "state": "QUASI_CLOSED", "index": 0, "datanode": "d2", "sequenceId":
10, "isEmpty": false, "origin": "o1"},
{ "state": "QUASI_CLOSED", "index": 0, "datanode": "d3", "sequenceId":
10, "isEmpty": false, "origin": "o1"},
- { "state": "QUASI_CLOSED", "index": 0, "datanode": "d4", "sequenceId":
10, "isEmpty": false, "origin": "o1", "operationalState": "IN_MAINTENANCE" },
- { "state": "QUASI_CLOSED", "index": 0, "datanode": "d5", "sequenceId":
10, "isEmpty": false, "origin": "o2"},
- { "state": "QUASI_CLOSED", "index": 0, "datanode": "d6", "sequenceId":
10, "isEmpty": false, "origin": "o2"},
- { "state": "QUASI_CLOSED", "index": 0, "datanode": "d7", "sequenceId":
10, "isEmpty": false, "origin": "o2", "operationalState": "IN_MAINTENANCE"}
+ { "state": "QUASI_CLOSED", "index": 0, "datanode": "d4", "sequenceId":
10, "isEmpty": false, "origin": "o1"},
+ { "state": "QUASI_CLOSED", "index": 0, "datanode": "d8", "sequenceId":
10, "isEmpty": false, "origin": "o1", "operationalState": "IN_MAINTENANCE" },
+ { "state": "QUASI_CLOSED", "index": 0, "datanode": "d5", "sequenceId":
8, "isEmpty": false, "origin": "o2"},
+ { "state": "QUASI_CLOSED", "index": 0, "datanode": "d6", "sequenceId":
8, "isEmpty": false, "origin": "o2"},
+ { "state": "QUASI_CLOSED", "index": 0, "datanode": "d7", "sequenceId":
8, "isEmpty": false, "origin": "o2", "operationalState": "IN_MAINTENANCE"}
],
"expectation": { "underReplicated": 0, "underReplicatedQueue": 0,
"quasiClosedStuckOverReplicated": 1, "overReplicatedQueue": 1,
"quasiClosedStuck": 1, "unhealthy": 0 },
"checkCommands": [],
"commands": [
- { "type": "deleteContainerCommand", "datanode": "d1|d2|d3" }
+ { "type": "deleteContainerCommand", "datanode": "d1|d2|d3|d4" }
]
},
{ "description": "Quasi-Closed stuck two origins over replicated with
stale", "containerState": "QUASI_CLOSED", "replicationConfig": "RATIS:THREE",
"sequenceId": 10,
@@ -257,9 +264,9 @@
{ "state": "QUASI_CLOSED", "index": 0, "datanode": "d1", "sequenceId":
10, "isEmpty": false, "origin": "o1"},
{ "state": "QUASI_CLOSED", "index": 0, "datanode": "d2", "sequenceId":
10, "isEmpty": false, "origin": "o1"},
{ "state": "QUASI_CLOSED", "index": 0, "datanode": "d3", "sequenceId":
10, "isEmpty": false, "origin": "o1", "healthState": "STALE" },
- { "state": "QUASI_CLOSED", "index": 0, "datanode": "d4", "sequenceId":
10, "isEmpty": false, "origin": "o2"},
- { "state": "QUASI_CLOSED", "index": 0, "datanode": "d5", "sequenceId":
10, "isEmpty": false, "origin": "o2"},
- { "state": "QUASI_CLOSED", "index": 0, "datanode": "d6", "sequenceId":
10, "isEmpty": false, "origin": "o2", "healthState": "STALE" }
+ { "state": "QUASI_CLOSED", "index": 0, "datanode": "d4", "sequenceId":
8, "isEmpty": false, "origin": "o2"},
+ { "state": "QUASI_CLOSED", "index": 0, "datanode": "d5", "sequenceId":
8, "isEmpty": false, "origin": "o2"},
+ { "state": "QUASI_CLOSED", "index": 0, "datanode": "d6", "sequenceId":
8, "isEmpty": false, "origin": "o2", "healthState": "STALE" }
],
"expectation": { "underReplicated": 0, "underReplicatedQueue": 0,
"quasiClosedStuckOverReplicated": 1, "overReplicatedQueue": 1,
"quasiClosedStuck": 1, "unhealthy": 0 },
"checkCommands": [],
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
