This is an automated email from the ASF dual-hosted git repository.
siddhant pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 31d0250c45 HDDS-9340. ReplicationManager: Handle all UNHEALTHY
replicas of a CLOSING Ratis container (#5363)
31d0250c45 is described below
commit 31d0250c45c1b2160058848b0c3893f6d3d629f4
Author: Stephen O'Donnell <[email protected]>
AuthorDate: Thu Sep 28 13:09:00 2023 +0100
HDDS-9340. ReplicationManager: Handle all UNHEALTHY replicas of a CLOSING
Ratis container (#5363)
---
.../health/ClosingContainerHandler.java | 17 ++++++
.../health/TestClosingContainerHandler.java | 69 ++++++++++++++++++++++
2 files changed, 86 insertions(+)
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ClosingContainerHandler.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ClosingContainerHandler.java
index 27ef6032a0..428d12e28e 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ClosingContainerHandler.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/health/ClosingContainerHandler.java
@@ -76,13 +76,30 @@ public class ClosingContainerHandler extends AbstractCheck {
containerInfo.containerID());
}
+ boolean allUnhealthy = true;
for (ContainerReplica replica : request.getContainerReplicas()) {
if (replica.getState() != ContainerReplicaProto.State.UNHEALTHY) {
+ allUnhealthy = false;
replicationManager.sendCloseContainerReplicaCommand(
containerInfo, replica.getDatanodeDetails(), forceClose);
}
}
+ // Moving a RATIS container that has only unhealthy replicas to
QUASI_CLOSED
+ // so the unhealthy replicas will be replicated if needed.
+ if (allUnhealthy && !request.getContainerReplicas().isEmpty()) {
+ LifeCycleEvent event = LifeCycleEvent.QUASI_CLOSE;
+ if (containerInfo.getReplicationConfig().getReplicationType()
+ == ReplicationType.EC) {
+ event = LifeCycleEvent.CLOSE;
+ }
+
+ LOG.debug("Container {} has only unhealthy replicas and is closing, so "
+ + "executing the {} event.", containerInfo, event);
+ replicationManager.updateContainerState(
+ containerInfo.containerID(), event);
+ }
+
/*
* Empty containers in CLOSING state should be CLOSED.
*
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestClosingContainerHandler.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestClosingContainerHandler.java
index b4f5a18bb9..a89730424a 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestClosingContainerHandler.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/health/TestClosingContainerHandler.java
@@ -48,6 +48,7 @@ import java.util.Set;
import java.util.stream.Stream;
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent.CLOSE;
+import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent.QUASI_CLOSE;
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.CLOSED;
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.CLOSING;
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.EC;
@@ -262,6 +263,74 @@ public class TestClosingContainerHandler {
.updateContainerState(containerInfo.containerID(), CLOSE);
}
+ @Test
+ public void testClosingRatisWithUnhealthyReplicas() {
+ ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo(
+ RATIS_REPLICATION_CONFIG, 1, CLOSING);
+ Set<ContainerReplica> containerReplicas = ReplicationTestUtil
+ .createReplicas(containerInfo.containerID(),
+ ContainerReplicaProto.State.UNHEALTHY, 0, 0, 0);
+
+ ReplicationManagerReport report = new ReplicationManagerReport();
+
+ ContainerCheckRequest request = new ContainerCheckRequest.Builder()
+ .setPendingOps(Collections.emptyList())
+ .setReport(report)
+ .setContainerInfo(containerInfo)
+ .setContainerReplicas(containerReplicas)
+ .build();
+ subject.handle(request);
+
+ Mockito.verify(replicationManager, Mockito.times(1))
+ .updateContainerState(containerInfo.containerID(), QUASI_CLOSE);
+
+ Mockito.clearInvocations(replicationManager);
+
+ // Now add an open container. This time, the container should not move to
+ // quasi-closed, and a close should be sent for the open replica.
+ containerReplicas.addAll(ReplicationTestUtil
+ .createReplicas(containerInfo.containerID(),
+ ContainerReplicaProto.State.OPEN, 0));
+
+ assertAndVerify(request, true, 1);
+ Mockito.verify(replicationManager, Mockito.times(0))
+ .updateContainerState(containerInfo.containerID(), QUASI_CLOSE);
+ }
+
+ @Test
+ public void testClosingECWithUnhealthyReplicas() {
+ ContainerInfo containerInfo = ReplicationTestUtil.createContainerInfo(
+ EC_REPLICATION_CONFIG, 1, CLOSING);
+ Set<ContainerReplica> containerReplicas = ReplicationTestUtil
+ .createReplicas(containerInfo.containerID(),
+ ContainerReplicaProto.State.UNHEALTHY, 1, 2, 3, 4, 5);
+
+ ReplicationManagerReport report = new ReplicationManagerReport();
+
+ ContainerCheckRequest request = new ContainerCheckRequest.Builder()
+ .setPendingOps(Collections.emptyList())
+ .setReport(report)
+ .setContainerInfo(containerInfo)
+ .setContainerReplicas(containerReplicas)
+ .build();
+ subject.handle(request);
+
+ Mockito.verify(replicationManager, Mockito.times(1))
+ .updateContainerState(containerInfo.containerID(), CLOSE);
+
+ Mockito.clearInvocations(replicationManager);
+
+ // Now add an open container. This time, the container should not move to
+ // quasi-closed, and a close should be sent for the open replica.
+ containerReplicas.addAll(ReplicationTestUtil
+ .createReplicas(containerInfo.containerID(),
+ ContainerReplicaProto.State.OPEN, 1));
+
+ assertAndVerify(request, true, 1);
+ Mockito.verify(replicationManager, Mockito.times(0))
+ .updateContainerState(containerInfo.containerID(), CLOSE);
+ }
+
/**
* Close commands should be sent for Open or Closing replicas.
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
