This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 2b07f482f0 HDDS-8902. Close open container immediately on ICR of
unhealthy replica (#5212)
2b07f482f0 is described below
commit 2b07f482f04e41e3d9f7f30c5f77bb92a2a5a8ff
Author: Stephen O'Donnell <[email protected]>
AuthorDate: Wed Aug 23 22:45:56 2023 +0100
HDDS-8902. Close open container immediately on ICR of unhealthy replica
(#5212)
---
.../container/AbstractContainerReportHandler.java | 9 +++--
.../TestIncrementalContainerReportHandler.java | 44 ++++++++++++++++++++++
.../hdds/scm/pipeline/MockPipelineManager.java | 14 ++++++-
3 files changed, 61 insertions(+), 6 deletions(-)
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java
index 004e32a2f2..7e163ac306 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/AbstractContainerReportHandler.java
@@ -257,10 +257,11 @@ public class AbstractContainerReportHandler {
* any other state.
*/
if (replica.getState() != State.OPEN) {
- logger.warn("Container {} is in OPEN state, but the datanode {} " +
- "reports an {} replica.", containerId,
- datanode, replica.getState());
- // Should we take some action?
+ logger.info("Moving OPEN container {} to CLOSING state, datanode {} " +
+ "reported {} replica with index {}.", containerId, datanode,
+ replica.getState(), replica.getReplicaIndex());
+ containerManager.updateContainerState(containerId,
+ LifeCycleEvent.FINALIZE);
}
break;
case CLOSING:
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestIncrementalContainerReportHandler.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestIncrementalContainerReportHandler.java
index 3433e73299..cc2620b4f0 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestIncrementalContainerReportHandler.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestIncrementalContainerReportHandler.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdds.scm.container;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.client.RatisReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -41,6 +42,7 @@ import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.node.SCMNodeManager;
import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
import org.apache.hadoop.hdds.scm.pipeline.MockPipelineManager;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import
org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.ContainerReportFromDatanode;
@@ -82,6 +84,7 @@ import java.util.stream.IntStream;
import static
org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails;
import static
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.CLOSED;
+import static
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.UNHEALTHY;
import static org.apache.hadoop.hdds.scm.HddsTestUtils.getContainer;
import static org.apache.hadoop.hdds.scm.HddsTestUtils.getECContainer;
import static org.apache.hadoop.hdds.scm.HddsTestUtils.getReplicas;
@@ -426,6 +429,47 @@ public class TestIncrementalContainerReportHandler {
containerManager.getContainer(container.containerID()).getState());
}
+ @Test
+ public void testOpenWithUnhealthyReplica() throws IOException {
+ final IncrementalContainerReportHandler reportHandler =
+ new IncrementalContainerReportHandler(
+ nodeManager, containerManager, scmContext);
+
+ RatisReplicationConfig replicationConfig =
+ RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE);
+ Pipeline pipeline = pipelineManager.createPipeline(replicationConfig);
+ List<DatanodeDetails> nodes = pipeline.getNodes();
+
+ final DatanodeDetails datanodeOne = nodes.get(0);
+ final DatanodeDetails datanodeTwo = nodes.get(1);
+ final DatanodeDetails datanodeThree = nodes.get(2);
+
+ final ContainerInfo container = getContainer(LifeCycleState.OPEN,
+ pipeline.getId());
+
+ nodeManager.register(datanodeOne, null, null);
+ nodeManager.register(datanodeTwo, null, null);
+ nodeManager.register(datanodeThree, null, null);
+ final Set<ContainerReplica> containerReplicas = getReplicas(
+ container.containerID(), ContainerReplicaProto.State.OPEN,
+ datanodeOne, datanodeTwo, datanodeThree);
+
+ containerStateManager.addContainer(container.getProtobuf());
+ containerReplicas.forEach(r ->
containerStateManager.updateContainerReplica(
+ container.containerID(), r));
+
+ final IncrementalContainerReportProto containerReport =
+ getIncrementalContainerReportProto(container.containerID(),
+ UNHEALTHY,
+ datanodeThree.getUuidString());
+ final IncrementalContainerReportFromDatanode icr =
+ new IncrementalContainerReportFromDatanode(
+ datanodeThree, containerReport);
+ reportHandler.onMessage(icr, publisher);
+ Assertions.assertEquals(LifeCycleState.CLOSING,
+ containerManager.getContainer(container.containerID()).getState());
+ }
+
@Test
public void testDeleteContainer() throws IOException, TimeoutException,
NodeNotFoundException {
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockPipelineManager.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockPipelineManager.java
index 3fedeff9c6..f00aa7f6c1 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockPipelineManager.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/MockPipelineManager.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.hdds.scm.pipeline;
+import com.google.common.collect.ImmutableList;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
@@ -70,8 +71,17 @@ public class MockPipelineManager implements PipelineManager {
public Pipeline createPipeline(ReplicationConfig replicationConfig,
List<DatanodeDetails> excludedNodes, List<DatanodeDetails> favoredNodes)
throws IOException {
- Pipeline pipeline = buildECPipeline(replicationConfig, excludedNodes,
- favoredNodes);
+ Pipeline pipeline;
+ if (replicationConfig.getReplicationType()
+ == HddsProtos.ReplicationType.EC) {
+ pipeline = buildECPipeline(
+ replicationConfig, excludedNodes, favoredNodes);
+ } else {
+ pipeline = createPipeline(replicationConfig,
+ ImmutableList.of(MockDatanodeDetails.randomDatanodeDetails(),
+ MockDatanodeDetails.randomDatanodeDetails(),
+ MockDatanodeDetails.randomDatanodeDetails()));
+ }
stateManager.addPipeline(pipeline.getProtobufMessage(
ClientVersion.CURRENT_VERSION));
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
