This is an automated email from the ASF dual-hosted git repository.
devmadhuu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 46519f7fddd HDDS-15308. Improve ICR/FCR-driven container state
recovery by plugging DN report processing gaps in Recon. (#10326)
46519f7fddd is described below
commit 46519f7fdddb6f1c8ec1035f3b88f46ee3abe5cb
Author: Devesh Kumar Singh <[email protected]>
AuthorDate: Wed Jun 10 09:35:33 2026 +0530
HDDS-15308. Improve ICR/FCR-driven container state recovery by plugging DN
report processing gaps in Recon. (#10326)
---
.../ozone/recon/TestNSSummaryMemoryLeak.java | 92 +++++++++++---------
.../ozone/recon/fsck/ReconReplicationManager.java | 2 +-
.../ozone/recon/scm/ReconContainerManager.java | 28 +++---
.../scm/ReconStorageContainerManagerFacade.java | 13 ---
.../recon/scm/ReconStorageContainerSyncHelper.java | 25 ++++--
.../ozone/recon/scm/TestReconContainerManager.java | 99 +++++++++++++++++++++-
...TestReconIncrementalContainerReportHandler.java | 59 ++++++++++++-
7 files changed, 241 insertions(+), 77 deletions(-)
diff --git
a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryMemoryLeak.java
b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryMemoryLeak.java
index 5bdfe32aa02..fdb0a3c1204 100644
---
a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryMemoryLeak.java
+++
b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/TestNSSummaryMemoryLeak.java
@@ -22,6 +22,8 @@
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL;
+import static
org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_OM_SNAPSHOT_TASK_INITIAL_DELAY;
+import static
org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_OM_SNAPSHOT_TASK_INTERVAL_DELAY;
import static org.assertj.core.api.Assertions.assertThat;
import java.io.IOException;
@@ -45,8 +47,9 @@
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager;
import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl;
+import org.apache.hadoop.ozone.recon.tasks.NSSummaryTask;
+import org.apache.hadoop.ozone.recon.tasks.ReconOmTask;
import org.apache.ozone.test.GenericTestUtils;
-import org.apache.ratis.RaftTestUtil;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
@@ -133,6 +136,8 @@ public static void init() throws Exception {
// Configure delays for testing
conf.setInt(OZONE_DIR_DELETING_SERVICE_INTERVAL, 1000000);
conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 10000000,
TimeUnit.MILLISECONDS);
+ conf.setTimeDuration(OZONE_RECON_OM_SNAPSHOT_TASK_INITIAL_DELAY, 1,
TimeUnit.DAYS);
+ conf.setTimeDuration(OZONE_RECON_OM_SNAPSHOT_TASK_INTERVAL_DELAY, 1,
TimeUnit.DAYS);
conf.setBoolean(OZONE_ACL_ENABLED, true);
recon = new ReconService(conf);
@@ -234,10 +239,9 @@ public void testNSSummaryCleanupOnHardDelete() throws
Exception {
// Trigger hard delete by clearing deleted tables
// This simulates the background process that hard deletes entries
simulateHardDelete(omMetadataManager);
- syncDataFromOM();
- // Verify memory leak fix - NSSummary entries should be cleaned up
- verifyNSSummaryCleanup(omMetadataManager, namespaceSummaryManager);
+ // Verify cleanup after hard delete simulation.
+ verifyNSSummaryCleanup(omMetadataManager, "memoryLeakTest");
LOG.info("NSSummary memory leak fix test completed successfully");
}
@@ -266,13 +270,11 @@ public void testNSSummaryCleanupOnHardDelete() throws
Exception {
* <li>Total: 1051 objects that will have NSSummary entries</li>
* </ul>
*
- * <p><b>Memory Usage Monitoring:</b>
- * <p>This test monitors memory usage before and after the deletion to
validate that
- * the memory leak fix prevents excessive memory consumption. The test
performs:
+ * <p><b>NSSummary Cleanup Validation:</b>
+ * <p>This test validates that NSSummary cleanup completes for a larger
+ * directory structure. The test performs:
* <ul>
- * <li>Memory measurement before deletion</li>
* <li>Directory structure deletion and hard delete simulation</li>
- * <li>Garbage collection and memory measurement after cleanup</li>
* <li>Verification that NSSummary entries are properly cleaned up</li>
* </ul>
*
@@ -291,36 +293,25 @@ public void testMemoryLeakWithLargeStructure() throws
Exception {
createDirectoryStructure(largeTestDir, numSubdirs, filesPerDir);
syncDataFromOM();
-
- // Get current memory usage
- Runtime runtime = Runtime.getRuntime();
- long memoryBefore = runtime.totalMemory() - runtime.freeMemory();
+
+ OzoneManagerServiceProviderImpl omServiceProvider =
(OzoneManagerServiceProviderImpl)
+ recon.getReconServer().getOzoneManagerServiceProvider();
+ ReconOMMetadataManager omMetadataManager =
+ (ReconOMMetadataManager)
omServiceProvider.getOMMetadataManagerInstance();
+ ReconNamespaceSummaryManager namespaceSummaryManager =
+ recon.getReconServer().getReconNamespaceSummaryManager();
+ verifyNSSummaryEntriesExist(omMetadataManager, namespaceSummaryManager,
+ numSubdirs);
// Delete and verify cleanup
fs.delete(largeTestDir, true);
syncDataFromOM();
// Simulate hard delete
- OzoneManagerServiceProviderImpl omServiceProvider =
(OzoneManagerServiceProviderImpl)
- recon.getReconServer().getOzoneManagerServiceProvider();
- ReconOMMetadataManager omMetadataManager =
- (ReconOMMetadataManager)
omServiceProvider.getOMMetadataManagerInstance();
-
simulateHardDelete(omMetadataManager);
- syncDataFromOM();
-
- // Force garbage collection
- RaftTestUtil.gc();
-
- // Verify memory cleanup
- long memoryAfter = runtime.totalMemory() - runtime.freeMemory();
- LOG.info("Memory usage - Before: {} bytes, After: {} bytes", memoryBefore,
memoryAfter);
- assertThat(memoryAfter).isLessThanOrEqualTo(memoryBefore);
- // Verify NSSummary cleanup
- ReconNamespaceSummaryManager namespaceSummaryManager =
- recon.getReconServer().getReconNamespaceSummaryManager();
- verifyNSSummaryCleanup(omMetadataManager, namespaceSummaryManager);
+ // Verify cleanup after hard delete simulation.
+ verifyNSSummaryCleanup(omMetadataManager, "largeMemoryLeakTest");
LOG.info("Large structure memory leak test completed successfully");
}
@@ -380,10 +371,13 @@ private void createDirectoryStructure(Path rootDir, int
numSubdirs, int filesPer
*
* @throws IOException if synchronization fails
*/
- private void syncDataFromOM() throws IOException {
+ private void syncDataFromOM() throws Exception {
OzoneManagerServiceProviderImpl impl = (OzoneManagerServiceProviderImpl)
recon.getReconServer().getOzoneManagerServiceProvider();
impl.syncDataFromOM();
+ GenericTestUtils.waitFor(
+ () -> NSSummaryTask.getRebuildState() !=
NSSummaryTask.RebuildState.RUNNING,
+ 100, 60000);
}
private void verifyNSSummaryEntriesExist(ReconOMMetadataManager
omMetadataManager,
@@ -445,15 +439,15 @@ private void
verifyEntriesInDeletedTables(ReconOMMetadataManager omMetadataManag
* <p>This simulation:
* <ol>
* <li>Iterates through all entries in deletedDirTable</li>
- * <li>Deletes each entry to trigger the memory leak fix</li>
- * <li>The deletion triggers {@code
NSSummaryTaskWithFSO.handleUpdateOnDeletedDirTable()}</li>
- * <li>Which in turn cleans up the corresponding NSSummary entries</li>
+ * <li>Deletes each entry from the deleted directory table</li>
+ * <li>Reprocesses NSSummary from the current Recon OM metadata
snapshot</li>
* </ol>
*
* @param omMetadataManager the metadata manager containing the deleted
tables
* @throws IOException if table operations fail
*/
- private void simulateHardDelete(ReconOMMetadataManager omMetadataManager)
throws IOException {
+ private void simulateHardDelete(ReconOMMetadataManager omMetadataManager)
+ throws IOException {
// Simulate hard delete by clearing deleted tables
Table<String, OmKeyInfo> deletedDirTable =
omMetadataManager.getDeletedDirTable();
@@ -464,29 +458,43 @@ private void simulateHardDelete(ReconOMMetadataManager
omMetadataManager) throws
deletedDirTable.delete(kv.getKey());
}
}
+ reprocessNSSummary(omMetadataManager);
+ }
+
+ private void reprocessNSSummary(ReconOMMetadataManager omMetadataManager) {
+ ReconOmTask nsSummaryTask = recon.getReconServer().getReconTaskController()
+ .getRegisteredTasks().get("NSSummaryTask");
+ assertThat(nsSummaryTask).isNotNull();
+ ReconOmTask.TaskResult result = nsSummaryTask.reprocess(omMetadataManager);
+ assertThat(result.isTaskSuccess()).isTrue();
}
private void verifyNSSummaryCleanup(ReconOMMetadataManager omMetadataManager,
- ReconNamespaceSummaryManager namespaceSummaryManager) throws Exception {
+ String path) throws Exception {
// Wait for cleanup to complete
GenericTestUtils.waitFor(() -> {
try {
- // Check that deleted directories don't have NSSummary entries
+ // Check that simulated hard delete drained the deleted directory
table.
Table<String, OmDirectoryInfo> dirTable =
omMetadataManager.getDirectoryTable();
+ Table<String, OmKeyInfo> deletedDirTable =
omMetadataManager.getDeletedDirTable();
+
+ if (omMetadataManager.countRowsInTable(deletedDirTable) != 0) {
+ return false;
+ }
// Verify that the main test directory is no longer in the directory
table
try (Table.KeyValueIterator<String, OmDirectoryInfo> iterator =
dirTable.iterator()) {
while (iterator.hasNext()) {
Table.KeyValue<String, OmDirectoryInfo> kv = iterator.next();
- String path = kv.getKey();
- if (path.contains("memoryLeakTest")) {
- LOG.info("Found test directory still in table: {}", path);
+ String key = kv.getKey();
+ if (key.contains(path)) {
+ LOG.info("Found test directory still in table: {}", key);
return false;
}
}
}
-
+
return true;
} catch (Exception e) {
LOG.error("Error verifying cleanup", e);
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ReconReplicationManager.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ReconReplicationManager.java
index 8f56ddc8f2f..09005cadb18 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ReconReplicationManager.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/fsck/ReconReplicationManager.java
@@ -322,7 +322,7 @@ public synchronized void processAll() {
processedCount++;
if (processedCount % logEvery == 0 || processedCount ==
containers.size()) {
- LOG.info("Processed {}/{} containers", processedCount,
containers.size());
+ LOG.debug("Processed {}/{} containers", processedCount,
containers.size());
}
} catch (ContainerNotFoundException e) {
LOG.error("Container {} not found", container.getContainerID(), e);
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java
index 4057093c3aa..25d8543ab27 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java
@@ -201,6 +201,7 @@ public void checkAndAddNewContainerBatch(
void transitionOpenToClosing(ContainerID containerID, ContainerInfo
containerInfo)
throws IOException, InvalidStateTransitionException {
PipelineID pipelineID = containerInfo.getPipelineID();
+ updateContainerState(containerID, FINALIZE); // OPEN → CLOSING
if (pipelineID != null) {
int curCnt = pipelineToOpenContainer.getOrDefault(pipelineID, 0);
if (curCnt == 1) {
@@ -209,12 +210,20 @@ void transitionOpenToClosing(ContainerID containerID,
ContainerInfo containerInf
pipelineToOpenContainer.put(pipelineID, curCnt - 1);
}
}
- updateContainerState(containerID, FINALIZE); // OPEN → CLOSING
}
/**
- * Check if an OPEN container should move to CLOSING based on a healthy
- * non-OPEN DN replica report.
+ * Check if Recon's container lifecycle state needs the Recon-specific
+ * pre-processing required before SCM's shared report handler processes the
+ * replica.
+ *
+ * <p>Recon only handles OPEN to CLOSING here to keep the per-pipeline open
+ * container count accurate. All other known-container lifecycle transitions
+ * are left to SCM's common ICR/FCR state machine, which is invoked after
this
+ * method by Recon's report handlers.
+ *
+ * @param containerID containerID to check
+ * @param replicaState replica state reported by a DataNode
*/
private void checkContainerStateAndUpdate(ContainerID containerID,
ContainerReplicaProto.State
replicaState)
@@ -222,15 +231,12 @@ private void checkContainerStateAndUpdate(ContainerID
containerID,
ContainerInfo containerInfo = getContainer(containerID);
HddsProtos.LifeCycleState reconState = containerInfo.getState();
- if (reconState != HddsProtos.LifeCycleState.OPEN
- || replicaState == ContainerReplicaProto.State.OPEN
- || !isHealthy(replicaState)) {
- return;
+ if (reconState == HddsProtos.LifeCycleState.OPEN
+ && replicaState != ContainerReplicaProto.State.OPEN &&
isHealthy(replicaState)) {
+ LOG.info("Container {} has state OPEN, but given state is {}.",
+ containerID, replicaState);
+ transitionOpenToClosing(containerID, containerInfo);
}
-
- LOG.info("Container {} is OPEN in Recon but DN reports replica state {}. "
- + "Moving to CLOSING.", containerID, replicaState);
- transitionOpenToClosing(containerID, containerInfo);
}
private boolean isHealthy(ContainerReplicaProto.State replicaState) {
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
index 1712e711fe0..679e3c3c71e 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
@@ -896,19 +896,6 @@ public boolean triggerTargetedSCMContainerSync() {
}
}
- public boolean syncWithSCMContainerInfo() {
- if (isSyncDataFromSCMRunning.compareAndSet(false, true)) {
- try {
- return runTargetedSyncWithMetrics();
- } finally {
- isSyncDataFromSCMRunning.compareAndSet(true, false);
- }
- } else {
- LOG.debug("SCM DB sync is already running.");
- return false;
- }
- }
-
private boolean runTargetedSyncWithMetrics() {
long startTime = Time.monotonicNow();
containerSyncMetrics.setTargetedSyncStatus(
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerSyncHelper.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerSyncHelper.java
index 9a1aa48a1e1..bbd03aab0d5 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerSyncHelper.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerSyncHelper.java
@@ -90,6 +90,7 @@ class ReconStorageContainerSyncHelper {
* (DELETED ID list).
*/
private static final long CONTAINER_ID_PROTO_SIZE_BYTES = 12;
+ private static final long DELETED_SYNC_TRANSITION_LOG_SAMPLE_INTERVAL =
1000L;
/**
* Conservative wire-size upper bound for one {@code ContainerWithPipeline}
@@ -365,6 +366,7 @@ private boolean syncDeletedContainers() {
OZONE_RECON_SCM_DELETED_CONTAINER_CHECK_BATCH_SIZE_DEFAULT);
int batchSize = (int) getContainerCountPerCall(configuredBatch);
int retiredCount = 0;
+ long processedCount = 0;
// Existing Recon containers need only the ID to retire to DELETED. Fetch
// full ContainerInfo only for IDs absent from Recon, where we must add a
@@ -380,7 +382,8 @@ private boolean syncDeletedContainers() {
if (page == null || page.isEmpty()) {
break;
}
- retiredCount += processDeletedPage(page);
+ retiredCount += processDeletedPage(page, processedCount);
+ processedCount += page.size();
start = ContainerID.valueOf(
page.get(page.size() - 1).getId() + 1);
}
@@ -403,9 +406,12 @@ private boolean syncDeletedContainers() {
* <li>If already DELETED in Recon: no-op.</li>
* </ul>
*/
- private int processDeletedPage(List<ContainerID> page) {
+ private int processDeletedPage(List<ContainerID> page,
+ long processedCountBeforePage) {
int retiredCount = 0;
+ long processedCount = processedCountBeforePage;
for (ContainerID containerID : page) {
+ processedCount++;
if (!containerManager.containerExist(containerID)) {
if (addContainerInfoFallback(containerID,
HddsProtos.LifeCycleState.DELETED, "DELETED sync")) {
@@ -417,7 +423,7 @@ private int processDeletedPage(List<ContainerID> page) {
ContainerInfo reconInfo = containerManager.getContainer(containerID);
if (reconInfo.getState() != HddsProtos.LifeCycleState.DELETED) {
retireContainerToDeleted(containerID, reconInfo,
- HddsProtos.LifeCycleState.DELETED);
+ HddsProtos.LifeCycleState.DELETED, processedCount);
retiredCount++;
}
// reconState == DELETED: already terminal, nothing to do.
@@ -464,10 +470,13 @@ private int processDeletedPage(List<ContainerID> page) {
* @param reconInfo current Recon snapshot of the container (used for
* OPEN→CLOSING transition and log messages)
* @param scmState always DELETED (passed through to log messages)
+ * @param processedCount current number of SCM DELETED IDs scanned in this
+ * sync cycle
*/
private void retireContainerToDeleted(ContainerID containerID,
ContainerInfo reconInfo,
- HddsProtos.LifeCycleState scmState) {
+ HddsProtos.LifeCycleState scmState,
+ long processedCount) {
try {
HddsProtos.LifeCycleState reconState = reconInfo.getState();
@@ -486,9 +495,11 @@ private void retireContainerToDeleted(ContainerID
containerID,
// DELETING → DELETED.
containerManager.updateContainerState(containerID, CLEANUP);
- LOG.info("DELETED sync: container {} transitioned "
- + "{} → DELETED in Recon (SCM state: {}).",
- containerID, reconInfo.getState(), scmState);
+ if (processedCount % DELETED_SYNC_TRANSITION_LOG_SAMPLE_INTERVAL == 0) {
+ LOG.debug("DELETED sync: container {} transitioned "
+ + "{} → DELETED in Recon (SCM state: {}).",
+ containerID, reconInfo.getState(), scmState);
+ }
} catch (InvalidStateTransitionException | IOException e) {
LOG.warn("DELETED sync: failed to retire container {} "
+ "from {} toward DELETED.", containerID, reconInfo.getState(), e);
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java
index e1e89f180be..8eade310b9b 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconContainerManager.java
@@ -25,7 +25,10 @@
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
import java.io.IOException;
import java.util.LinkedList;
@@ -43,6 +46,7 @@
import org.apache.hadoop.hdds.scm.container.ContainerChecksums;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
import org.apache.hadoop.hdds.scm.container.ContainerReplica;
import
org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.ha.SCMHAManager;
@@ -175,12 +179,103 @@ public void testUpdateContainerStateFromOpen() throws
Exception {
DatanodeDetails datanodeDetails = randomDatanodeDetails();
- // First report with "CLOSED" replica state moves container state to
+ // First report with "CLOSING" replica state moves container state to
// "CLOSING".
- getContainerManager().checkAndAddNewContainer(containerID, State.CLOSED,
+ getContainerManager().checkAndAddNewContainer(containerID, State.CLOSING,
datanodeDetails);
assertEquals(CLOSING,
getContainerManager().getContainer(containerID).getState());
+ assertFalse(getContainerManager().getPipelineToOpenContainer()
+ .containsKey(containerWithPipeline.getPipeline().getId()));
+ }
+
+ @Test
+ public void testOpenContainerTransitionsToClosingWithoutScmLookup()
+ throws Exception {
+ ContainerWithPipeline openContainer =
+ getTestContainer(113L, LifeCycleState.OPEN);
+ ContainerID containerID = openContainer.getContainerInfo().containerID();
+ getContainerManager().addNewContainer(openContainer);
+
+ getContainerManager().checkAndAddNewContainer(containerID, State.CLOSED,
+ randomDatanodeDetails());
+
+ assertEquals(CLOSING,
+ getContainerManager().getContainer(containerID).getState());
+ assertFalse(getContainerManager().getPipelineToOpenContainer()
+ .containsKey(openContainer.getPipeline().getId()));
+ verify(getContainerManager().getScmClient(), never())
+ .getContainerWithPipeline(containerID.getId());
+ }
+
+ @Test
+ public void testTransitionOpenToClosingDoesNotDecrementCountOnFailure()
+ throws Exception {
+ ContainerWithPipeline missingContainer =
+ getTestContainer(114L, LifeCycleState.OPEN);
+ ContainerID containerID =
+ missingContainer.getContainerInfo().containerID();
+ Pipeline pipeline = missingContainer.getPipeline();
+ getContainerManager().getPipelineToOpenContainer().put(pipeline.getId(),
1);
+
+ assertThrows(ContainerNotFoundException.class,
+ () -> getContainerManager().transitionOpenToClosing(containerID,
+ missingContainer.getContainerInfo()));
+
+ assertEquals(1,
+ getContainerManager().getPipelineToOpenContainer()
+ .get(pipeline.getId()));
+ }
+
+ @Test
+ public void testOpenContainerNotUpdatedFromUnhealthyReplicaReports()
+ throws Exception {
+ for (State replicaState : new State[] {
+ State.UNHEALTHY, State.INVALID, State.DELETED}) {
+ ContainerWithPipeline containerWithPipeline =
+ getTestContainer(120L + replicaState.ordinal(), LifeCycleState.OPEN);
+ ContainerID containerID =
+ containerWithPipeline.getContainerInfo().containerID();
+ getContainerManager().addNewContainer(containerWithPipeline);
+
+ getContainerManager().checkAndAddNewContainer(containerID, replicaState,
+ randomDatanodeDetails());
+
+ assertEquals(LifeCycleState.OPEN,
+ getContainerManager().getContainer(containerID).getState());
+ assertTrue(getContainerManager().getPipelineToOpenContainer()
+ .containsKey(containerWithPipeline.getPipeline().getId()));
+ }
+ }
+
+ @Test
+ public void testClosingContainerNotUpdatedFromUnhealthyReplicaReport()
+ throws Exception {
+ ContainerWithPipeline closingContainer = getTestContainer(105L, CLOSING);
+ ContainerID containerID =
closingContainer.getContainerInfo().containerID();
+ getContainerManager().addNewContainer(closingContainer);
+
+ getContainerManager().checkAndAddNewContainer(containerID, State.UNHEALTHY,
+ randomDatanodeDetails());
+
+ assertEquals(CLOSING,
+ getContainerManager().getContainer(containerID).getState());
+ }
+
+ @Test
+ public void testOtherReconStatesDoNotInferDnReplicaTransition()
+ throws Exception {
+ ContainerWithPipeline closedContainer = getTestContainer(112L, CLOSED);
+ ContainerID containerID = closedContainer.getContainerInfo().containerID();
+ getContainerManager().addNewContainer(closedContainer);
+
+ getContainerManager().checkAndAddNewContainer(containerID,
+ State.CLOSING, randomDatanodeDetails());
+
+ assertEquals(CLOSED,
+ getContainerManager().getContainer(containerID).getState());
+ verify(getContainerManager().getScmClient(), never())
+ .getContainerWithPipeline(containerID.getId());
}
ContainerInfo newContainerInfo(long containerId, Pipeline pipeline) {
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconIncrementalContainerReportHandler.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconIncrementalContainerReportHandler.java
index 78f9bbceafc..c82eabb92e7 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconIncrementalContainerReportHandler.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconIncrementalContainerReportHandler.java
@@ -24,6 +24,8 @@
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.IOException;
@@ -129,6 +131,7 @@ public void testProcessICRStateMismatch()
containerId++, OPEN);
ContainerID containerID =
containerWithPipeline.getContainerInfo().containerID();
+ LifeCycleState expectedState = getContainerStateFromReplicaState(state);
ReconContainerManager containerManager = getContainerManager();
containerManager.addNewContainer(containerWithPipeline);
@@ -155,12 +158,46 @@ public void testProcessICRStateMismatch()
assertTrue(containerManager.containerExist(containerID));
assertEquals(1,
containerManager.getContainerReplicas(containerID).size());
- LifeCycleState expectedState = getContainerStateFromReplicaState(state);
LifeCycleState actualState =
containerManager.getContainer(containerID).getState();
assertEquals(expectedState, actualState,
String.format("Expecting %s in container state for replica state %s",
expectedState, state));
+ verify(containerManager.getScmClient(), never())
+ .getContainerWithPipeline(containerID.getId());
+ }
+ }
+
+ @Test
+ public void testClosingContainerAdvancesViaScmHandlerWithoutScmLookup()
+ throws IOException, NodeNotFoundException, TimeoutException {
+ long containerId = 200;
+ for (State state : Arrays.asList(State.QUASI_CLOSED, State.CLOSED)) {
+ ContainerWithPipeline containerWithPipeline =
+ getTestContainer(containerId++, LifeCycleState.CLOSING);
+ ContainerID containerID =
+ containerWithPipeline.getContainerInfo().containerID();
+ LifeCycleState expectedState = getContainerStateFromReplicaState(state);
+ ReconContainerManager containerManager = getContainerManager();
+ containerManager.addNewContainer(containerWithPipeline);
+
+ DatanodeDetails datanodeDetails =
+ containerWithPipeline.getPipeline().getFirstNode();
+ ReconIncrementalContainerReportHandler reconIcr =
+ new ReconIncrementalContainerReportHandler(
+ getNodeManagerMock(datanodeDetails), containerManager,
+ SCMContext.emptyContext());
+ IncrementalContainerReportFromDatanode reportMock =
+ getReportMock(containerID, state, datanodeDetails);
+
+ reconIcr.onMessage(reportMock, mock(EventPublisher.class));
+
+ assertEquals(expectedState,
+ containerManager.getContainer(containerID).getState(),
+ String.format("Expecting %s in container state for replica state %s",
+ expectedState, state));
+ verify(containerManager.getScmClient(), never())
+ .getContainerWithPipeline(containerID.getId());
}
}
@@ -199,6 +236,26 @@ private LifeCycleState getContainerStateFromReplicaState(
}
}
+ private static NodeManager getNodeManagerMock(DatanodeDetails
datanodeDetails)
+ throws NodeNotFoundException {
+ NodeManager nodeManagerMock = mock(NodeManager.class);
+ when(nodeManagerMock.getNode(any(DatanodeID.class)))
+ .thenReturn(datanodeDetails);
+ return nodeManagerMock;
+ }
+
+ private static IncrementalContainerReportFromDatanode getReportMock(
+ ContainerID containerID, State state, DatanodeDetails datanodeDetails) {
+ IncrementalContainerReportFromDatanode reportMock =
+ mock(IncrementalContainerReportFromDatanode.class);
+ when(reportMock.getDatanodeDetails()).thenReturn(datanodeDetails);
+ IncrementalContainerReportProto containerReport =
+ getIncrementalContainerReportProto(containerID, state,
+ datanodeDetails.getUuidString());
+ when(reportMock.getReport()).thenReturn(containerReport);
+ return reportMock;
+ }
+
private static IncrementalContainerReportProto
getIncrementalContainerReportProto(final ContainerID containerId,
final State state,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
