This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new b97a11bed33 HDDS-15651. Test case for DiskBalancer when
markContainerForDelete fails (#10593)
b97a11bed33 is described below
commit b97a11bed338d19945be5935cbeb5f9d53faeabd
Author: Arun Sarin <[email protected]>
AuthorDate: Sat Jul 4 21:55:52 2026 +0530
HDDS-15651. Test case for DiskBalancer when markContainerForDelete fails
(#10593)
---
.../diskbalancer/TestDiskBalancerTask.java | 110 +++++++++++++++++++++
1 file changed, 110 insertions(+)
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerTask.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerTask.java
index ef9dc8f57b8..e15eaa3a293 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerTask.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerTask.java
@@ -20,6 +20,7 @@
import static
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CONTAINER_INTERNAL_ERROR;
import static
org.apache.hadoop.ozone.container.common.ContainerTestUtils.createDbInstancesForTestIfNeeded;
import static
org.apache.hadoop.ozone.container.diskbalancer.DiskBalancerService.DISK_BALANCER_DIR;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
@@ -504,6 +505,115 @@ public void
moveFailsDuringInMemoryUpdate(ContainerTestVersionInfo versionInfo)
assertEquals(initialSourceDelta,
diskBalancerService.getDeltaSizes().get(sourceVolume));
}
+ /**
+ * When markContainerForDelete fails after import and ContainerSet update,
+ * the move is still reported as success, the destination replica is active,
the source
+ * replica is queued for lazy deletion, and cleanup removes it after the
delay.
+ */
+ @ContainerTestVersionInfo.ContainerTest
+ public void moveSucceedsWhenMarkContainerForDeleteFails(
+ ContainerTestVersionInfo versionInfo)
+ throws IOException, InterruptedException, TimeoutException {
+ setLayoutAndSchemaForTest(versionInfo);
+ long delay = 2_000L;
+ diskBalancerService.setReplicaDeletionDelay(delay);
+
+ KeyValueContainer container = createContainer(CONTAINER_ID, sourceVolume,
State.CLOSED);
+ File oldContainerDir = new
File(container.getContainerData().getContainerPath());
+ Path destDirPath = Paths.get(
+ KeyValueContainerLocationUtil.getBaseContainerLocation(
+ destVolume.getHddsRootDir().toString(), scmId, CONTAINER_ID));
+ assertThat(destDirPath.toFile())
+ .as("Destination container should not exist before task execution")
+ .doesNotExist();
+
+ KeyValueContainer spyContainer = spy(container);
+ containerSet.removeContainer(CONTAINER_ID);
+ containerSet.addContainer(spyContainer);
+ doThrow(new RuntimeException("simulated markContainerForDelete failure"))
+ .when(spyContainer).markContainerForDelete();
+
+ LogCapturer serviceLog =
GenericTestUtils.LogCapturer.captureLogs(DiskBalancerService.class);
+ DiskBalancerService.DiskBalancerTask task = getTask();
+ task.call();
+
+ assertThat(serviceLog.getOutput())
+ .contains("Failed to mark the old container " + CONTAINER_ID + " for
delete");
+ assertThat(serviceLog.getOutput())
+ .as("move should not roll back when markContainerForDelete fails")
+ .doesNotContain("Rolling back move");
+
+ assertEquals(1, diskBalancerService.getMetrics().getSuccessCount());
+ assertEquals(0, diskBalancerService.getMetrics().getFailureCount());
+ assertEquals(CONTAINER_SIZE,
diskBalancerService.getMetrics().getSuccessBytes());
+
+ Container activeReplica = containerSet.getContainer(CONTAINER_ID);
+ assertThat(activeReplica).isNotSameAs(spyContainer);
+ assertEquals(destVolume, activeReplica.getContainerData().getVolume());
+ assertThat(new
File(activeReplica.getContainerData().getContainerPath())).exists();
+ assertThat(oldContainerDir)
+ .as("Source replica should remain on disk until lazy deletion runs")
+ .exists();
+ assertEquals(1, diskBalancerService.getPendingDeletionQueueSize(),
+ "Source replica should be queued for lazy deletion after mark
failure");
+
+ clock.fastForward(delay);
+ diskBalancerService.cleanupPendingDeletionContainers();
+
+ assertThat(oldContainerDir)
+ .as("Source replica should be removed after lazy deletion delay")
+ .doesNotExist();
+ assertEquals(0, diskBalancerService.getPendingDeletionQueueSize());
+ }
+
+ /**
+ * When lazy deletion fails, the pending queue entry is dropped and
+ * the source replica is not retried for deletion.
+ */
+ @ContainerTestVersionInfo.ContainerTest
+ public void lazyDeletionFailureDoesNotRetry(
+ ContainerTestVersionInfo versionInfo) throws Exception {
+ setLayoutAndSchemaForTest(versionInfo);
+ long delay = 2_000L;
+ diskBalancerService.setReplicaDeletionDelay(delay);
+
+ Container container = createContainer(CONTAINER_ID, sourceVolume,
State.CLOSED);
+ File oldContainerDir = new
File(container.getContainerData().getContainerPath());
+
+ DiskBalancerService.DiskBalancerTask task = getTask();
+ task.call();
+
+ assertEquals(1, diskBalancerService.getMetrics().getSuccessCount());
+ assertEquals(1, diskBalancerService.getPendingDeletionQueueSize());
+ assertThat(oldContainerDir).exists();
+
+ clock.fastForward(delay);
+
+ LogCapturer serviceLog =
GenericTestUtils.LogCapturer.captureLogs(DiskBalancerService.class);
+ try (MockedStatic<KeyValueContainerUtil> mockedUtil =
+ mockStatic(KeyValueContainerUtil.class,
Mockito.CALLS_REAL_METHODS)) {
+ mockedUtil.when(() -> KeyValueContainerUtil.removeContainer(
+ any(KeyValueContainerData.class), any(OzoneConfiguration.class)))
+ .thenThrow(new IOException("simulated lazy deletion failure"));
+
+ diskBalancerService.cleanupPendingDeletionContainers();
+
+ assertThat(oldContainerDir)
+ .as("Source replica should remain when lazy deletion fails")
+ .exists();
+ assertEquals(0, diskBalancerService.getPendingDeletionQueueSize(),
+ "Failed deletion should be removed from the pending queue");
+ assertThat(serviceLog.getOutput())
+ .contains("Failed to delete old container " + CONTAINER_ID);
+ assertThat(serviceLog.getOutput()).contains("background scanners");
+ }
+
+ diskBalancerService.cleanupPendingDeletionContainers();
+ assertThat(oldContainerDir)
+ .as("Source replica should not be retried after lazy deletion failure")
+ .exists();
+ }
+
@ContainerTestVersionInfo.ContainerTest
public void moveFailsDuringOldContainerRemove(ContainerTestVersionInfo
versionInfo) throws IOException {
setLayoutAndSchemaForTest(versionInfo);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
