This is an automated email from the ASF dual-hosted git repository.
smengcl pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 031e76aa2c9 HDDS-16933. Speed up TestKeyValueContainer,
TestEventBufferOverflow, TestBlockInputStream and TestEventQueue (#11481)
031e76aa2c9 is described below
commit 031e76aa2c9a60b67185390b5b33499cbbb82e43
Author: Siyao Meng <[email protected]>
AuthorDate: Tue Oct 13 00:11:08 2026 -0700
HDDS-16933. Speed up TestKeyValueContainer, TestEventBufferOverflow,
TestBlockInputStream and TestEventQueue (#11481)
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
.../hadoop/hdds/scm/storage/TestBlockInputStream.java | 17 +++++++----------
.../container/keyvalue/TestKeyValueContainer.java | 4 +---
.../hadoop/hdds/server/events/TestEventQueue.java | 2 +-
.../ozone/recon/tasks/TestEventBufferOverflow.java | 18 +++++++++++-------
4 files changed, 20 insertions(+), 21 deletions(-)
diff --git
a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java
b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java
index 22416df3af4..506d26ac1d8 100644
---
a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java
+++
b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java
@@ -93,6 +93,7 @@ public class TestBlockInputStream {
private Function<BlockID, BlockLocationInfo> refreshFunction;
private OzoneConfiguration conf = new OzoneConfiguration();
+ private OzoneClientConfig clientConfig;
@BeforeEach
@SuppressWarnings("unchecked")
@@ -101,8 +102,9 @@ public void setup() throws Exception {
BlockID blockID = new BlockID(new ContainerBlockID(1, 1));
checksum = new Checksum(ChecksumType.NONE, CHUNK_SIZE);
createChunkList(5);
- OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class);
+ clientConfig = conf.getObject(OzoneClientConfig.class);
clientConfig.setChecksumVerify(false);
+ clientConfig.setReadRetryInterval(0);
Pipeline pipeline = MockPipeline.createSingleNodePipeline();
blockStream = new DummyBlockInputStream(blockID, blockSize, pipeline, null,
@@ -275,14 +277,15 @@ public void testRefreshPipelineFunction() throws
Exception {
BlockID blockID = new BlockID(new ContainerBlockID(1, 1));
AtomicBoolean isRefreshed = new AtomicBoolean();
createChunkList(5);
- OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class);
- clientConfig.setChecksumVerify(false);
+ // default retry interval, the retry delay is logged only if it is positive
+ OzoneClientConfig retryDelayConfig =
conf.getObject(OzoneClientConfig.class);
+ retryDelayConfig.setChecksumVerify(false);
try (BlockInputStream blockInputStreamWithRetry =
new DummyBlockInputStreamWithRetry(blockID, blockSize,
MockPipeline.createSingleNodePipeline(), null,
null, chunks, chunkDataMap, isRefreshed, null,
- clientConfig)) {
+ retryDelayConfig)) {
assertFalse(isRefreshed.get());
seekAndVerify(50);
byte[] b = new byte[200];
@@ -433,8 +436,6 @@ private static ChunkInputStream
throwingChunkInputStream(IOException ex,
private BlockInputStream createSubject(BlockID blockID, Pipeline pipeline,
ChunkInputStream stream) throws IOException {
- OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class);
- clientConfig.setChecksumVerify(false);
return new DummyBlockInputStream(blockID, blockSize, pipeline, null,
null, refreshFunction, chunks, null, clientConfig) {
@Override
@@ -493,8 +494,6 @@ public void
testRefreshOnReadFailureAfterUnbuffer(IOException ex)
.thenReturn(blockLocationInfo);
when(blockLocationInfo.getPipeline()).thenReturn(newPipeline);
- OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class);
- clientConfig.setChecksumVerify(false);
BlockInputStream subject = new BlockInputStream(
new BlockLocationInfo(new
BlockLocationInfo.Builder().setBlockID(blockID).setLength(blockSize)),
pipeline, null, clientFactory, refreshFunction,
@@ -583,8 +582,6 @@ private BlockInputStream createLargeBlockStream() throws
Exception {
chunkDataMap.put(chunkName, blockData);
chunks.add(chunkInfo);
- OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class);
- clientConfig.setChecksumVerify(false);
Pipeline pipeline = MockPipeline.createSingleNodePipeline();
return new DummyBlockInputStream(blockID, blockSize, pipeline, null,
null, refreshFunction, chunks, chunkDataMap, clientConfig);
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
index c0dff1dbeed..409fcfc92fb 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
@@ -987,7 +987,7 @@ void testAutoCompactionSmallSstFile(
long containerId = startContainerId;
// Create containers on each volume and export all container on new volume
- int count = 200;
+ int count = 10;
long numberOfKeysToWrite = 500;
KeyValueContainerData containerData;
KeyValueContainer container;
@@ -1057,8 +1057,6 @@ void testAutoCompactionSmallSstFile(
// After moving away from using Table.loadFromFile(), no SST files are
generated unless the db is force flushed
assertEquals(0, fileMetaDataList1.size());
hddsVolume.compactDb();
- // Sleep a while to wait for compaction to complete
- Thread.sleep(7000);
List<LiveFileMetaData> fileMetaDataList2 =
((RDBStore)(dnStore.getStore())).getDb().getLiveFilesMetaData();
assertThat(fileMetaDataList2).hasSizeLessThanOrEqualTo(fileMetaDataList1.size());
diff --git
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/events/TestEventQueue.java
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/events/TestEventQueue.java
index 5cbd5fe0f37..474cdd89cb6 100644
---
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/events/TestEventQueue.java
+++
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/events/TestEventQueue.java
@@ -160,7 +160,7 @@ public class TestHandler implements EventHandler {
@Override
public void onMessage(Object payload, EventPublisher publisher) {
try {
- Thread.sleep(2000);
+ Thread.sleep(500);
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
}
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestEventBufferOverflow.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestEventBufferOverflow.java
index 77fc6cdbb07..9d6fa9810ef 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestEventBufferOverflow.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestEventBufferOverflow.java
@@ -54,6 +54,7 @@
import
org.apache.hadoop.ozone.recon.tasks.updater.ReconTaskStatusUpdaterManager;
import org.apache.ozone.recon.schema.generated.tables.daos.ReconTaskStatusDao;
import org.apache.ozone.test.GenericTestUtils;
+import org.apache.ozone.test.MockClock;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -387,11 +388,12 @@ public void testCheckpointCreationFailureAndRetry()
throws Exception {
when(reconDbProvider.getDbStore()).thenReturn(mock(DBStore.class));
when(reconDbProvider.getStagedReconDBProvider()).thenReturn(reconDbProvider);
+ MockClock testClock = MockClock.newInstance();
ReconTaskControllerImpl reconTaskController = new ReconTaskControllerImpl(
config, new HashSet<>(), taskStatusUpdaterManager,
reconDbProvider, mock(ReconContainerMetadataManager.class),
mock(ReconNamespaceSummaryManager.class),
- mock(ReconGlobalStatsManager.class),
mock(ReconFileMetadataManager.class));
+ mock(ReconGlobalStatsManager.class),
mock(ReconFileMetadataManager.class), testClock);
// Set up a mock OM metadata manager
ReconOMMetadataManager mockOMMetadataManager =
mock(ReconOMMetadataManager.class);
@@ -409,7 +411,7 @@ reconDbProvider, mock(ReconContainerMetadataManager.class),
// Attempts 1-6: should return RETRY_LATER
for (int i = 1; i <= 6; i++) {
if (i > 1) {
- Thread.sleep(2100); // Wait for retry delay
+ testClock.fastForward(2100); // Wait for retry delay
}
result = controllerSpy.queueReInitializationEvent(
ReconTaskReInitializationEvent.ReInitializationReason.BUFFER_OVERFLOW);
@@ -418,7 +420,7 @@ reconDbProvider, mock(ReconContainerMetadataManager.class),
}
// Attempt 7: should return MAX_RETRIES_EXCEEDED
- Thread.sleep(2100);
+ testClock.fastForward(2100);
result = controllerSpy.queueReInitializationEvent(
ReconTaskReInitializationEvent.ReInitializationReason.BUFFER_OVERFLOW);
assertEquals(ReconTaskController.ReInitializationResult.MAX_RETRIES_EXCEEDED,
result,
@@ -449,11 +451,12 @@ public void testRetryMechanismWithFullSnapshotFallback()
throws Exception {
when(reconDbProvider.getDbStore()).thenReturn(mock(DBStore.class));
when(reconDbProvider.getStagedReconDBProvider()).thenReturn(reconDbProvider);
+ MockClock testClock = MockClock.newInstance();
ReconTaskControllerImpl reconTaskController = new ReconTaskControllerImpl(
config, new HashSet<>(), taskStatusUpdaterManager,
reconDbProvider, mock(ReconContainerMetadataManager.class),
mock(ReconNamespaceSummaryManager.class),
- mock(ReconGlobalStatsManager.class),
mock(ReconFileMetadataManager.class));
+ mock(ReconGlobalStatsManager.class),
mock(ReconFileMetadataManager.class), testClock);
ReconOMMetadataManager mockOMMetadataManager =
mock(ReconOMMetadataManager.class);
reconTaskController.updateOMMetadataManager(mockOMMetadataManager);
@@ -473,7 +476,7 @@ reconDbProvider, mock(ReconContainerMetadataManager.class),
for (int attempt = 1; attempt <= maxReinitQueueFailures; attempt++) {
if (attempt > 1) {
// Wait for delay between iterations
- Thread.sleep(2100);
+ testClock.fastForward(2100);
}
ReconTaskController.ReInitializationResult result =
controllerSpy.queueReInitializationEvent(
@@ -519,11 +522,12 @@ public void testSuccessfulCheckpointAfterFailures()
throws Exception {
when(reconDbProvider.getDbStore()).thenReturn(mock(DBStore.class));
when(reconDbProvider.getStagedReconDBProvider()).thenReturn(reconDbProvider);
+ MockClock testClock = MockClock.newInstance();
ReconTaskControllerImpl reconTaskController = new ReconTaskControllerImpl(
config, new HashSet<>(), taskStatusUpdaterManager,
reconDbProvider, mock(ReconContainerMetadataManager.class),
mock(ReconNamespaceSummaryManager.class),
- mock(ReconGlobalStatsManager.class),
mock(ReconFileMetadataManager.class));
+ mock(ReconGlobalStatsManager.class),
mock(ReconFileMetadataManager.class), testClock);
// Set up properly mocked ReconOMMetadataManager with required dependencies
ReconOMMetadataManager mockOMMetadataManager =
mock(ReconOMMetadataManager.class);
@@ -558,7 +562,7 @@ reconDbProvider, mock(ReconContainerMetadataManager.class),
}
// Second attempt - should return RETRY_LATER (after delay)
- Thread.sleep(2100);
+ testClock.fastForward(2100);
ReconTaskController.ReInitializationResult result2 =
controllerSpy.queueReInitializationEvent(
ReconTaskReInitializationEvent.ReInitializationReason.BUFFER_OVERFLOW);
if (result2 != ReconTaskController.ReInitializationResult.SUCCESS) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
