This is an automated email from the ASF dual-hosted git repository.
kfaraz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new cd4c536e5d2 Improve runtime of indexing-service UTs (#18482)
cd4c536e5d2 is described below
commit cd4c536e5d29e676da97f630668d49cd26c2f1bc
Author: Kashif Faraz <[email protected]>
AuthorDate: Fri Sep 5 00:09:55 2025 +0530
Improve runtime of indexing-service UTs (#18482)
Changes:
- Increase number of task runner threads
- Increase `maxNumConcurrentSubTasks` to increase task parallelism
- Reduce the total runtime of the two slowest UTs from 15 mins to 5 mins
---
.../druid/indexing/common/task/IngestionTestBase.java | 11 +++++++++--
.../parallel/AbstractParallelIndexSupervisorTaskTest.java | 2 +-
.../HashPartitionMultiPhaseParallelIndexingTest.java | 14 +++++++-------
.../RangePartitionMultiPhaseParallelIndexingTest.java | 10 +++++-----
4 files changed, 22 insertions(+), 15 deletions(-)
diff --git
a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IngestionTestBase.java
b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IngestionTestBase.java
index 80c93cbdf58..7444038a5c7 100644
---
a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IngestionTestBase.java
+++
b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IngestionTestBase.java
@@ -131,6 +131,7 @@ public abstract class IngestionTestBase extends
InitializedNullHandlingTest
private SegmentCacheManagerFactory segmentCacheManagerFactory;
private TaskStorage taskStorage;
private IndexerSQLMetadataStorageCoordinator storageCoordinator;
+ private TaskActionToolbox taskActionToolbox;
private SegmentsMetadataManager segmentsMetadataManager;
private GlobalTaskLockbox lockbox;
private File baseDir;
@@ -194,6 +195,7 @@ public abstract class IngestionTestBase extends
InitializedNullHandlingTest
segmentCacheManagerFactory = new
SegmentCacheManagerFactory(TestIndex.INDEX_IO, getObjectMapper());
reportsFile = temporaryFolder.newFile();
dataSegmentKiller = new TestDataSegmentKiller();
+ taskActionToolbox = createTaskActionToolbox();
segmentMetadataCache.start();
segmentMetadataCache.becomeLeader();
@@ -278,7 +280,7 @@ public abstract class IngestionTestBase extends
InitializedNullHandlingTest
return dataSegmentKiller;
}
- public TaskActionToolbox createTaskActionToolbox()
+ private TaskActionToolbox createTaskActionToolbox()
{
storageCoordinator.start();
return new TaskActionToolbox(
@@ -291,6 +293,11 @@ public abstract class IngestionTestBase extends
InitializedNullHandlingTest
);
}
+ public TaskActionToolbox getTaskActionToolbox()
+ {
+ return taskActionToolbox;
+ }
+
public TaskToolbox createTaskToolbox(TaskConfig config, Task task,
SupervisorManager supervisorManager)
{
CentralizedDatasourceSchemaConfig centralizedDatasourceSchemaConfig =
CentralizedDatasourceSchemaConfig.enabled(true);
@@ -417,7 +424,7 @@ public abstract class IngestionTestBase extends
InitializedNullHandlingTest
private TestLocalTaskActionClient(Task task)
{
- super(task, taskStorage, createTaskActionToolbox());
+ super(task, taskStorage, getTaskActionToolbox());
}
@Override
diff --git
a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractParallelIndexSupervisorTaskTest.java
b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractParallelIndexSupervisorTaskTest.java
index fac3feff14c..664ffa5176b 100644
---
a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractParallelIndexSupervisorTaskTest.java
+++
b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractParallelIndexSupervisorTaskTest.java
@@ -318,7 +318,7 @@ public class AbstractParallelIndexSupervisorTaskTest
extends IngestionTestBase
SimpleThreadingTaskRunner(String threadNameBase)
{
- service = MoreExecutors.listeningDecorator(Execs.multiThreaded(5,
threadNameBase + "-%d"));
+ service = MoreExecutors.listeningDecorator(Execs.multiThreaded(20,
threadNameBase + "-%d"));
taskKiller.scheduleAtFixedRate(
() -> {
for (TaskContainer taskContainer : tasks.values()) {
diff --git
a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/HashPartitionMultiPhaseParallelIndexingTest.java
b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/HashPartitionMultiPhaseParallelIndexingTest.java
index 53df996e805..02797bc1067 100644
---
a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/HashPartitionMultiPhaseParallelIndexingTest.java
+++
b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/HashPartitionMultiPhaseParallelIndexingTest.java
@@ -96,15 +96,15 @@ public class HashPartitionMultiPhaseParallelIndexingTest
extends AbstractMultiPh
public static Iterable<Object[]> constructorFeeder()
{
return ImmutableList.of(
- new Object[]{LockGranularity.TIME_CHUNK, false, 2, INTERVAL_TO_INDEX,
2},
- new Object[]{LockGranularity.TIME_CHUNK, true, 2, INTERVAL_TO_INDEX,
2},
- new Object[]{LockGranularity.TIME_CHUNK, true, 2, null, 2},
+ new Object[]{LockGranularity.TIME_CHUNK, false, 10, INTERVAL_TO_INDEX,
2},
+ new Object[]{LockGranularity.TIME_CHUNK, true, 10, INTERVAL_TO_INDEX,
2},
+ new Object[]{LockGranularity.TIME_CHUNK, true, 10, null, 2},
new Object[]{LockGranularity.TIME_CHUNK, true, 1, INTERVAL_TO_INDEX,
2},
- new Object[]{LockGranularity.SEGMENT, true, 2, INTERVAL_TO_INDEX, 2},
- new Object[]{LockGranularity.TIME_CHUNK, true, 2, INTERVAL_TO_INDEX,
null},
- new Object[]{LockGranularity.TIME_CHUNK, true, 2, null, null},
+ new Object[]{LockGranularity.SEGMENT, true, 10, INTERVAL_TO_INDEX, 2},
+ new Object[]{LockGranularity.TIME_CHUNK, true, 10, INTERVAL_TO_INDEX,
null},
+ new Object[]{LockGranularity.TIME_CHUNK, true, 10, null, null},
new Object[]{LockGranularity.TIME_CHUNK, true, 1, INTERVAL_TO_INDEX,
null},
- new Object[]{LockGranularity.SEGMENT, true, 2, INTERVAL_TO_INDEX, null}
+ new Object[]{LockGranularity.SEGMENT, true, 10, INTERVAL_TO_INDEX,
null}
);
}
diff --git
a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/RangePartitionMultiPhaseParallelIndexingTest.java
b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/RangePartitionMultiPhaseParallelIndexingTest.java
index 82eb3ebb168..38cb488669b 100644
---
a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/RangePartitionMultiPhaseParallelIndexingTest.java
+++
b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/RangePartitionMultiPhaseParallelIndexingTest.java
@@ -115,12 +115,12 @@ public class RangePartitionMultiPhaseParallelIndexingTest
extends AbstractMultiP
public static Iterable<Object[]> constructorFeeder()
{
return ImmutableList.of(
- new Object[]{LockGranularity.TIME_CHUNK, !USE_INPUT_FORMAT_API, 2,
!USE_MULTIVALUE_DIM, INTERVAL_TO_INDEX},
- new Object[]{LockGranularity.TIME_CHUNK, USE_INPUT_FORMAT_API, 2,
!USE_MULTIVALUE_DIM, INTERVAL_TO_INDEX},
- new Object[]{LockGranularity.TIME_CHUNK, USE_INPUT_FORMAT_API, 2,
!USE_MULTIVALUE_DIM, null},
- new Object[]{LockGranularity.SEGMENT, USE_INPUT_FORMAT_API, 2,
!USE_MULTIVALUE_DIM, INTERVAL_TO_INDEX},
+ new Object[]{LockGranularity.TIME_CHUNK, !USE_INPUT_FORMAT_API, 10,
!USE_MULTIVALUE_DIM, INTERVAL_TO_INDEX},
+ new Object[]{LockGranularity.TIME_CHUNK, USE_INPUT_FORMAT_API, 10,
!USE_MULTIVALUE_DIM, INTERVAL_TO_INDEX},
+ new Object[]{LockGranularity.TIME_CHUNK, USE_INPUT_FORMAT_API, 10,
!USE_MULTIVALUE_DIM, null},
+ new Object[]{LockGranularity.SEGMENT, USE_INPUT_FORMAT_API, 10,
!USE_MULTIVALUE_DIM, INTERVAL_TO_INDEX},
new Object[]{LockGranularity.SEGMENT, USE_INPUT_FORMAT_API, 1,
!USE_MULTIVALUE_DIM, INTERVAL_TO_INDEX}, // will spawn subtask
- new Object[]{LockGranularity.SEGMENT, USE_INPUT_FORMAT_API, 2,
USE_MULTIVALUE_DIM, INTERVAL_TO_INDEX} // expected to fail
+ new Object[]{LockGranularity.SEGMENT, USE_INPUT_FORMAT_API, 10,
USE_MULTIVALUE_DIM, INTERVAL_TO_INDEX} // expected to fail
);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
