This is an automated email from the ASF dual-hosted git repository.
gian pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git.
from ac6d703 Support inputFormat and inputSource for sampler (#8901)
add ff62173 Refactor parallel indexing perfect rollup partitioning (#8852)
No new revisions were added by this update.
Summary of changes:
.../druid/data/input/HandlingInputRowIterator.java | 92 ++++
.../indexer/partitions/DynamicPartitionsSpec.java | 6 +
.../indexer/partitions/HashedPartitionsSpec.java | 8 +
.../druid/indexer/partitions/PartitionsSpec.java | 26 +
.../partitions/SingleDimensionPartitionsSpec.java | 12 +-
.../data/input/HandlingInputRowIteratorTest.java | 179 +++++++
.../druid/testing/junit/LoggerCaptureRule.java | 11 +-
indexing-service/pom.xml | 5 +
.../common/task/AbstractBatchIndexTask.java | 6 +
.../common/task/CachingLocalSegmentAllocator.java | 102 +---
.../indexing/common/task/HadoopIndexTask.java | 6 -
.../HashPartitionCachingLocalSegmentAllocator.java | 114 ++++
.../druid/indexing/common/task/IndexTask.java | 34 +-
.../indexing/common/task/InputSourceProcessor.java | 44 +-
.../apache/druid/indexing/common/task/Task.java | 8 +-
...pec.java => GeneratedHashPartitionsReport.java} | 23 +-
.../batch/parallel/GeneratedPartitionsReport.java | 17 +-
...rgeIOConfig.java => HashPartitionLocation.java} | 36 +-
.../{PartitionStat.java => HashPartitionStat.java} | 82 +--
...> InputSourceSplitParallelIndexTaskRunner.java} | 77 +--
.../batch/parallel/ParallelIndexPhaseRunner.java | 2 +-
.../parallel/ParallelIndexSupervisorTask.java | 138 +++--
.../batch/parallel/ParallelIndexTaskRunner.java | 2 +-
.../batch/parallel/ParallelIndexTuningConfig.java | 4 +-
...HashSegmentGenerateParallelIndexTaskRunner.java | 115 ++++
.../parallel/PartialHashSegmentGenerateTask.java | 173 ++++++
...c.java => PartialHashSegmentMergeIOConfig.java} | 19 +-
...a => PartialHashSegmentMergeIngestionSpec.java} | 8 +-
...alHashSegmentMergeParallelIndexTaskRunner.java} | 37 +-
.../parallel/PartialHashSegmentMergeTask.java | 114 ++++
.../batch/parallel/PartialSegmentGenerateTask.java | 196 ++-----
.../parallel/PartialSegmentMergeIOConfig.java | 12 +-
.../parallel/PartialSegmentMergeIngestionSpec.java | 15 +-
.../batch/parallel/PartialSegmentMergeTask.java | 176 ++----
.../task/batch/parallel/PartitionLocation.java | 63 ++-
.../common/task/batch/parallel/PartitionStat.java | 59 +-
.../batch/parallel/PerfectRollupWorkerTask.java | 122 +++++
.../task/batch/parallel/SinglePhaseSubTask.java | 6 -
.../common/task/batch/parallel/SubTaskReport.java | 2 +-
.../DefaultIndexTaskInputRowIteratorBuilder.java | 149 ++++++
.../iterator/IndexTaskInputRowIteratorBuilder.java | 61 +++
.../apache/druid/indexing/common/TestUtils.java | 56 +-
.../indexing/common/task/IndexTaskSerdeTest.java | 2 +-
.../common/task/NoopIndexTaskClientFactory.java | 4 +-
.../AbstractMultiPhaseParallelIndexingTest.java | 284 ++++++++++
.../AbstractParallelIndexSupervisorTaskTest.java | 29 +-
.../GeneratedHashPartitionsReportTest.java | 59 ++
...hPartitionCachingLocalSegmentAllocatorTest.java | 145 +++++
...ashPartitionMultiPhaseParallelIndexingTest.java | 335 ++++++++++++
.../task/batch/parallel/HashPartitionStatTest.java | 59 ++
.../parallel/MultiPhaseParallelIndexingTest.java | 593 ---------------------
.../ParallelIndexSupervisorTaskSerdeTest.java | 48 +-
.../parallel/ParallelIndexTestingFactory.java | 247 +++++++++
.../PartialHashSegmentGenerateTaskTest.java | 74 +++
.../PartialHashSegmentMergeIOConfigTest.java | 54 ++
.../PartialHashSegmentMergeIngestionSpecTest.java | 68 +++
.../parallel/PartialHashSegmentMergeTaskTest.java | 90 ++++
.../parallel/PerfectRollupWorkerTaskTest.java | 189 +++++++
...efaultIndexTaskInputRowIteratorBuilderTest.java | 239 +++++++++
...xTaskInputRowIteratorBuilderTestingFactory.java | 162 ++++++
.../tests/indexer/AbstractITBatchIndexTest.java | 8 +-
.../druid/tests/indexer/ITParallelIndexTest.java | 12 +-
.../java/org/apache/druid/jackson/JodaStuff.java | 10 +
.../java/org/apache/druid/segment/TestHelper.java | 19 +
.../realtime/firehose/InlineFirehoseFactory.java | 2 +
65 files changed, 3773 insertions(+), 1376 deletions(-)
create mode 100644
core/src/main/java/org/apache/druid/data/input/HandlingInputRowIterator.java
create mode 100644
core/src/test/java/org/apache/druid/data/input/HandlingInputRowIteratorTest.java
create mode 100644
indexing-service/src/main/java/org/apache/druid/indexing/common/task/HashPartitionCachingLocalSegmentAllocator.java
copy
indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/{PartialSegmentMergeIngestionSpec.java
=> GeneratedHashPartitionsReport.java} (61%)
copy
indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/{PartialSegmentMergeIOConfig.java
=> HashPartitionLocation.java} (57%)
copy
indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/{PartitionStat.java
=> HashPartitionStat.java} (53%)
rename
indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/{PartialSegmentGenerateParallelIndexTaskRunner.java
=> InputSourceSplitParallelIndexTaskRunner.java} (69%)
create mode 100644
indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialHashSegmentGenerateParallelIndexTaskRunner.java
create mode 100644
indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialHashSegmentGenerateTask.java
copy
indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/{PartialSegmentMergeIngestionSpec.java
=> PartialHashSegmentMergeIOConfig.java} (65%)
copy
indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/{PartialSegmentMergeIngestionSpec.java
=> PartialHashSegmentMergeIngestionSpec.java} (80%)
rename
indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/{PartialSegmentMergeParallelIndexTaskRunner.java
=> PartialHashSegmentMergeParallelIndexTaskRunner.java} (68%)
create mode 100644
indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialHashSegmentMergeTask.java
create mode 100644
indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PerfectRollupWorkerTask.java
create mode 100644
indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/iterator/DefaultIndexTaskInputRowIteratorBuilder.java
create mode 100644
indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/iterator/IndexTaskInputRowIteratorBuilder.java
create mode 100644
indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractMultiPhaseParallelIndexingTest.java
create mode 100644
indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/GeneratedHashPartitionsReportTest.java
create mode 100644
indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/HashPartitionCachingLocalSegmentAllocatorTest.java
create mode 100644
indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/HashPartitionMultiPhaseParallelIndexingTest.java
create mode 100644
indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/HashPartitionStatTest.java
delete mode 100644
indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/MultiPhaseParallelIndexingTest.java
create mode 100644
indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexTestingFactory.java
create mode 100644
indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/PartialHashSegmentGenerateTaskTest.java
create mode 100644
indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/PartialHashSegmentMergeIOConfigTest.java
create mode 100644
indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/PartialHashSegmentMergeIngestionSpecTest.java
create mode 100644
indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/PartialHashSegmentMergeTaskTest.java
create mode 100644
indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/PerfectRollupWorkerTaskTest.java
create mode 100644
indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/iterator/DefaultIndexTaskInputRowIteratorBuilderTest.java
create mode 100644
indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/iterator/IndexTaskInputRowIteratorBuilderTestingFactory.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
