This is an automated email from the ASF dual-hosted git repository.
gian pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git.
from d628beb Make supervisor API similar to submit task API (#8810)
add ac6d703 Support inputFormat and inputSource for sampler (#8901)
No new revisions were added by this update.
Summary of changes:
.../druid/data/input/AbstractInputSource.java | 6 +-
.../java/org/apache/druid/data/input/Firehose.java | 18 +-
.../input/FirehoseFactoryToInputSourceAdaptor.java | 5 +
.../apache/druid/data/input/InputEntityReader.java | 11 +-
.../org/apache/druid/data/input/InputFormat.java | 4 +-
.../druid/data/input/InputRowListPlusJson.java | 113 --
.../data/input/InputRowListPlusRawValues.java | 96 ++
.../org/apache/druid/data/input/InputSource.java | 6 +-
.../apache/druid/data/input/InputSourceReader.java | 2 +-
.../data/input/IntermediateRowParsingReader.java | 30 +-
.../apache/druid/data/input/impl/ByteEntity.java | 5 +
.../apache/druid/data/input/impl/CsvReader.java | 18 +-
.../data/input/impl/FileIteratingFirehose.java | 9 +-
.../impl/FirehoseToInputSourceReaderAdaptor.java | 12 +-
.../druid/data/input/impl/InlineInputSource.java} | 35 +-
.../input/impl/InputEntityIteratingReader.java | 21 +-
.../apache/druid/data/input/impl/JsonReader.java | 6 +-
.../druid/data/input/impl/RegexInputFormat.java} | 30 +-
.../impl/{CsvReader.java => RegexReader.java} | 94 +-
.../data/input/impl/StringInputRowParser.java | 4 +-
.../data/input/impl/TimedShutoffInputSource.java} | 43 +-
.../input/impl/TimedShutoffInputSourceReader.java | 137 +++
.../druid/java/util/common/CloseableIterators.java | 3 +
.../druid/java/util/common/parsers/CSVParser.java | 9 +-
.../util/common/parsers/CloseableIterator.java | 82 +-
.../input/impl/TimedShutoffInputSourceTest.java | 57 +
.../util/common/parsers/CloseableIteratorTest.java | 102 ++
.../IncrementalPublishingKafkaIndexTaskRunner.java | 4 +-
.../druid/indexing/kafka/KafkaIndexTask.java | 2 +-
.../indexing/kafka/KafkaIndexTaskIOConfig.java | 13 +-
.../druid/indexing/kafka/KafkaSamplerSpec.java | 50 +-
.../indexing/kafka/supervisor/KafkaSupervisor.java | 5 +-
.../kafka/supervisor/KafkaSupervisorIOConfig.java | 3 +
.../druid/indexing/kafka/KafkaIOConfigTest.java | 3 +-
.../druid/indexing/kafka/KafkaIndexTaskTest.java | 231 ++--
.../druid/indexing/kafka/KafkaSamplerSpecTest.java | 163 +--
.../kafka/supervisor/KafkaSupervisorTest.java | 41 +-
.../druid/indexing/kinesis/KinesisIndexTask.java | 3 +-
.../indexing/kinesis/KinesisIndexTaskIOConfig.java | 7 +-
.../indexing/kinesis/KinesisIndexTaskRunner.java | 2 +-
.../druid/indexing/kinesis/KinesisSamplerSpec.java | 67 +-
.../kinesis/supervisor/KinesisSupervisor.java | 3 +
.../supervisor/KinesisSupervisorIOConfig.java | 3 +
.../indexing/kinesis/KinesisIOConfigTest.java | 1 +
.../indexing/kinesis/KinesisIndexTaskTest.java | 168 ++-
.../indexing/kinesis/KinesisSamplerSpecTest.java | 183 ++-
.../kinesis/supervisor/KinesisSupervisorTest.java | 37 +-
.../druid/indexing/common/task/IndexTask.java | 5 +
.../indexing/overlord/sampler/FirehoseSampler.java | 297 -----
.../overlord/sampler/IndexTaskSamplerSpec.java | 54 +-
.../overlord/sampler/InputSourceSampler.java | 227 ++++
.../indexing/overlord/sampler/SamplerCache.java | 179 ---
.../indexing/overlord/sampler/SamplerConfig.java | 43 +-
.../indexing/overlord/sampler/SamplerModule.java | 3 +-
.../indexing/overlord/sampler/SamplerResponse.java | 36 +-
.../seekablestream/RecordSupplierInputSource.java | 158 +++
.../SeekableStreamIndexTaskIOConfig.java | 20 +-
.../SeekableStreamIndexTaskRunner.java | 60 +-
.../seekablestream/SeekableStreamSamplerSpec.java | 204 ++--
.../SeekableStreamSupervisorIOConfig.java | 25 +
.../apache/druid/indexing/common/TestFirehose.java | 86 +-
.../druid/indexing/overlord/TaskLifecycleTest.java | 6 +-
.../sampler/CsvInputSourceSamplerTest.java | 158 +++
.../overlord/sampler/FirehoseSamplerTest.java | 1202 -------------------
.../overlord/sampler/IndexTaskSamplerSpecTest.java | 50 +-
.../overlord/sampler/InputSourceSamplerTest.java | 1249 ++++++++++++++++++++
.../overlord/sampler/SamplerCacheTest.java | 177 ---
.../overlord/sampler/SamplerResponseTest.java | 10 +-
.../overlord/sampler/SamplerTestUtils.java | 34 +-
.../RecordSupplierInputSourceTest.java | 219 ++++
.../SeekableStreamIndexTaskTestBase.java | 112 +-
.../SeekableStreamSupervisorStateTest.java | 33 +-
.../tests/indexer/AbstractKafkaIndexerTest.java | 13 +-
.../tests/indexer/ITKafkaIndexingServiceTest.java | 24 +-
.../ITKafkaIndexingServiceTransactionalTest.java | 24 +-
...son => kafka_supervisor_spec_input_format.json} | 27 +-
...on => kafka_supervisor_spec_legacy_parser.json} | 0
.../segment/incremental/IncrementalIndex.java | 2 +-
.../incremental/IncrementalIndexSchema.java | 12 -
.../druid/segment/transform/TransformSpec.java | 8 +-
.../druid/segment/transform/Transformer.java | 29 +-
...der.java => TransformingInputEntityReader.java} | 12 +-
...der.java => TransformingInputSourceReader.java} | 8 +-
.../apache/druid/segment/indexing/DataSchema.java | 4 +-
.../granularity/ArbitraryGranularitySpec.java | 3 +-
.../segment/realtime/firehose/InlineFirehose.java | 9 +-
.../realtime/firehose/InlineFirehoseFactory.java | 2 +-
.../firehose/TimedShutoffFirehoseFactory.java | 21 +-
.../realtime/firehose/InlineFirehoseTest.java | 46 +-
89 files changed, 3823 insertions(+), 3015 deletions(-)
delete mode 100644
core/src/main/java/org/apache/druid/data/input/InputRowListPlusJson.java
create mode 100644
core/src/main/java/org/apache/druid/data/input/InputRowListPlusRawValues.java
copy core/src/{test/java/org/apache/druid/data/input/impl/NoopInputSource.java
=> main/java/org/apache/druid/data/input/impl/InlineInputSource.java} (62%)
copy core/src/{test/java/org/apache/druid/data/input/impl/NoopInputFormat.java
=> main/java/org/apache/druid/data/input/impl/RegexInputFormat.java} (63%)
copy core/src/main/java/org/apache/druid/data/input/impl/{CsvReader.java =>
RegexReader.java} (52%)
copy core/src/{test/java/org/apache/druid/data/input/impl/NoopInputSource.java
=> main/java/org/apache/druid/data/input/impl/TimedShutoffInputSource.java}
(57%)
create mode 100644
core/src/main/java/org/apache/druid/data/input/impl/TimedShutoffInputSourceReader.java
create mode 100644
core/src/test/java/org/apache/druid/data/input/impl/TimedShutoffInputSourceTest.java
create mode 100644
core/src/test/java/org/apache/druid/java/util/common/parsers/CloseableIteratorTest.java
delete mode 100644
indexing-service/src/main/java/org/apache/druid/indexing/overlord/sampler/FirehoseSampler.java
create mode 100644
indexing-service/src/main/java/org/apache/druid/indexing/overlord/sampler/InputSourceSampler.java
delete mode 100644
indexing-service/src/main/java/org/apache/druid/indexing/overlord/sampler/SamplerCache.java
create mode 100644
indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/RecordSupplierInputSource.java
create mode 100644
indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/CsvInputSourceSamplerTest.java
delete mode 100644
indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/FirehoseSamplerTest.java
create mode 100644
indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/InputSourceSamplerTest.java
delete mode 100644
indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/SamplerCacheTest.java
copy
processing/src/main/java/org/apache/druid/query/context/DefaultResponseContext.java
=>
indexing-service/src/test/java/org/apache/druid/indexing/overlord/sampler/SamplerTestUtils.java
(60%)
create mode 100644
indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/RecordSupplierInputSourceTest.java
copy integration-tests/src/test/resources/indexer/{kafka_supervisor_spec.json
=> kafka_supervisor_spec_input_format.json} (65%)
rename
integration-tests/src/test/resources/indexer/{kafka_supervisor_spec.json =>
kafka_supervisor_spec_legacy_parser.json} (100%)
copy
processing/src/main/java/org/apache/druid/segment/transform/{TransformingReader.java
=> TransformingInputEntityReader.java} (76%)
rename
processing/src/main/java/org/apache/druid/segment/transform/{TransformingReader.java
=> TransformingInputSourceReader.java} (82%)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
