This is an automated email from the ASF dual-hosted git repository.
snlee pushed a change to branch release-0.8.0-rc
in repository https://gitbox.apache.org/repos/asf/pinot.git.
from 3503074 Add system info on instance config for servers. (#7093)
add 3976eac Move KafkaJsonDecoder code to pinot-json (#7021)
add 47f1d93 Add `PinotHelixResourceManager` to `TableConfigTuner.init()`
method. (#7101)
add bbdd615 Added support for additional mvn options (#7103)
add 0981271 Add minion metrics of task queueing time and task numbers
(#7099)
add 2475089 Fix maven options (#7105)
add 47a75e5 Remove parallel builds in compat scripts (#7108)
add ea3566f Fix the memory issue for selection query with large limit
(#7112)
add f38609b Clean up SegmentMetadata and ColumnMetadata (#7104)
add f809e50 SegmentProcessorFramework Enhancement (#7092)
add 0fb9209 Implement Append merger for partial upsert (#7087)
add 0255757 Fix build (#7115)
add 423ac35 Validate the length of preprocessing operation from custom
config (#7119)
add 70308f8 Fixing pinot k8s helm for jdk11 (#7118)
add 84c6a6e Add integration test for Kinesis realtime connector (#7016)
add d4d0b93 Cancel running Kinesis consumer tasks when timeout occurs
(#7109)
add f9f58c7 Allow updating controller and broker helix hostname (#7064)
add 1ea4260 adding json-bigInt package to parse long values in json
(#7120)
add 3c033a6 Cleanup PinotHelixResourceManager and move the default tag
handling to the starter (#7123)
add 0eb0f89 Fix UpsertConfig JSON deserialization. (#7125)
add e656e16 Fix custom instance id for controller/broker/minion (#7127)
add 1956c07 Add timestamp datatype support in JDBC (#7117)
add b99e804 Fixing Controller/Broker/Server/Minion zk and cluster name
configs in PinotServiceMananger when start using config file (#7131)
add 4fdf01b Fixing init controller conf for StartControllerCommand (#7132)
add a251c42 Adding ability to check ingestion status for Offline Pinot
table (#7070)
add 4e88da6 Add datetime function with 2 arguments (#7116)
add 0102227 Fix package for ReplicationUtils. (#7135)
add c42aef6 Validate parsing multi value delimiter as unicode (#7133)
add 8b5c0ac Release helm pinot 0.2.4 (#7136)
add 88bff60 Add support to skip kinesis test if docker is not available
(#7140)
add 1a12510 Move minion metrics definition to pinot-common package (#7107)
add 7fb65f0 Update ExecutionStats.java (#7146)
add f0675af Switch to using 'authority' instead of 'host' when building
parent URI in PinotFS::move (#7037)
add 0500c12 Fixing pinot-admin script to parse env var PLUGINS_INCLUDE
(#7145)
add 52d733c Provide HTTP client to kinesis builder (#7148)
add 7233e2c Support Dictionary Based Plan For DISTINCT (#7141)
add 0447e74 Fix the exception message for segment reset (#7154)
add f04078e Upgrade avro-mapred to 1.9.2 so that all avro dependencies
are 1.9.2. (#7111)
add e17029f Wire TaskMetricsEmitter to controller starter (#7126)
add 1ef50f7 Add controller/server API to fetch aggregated segment
metadata (#7102)
add db5dfd7 Fix race condition in MinionInstancesCleanupTask (#7122)
add de2f0e0 Enable parallel builds for compat checker (#7149)
add 34e8257 Add systemResourceInfo to getInstance API (#7160)
add 6d0ab91 Fix typo in SegmentMetadataFetcher. (#7162)
add dbaac00 Fix errors happening due to parallel builds (#7172)
add 480395f Add Apache Pulsar low level and high level connector (#7026)
add a766c65 Updated CONTRIBUTING.md with Code Formatting Guidelines
(#7170)
add db5561a Improve the segment aggregated metadata controller/server API
(#7157)
add 738a584 Extract time handling for SegmentProcessorFramework (#7158)
add b6907fc add groovy unit test for sha256 data masking (#7179)
add fe83e95 Added a query param to /tables for getting sorted table names
based on time metadata (#7142)
No new revisions were added by this update.
Summary of changes:
CONTRIBUTING.md | 5 +
compatibility-verifier/checkoutAndBuild.sh | 222 +++--
compatibility-verifier/compCheck.sh | 21 +
kubernetes/helm/README-dev.md | 4 +
kubernetes/helm/index.yaml | 28 +
kubernetes/helm/pinot-0.2.4.tgz | Bin 0 -> 26536 bytes
kubernetes/helm/pinot/Chart.yaml | 4 +-
.../helm/pinot/templates/controller/configmap.yaml | 2 +-
.../helm/pinot/templates/minion/configmap.yaml | 2 +-
kubernetes/helm/pinot/values.yaml | 10 +-
kubernetes/helm/presto/Chart.yaml | 4 +-
.../broker/broker/helix/BaseBrokerStarter.java | 88 +-
.../broker/HelixBrokerStarterHostnamePortTest.java | 115 +++
.../org/apache/pinot/client/ExecutionStats.java | 2 +-
pinot-clients/pinot-jdbc-client/pom.xml | 34 +
.../pinot/client/PinotPreparedStatement.java | 35 +-
.../org/apache/pinot/client/PinotStatement.java | 30 +
.../org/apache/pinot/client/utils/DriverUtils.java | 16 +
.../pinot/client/PinotPreparedStatementTest.java | 18 +-
.../common/function/scalar/DateTimeFunctions.java | 11 +
.../pinot/common/metrics/ControllerGauge.java | 5 +-
.../apache/pinot/common}/metrics/MinionGauge.java | 4 +-
.../apache/pinot/common}/metrics/MinionMeter.java | 3 +-
.../pinot/common}/metrics/MinionMetrics.java | 3 +-
.../pinot/common}/metrics/MinionQueryPhase.java | 6 +-
.../apache/pinot/common}/metrics/MinionTimer.java | 3 +-
.../restlet/resources/TableMetadataInfo.java | 43 +
.../pinot/common/utils/helix/HelixHelper.java | 70 +-
.../common/utils/helix/LeadControllerUtils.java | 9 +-
.../common/utils/config/TableConfigSerDeTest.java | 2 +-
.../pinot/common/utils/helix/HelixHelperTest.java | 78 ++
.../pinot/controller/BaseControllerStarter.java | 56 +-
.../apache/pinot/controller/ControllerConf.java | 34 +-
.../pinot/controller/LeadControllerManager.java | 16 +-
.../pinot/controller/api/debug/TableDebugInfo.java | 11 +-
.../resources/PinotInstanceRestletResource.java | 6 +
.../api/resources/PinotQueryResource.java | 2 +-
.../api/resources/PinotSegmentRestletResource.java | 65 +-
.../api/resources/PinotTableRestletResource.java | 155 +++-
.../api/resources/PinotTaskRestletResource.java | 9 +
.../controller/api/resources/ResourceUtils.java | 43 +
.../api/resources/TableConfigsRestletResource.java | 2 +-
.../api/resources/TableDebugResource.java | 28 +-
.../helix/core/PinotHelixResourceManager.java | 69 +-
.../core/minion/MinionInstancesCleanupTask.java | 37 +-
.../core/minion/PinotHelixTaskResourceManager.java | 35 +
.../helix/core/minion/TaskMetricsEmitter.java | 32 +-
.../realtime/PinotLLCRealtimeSegmentManager.java | 100 ++-
.../helix/core/util/ZKMetadataUtils.java | 13 +-
.../tuner/NoOpTableTableConfigTuner.java | 3 +-
.../controller/tuner/RealTimeAutoIndexTuner.java | 3 +-
.../pinot/controller/tuner/TableConfigTuner.java | 14 +-
.../controller/tuner/TableConfigTunerUtils.java | 5 +-
.../util/ServerSegmentMetadataReader.java | 88 ++
.../util/TableIngestionStatusHelper.java | 89 ++
.../pinot/controller/util/TableMetadataReader.java | 21 +
.../src/main/resources/app/components/SideBar.tsx | 2 +-
.../src/main/resources/app/requests/index.ts | 4 +-
.../src/main/resources/app/styles/styles.css | 2 +-
.../main/resources/app/utils/PinotMethodUtils.ts | 2 +
.../src/main/resources/app/utils/axios-config.ts | 4 +-
.../src/main/resources/package-lock.json | 13 +
pinot-controller/src/main/resources/package.json | 1 +
.../pinot/controller/ControllerStarterTest.java | 110 +++
.../controller/LeadControllerManagerTest.java | 13 +-
.../controller/api/PinotSegmentsMetadataTest.java | 4 -
...ControllerPeriodicTaskStarterStatelessTest.java | 2 +-
.../controller/helix/PinotResourceManagerTest.java | 2 +-
.../PinotLLCRealtimeSegmentManagerTest.java | 7 +-
.../helix/core/realtime/SegmentCompletionTest.java | 3 +-
.../tuner/RealTimeAutoIndexTunerTest.java | 2 +-
.../pinot/controller/tuner/TunerRegistryTest.java | 2 +-
.../controller/utils/SegmentMetadataMockUtils.java | 6 +-
.../apache/pinot/core/common/MinionConstants.java | 12 +-
.../pinot/core/minion/MergeRollupConverter.java | 186 -----
.../pinot/core/minion/RawIndexConverter.java | 13 +-
.../core/minion/rollup/RollupRecordAggregator.java | 11 +-
.../query/DictionaryBasedDistinctOperator.java | 147 ++++
.../function/DateTruncTransformFunction.java | 9 +-
.../core/plan/DictionaryBasedDistinctPlanNode.java | 67 ++
.../core/plan/maker/InstancePlanMakerImplV2.java | 32 +-
.../query/selection/SelectionOperatorUtils.java | 9 +-
.../MaxValueAggregator.java | 2 +-
.../MinValueAggregator.java | 2 +-
.../SumValueAggregator.java | 2 +-
.../{collector => aggregator}/ValueAggregator.java | 2 +-
.../ValueAggregatorFactory.java | 26 +-
.../segment/processing/collector/Collector.java | 56 --
.../processing/collector/CollectorConfig.java | 109 ---
.../processing/collector/CollectorFactory.java | 61 --
.../processing/collector/ConcatCollector.java | 153 ----
.../processing/collector/GenericRowSorter.java | 95 ---
.../processing/collector/RollupCollector.java | 174 ----
.../segment/processing/framework/MergeType.java | 23 +-
.../processing/framework/SegmentConfig.java | 26 +-
.../processing/framework/SegmentMapperConfig.java | 79 --
.../framework/SegmentProcessorConfig.java | 112 +--
.../framework/SegmentProcessorFramework.java | 94 ++-
.../processing/framework/SegmentReducer.java | 129 ---
.../processing/framework/SegmentReducerConfig.java | 61 --
.../genericrow/GenericRowDeserializer.java | 221 +++--
.../genericrow/GenericRowFileManager.java | 28 +-
.../genericrow/GenericRowFileReader.java | 34 +-
.../genericrow/GenericRowFileRecordReader.java | 127 +++
.../genericrow/GenericRowSerializer.java | 2 +
.../{framework => mapper}/SegmentMapper.java | 85 +-
.../partitioner/ColumnValuePartitioner.java | 1 -
.../processing/partitioner/NoOpPartitioner.java | 1 -
.../processing/partitioner}/Partitioner.java | 2 +-
.../processing/partitioner/PartitionerFactory.java | 1 -
.../partitioner/RoundRobinPartitioner.java | 1 -
.../partitioner/TableConfigPartitioner.java | 1 -
.../partitioner/TransformFunctionPartitioner.java | 1 -
.../ConcatReducer.java} | 20 +-
.../segment/processing/reducer/DedupReducer.java | 89 ++
.../core/segment/processing/reducer/Reducer.java | 13 +-
.../segment/processing/reducer/ReducerFactory.java | 48 ++
.../segment/processing/reducer/RollupReducer.java | 172 ++++
.../processing/timehandler/EpochTimeHandler.java | 69 ++
.../processing/timehandler/NoOpTimeHandler.java | 13 +-
.../processing/timehandler/TimeHandler.java | 22 +-
.../processing/timehandler/TimeHandlerConfig.java | 106 +++
.../processing/timehandler/TimeHandlerFactory.java | 57 ++
.../processing/utils/SegmentProcessingUtils.java | 84 --
.../processing/utils/SegmentProcessorUtils.java | 98 +++
.../processing/utils/SortOrderComparator.java | 72 --
.../pinot/core/transport/ServerInstance.java | 11 +-
.../server/realtime/ControllerLeaderLocator.java | 55 +-
.../data/function/GroovyFunctionEvaluatorTest.java | 4 +
.../processing/framework/CollectorTest.java | 306 -------
.../processing/framework/GenericRowSorterTest.java | 77 --
.../processing/framework/PartitionerTest.java | 2 +-
.../segment/processing/framework/ReducerTest.java | 565 +++++++++++++
.../processing/framework/SegmentMapperTest.java | 210 ++---
.../framework/SegmentProcessingFrameworkTest.java | 551 -------------
.../framework/SegmentProcessorFrameworkTest.java | 673 +++++++++++++++
.../processing/framework/SegmentReducerTest.java | 205 -----
.../processing/genericrow/GenericRowSerDeTest.java | 40 +-
.../apache/pinot/queries/DistinctQueriesTest.java | 66 +-
...erSegmentAggregationSingleValueQueriesTest.java | 3 +-
.../realtime/ControllerLeaderLocatorTest.java | 41 +-
pinot-integration-tests/pom.xml | 14 +
.../org/apache/pinot/compat/tests/StreamOp.java | 11 +-
.../tests/BaseClusterIntegrationTest.java | 2 +-
...vertToRawIndexMinionClusterIntegrationTest.java | 6 +-
.../tests/OfflineClusterIntegrationTest.java | 59 +-
.../tests/RealtimeKinesisIntegrationTest.java | 483 +++++++++++
.../ControllerLeaderLocatorIntegrationTest.java | 16 +-
.../kinesis/airlineStats_data_reduced.json | 200 +++++
.../kinesis/airlineStats_data_reduced.schema | 32 +
.../org/apache/pinot/minion/BaseMinionStarter.java | 51 +-
.../org/apache/pinot/minion/MinionContext.java | 2 +-
.../minion/taskfactory/TaskFactoryRegistry.java | 46 +-
.../pinot/perf/BenchmarkOfflineIndexReader.java | 2 +-
.../v0_deprecated/pinot-hadoop/pom.xml | 1 -
.../utils/preprocess/DataPreprocessingUtils.java | 8 +-
.../inputformat/json/JSONMessageDecoder.java} | 6 +-
.../merge_rollup/MergeRollupTaskExecutor.java | 63 +-
.../tasks/merge_rollup/MergeRollupTaskUtils.java | 40 +-
.../RealtimeToOfflineSegmentsTaskExecutor.java | 234 ++----
.../RealtimeToOfflineSegmentsTaskGenerator.java | 43 +-
.../merge_rollup/MergeRollupTaskUtilsTest.java | 16 +-
.../RealtimeToOfflineSegmentsTaskExecutorTest.java | 29 +-
...RealtimeToOfflineSegmentsTaskGeneratorTest.java | 8 +-
.../pinot-kafka-base/pom.xml | 1 -
.../stream/kafka/KafkaJSONMessageDecoder.java | 58 +-
.../pinot/plugin/stream/kinesis/KinesisConfig.java | 6 +
.../stream/kinesis/KinesisConnectionHandler.java | 26 +-
.../plugin/stream/kinesis/KinesisConsumer.java | 28 +-
.../plugin/stream/kinesis/KinesisConsumerTest.java | 5 +-
.../pinot-stream-ingestion/pinot-pulsar/pom.xml | 375 +++++++++
.../stream/pulsar/MessageIdStreamOffset.java | 72 ++
.../pulsar/MessageIdStreamOffsetFactory.java | 35 +-
.../pinot/plugin/stream/pulsar/PulsarConfig.java | 83 ++
.../stream/pulsar/PulsarConsumerFactory.java | 66 ++
.../plugin/stream/pulsar/PulsarMessageBatch.java | 86 ++
.../PulsarPartitionLevelConnectionHandler.java | 79 ++
.../pulsar/PulsarPartitionLevelConsumer.java | 138 ++++
.../stream/pulsar/PulsarStreamLevelConsumer.java | 113 +++
.../pulsar/PulsarStreamLevelConsumerManager.java | 175 ++++
.../pulsar/PulsarStreamMetadataProvider.java | 158 ++++
.../plugin/stream/pulsar/PulsarConsumerTest.java | 191 +++++
.../stream/pulsar/PulsarStandaloneCluster.java | 132 +++
.../src/test/resources/standalone.properties | 898 +++++++++++++++++++++
pinot-plugins/pinot-stream-ingestion/pom.xml | 1 +
.../indexsegment/immutable/EmptyIndexSegment.java | 6 +-
.../immutable/ImmutableSegmentImpl.java | 2 +-
.../immutable/ImmutableSegmentLoader.java | 4 +-
.../indexsegment/mutable/MutableSegmentImpl.java | 6 +-
.../index/column/BaseVirtualColumnProvider.java | 12 +-
.../DefaultNullValueVirtualColumnProvider.java | 8 +-
.../index/column/PhysicalColumnIndexContainer.java | 2 +-
.../converter/SegmentV1V2ToV3FormatConverter.java | 2 +-
.../segment/index/datasource/EmptyDataSource.java | 2 +-
.../index/datasource/ImmutableDataSource.java | 2 +-
.../local/segment/index/loader/LoaderUtils.java | 5 +-
.../loader/bloomfilter/BloomFilterHandler.java | 4 +-
.../ColumnMinMaxValueGenerator.java | 2 +-
.../defaultcolumn/BaseDefaultColumnHandler.java | 23 +-
.../defaultcolumn/DefaultColumnHandlerFactory.java | 2 +-
.../index/loader/invertedindex/H3IndexHandler.java | 4 +-
.../loader/invertedindex/InvertedIndexHandler.java | 8 +-
.../loader/invertedindex/JsonIndexHandler.java | 7 +-
.../invertedindex/LuceneFSTIndexHandler.java | 8 +-
.../loader/invertedindex/RangeIndexHandler.java | 8 +-
.../loader/invertedindex/TextIndexHandler.java | 11 +-
.../local/segment/store/FilePerIndexDirectory.java | 36 +-
.../segment/store/SegmentLocalFSDirectory.java | 8 +-
.../virtualcolumn/DocIdVirtualColumnProvider.java | 9 +-
.../virtualcolumn/VirtualColumnProvider.java | 2 +-
.../local/startree/StarTreeBuilderUtils.java | 4 +-
.../startree/v2/builder/MultipleTreesBuilder.java | 5 +-
.../v2/builder/StarTreeV2BuilderConfig.java | 6 +-
.../startree/v2/store/StarTreeLoaderUtils.java | 2 +-
.../{IncrementMerger.java => AppendMerger.java} | 29 +-
.../local/upsert/merger/IncrementMerger.java | 4 +
.../local/upsert/merger/OverwriteMerger.java | 4 +
.../upsert/merger/PartialUpsertMergerFactory.java | 12 +-
.../{IncrementMerger.java => UnionMerger.java} | 33 +-
.../segment/local/utils/ReplicationUtils.java | 2 +-
.../segment/local/utils/TableConfigUtils.java | 10 +-
.../mutable/MutableSegmentImplUpsertTest.java | 4 +-
.../local/segment/creator/IntArraysTest.java | 6 +-
.../local/segment/index/ColumnMetadataTest.java | 59 +-
.../segment/index/SegmentMetadataImplTest.java | 6 +-
.../DefaultNullValueVirtualColumnProviderTest.java | 208 +++--
.../SegmentV1V2ToV3FormatConverterTest.java | 4 +-
.../SegmentGenerationWithBytesTypeTest.java | 6 +-
.../index/creator/SegmentPartitionTest.java | 10 +-
.../local/segment/index/loader/LoaderTest.java | 38 +-
.../index/loader/SegmentPreProcessorTest.java | 194 ++---
.../store/ColumnIndexDirectoryTestHelper.java | 54 +-
.../store/SingleFileIndexDirectoryTest.java | 2 +-
.../v2/builder/StarTreeV2BuilderConfigTest.java | 2 +-
.../upsert/merger/PartialUpsertMergerTest.java | 59 ++
.../segment/local/utils/TableConfigUtilsTest.java | 21 +-
.../apache/pinot/segment/spi/ColumnMetadata.java | 90 +++
.../apache/pinot/segment/spi/SegmentMetadata.java | 59 +-
.../org/apache/pinot/segment/spi/V1Constants.java | 18 +-
.../spi/creator/SegmentGeneratorConfig.java | 27 +-
.../creator/name/SimpleSegmentNameGenerator.java | 10 +-
.../segment/spi/index/metadata/ColumnMetadata.java | 623 --------------
.../spi/index/metadata/ColumnMetadataImpl.java | 390 +++++++++
.../spi/index/metadata/SegmentMetadataImpl.java | 284 ++-----
.../api/resources/SegmentMetadataFetcher.java | 8 +-
.../pinot/server/api/resources/TablesResource.java | 112 ++-
.../server/starter/helix/BaseServerStarter.java | 115 ++-
.../pinot/server/api/TablesResourceTest.java | 48 +-
.../apache/pinot/spi/config/table/TableStatus.java | 3 +-
.../pinot/spi/config/table/UpsertConfig.java | 20 +-
.../java/org/apache/pinot/spi/data/FieldSpec.java | 2 +-
.../apache/pinot/spi/data/readers/GenericRow.java | 6 +-
.../org/apache/pinot/spi/filesystem/PinotFS.java | 18 +-
.../org/apache/pinot/spi/plugin/PluginManager.java | 4 +-
.../apache/pinot/spi/utils/CommonConstants.java | 9 +
.../pinot/spi/config/table/UpsertConfigTest.java | 2 +-
.../apache/pinot/spi/filesystem/PinotFSTest.java | 180 +++++
.../ingestion/batch/IngestionJobLauncherTest.java | 9 +
.../test/resources/ingestion_job_spec_unicode.yaml | 48 ++
.../pinot/tools/PartialUpsertQuickStart.java | 23 +-
.../command/SegmentProcessorFrameworkCommand.java | 8 +-
.../tools/admin/command/StartBrokerCommand.java | 14 +-
.../admin/command/StartControllerCommand.java | 7 +-
.../tools/admin/command/StartMinionCommand.java | 7 +-
.../tools/admin/command/StartServerCommand.java | 7 +-
.../anonymizer/ArrayBasedGlobalDictionaries.java | 2 +-
.../pinot/tools/anonymizer/GlobalDictionaries.java | 2 +-
.../anonymizer/MapBasedGlobalDictionaries.java | 2 +-
.../anonymizer/PinotDataAndQueryAnonymizer.java | 2 +-
.../apache/pinot/tools/scan/query/Projection.java | 2 +-
.../tools/scan/query/SegmentQueryProcessor.java | 3 +-
.../processor/SegmentProcessorFrameworkSpec.java | 45 +-
.../src/main/resources/appAssemblerScriptTemplate | 5 +-
...t_partial_meetupRsvp_realtime_table_config.json | 4 +-
.../upsert_partial_meetupRsvp_schema.json | 67 ++
pom.xml | 3 +-
276 files changed, 9975 insertions(+), 5239 deletions(-)
create mode 100644 kubernetes/helm/pinot-0.2.4.tgz
create mode 100644
pinot-broker/src/test/java/org/apache/pinot/broker/broker/HelixBrokerStarterHostnamePortTest.java
rename {pinot-minion/src/main/java/org/apache/pinot/minion =>
pinot-common/src/main/java/org/apache/pinot/common}/metrics/MinionGauge.java
(93%)
rename {pinot-minion/src/main/java/org/apache/pinot/minion =>
pinot-common/src/main/java/org/apache/pinot/common}/metrics/MinionMeter.java
(94%)
copy {pinot-minion/src/main/java/org/apache/pinot/minion =>
pinot-common/src/main/java/org/apache/pinot/common}/metrics/MinionMetrics.java
(94%)
rename {pinot-minion/src/main/java/org/apache/pinot/minion =>
pinot-common/src/main/java/org/apache/pinot/common}/metrics/MinionQueryPhase.java
(90%)
rename {pinot-minion/src/main/java/org/apache/pinot/minion =>
pinot-common/src/main/java/org/apache/pinot/common}/metrics/MinionTimer.java
(92%)
create mode 100644
pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableMetadataInfo.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/utils/helix/HelixHelperTest.java
create mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/ResourceUtils.java
create mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/util/TableIngestionStatusHelper.java
create mode 100644
pinot-controller/src/test/java/org/apache/pinot/controller/ControllerStarterTest.java
delete mode 100644
pinot-core/src/main/java/org/apache/pinot/core/minion/MergeRollupConverter.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/operator/query/DictionaryBasedDistinctOperator.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/plan/DictionaryBasedDistinctPlanNode.java
rename
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/{collector =>
aggregator}/MaxValueAggregator.java (96%)
rename
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/{collector =>
aggregator}/MinValueAggregator.java (96%)
rename
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/{collector =>
aggregator}/SumValueAggregator.java (96%)
rename
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/{collector =>
aggregator}/ValueAggregator.java (94%)
rename
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/{collector =>
aggregator}/ValueAggregatorFactory.java (68%)
delete mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/collector/Collector.java
delete mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/collector/CollectorConfig.java
delete mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/collector/CollectorFactory.java
delete mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/collector/ConcatCollector.java
delete mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/collector/GenericRowSorter.java
delete mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/collector/RollupCollector.java
copy
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/Partitioner.java
=>
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/framework/MergeType.java
(68%)
delete mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/framework/SegmentMapperConfig.java
delete mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/framework/SegmentReducer.java
delete mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/framework/SegmentReducerConfig.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowFileRecordReader.java
rename
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/{framework =>
mapper}/SegmentMapper.java (64%)
copy {pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition
=>
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/partitioner}/Partitioner.java
(94%)
copy
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/{partitioner/NoOpPartitioner.java
=> reducer/ConcatReducer.java} (64%)
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/reducer/DedupReducer.java
copy
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/Partitioner.java
=>
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/reducer/Reducer.java
(68%)
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/reducer/ReducerFactory.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/reducer/RollupReducer.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/timehandler/EpochTimeHandler.java
copy
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/Partitioner.java
=>
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/timehandler/NoOpTimeHandler.java
(75%)
rename
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/partition/Partitioner.java
=>
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/timehandler/TimeHandler.java
(55%)
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/timehandler/TimeHandlerConfig.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/timehandler/TimeHandlerFactory.java
delete mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/utils/SegmentProcessingUtils.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/utils/SegmentProcessorUtils.java
delete mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/utils/SortOrderComparator.java
delete mode 100644
pinot-core/src/test/java/org/apache/pinot/core/segment/processing/framework/CollectorTest.java
delete mode 100644
pinot-core/src/test/java/org/apache/pinot/core/segment/processing/framework/GenericRowSorterTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/segment/processing/framework/ReducerTest.java
delete mode 100644
pinot-core/src/test/java/org/apache/pinot/core/segment/processing/framework/SegmentProcessingFrameworkTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/segment/processing/framework/SegmentProcessorFrameworkTest.java
delete mode 100644
pinot-core/src/test/java/org/apache/pinot/core/segment/processing/framework/SegmentReducerTest.java
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/RealtimeKinesisIntegrationTest.java
create mode 100644
pinot-integration-tests/src/test/resources/kinesis/airlineStats_data_reduced.json
create mode 100644
pinot-integration-tests/src/test/resources/kinesis/airlineStats_data_reduced.schema
copy
pinot-plugins/{pinot-stream-ingestion/pinot-kafka-base/src/main/java/org/apache/pinot/plugin/stream/kafka/KafkaJSONMessageDecoder.java
=>
pinot-input-format/pinot-json/src/main/java/org/apache/pinot/plugin/inputformat/json/JSONMessageDecoder.java}
(93%)
create mode 100644 pinot-plugins/pinot-stream-ingestion/pinot-pulsar/pom.xml
create mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-pulsar/src/main/java/org/apache/pinot/plugin/stream/pulsar/MessageIdStreamOffset.java
rename
pinot-minion/src/main/java/org/apache/pinot/minion/metrics/MinionMetrics.java
=>
pinot-plugins/pinot-stream-ingestion/pinot-pulsar/src/main/java/org/apache/pinot/plugin/stream/pulsar/MessageIdStreamOffsetFactory.java
(50%)
create mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-pulsar/src/main/java/org/apache/pinot/plugin/stream/pulsar/PulsarConfig.java
create mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-pulsar/src/main/java/org/apache/pinot/plugin/stream/pulsar/PulsarConsumerFactory.java
create mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-pulsar/src/main/java/org/apache/pinot/plugin/stream/pulsar/PulsarMessageBatch.java
create mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-pulsar/src/main/java/org/apache/pinot/plugin/stream/pulsar/PulsarPartitionLevelConnectionHandler.java
create mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-pulsar/src/main/java/org/apache/pinot/plugin/stream/pulsar/PulsarPartitionLevelConsumer.java
create mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-pulsar/src/main/java/org/apache/pinot/plugin/stream/pulsar/PulsarStreamLevelConsumer.java
create mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-pulsar/src/main/java/org/apache/pinot/plugin/stream/pulsar/PulsarStreamLevelConsumerManager.java
create mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-pulsar/src/main/java/org/apache/pinot/plugin/stream/pulsar/PulsarStreamMetadataProvider.java
create mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-pulsar/src/test/java/org/apache/pinot/plugin/stream/pulsar/PulsarConsumerTest.java
create mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-pulsar/src/test/java/org/apache/pinot/plugin/stream/pulsar/PulsarStandaloneCluster.java
create mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-pulsar/src/test/resources/standalone.properties
copy
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/merger/{IncrementMerger.java
=> AppendMerger.java} (57%)
copy
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/upsert/merger/{IncrementMerger.java
=> UnionMerger.java} (55%)
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/upsert/merger/PartialUpsertMergerTest.java
create mode 100644
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/ColumnMetadata.java
delete mode 100644
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/ColumnMetadata.java
create mode 100644
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/ColumnMetadataImpl.java
create mode 100644
pinot-spi/src/test/java/org/apache/pinot/spi/filesystem/PinotFSTest.java
create mode 100644 pinot-spi/src/test/resources/ingestion_job_spec_unicode.yaml
create mode 100644
pinot-tools/src/main/resources/examples/stream/meetupRsvp/upsert_partial_meetupRsvp_schema.json
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
