This is an automated email from the ASF dual-hosted git repository.
jackie pushed a change to branch h3-index
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
from 35f1257 Add real-time H3 index reader (#6400)
add d1373cf H3 index cleanup merge (#6401)
No new revisions were added by this update.
Summary of changes:
.github/workflows/scripts/.pinot_quickstart.sh | 27 +
.gitignore | 1 -
.../images/pinot/etc/conf/pinot-broker-log4j2.xml | 62 +
.../pinot/etc/conf/pinot-controller-log4j2.xml | 61 +
.../images/pinot/etc/conf/pinot-server-log4j2.xml | 61 +
kubernetes/helm/index.yaml | 6 +-
kubernetes/helm/pinot/README.md | 4 +-
.../helm/pinot/pinot-realtime-quickstart.yml | 684 +-
kubernetes/helm/pinot/requirements.lock | 2 +-
kubernetes/helm/pinot/requirements.yaml | 2 +-
kubernetes/helm/thirdeye/Chart.lock | 6 +-
kubernetes/helm/thirdeye/Chart.yaml | 3 +-
.../antlr4/org/apache/pinot/pql/parsers/PQL2.g4 | 4 +
.../apache/pinot/common/minion/MinionClient.java | 2 +-
.../pinot/common/request/AggregationInfo.java | 52 +-
.../apache/pinot/common/request/BrokerRequest.java | 353 +-
.../apache/pinot/common/request/DataSource.java | 11 +-
.../apache/pinot/common/request/Expression.java | 35 +-
.../pinot/common/request/ExpressionType.java | 4 +-
.../pinot/common/request/FilterOperator.java | 11 +-
.../apache/pinot/common/request/FilterQuery.java | 46 +-
.../pinot/common/request/FilterQueryMap.java | 11 +-
.../org/apache/pinot/common/request/Function.java | 21 +-
.../org/apache/pinot/common/request/GroupBy.java | 85 +-
.../pinot/common/request/HavingFilterQuery.java | 46 +-
.../pinot/common/request/HavingFilterQueryMap.java | 11 +-
.../apache/pinot/common/request/Identifier.java | 16 +-
.../pinot/common/request/InstanceRequest.java | 76 +-
.../org/apache/pinot/common/request/Literal.java | 4 +-
.../apache/pinot/common/request/PinotQuery.java | 61 +-
.../apache/pinot/common/request/QuerySource.java | 11 +-
.../org/apache/pinot/common/request/QueryType.java | 31 +-
.../org/apache/pinot/common/request/Selection.java | 94 +-
.../apache/pinot/common/request/SelectionSort.java | 16 +-
.../pinot/common/response/ProcessingException.java | 283 +-
.../apache/pinot/parsers/utils/ParserUtils.java | 1 +
.../parsers/PinotQuery2BrokerRequestConverter.java | 3 +-
.../pinot/pql/parsers/pql2/ast/FilterKind.java | 3 +-
.../apache/pinot/sql/parsers/CalciteSqlParser.java | 21 +-
pinot-common/src/thrift/request.thrift | 3 +-
.../api/resources/PinotSegmentRestletResource.java | 65 +-
.../helix/core/PinotHelixResourceManager.java | 155 +
.../SegmentGenerationAndPushTaskGenerator.java | 315 +
.../minion/generator/TaskGeneratorRegistry.java | 1 +
.../pinot/controller/util/FileIngestionUtils.java | 2 +-
.../org/apache/pinot/core/common/DataSource.java | 8 +
.../apache/pinot/core/common/MinionConstants.java | 6 +
.../core/data/partition/PartitionFunction.java | 5 +-
.../data/recordtransformer/RecordTransformer.java | 3 +-
.../generator/SegmentGeneratorConfig.java | 38 +-
.../indexsegment/mutable/MutableSegmentImpl.java | 50 +-
.../io/util/VarLengthBytesValueReaderWriter.java | 241 -
.../pinot/core/io/util/VarLengthValueReader.java | 121 +
.../pinot/core/io/util/VarLengthValueWriter.java | 133 +
.../core/operator/filter/FilterOperatorUtils.java | 9 +-
.../operator/filter/JsonMatchFilterOperator.java | 59 +
.../org/apache/pinot/core/plan/FilterPlanNode.java | 27 +-
.../{Predicate.java => JsonMatchPredicate.java} | 57 +-
.../query/request/context/predicate/Predicate.java | 2 +-
.../context/utils/QueryContextConverterUtils.java | 4 +
.../segment/creator/ColumnIndexCreationInfo.java | 3 +-
.../core/segment/creator/ColumnStatistics.java | 3 +-
...eationDataSource.java => JsonIndexCreator.java} | 21 +-
.../segment/creator/SegmentCreationDataSource.java | 3 +-
.../pinot/core/segment/creator/SegmentCreator.java | 6 +-
.../creator/SegmentIndexCreationDriver.java | 3 +-
.../segment/creator/SegmentIndexCreationInfo.java | 5 +-
.../creator/SegmentPreIndexStatsContainer.java | 5 +-
.../creator/impl/SegmentColumnarIndexCreator.java | 93 +-
.../creator/impl/SegmentDictionaryCreator.java | 41 +-
.../core/segment/creator/impl/V1Constants.java | 2 +
.../impl/inv/BitmapInvertedIndexWriter.java | 90 +
.../inv/OffHeapBitmapInvertedIndexCreator.java | 43 +-
.../impl/inv/OnHeapBitmapInvertedIndexCreator.java | 45 +-
.../impl/inv/json/BaseJsonIndexCreator.java | 173 +
.../impl/inv/json/OffHeapJsonIndexCreator.java | 282 +
.../impl/inv/json/OnHeapJsonIndexCreator.java | 60 +
.../segment/index/column/ColumnIndexContainer.java | 14 +-
.../index/column/PhysicalColumnIndexContainer.java | 53 +-
.../converter/SegmentV1V2ToV3FormatConverter.java | 62 +-
.../segment/index/datasource/BaseDataSource.java | 13 +-
.../index/datasource/ImmutableDataSource.java | 4 +-
.../index/datasource/MutableDataSource.java | 22 +-
.../segment/index/loader/IndexLoadingConfig.java | 10 +
.../segment/index/loader/SegmentPreProcessor.java | 14 +-
.../defaultcolumn/BaseDefaultColumnHandler.java | 4 +-
.../index/loader/invertedindex/H3IndexHandler.java | 27 +-
.../loader/invertedindex/JsonIndexHandler.java | 176 +
.../index/readers/BaseImmutableDictionary.java | 17 +-
.../segment/index/readers/JsonIndexReader.java | 243 +
.../pinot/core/segment/memory/PinotByteBuffer.java | 15 +-
.../core/segment/name/SegmentNameGenerator.java | 3 +-
.../pinot/core/segment/store/ColumnIndexType.java | 3 +-
.../core/segment/store/FilePerIndexDirectory.java | 14 +-
.../pinot/core/segment/store/SegmentDirectory.java | 25 +-
.../segment/store/SegmentLocalFSDirectory.java | 10 -
.../virtualcolumn/VirtualColumnIndexContainer.java | 6 +
.../core/startree/v2/store/StarTreeDataSource.java | 4 +-
.../util/VarLengthBytesValueReaderWriterTest.java | 172 -
.../io/util/VarLengthValueReaderWriterTest.java | 117 +
.../pinot/core/segment/index/JsonIndexTest.java | 205 +
.../segment/store/SegmentLocalFSDirectoryTest.java | 2 +-
.../tests/AdminConsoleIntegrationTest.java | 13 +
pinot-minion/pom.xml | 5 +
.../executor/SegmentGenerationAndPushResult.java | 91 +
.../SegmentGenerationAndPushTaskExecutor.java | 306 +
...egmentGenerationAndPushTaskExecutorFactory.java | 15 +-
.../executor/TaskExecutorFactoryRegistry.java | 1 +
.../ingestion/batch/common/SegmentPushUtils.java | 44 +-
.../v0_deprecated/pinot-spark/pom.xml | 282 +-
.../src/test/java/SegmentCreationSparkTest.java | 141 +
.../src/test/resources}/test_sample_data.csv | 0
.../apache/pinot/spi/config/BaseJsonConfig.java | 3 +-
.../pinot/spi/config/table/IndexingConfig.java | 9 +
.../java/org/apache/pinot/spi/data/FieldSpec.java | 3 +-
.../pinot/spi/data/IngestionSchemaValidator.java | 4 +-
.../java/org/apache/pinot/spi/data/Schema.java | 5 +-
.../apache/pinot/spi/data/TimeGranularitySpec.java | 5 +-
.../apache/pinot/spi/data/readers/GenericRow.java | 3 +-
.../pinot/spi/data/readers/RecordExtractor.java | 3 +-
.../pinot/spi/data/readers/RecordReader.java | 3 +-
.../spi/ingestion/batch/BatchConfigProperties.java | 1 -
.../batch/spec/SegmentGenerationJobSpec.java | 13 +
.../org/apache/pinot/spi/plugin/PluginManager.java | 44 +
.../pinot/spi/utils/IngestionConfigUtils.java | 8 +-
.../java/org/apache/pinot/spi/utils/JsonUtils.java | 191 +-
.../org/apache/pinot/spi/utils/JsonUtilsTest.java | 145 +
pinot-tools/pom.xml | 22 +
.../pinot/tools/BatchQuickstartWithMinion.java | 18 +-
.../org/apache/pinot/tools/BootstrapTableTool.java | 140 +-
.../org/apache/pinot/tools/GenericQuickstart.java | 9 +-
.../apache/pinot/tools/JsonIndexQuickStart.java | 89 +
.../java/org/apache/pinot/tools/Quickstart.java | 17 +-
.../tools/admin/command/QuickStartCommand.java | 7 +
.../tools/admin/command/QuickstartRunner.java | 48 +-
.../tools/admin/command/StartMinionCommand.java | 20 +
.../githubEvents_offline_table_config.json | 27 +
.../batch/githubEvents/githubEvents_schema.json | 40 +
.../batch/githubEvents/ingestionJobSpec.yaml | 139 +
.../githubEvents/rawdata/githubEvents_data.json | 10000 +++++++++++++++++++
.../batch/githubEvents/sparkIngestionJobSpec.yaml | 147 +
.../airlineStats_offline_table_config.json | 44 +
.../batch}/airlineStats/airlineStats_schema.json | 0
.../baseballStats_offline_table_config.json | 43 +
.../batch/baseballStats/baseballStats_schema.json | 0
.../baseballStats/rawdata/baseballStats_data.csv | 0
pom.xml | 2 +-
.../installation/cloud/on-premise.md | 4 +-
148 files changed, 15825 insertions(+), 1977 deletions(-)
create mode 100644 docker/images/pinot/etc/conf/pinot-broker-log4j2.xml
create mode 100644 docker/images/pinot/etc/conf/pinot-controller-log4j2.xml
create mode 100644 docker/images/pinot/etc/conf/pinot-server-log4j2.xml
create mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/generator/SegmentGenerationAndPushTaskGenerator.java
delete mode 100644
pinot-core/src/main/java/org/apache/pinot/core/io/util/VarLengthBytesValueReaderWriter.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/io/util/VarLengthValueReader.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/io/util/VarLengthValueWriter.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/operator/filter/JsonMatchFilterOperator.java
copy
pinot-core/src/main/java/org/apache/pinot/core/query/request/context/predicate/{Predicate.java
=> JsonMatchPredicate.java} (50%)
copy
pinot-core/src/main/java/org/apache/pinot/core/segment/creator/{SegmentCreationDataSource.java
=> JsonIndexCreator.java} (69%)
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/inv/BitmapInvertedIndexWriter.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/inv/json/BaseJsonIndexCreator.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/inv/json/OffHeapJsonIndexCreator.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/inv/json/OnHeapJsonIndexCreator.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/index/loader/invertedindex/JsonIndexHandler.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/segment/index/readers/JsonIndexReader.java
delete mode 100644
pinot-core/src/test/java/org/apache/pinot/core/io/util/VarLengthBytesValueReaderWriterTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/io/util/VarLengthValueReaderWriterTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/segment/index/JsonIndexTest.java
create mode 100644
pinot-minion/src/main/java/org/apache/pinot/minion/executor/SegmentGenerationAndPushResult.java
create mode 100644
pinot-minion/src/main/java/org/apache/pinot/minion/executor/SegmentGenerationAndPushTaskExecutor.java
copy
pinot-core/src/main/java/org/apache/pinot/core/segment/creator/SegmentIndexCreationInfo.java
=>
pinot-minion/src/main/java/org/apache/pinot/minion/executor/SegmentGenerationAndPushTaskExecutorFactory.java
(76%)
create mode 100644
pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/src/test/java/SegmentCreationSparkTest.java
copy {pinot-core/src/test/resources/data =>
pinot-plugins/pinot-batch-ingestion/v0_deprecated/pinot-spark/src/test/resources}/test_sample_data.csv
(100%)
create mode 100644
pinot-spi/src/test/java/org/apache/pinot/spi/utils/JsonUtilsTest.java
copy
pinot-core/src/main/java/org/apache/pinot/core/segment/creator/SegmentIndexCreationInfo.java
=>
pinot-tools/src/main/java/org/apache/pinot/tools/BatchQuickstartWithMinion.java
(68%)
create mode 100644
pinot-tools/src/main/java/org/apache/pinot/tools/JsonIndexQuickStart.java
create mode 100644
pinot-tools/src/main/resources/examples/batch/githubEvents/githubEvents_offline_table_config.json
create mode 100644
pinot-tools/src/main/resources/examples/batch/githubEvents/githubEvents_schema.json
create mode 100644
pinot-tools/src/main/resources/examples/batch/githubEvents/ingestionJobSpec.yaml
create mode 100644
pinot-tools/src/main/resources/examples/batch/githubEvents/rawdata/githubEvents_data.json
create mode 100644
pinot-tools/src/main/resources/examples/batch/githubEvents/sparkIngestionJobSpec.yaml
create mode 100644
pinot-tools/src/main/resources/examples/minions/batch/airlineStats/airlineStats_offline_table_config.json
copy pinot-tools/src/main/resources/examples/{stream =>
minions/batch}/airlineStats/airlineStats_schema.json (100%)
create mode 100644
pinot-tools/src/main/resources/examples/minions/batch/baseballStats/baseballStats_offline_table_config.json
copy pinot-tools/src/main/resources/examples/{ =>
minions}/batch/baseballStats/baseballStats_schema.json (100%)
copy pinot-tools/src/main/resources/examples/{ =>
minions}/batch/baseballStats/rawdata/baseballStats_data.csv (100%)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
