This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a change to branch xiangfu0/agg-rewrite/03-mode
in repository https://gitbox.apache.org/repos/asf/pinot.git
from ff11dd135a8 Merge xiangfu0/agg-rewrite/01-first-last-with-time into
xiangfu0/agg-rewrite/03-mode
add 91fbab84f5d Move the segment assignment partition column to
SegmentAssignmentConfig (#19958)
add b02a7651269 Fix broker routing ignoring a shutting down server while
another server is excluded (#19997)
add 8f20062cd13 Give the dimension table tests their own temp directory
(#20004)
add 77ec2fbf1cf Fold MSE literal function calls with runtime value types
(#19948)
add af3c64f7eb7 Make sorted ORDER BY late materialization automatic
(#20003)
add 678c8b07a90 Hold the consumer semaphore while a replace reverts upsert
metadata (#19817)
add b3bf808a371 Bump org.eclipse.sisu:sisu-maven-plugin from 1.1.0 to
1.1.1 (#20013)
add b5631229eae Bump org.roaringbitmap:RoaringBitmap from 1.6.24 to 1.6.25
(#20011)
add b047e331c11 Bump io.grpc:grpc-bom from 1.85.0 to 1.85.1 (#20014)
add f57c2285ec2 Validate segment assignment for strict replica-group
routing (#20006)
add b99203c6026 Reuse the outer local repo in the assembly-descriptor IT
and report its real error (#19885)
add f1bdf98e6ea Read Parquet unsigned 32-bit integers as long (#19999)
add 60b13a0640b Treat a segment whose partition function cannot be built
as unpartitioned in broker routing (#19953)
add 230045f26d4 Rewrite arrayContains on multi-value columns into an
indexable EQUALS filter (#20000)
add f290065ed21 Add -outputFormat and -outputFile options to PostQuery CLI
command (#19610)
add 8cf16156a44 Remove the kept new segment when a resumed pauseless
commit start completes without it (#19995)
add adb87d861c8 Avoid redundant local input copies and support Zstandard
batch archives (#19898)
add e106c097f46 Bump software.amazon.awssdk:bom from 2.56.0 to 2.57.0
(#20012)
add dc759c51e66 Support FIRST_WITH_TIME and LAST_WITH_TIME without a data
type (#19986)
add efd180e4bb2 Merge master into xiangfu0/agg-rewrite/03-mode
No new revisions were added by this update.
Summary of changes:
docker/images/pinot/Dockerfile | 2 +-
docker/images/pinot/Dockerfile.build | 2 +-
docker/images/pinot/Dockerfile.package | 2 +-
.../routing/manager/BaseBrokerRoutingManager.java | 21 +-
.../segmentpartition/SegmentPartitionUtils.java | 32 +-
.../routing/manager/BrokerRoutingManagerTest.java | 30 ++
.../SegmentPartitionUtilsTest.java | 24 +
.../SinglePartitionColumnSegmentPrunerTest.java | 13 +
.../assignment/InstanceAssignmentConfigUtils.java | 4 +-
.../common/function/TransformFunctionType.java | 3 +-
.../common/utils/config/QueryOptionsUtils.java | 8 -
.../common/utils/TarCompressionUtilsTest.java | 4 +-
.../utils/config/TableConfigSerDeUtilsTest.java | 46 +-
.../api/resources/TableConfigValidationUtils.java | 58 ++-
.../api/upload/SegmentValidationUtils.java | 56 +--
.../helix/core/PinotHelixResourceManager.java | 89 ++++
.../strategy/SegmentAssignmentStrategyFactory.java | 74 ++-
.../realtime/PinotLLCRealtimeSegmentManager.java | 82 ++-
.../resources/TableConfigValidationUtilsTest.java | 76 +++
.../api/upload/SegmentValidationUtilsTest.java | 8 +-
.../controller/helix/PinotResourceManagerTest.java | 2 -
.../PinotHelixResourceManagerStatelessTest.java | 135 +++++
.../instance/InstanceAssignmentTest.java | 3 +-
.../RealtimeReplicaGroupSegmentAssignmentTest.java | 11 +-
.../StrictRealtimeSegmentAssignmentTest.java | 9 +-
.../ReplicaGroupSegmentAssignmentStrategyTest.java | 18 +-
...nReplicaGroupSegmentAssignmentStrategyTest.java | 14 +-
.../RoundRobinSegmentAssignmentStrategyTest.java | 3 +-
.../SegmentAssignmentStrategyFactoryTest.java | 9 +-
.../PinotLLCRealtimeSegmentManagerTest.java | 166 ++++++
.../realtime/RealtimeSegmentDataManager.java | 57 ++-
.../pinot/core/function/scalar/IdSetFunctions.java | 19 +-
.../query/LinearSelectionOrderByOperator.java | 154 ++----
.../SelectionPartiallyOrderedByDescOperation.java | 16 +-
.../SelectionPartiallyOrderedByLinearOperator.java | 10 +-
.../apache/pinot/core/plan/SelectionPlanNode.java | 57 +--
.../pinot/core/query/optimizer/QueryOptimizer.java | 10 +-
.../filter/ArrayContainsFilterOptimizer.java | 166 ++++++
.../realtime/RealtimeSegmentDataManagerTest.java | 26 +-
.../core/function/scalar/IdSetFunctionsTest.java | 7 +-
.../core/query/optimizer/QueryOptimizerTest.java | 48 +-
.../ArrayContainsMultiValueQueriesTest.java | 115 +++++
.../org/apache/pinot/queries/IdSetQueriesTest.java | 37 ++
...nnerSegmentSelectionSingleValueQueriesTest.java | 113 ++---
.../queries/NullHandlingEnabledQueriesTest.java | 71 ++-
.../queries/TextMatchTransformFunctionTest.java | 12 +-
.../tests/BaseClusterIntegrationTest.java | 12 +-
.../tests/BaseDedupIntegrationTest.java | 2 -
.../tests/DedupPreloadIntegrationTest.java | 2 -
.../tests/custom/OfflineUpsertTableTest.java | 17 +-
.../tests/custom/VectorUpsertTableTest.java | 2 -
.../simpleMeetup_realtime_table_config.json | 6 +-
pinot-plugins/assembly-descriptor/pom.xml | 14 +
.../src/it/projects/simple-assembly/pom.xml | 19 +-
.../src/it/projects/simple-assembly/verify.groovy | 12 +-
.../batch/common/BaseSegmentPushJobRunner.java | 11 +-
.../batch/hadoop/HadoopSegmentCreationMapper.java | 3 +-
.../hadoop/HadoopSegmentMetadataPushJobRunner.java | 18 +-
.../hadoop/HadoopSegmentTarPushJobRunner.java | 14 +-
.../hadoop/HadoopSegmentUriPushJobRunner.java | 15 +-
.../spark3/SparkSegmentGenerationJobRunner.java | 3 +-
.../spark3/SparkSegmentMetadataPushJobRunner.java | 16 +-
.../BaseSparkSegmentMetadataPushJobRunner.java | 10 +-
.../common/BaseSparkSegmentTarPushJobRunner.java | 10 +-
.../common/BaseSparkSegmentUriPushJobRunner.java | 14 +-
.../BaseSparkSegmentMetadataPushJobRunnerTest.java | 125 +++++
.../standalone/SegmentGenerationJobRunner.java | 23 +-
.../standalone/SegmentGenerationJobRunnerTest.java | 151 ++++++
.../parquet/ParquetAvroRecordExtractor.java | 37 +-
.../parquet/ParquetAvroRecordReader.java | 2 +-
.../parquet/ParquetNativeRecordExtractor.java | 9 +-
.../plugin/inputformat/parquet/ParquetUtils.java | 34 ++
.../parquet/ParquetReaderLogicalTypesTest.java | 85 ++++
.../segmentuploader/SegmentUploaderDefault.java | 2 +-
.../rel/rules/PinotEvaluateLiteralRule.java | 82 ++-
.../resources/queries/LiteralEvaluationPlans.json | 108 ++++
.../src/test/resources/queries/IdSet.json | 7 +-
.../test/resources/queries/LiteralExpressions.json | 33 ++
.../segment/local/utils/SegmentPushUtils.java | 39 +-
.../segment/local/utils/TableConfigUtils.java | 322 ++++++++++--
.../segment/local/utils/SegmentPushUtilsTest.java | 140 ++++++
.../segment/local/utils/TableConfigUtilsTest.java | 555 +++++++++++++++++++--
.../config/table/ReplicaGroupStrategyConfig.java | 4 +-
.../SegmentsValidationAndRetentionConfig.java | 9 +-
.../table/assignment/InstanceAssignmentConfig.java | 35 +-
.../InstanceReplicaGroupPartitionConfig.java | 9 +-
.../table/assignment/SegmentAssignmentConfig.java | 22 +-
.../pinot/spi/ingestion/batch/spec/Constants.java | 20 +
.../batch/spec/SegmentGenerationJobSpec.java | 20 +
.../apache/pinot/spi/utils/CommonConstants.java | 23 +-
.../spi/utils/builder/TableConfigBuilder.java | 3 +-
pinot-tools/pom.xml | 4 +
.../command/LaunchBackfillIngestionJobCommand.java | 35 +-
.../tools/admin/command/PostQueryCommand.java | 130 ++++-
.../userAttributes_offline_table_config.json | 3 +-
.../userFactEvents_offline_table_config.json | 6 +-
.../userGroups_offline_table_config.json | 6 +-
...upsertJsonMeetupRsvp_realtime_table_config.json | 1 -
.../upsertMeetupRsvp_realtime_table_config.json | 1 -
...ertPartialMeetupRsvp_realtime_table_config.json | 1 -
.../LaunchBackfillIngestionJobCommandTest.java | 59 +++
.../tools/admin/command/PostQueryCommandTest.java | 330 ++++++++++++
pom.xml | 8 +-
103 files changed, 3776 insertions(+), 829 deletions(-)
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/ArrayContainsFilterOptimizer.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/queries/ArrayContainsMultiValueQueriesTest.java
create mode 100644
pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-base/src/test/java/org/apache/pinot/plugin/ingestion/batch/spark/common/BaseSparkSegmentMetadataPushJobRunnerTest.java
create mode 100644
pinot-tools/src/test/java/org/apache/pinot/tools/admin/command/LaunchBackfillIngestionJobCommandTest.java
create mode 100644
pinot-tools/src/test/java/org/apache/pinot/tools/admin/command/PostQueryCommandTest.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
