This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a change to branch xiangfu0/codex1/sorted-exchange-core
in repository https://gitbox.apache.org/repos/asf/pinot.git
omit e66ef07ee41 Preserve partition workers for complete-input window sorts
omit e4db9b59a80 Align stage-equivalence field coverage with receive nodes
omit 3843f0795d1 Remove the extra merge-cursor blank line
omit ba0498c33f7 Keep complete window input when fallback stages use older
servers
omit f12c89c40e7 Complete ordered-window review fixes
omit 8f0b844868e Address ordered-window merge review feedback
omit 192a34790f2 [test] Verify homogeneous snapshot builds disable k-way
merge
omit 069c022bbff Align window input sort plan expectations with complete
input
omit f46c09fa0bb Retain complete input for global window receiver sorts
omit 80ca7ec3e97 [MSE] Express ordered streams with a k-way merge exchange
omit 6ab29db2027 [build] Include Zstd API annotations for compiler
validation
omit 85d19bca2bb [MSE] Use current planner test APIs
omit 712a9c74c2d [MSE] Simplify sorted mailbox merging
omit 7a879e469db [MSE] Speed up four-sender sorted window merge
omit 862715533a5 [MSE] Address sender-sorted window review feedback
omit bae0b0c8283 [MSE] Minimize sender-sorted merge changes
omit de8485d8a86 [MSE] Avoid eager sort buffer allocation
omit 9a42c49a9dc [MSE] Limit sender-sorted merge to global windows
omit d6487b0b501 [MSE] Optimize sorted mailbox merge
omit d107d91460c [MSE] Merge sender-sorted streams for ordered windows
add 91fbab84f5d Move the segment assignment partition column to
SegmentAssignmentConfig (#19958)
add b02a7651269 Fix broker routing ignoring a shutting down server while
another server is excluded (#19997)
add 8f20062cd13 Give the dimension table tests their own temp directory
(#20004)
add 77ec2fbf1cf Fold MSE literal function calls with runtime value types
(#19948)
add af3c64f7eb7 Make sorted ORDER BY late materialization automatic
(#20003)
add 678c8b07a90 Hold the consumer semaphore while a replace reverts upsert
metadata (#19817)
add b3bf808a371 Bump org.eclipse.sisu:sisu-maven-plugin from 1.1.0 to
1.1.1 (#20013)
add b5631229eae Bump org.roaringbitmap:RoaringBitmap from 1.6.24 to 1.6.25
(#20011)
add b047e331c11 Bump io.grpc:grpc-bom from 1.85.0 to 1.85.1 (#20014)
add f57c2285ec2 Validate segment assignment for strict replica-group
routing (#20006)
add b99203c6026 Reuse the outer local repo in the assembly-descriptor IT
and report its real error (#19885)
add f1bdf98e6ea Read Parquet unsigned 32-bit integers as long (#19999)
add 60b13a0640b Treat a segment whose partition function cannot be built
as unpartitioned in broker routing (#19953)
add 230045f26d4 Rewrite arrayContains on multi-value columns into an
indexable EQUALS filter (#20000)
add f290065ed21 Add -outputFormat and -outputFile options to PostQuery CLI
command (#19610)
add 8cf16156a44 Remove the kept new segment when a resumed pauseless
commit start completes without it (#19995)
add adb87d861c8 Avoid redundant local input copies and support Zstandard
batch archives (#19898)
add e106c097f46 Bump software.amazon.awssdk:bom from 2.56.0 to 2.57.0
(#20012)
add dc759c51e66 Support FIRST_WITH_TIME and LAST_WITH_TIME without a data
type (#19986)
add d3893e27874 Parse validationTypesToSkip once and apply it consistently
(#20019)
add ac12baf5be9 Add jsonExtractScalar scalar function for
JSON_EXTRACT_SCALAR (#18979)
add 083f2937380 Meter failed ERROR-segment repairs per table (#20018)
add e3b4e41cb29 Track partition assignment in IdealState for strict
replica-group tables (#19916)
add 62958d8ff1b Throw FileNotFoundException from
GcsPinotFS.copyToLocalFile for a missing object (#20026)
add aea5680a40b Detect servers that stop answering queries but keep their
connection open (#19452)
add c5ac29be0e6 Refresh MSE gRPC channels when a server restarts (#19875)
add ad6b54f7b38 Enable query fingerprinting by default and reload cluster
overrides (#20038)
add bff525d98ea Keep source nulls when deriving a column at segment load
(#19955)
add c5b8107fa6a Return the cause chain when an /ingestFromFile record
fails (#19716)
add efffb312091 Bump org.webjars:swagger-ui from 5.33.2 to 5.34.0 (#20045)
add 332a350000e Bump org.apache.pulsar:pulsar-bom from 5.0.0 to 5.0.1
(#20046)
add 8ab072694fe Bump reactor-netty.version from 1.3.7 to 1.3.8 (#20047)
add 452de38982a Bump com.gradle:develocity-maven-extension from 2.6.0 to
2.6.1 (#20049)
add 531bcd02591 Bump com.github.jnr:jnr-posix from 3.2.4 to 3.2.5 (#20053)
add 93978b69b20 Bump software.amazon.awssdk:bom from 2.57.0 to 2.57.3
(#20050)
add 2fd3001a330 Bump com.github.jnr:jnr-ffi from 2.3.3 to 2.3.4 (#20051)
add 960190cbe91 Bump at.yawk.lz4:lz4-java from 1.12.0 to 1.12.1 (#20052)
add 7e772548357 Bump actions/setup-python from 7.0.0 to 7.1.0 (#20055)
add c051ac7643b Bump docker/login-action from 4.6.0 to 4.7.0 (#20056)
add 9c48dac9d7b Close the posting list stream in OffHeapJsonIndexCreator
(#19925)
add 401528f2cc2 Bump io.projectreactor:reactor-core from 3.8.7 to 3.8.8
(#20054)
add d085f946779 Recover unhealthy MSE workers that are not servers in the
routing manager (#20005)
add 32bc2ee2da7 Prebuild Lucene doc ID mappings from numeric doc values
(#19897)
add 19dec509431 Reject the join distribution types that answer the join
wrong (#19862)
add 9c8b2901d3e Require consecutive readiness checks in the
integration-test base (#20015)
add 7835c7dc378 Support MODE on STRING, BOOLEAN and TIMESTAMP inputs
(#19991)
add b58776bca94 Return the input's logical type from ANY_VALUE (#19990)
add 3381db70831 Support is_leaf_return_final_result hint for aggregation
without GROUP BY (#20063)
add eeebb520118 Preserve wide Parquet INT96 timestamps (#20060)
add 3a83ea5f587 Fix NPE when sorting a no-dictionary sorted column that is
an aggregation key (#20002)
add ee7f50ac767 Retry task deletion when Helix fails to remove the job
context (#20009)
add 52d9c8027b8 Add scoped buffer reads for compressed forward indexes
(#20069)
add fcd05cca42d Fix compiled aggregation inputs after timestamp index
rewrites (#20035)
add e94cf06c161 Update DOAP metadata for Apache Pinot 1.6.0 (#20078)
add c03190564e6 Remove expired plan maker constant aliases (#20079)
add 6e0a0a1b35a Support ARRAY_AGG without a data type (#19988)
add 0de3f5e3ffc Cover ANY_VALUE block reads across all stored types
(#19905)
add 3f6fef49e05 Send v1 DataTable responses to Netty and gRPC as pages
instead of one byte[] (#19943)
add ad97f19f7d6 Bump dnsjava:dnsjava from 3.6.5 to 3.7.0 (#20087)
add c6c5aee5e47 Bump org.jacoco:jacoco-maven-plugin from 0.8.15 to 0.8.16
(#20086)
add 7341df07c8d Preserve live instance fields during controller updates
(#20064)
add 577dc1fd937 Bump com.google.errorprone:error_prone_annotations from
2.50.0 to 2.51.0 (#20048)
add e58b6fc2e6a Fix MSE leaf final results for array and raw sketch
aggregations (#20074)
add c91d0e8378d Fix the shared test query server answering every query
with an error (#20096)
add 20cac709644 Add per-table hybrid retention strategy with cluster-level
fallback (#20057)
add 481d4768b72 Revert "Add per-table hybrid retention strategy with
cluster-level fallback (…" (#20101)
add baf1c493137 [MSE] Merge sender-sorted streams for ordered windows
add 7f360018b3f [MSE] Optimize sorted mailbox merge
add 06223c4a090 [MSE] Limit sender-sorted merge to global windows
add cd200a52a73 [MSE] Avoid eager sort buffer allocation
add 15963008a34 [MSE] Minimize sender-sorted merge changes
add eaae4f9c621 [MSE] Address sender-sorted window review feedback
add 8d222b8cdc2 [MSE] Speed up four-sender sorted window merge
add fd01b62a2f2 [MSE] Simplify sorted mailbox merging
add 9205e896d33 [MSE] Use current planner test APIs
add ead7520cede [build] Include Zstd API annotations for compiler
validation
add 9bc53a4ab67 [MSE] Express ordered streams with a k-way merge exchange
add d5f5d3d766c Retain complete input for global window receiver sorts
add 1cf85058a9a Align window input sort plan expectations with complete
input
add 067bdcb5c75 [test] Verify homogeneous snapshot builds disable k-way
merge
add 3194d04f5b1 Address ordered-window merge review feedback
add f0f839c6b9f Complete ordered-window review fixes
add 810ca14cfde Keep complete window input when fallback stages use older
servers
add e07e184497c Remove the extra merge-cursor blank line
add 80558b56997 Align stage-equivalence field coverage with receive nodes
add be31a33d114 Preserve partition workers for complete-input window sorts
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (e66ef07ee41)
\
N -- N -- N refs/heads/xiangfu0/codex1/sorted-exchange-core
(be31a33d114)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.github/workflows/build-all-pinot-docker-image.yml | 6 +-
.../workflows/build-pinot-base-docker-image.yml | 4 +-
.github/workflows/build-pinot-docker-image.yml | 2 +-
.github/workflows/build-superset-docker-image.yml | 2 +-
.../nightly-build-all-pinot-docker-image.yml | 6 +-
...htly-build-multi-arch-superset-docker-image.yml | 4 +-
.github/workflows/pr-flow-checks.yml | 2 +-
.github/workflows/pr-flow.yml | 4 +-
.mvn/extensions.xml | 2 +-
doap_Pinot.rdf | 6 +-
docker/images/pinot/Dockerfile | 2 +-
docker/images/pinot/Dockerfile.build | 2 +-
docker/images/pinot/Dockerfile.package | 2 +-
pinot-bom/pom.xml | 2 +-
.../broker/broker/helix/BaseBrokerStarter.java | 21 +-
.../requesthandler/BaseBrokerRequestHandler.java | 53 +-
.../BaseSingleStageBrokerRequestHandler.java | 25 +-
.../requesthandler/GrpcBrokerRequestHandler.java | 4 +-
.../requesthandler/InferAnyValueTypeProvider.java | 70 ++
.../MultiStageBrokerRequestHandler.java | 46 +-
.../SingleConnectionBrokerRequestHandler.java | 77 +-
.../routing/manager/BaseBrokerRoutingManager.java | 44 +-
.../segmentpartition/SegmentPartitionUtils.java | 32 +-
.../BaseSingleStageBrokerRequestHandlerTest.java | 94 ++-
.../InferAnyValueTypeProviderTest.java | 69 ++
.../MultiStageBrokerRequestHandlerTest.java | 188 ++++-
...ionBrokerRequestHandlerFailureDetectorTest.java | 454 +++++++++++
.../routing/manager/BrokerRoutingManagerTest.java | 63 +-
.../SegmentPartitionUtilsTest.java | 24 +
.../SinglePartitionColumnSegmentPrunerTest.java | 13 +
.../pinot/client/admin/TableAdminClient.java | 6 +-
.../assignment/InstanceAssignmentConfigUtils.java | 4 +-
.../assignment/PartitionAssignmentUtils.java | 117 +++
.../apache/pinot/common/datatable/DataTable.java | 11 +
.../pinot/common/datatable/DataTableImplV4.java | 89 ++-
...BaseExponentialBackoffRetryFailureDetector.java | 132 +++-
.../failuredetector/ConnectionFailureDetector.java | 9 +-
.../common/failuredetector/FailureDetector.java | 23 +
.../pinot/common/function/JsonPathCache.java | 11 +-
.../common/function/TransformFunctionType.java | 9 +-
.../common/function/scalar/JsonFunctions.java | 458 ++++++++++-
.../pinot/common/metadata/ZKMetadataProvider.java | 10 +-
.../pinot/common/metrics/ControllerMeter.java | 6 +-
.../pinot/common/request/InstanceRequest.java | 106 ++-
.../apache/pinot/common/utils/ArrayListUtils.java | 94 +--
.../org/apache/pinot/common/utils/DataSchema.java | 98 +--
.../apache/pinot/common/utils/ExceptionUtils.java | 45 ++
.../apache/pinot/common/utils/JsonNumberUtils.java | 207 +++++
.../common/utils/config/QueryOptionsUtils.java | 13 +-
.../apache/pinot/sql/parsers/CalciteSqlParser.java | 11 +
.../rewriter/CompileTimeFunctionsInvoker.java | 7 +
.../assignment/PartitionAssignmentUtilsTest.java | 65 ++
.../ConnectionFailureDetectorTest.java | 284 ++++++-
.../common/function/FastJsonPathExtractorTest.java | 21 +-
.../pinot/common/function/JsonFunctionsTest.java | 352 +++++++++
.../pinot/common/utils/ArrayListUtilsTest.java | 117 ---
.../pinot/common/utils/ExceptionUtilsTest.java | 129 +++
.../pinot/common/utils/JsonNumberUtilsTest.java | 86 +-
.../common/utils/TarCompressionUtilsTest.java | 4 +-
.../utils/config/TableConfigSerDeUtilsTest.java | 46 +-
.../pinot/sql/parsers/CalciteSqlCompilerTest.java | 17 +
.../pinot/sql/parsers/CalciteSqlParserTest.java | 14 +
pinot-common/src/thrift/request.thrift | 4 +
.../exception/ControllerApplicationException.java | 23 +-
.../resources/PinotIngestionRestletResource.java | 27 +-
.../api/resources/PinotTableRestletResource.java | 29 +-
.../api/resources/TableConfigValidationUtils.java | 109 ++-
.../api/resources/TableConfigsRestletResource.java | 43 +-
.../resources/WebApplicationExceptionMapper.java | 19 +-
.../api/upload/SegmentValidationUtils.java | 56 +-
.../helix/core/PinotHelixResourceManager.java | 165 +++-
.../assignment/segment/BaseSegmentAssignment.java | 174 +++-
.../BaseStrictRealtimeSegmentAssignment.java | 146 ----
.../MultiTierStrictRealtimeSegmentAssignment.java | 131 ---
.../segment/OfflineSegmentAssignment.java | 136 +++-
.../segment/RealtimeSegmentAssignment.java | 193 ++++-
.../core/assignment/segment/SegmentAssignment.java | 18 +-
.../segment/SegmentAssignmentFactory.java | 26 +-
.../assignment/segment/SegmentAssignmentUtils.java | 11 +
.../SingleTierStrictRealtimeSegmentAssignment.java | 75 --
.../core/assignment/segment/TableAssignment.java | 59 ++
.../strategy/SegmentAssignmentStrategyFactory.java | 74 +-
.../core/cleanup/StaleInstancesCleanupTask.java | 9 +-
.../core/minion/PinotHelixTaskResourceManager.java | 36 +-
.../realtime/PinotLLCRealtimeSegmentManager.java | 135 ++--
.../helix/core/rebalance/TableRebalancer.java | 876 ++++++++++++---------
.../pinot/controller/util/TaskConfigUtils.java | 24 +-
...PinotIngestionRestletResourceStatelessTest.java | 86 +-
.../api/TableConfigsRestletResourceTest.java | 7 -
.../ControllerApplicationExceptionTest.java | 36 +-
.../PinotIngestionRestletResourceTest.java | 140 ++++
.../resources/TableConfigValidationUtilsTest.java | 76 ++
.../WebApplicationExceptionMapperTest.java | 98 +++
.../api/upload/SegmentValidationUtilsTest.java | 8 +-
.../controller/helix/PinotResourceManagerTest.java | 2 -
...otHelixResourceManagerConfigValidationTest.java | 107 +++
.../PinotHelixResourceManagerStatelessTest.java | 152 ++++
.../instance/InstanceAssignmentTest.java | 3 +-
...NonReplicaGroupTieredSegmentAssignmentTest.java | 49 +-
...artitionPinnedOfflineSegmentAssignmentTest.java | 353 +++++++++
...rtitionPinnedRealtimeSegmentAssignmentTest.java | 704 +++++++++++++++++
...altimeNonReplicaGroupSegmentAssignmentTest.java | 82 +-
...NonReplicaGroupTieredSegmentAssignmentTest.java | 42 +-
.../RealtimeReplicaGroupSegmentAssignmentTest.java | 167 ++--
.../segment/SegmentAssignmentTestUtils.java | 10 +
.../StrictRealtimeSegmentAssignmentTest.java | 534 -------------
.../AllServersSegmentAssignmentStrategyTest.java | 37 +-
.../BalancedNumSegmentAssignmentStrategyTest.java | 40 +-
.../ReplicaGroupSegmentAssignmentStrategyTest.java | 166 ++--
...nReplicaGroupSegmentAssignmentStrategyTest.java | 57 +-
.../RoundRobinSegmentAssignmentStrategyTest.java | 27 +-
.../SegmentAssignmentStrategyFactoryTest.java | 9 +-
.../cleanup/StaleInstancesCleanupTaskTest.java | 29 +
.../minion/PinotHelixTaskResourceManagerTest.java | 54 ++
.../PinotLLCRealtimeSegmentManagerTest.java | 210 ++++-
.../TableRebalancerClusterStatelessTest.java | 43 +
.../helix/core/rebalance/TableRebalancerTest.java | 339 ++++++--
.../pinot/controller/util/TaskConfigUtilsTest.java | 16 +
.../apache/pinot/core/common/ObjectSerDeUtils.java | 98 ++-
.../common/datatable/DataTableBuilderUtils.java | 52 +-
.../realtime/RealtimeSegmentDataManager.java | 57 +-
.../apache/pinot/core/data/table/IndexedTable.java | 6 +-
.../pinot/core/data/table/IntermediateRecord.java | 7 +-
.../apache/pinot/core/data/table/TableResizer.java | 40 +-
.../pinot/core/function/scalar/IdSetFunctions.java | 19 +-
.../blocks/results/AggregationResultsBlock.java | 63 +-
.../query/LinearSelectionOrderByOperator.java | 154 ++--
.../SelectionPartiallyOrderedByDescOperation.java | 16 +-
.../SelectionPartiallyOrderedByLinearOperator.java | 10 +-
.../operator/streaming/StreamingResponseUtils.java | 20 +-
.../function/ArrayLiteralTransformFunction.java | 20 +
.../JsonExtractScalarTransformFunction.java | 123 +--
.../apache/pinot/core/plan/SelectionPlanNode.java | 57 +-
.../core/plan/maker/InstancePlanMakerImplV2.java | 233 ++++--
.../aggregation/function/AggregationFunction.java | 23 +-
.../function/AggregationFunctionUtils.java | 29 +-
.../function/AnyValueAggregationFunction.java | 263 ++++---
.../function/BaseAggregationFunction.java | 2 +-
.../BaseSingleInputAggregationFunction.java | 3 +-
...igDecimalValueWithTimeAggregationFunction.java} | 50 +-
...irstBytesValueWithTimeAggregationFunction.java} | 67 +-
...rstLastWithTimeAggregationFunctionProvider.java | 25 +
.../FirstLongValueWithTimeAggregationFunction.java | 16 +-
.../function/FirstWithTimeAggregationFunction.java | 12 +-
.../function/HistogramAggregationFunction.java | 9 +-
...igDecimalValueWithTimeAggregationFunction.java} | 50 +-
...LastBytesValueWithTimeAggregationFunction.java} | 67 +-
.../LastLongValueWithTimeAggregationFunction.java | 16 +-
.../function/LastWithTimeAggregationFunction.java | 12 +-
.../function/ModeAggregationFunction.java | 201 ++++-
.../function/array/ArrayAggBytesFunction.java | 4 +-
.../array/ArrayAggDistinctBytesFunction.java | 5 +-
.../array/BaseArrayAggBigDecimalFunction.java | 12 +-
.../function/array/BaseArrayAggBytesFunction.java | 3 +-
.../function/array/BaseArrayAggDoubleFunction.java | 9 +-
.../function/array/BaseArrayAggFloatFunction.java | 9 +-
.../function/array/BaseArrayAggFunction.java | 2 +-
.../function/array/BaseArrayAggIntFunction.java | 9 +-
.../function/array/BaseArrayAggLongFunction.java | 9 +-
.../function/array/BaseArrayAggStringFunction.java | 13 +-
.../array/SumArrayDoubleAggregationFunction.java | 7 +-
.../array/SumArrayLongAggregationFunction.java | 7 +-
.../function/funnel/BitmapMergeStrategy.java | 5 +-
.../funnel/FunnelCountAggregationFunction.java | 13 +-
.../FunnelCountAggregationFunctionFactory.java | 5 +-
.../aggregation/function/funnel/MergeStrategy.java | 3 +-
.../function/funnel/PartitionedMergeStrategy.java | 24 +-
.../function/funnel/SetMergeStrategy.java | 5 +-
.../function/funnel/ThetaSketchMergeStrategy.java | 5 +-
.../window/FunnelBaseAggregationFunction.java | 2 +-
...unnelEventsFunctionEvalAggregationFunction.java | 63 +-
.../window/FunnelMatchStepAggregationFunction.java | 18 +-
...FunnelStepDurationStatsAggregationFunction.java | 33 +-
.../pinot/core/query/optimizer/QueryOptimizer.java | 10 +-
.../filter/ArrayContainsFilterOptimizer.java | 166 ++++
.../AggregateFunctionRewriteOptimizer.java | 192 ++++-
.../query/reduce/AggregationDataTableReducer.java | 13 +-
.../core/query/reduce/GroupByDataTableReducer.java | 19 +-
.../core/query/request/context/QueryContext.java | 128 +--
.../context/utils/QueryContextConverterUtils.java | 11 +-
.../query/scheduler/BinaryWorkloadScheduler.java | 15 +-
.../core/query/scheduler/PriorityScheduler.java | 11 +-
.../pinot/core/query/scheduler/QueryScheduler.java | 119 ++-
.../query/scheduler/SchedulerQueryContext.java | 7 +-
.../core/query/scheduler/WorkloadScheduler.java | 9 +-
.../query/scheduler/fcfs/FCFSQueryScheduler.java | 7 +-
.../core/transport/InstanceRequestHandler.java | 53 +-
.../apache/pinot/core/transport/QueryRouter.java | 149 +++-
.../pinot/core/transport/ServerChannels.java | 47 +-
.../org/apache/pinot/core/util/NumberUtils.java | 122 +--
.../pinot/core/common/ObjectSerDeUtilsTest.java | 51 ++
.../core/common/datatable/DataTableSerDeTest.java | 23 +-
.../DimensionTableDataBytesPrimaryKeyTest.java | 4 +-
.../offline/DimensionTableDataManagerTest.java | 4 +-
.../realtime/RealtimeSegmentDataManagerTest.java | 26 +-
.../table/UnboundedConcurrentIndexedTableTest.java | 2 +-
.../pinot/core/function/FunctionRegistryTest.java | 2 +-
.../core/function/scalar/IdSetFunctionsTest.java | 7 +-
.../streaming/StreamingResponseUtilsTest.java | 72 ++
.../ArrayLiteralTransformFunctionTest.java | 29 +
.../JsonExtractScalarTransformFunctionTest.java | 34 +-
.../plan/maker/QueryOverrideWithHintsTest.java | 349 +++++++-
.../AggregationFunctionNullContractTest.java | 12 +-
.../function/AggregationFunctionUtilsTest.java | 25 +-
.../function/AnyValueAggregationFunctionTest.java | 170 +++-
.../function/ModeAggregationFunctionTest.java | 181 +++++
.../function/ValueAggregationNullHandlingTest.java | 20 +-
.../function/funnel/FunnelFinalResultTest.java | 48 +-
.../function/funnel/FunnelNullHandlingTest.java | 16 +-
.../core/query/optimizer/QueryOptimizerTest.java | 48 +-
.../AggregateFunctionRewriteOptimizerTest.java | 264 +++++++
.../scheduler/QuerySchedulerResponseApiTest.java | 98 +++
.../core/transport/InstanceRequestHandlerTest.java | 56 ++
.../pinot/core/transport/QueryRoutingTest.java | 353 ++++++++-
.../pinot/core/transport/QueryServerTestUtils.java | 89 +++
.../pinot/core/transport/ServerChannelsTest.java | 39 +-
.../apache/pinot/core/util/NumberUtilsTest.java | 12 +
.../apache/pinot/queries/AnyValueQueriesTest.java | 230 ++++++
.../pinot/queries/ArrayAggMvQueriesTest.java | 19 +
.../apache/pinot/queries/ArrayAggQueriesTest.java | 43 +
.../pinot/queries/ArrayAggStarTreeQueriesTest.java | 42 +-
.../ArrayContainsMultiValueQueriesTest.java | 115 +++
.../pinot/queries/FirstWithTimeQueriesTest.java | 148 ++++
.../org/apache/pinot/queries/IdSetQueriesTest.java | 37 +
...nnerSegmentSelectionSingleValueQueriesTest.java | 113 +--
.../pinot/queries/LastWithTimeQueriesTest.java | 148 ++++
.../org/apache/pinot/queries/ModeQueriesTest.java | 210 ++++-
.../queries/NullHandlingEnabledQueriesTest.java | 71 +-
.../queries/TextMatchTransformFunctionTest.java | 12 +-
.../apache/pinot/queries/TimestampQueriesTest.java | 70 ++
.../tests/BaseClusterIntegrationTest.java | 32 +-
.../tests/BaseDedupIntegrationTest.java | 2 -
.../tests/DedupPreloadIntegrationTest.java | 2 -
.../integration/tests/custom/JsonPathTest.java | 107 +++
.../tests/custom/OfflineUpsertTableTest.java | 17 +-
.../tests/custom/VectorUpsertTableTest.java | 2 -
.../simpleMeetup_realtime_table_config.json | 6 +-
.../resources/udf-test-results/all-functions.yaml | 3 +-
.../AbstractAggregationFunctionBenchmark.java | 8 +-
pinot-plugins/assembly-descriptor/pom.xml | 14 +
.../src/it/projects/simple-assembly/pom.xml | 19 +-
.../src/it/projects/simple-assembly/verify.groovy | 12 +-
.../batch/common/BaseSegmentPushJobRunner.java | 11 +-
.../batch/hadoop/HadoopSegmentCreationMapper.java | 3 +-
.../hadoop/HadoopSegmentMetadataPushJobRunner.java | 18 +-
.../hadoop/HadoopSegmentTarPushJobRunner.java | 14 +-
.../hadoop/HadoopSegmentUriPushJobRunner.java | 15 +-
.../spark3/SparkSegmentGenerationJobRunner.java | 3 +-
.../spark3/SparkSegmentMetadataPushJobRunner.java | 16 +-
.../BaseSparkSegmentMetadataPushJobRunner.java | 10 +-
.../common/BaseSparkSegmentTarPushJobRunner.java | 10 +-
.../common/BaseSparkSegmentUriPushJobRunner.java | 14 +-
.../BaseSparkSegmentMetadataPushJobRunnerTest.java | 125 +++
.../standalone/SegmentGenerationJobRunner.java | 23 +-
.../standalone/SegmentGenerationJobRunnerTest.java | 151 ++++
.../apache/pinot/plugin/filesystem/GcsPinotFS.java | 4 +-
.../filesystem/GcsPinotFSNullSafetyTest.java | 22 +
.../parquet/ParquetAvroRecordExtractor.java | 45 +-
.../parquet/ParquetAvroRecordReader.java | 22 +-
.../parquet/ParquetNativeRecordExtractor.java | 18 +-
.../parquet/ParquetNativeRecordReader.java | 3 +-
.../parquet/ParquetRecordReaderConfig.java | 1 +
.../plugin/inputformat/parquet/ParquetUtils.java | 76 +-
.../parquet/ParquetAvroRecordExtractorTest.java | 27 +-
.../parquet/ParquetNativeRecordExtractorTest.java | 85 +-
.../parquet/ParquetReaderLogicalTypesTest.java | 158 ++++
.../inputformat/parquet/ParquetTestUtils.java | 14 +
.../segmentuploader/SegmentUploaderDefault.java | 2 +-
.../pinot/calcite/rel/hint/PinotHintOptions.java | 8 +
.../PinotAggregateExchangeNodeInsertRule.java | 9 +
.../PinotAggregateTypeArgumentRewriteRule.java | 254 ++++++
.../PinotApproximateAggregateRewriteRule.java | 22 +-
.../rel/rules/PinotEvaluateLiteralRule.java | 135 +++-
.../rel/rules/PinotJoinExchangeNodeInsertRule.java | 134 +++-
.../pinot/calcite/rel/rules/PinotRuleUtils.java | 21 +
.../org/apache/pinot/query/QueryEnvironment.java | 24 +-
.../query/planner/logical/RexExpressionUtils.java | 50 +-
.../v2/opt/rules/AggregatePushdownRule.java | 9 +
.../rules/PinotJoinExchangeNodeInsertRuleTest.java | 143 ++++
.../query/ApproximateAggregateRewriteTest.java | 32 +
.../apache/pinot/query/QueryCompilationTest.java | 128 +++
.../pinot/query/QueryPlannerRuleOptionsTest.java | 24 +-
.../src/test/resources/queries/AggregatePlans.json | 242 ++++++
.../resources/queries/LiteralEvaluationPlans.json | 108 +++
.../apache/pinot/query/mailbox/MailboxService.java | 18 +-
.../query/mailbox/channel/ChannelManager.java | 73 +-
.../query/runtime/operator/AggregateOperator.java | 8 +-
.../query/runtime/operator/BaseJoinOperator.java | 8 +-
.../query/runtime/operator/FilterOperator.java | 4 +-
.../pinot/query/runtime/operator/LeafOperator.java | 45 +-
.../query/runtime/operator/LookupJoinOperator.java | 5 +-
.../operator/MultistageAggregationExecutor.java | 37 +-
.../operator/MultistageGroupByExecutor.java | 12 +-
.../query/runtime/operator/TransformOperator.java | 5 +-
.../query/runtime/operator/UnnestOperator.java | 4 +-
.../runtime/operator/operands/FilterOperand.java | 45 +-
.../runtime/operator/operands/FunctionOperand.java | 60 +-
.../operator/operands/TransformOperandFactory.java | 38 +-
.../query/runtime/operator/utils/SortUtils.java | 4 +-
.../query/runtime/operator/utils/TypeUtils.java | 44 --
.../plan/server/ServerPlanRequestVisitor.java | 40 +-
.../query/service/dispatch/DispatchClient.java | 21 +
.../query/service/dispatch/QueryDispatcher.java | 89 ++-
.../query/mailbox/channel/ChannelManagerTest.java | 111 ++-
.../runtime/operator/AggregateOperatorTest.java | 106 ++-
.../query/runtime/operator/LeafOperatorTest.java | 67 ++
.../SerializedIntermediateResultMergeTest.java | 2 +-
.../query/runtime/queries/QueryRunnerTest.java | 217 ++++-
.../query/runtime/queries/QueryRunnerTestBase.java | 9 +
.../service/dispatch/QueryDispatcherTest.java | 142 ++++
.../src/test/resources/queries/IdSet.json | 7 +-
.../test/resources/queries/LiteralExpressions.json | 33 +
.../test/resources/queries/NonEquiOuterJoins.json | 58 ++
.../src/test/resources/queries/QueryHints.json | 89 +++
.../src/test/resources/queries/UDFAggregates.json | 272 ++++++-
...StringLongPair.java => BigDecimalLongPair.java} | 27 +-
.../{StringLongPair.java => BytesLongPair.java} | 26 +-
.../indexsegment/mutable/MutableSegmentImpl.java | 51 +-
.../MultiColumnRealtimeLuceneTextIndex.java | 2 +-
.../invertedindex/RealtimeLuceneTextIndex.java | 2 +-
.../segment/creator/RecordProcessingException.java | 16 +-
.../RecordReaderSegmentCreationDataSource.java | 2 +-
.../impl/SegmentIndexCreationDriverImpl.java | 3 +-
.../impl/inv/json/OffHeapJsonIndexCreator.java | 25 +-
.../creator/impl/text/LuceneTextIndexCreator.java | 112 +--
.../defaultcolumn/BaseDefaultColumnHandler.java | 57 +-
.../forward/BaseChunkForwardIndexReader.java | 10 +-
.../forward/VarByteChunkForwardIndexReaderV4.java | 15 +-
.../index/readers/text/LuceneTextIndexReader.java | 164 +---
.../text/MultiColumnLuceneTextIndexReader.java | 4 +-
.../local/segment/store/TextIndexUtils.java | 68 ++
.../segment/local/utils/SegmentPushUtils.java | 39 +-
.../segment/local/utils/TableConfigUtils.java | 742 +++++++++++------
.../converter/RealtimeSegmentConverterTest.java | 142 ++++
.../creator/RecordProcessingExceptionTest.java | 143 ++++
.../segment/local/segment/index/JsonIndexTest.java | 97 +++
.../index/creator/LuceneTextIndexCreatorTest.java | 287 ++++++-
.../local/segment/index/loader/LoaderTest.java | 54 +-
.../index/loader/SegmentPreProcessorTest.java | 67 ++
.../TextIndexRollbackRecoveryTest.java | 2 +-
.../text/LuceneTextIndexCompatibleTest.java | 174 +++-
.../readers/text/LuceneTextIndexCountTest.java | 22 +-
.../segment/store/FilePerIndexDirectoryTest.java | 22 +-
.../store/SingleFileIndexDirectoryTest.java | 22 +-
.../segment/local/utils/SegmentPushUtilsTest.java | 140 ++++
.../segment/local/utils/TableConfigUtilsTest.java | 755 +++++++++++++++---
.../pinot/segment/spi/AggregationFunctionType.java | 26 +-
.../pinot/segment/spi/index/TextIndexConfig.java | 6 +-
.../pinot/segment/spi/memory/PinotDataBuffer.java | 12 +
.../server/starter/helix/BaseServerStarter.java | 10 +-
...ava => ServerMailboxChannelRefreshHandler.java} | 107 ++-
.../warmup/PageCacheWarmupServerQueryExecutor.java | 5 +-
...=> ServerMailboxChannelRefreshHandlerTest.java} | 155 +++-
.../PageCacheWarmupServerQueryExecutorTest.java | 7 +-
.../config/table/ReplicaGroupStrategyConfig.java | 4 +-
.../SegmentsValidationAndRetentionConfig.java | 9 +-
.../table/assignment/InstanceAssignmentConfig.java | 35 +-
.../InstanceReplicaGroupPartitionConfig.java | 9 +-
.../table/assignment/SegmentAssignmentConfig.java | 22 +-
.../pinot/spi/ingestion/batch/spec/Constants.java | 20 +
.../batch/spec/SegmentGenerationJobSpec.java | 20 +
.../apache/pinot/spi/utils/CommonConstants.java | 56 +-
.../pinot/spi/utils/StoredValueComparator.java | 59 ++
.../spi/utils/builder/TableConfigBuilder.java | 3 +-
pinot-tools/pom.xml | 4 +
.../command/LaunchBackfillIngestionJobCommand.java | 35 +-
.../tools/admin/command/PostQueryCommand.java | 130 ++-
.../userAttributes_offline_table_config.json | 3 +-
.../userFactEvents_offline_table_config.json | 6 +-
.../userGroups_offline_table_config.json | 6 +-
...upsertJsonMeetupRsvp_realtime_table_config.json | 1 -
.../upsertMeetupRsvp_realtime_table_config.json | 1 -
...ertPartialMeetupRsvp_realtime_table_config.json | 1 -
.../LaunchBackfillIngestionJobCommandTest.java | 59 ++
.../tools/admin/command/PostQueryCommandTest.java | 330 ++++++++
pom.xml | 28 +-
376 files changed, 19935 insertions(+), 5316 deletions(-)
create mode 100644
pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/InferAnyValueTypeProvider.java
create mode 100644
pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/InferAnyValueTypeProviderTest.java
create mode 100644
pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/SingleConnectionBrokerRequestHandlerFailureDetectorTest.java
create mode 100644
pinot-common/src/main/java/org/apache/pinot/common/assignment/PartitionAssignmentUtils.java
create mode 100644
pinot-common/src/main/java/org/apache/pinot/common/utils/JsonNumberUtils.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/assignment/PartitionAssignmentUtilsTest.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/utils/ExceptionUtilsTest.java
copy pinot-core/src/test/java/org/apache/pinot/core/util/NumberUtilsTest.java
=>
pinot-common/src/test/java/org/apache/pinot/common/utils/JsonNumberUtilsTest.java
(59%)
delete mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/assignment/segment/BaseStrictRealtimeSegmentAssignment.java
delete mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/assignment/segment/MultiTierStrictRealtimeSegmentAssignment.java
delete mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/assignment/segment/SingleTierStrictRealtimeSegmentAssignment.java
create mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/assignment/segment/TableAssignment.java
create mode 100644
pinot-controller/src/test/java/org/apache/pinot/controller/api/resources/PinotIngestionRestletResourceTest.java
create mode 100644
pinot-controller/src/test/java/org/apache/pinot/controller/api/resources/WebApplicationExceptionMapperTest.java
create mode 100644
pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/assignment/segment/PartitionPinnedOfflineSegmentAssignmentTest.java
create mode 100644
pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/assignment/segment/PartitionPinnedRealtimeSegmentAssignmentTest.java
delete mode 100644
pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/assignment/segment/StrictRealtimeSegmentAssignmentTest.java
copy
pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/{FirstLongValueWithTimeAggregationFunction.java
=> FirstBigDecimalValueWithTimeAggregationFunction.java} (64%)
copy
pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/{FirstFloatValueWithTimeAggregationFunction.java
=> FirstBytesValueWithTimeAggregationFunction.java} (57%)
copy
pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/{LastDoubleValueWithTimeAggregationFunction.java
=> LastBigDecimalValueWithTimeAggregationFunction.java} (64%)
copy
pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/{LastFloatValueWithTimeAggregationFunction.java
=> LastBytesValueWithTimeAggregationFunction.java} (57%)
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/query/optimizer/filter/ArrayContainsFilterOptimizer.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/operator/streaming/StreamingResponseUtilsTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/query/optimizer/statement/AggregateFunctionRewriteOptimizerTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/query/scheduler/QuerySchedulerResponseApiTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/transport/QueryServerTestUtils.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/queries/AnyValueQueriesTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/queries/ArrayContainsMultiValueQueriesTest.java
create mode 100644
pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-spark-base/src/test/java/org/apache/pinot/plugin/ingestion/batch/spark/common/BaseSparkSegmentMetadataPushJobRunnerTest.java
create mode 100644
pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotAggregateTypeArgumentRewriteRule.java
create mode 100644
pinot-query-planner/src/test/java/org/apache/pinot/calcite/rel/rules/PinotJoinExchangeNodeInsertRuleTest.java
copy
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/customobject/{StringLongPair.java
=> BigDecimalLongPair.java} (59%)
copy
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/customobject/{StringLongPair.java
=> BytesLongPair.java} (62%)
copy
pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/admin/PinotAdminException.java
=>
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/RecordProcessingException.java
(70%)
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/RecordProcessingExceptionTest.java
rename
pinot-server/src/main/java/org/apache/pinot/server/starter/helix/{ServerGrpcChannelBackoffResetHandler.java
=> ServerMailboxChannelRefreshHandler.java} (53%)
rename
pinot-server/src/test/java/org/apache/pinot/server/starter/helix/{ServerGrpcChannelBackoffResetHandlerTest.java
=> ServerMailboxChannelRefreshHandlerTest.java} (55%)
create mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/utils/StoredValueComparator.java
create mode 100644
pinot-tools/src/test/java/org/apache/pinot/tools/admin/command/LaunchBackfillIngestionJobCommandTest.java
create mode 100644
pinot-tools/src/test/java/org/apache/pinot/tools/admin/command/PostQueryCommandTest.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
