This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a change to branch xiangfu0/variant-split/03-segment-storage
in repository https://gitbox.apache.org/repos/asf/pinot.git
discard 9a360a81bd3 [VARIANT 3/8] VARIANT segment storage and
schema/table-config validation
discard 1551020ec71 [VARIANT 2/8] Add Variant codec, scalar functions and
VARIANT column type (pinot-common)
discard de2b399c4fb [VARIANT 1/8] Add VARIANT logical type foundation
(pinot-spi)
add e3b4e41cb29 Track partition assignment in IdealState for strict
replica-group tables (#19916)
add 62958d8ff1b Throw FileNotFoundException from
GcsPinotFS.copyToLocalFile for a missing object (#20026)
add aea5680a40b Detect servers that stop answering queries but keep their
connection open (#19452)
add c5ac29be0e6 Refresh MSE gRPC channels when a server restarts (#19875)
add ad6b54f7b38 Enable query fingerprinting by default and reload cluster
overrides (#20038)
add bff525d98ea Keep source nulls when deriving a column at segment load
(#19955)
add c5b8107fa6a Return the cause chain when an /ingestFromFile record
fails (#19716)
add efffb312091 Bump org.webjars:swagger-ui from 5.33.2 to 5.34.0 (#20045)
add 332a350000e Bump org.apache.pulsar:pulsar-bom from 5.0.0 to 5.0.1
(#20046)
add 8ab072694fe Bump reactor-netty.version from 1.3.7 to 1.3.8 (#20047)
add 452de38982a Bump com.gradle:develocity-maven-extension from 2.6.0 to
2.6.1 (#20049)
add 531bcd02591 Bump com.github.jnr:jnr-posix from 3.2.4 to 3.2.5 (#20053)
add 93978b69b20 Bump software.amazon.awssdk:bom from 2.57.0 to 2.57.3
(#20050)
add 2fd3001a330 Bump com.github.jnr:jnr-ffi from 2.3.3 to 2.3.4 (#20051)
add 960190cbe91 Bump at.yawk.lz4:lz4-java from 1.12.0 to 1.12.1 (#20052)
add 7e772548357 Bump actions/setup-python from 7.0.0 to 7.1.0 (#20055)
add c051ac7643b Bump docker/login-action from 4.6.0 to 4.7.0 (#20056)
add 9c48dac9d7b Close the posting list stream in OffHeapJsonIndexCreator
(#19925)
add 401528f2cc2 Bump io.projectreactor:reactor-core from 3.8.7 to 3.8.8
(#20054)
add d085f946779 Recover unhealthy MSE workers that are not servers in the
routing manager (#20005)
add 32bc2ee2da7 Prebuild Lucene doc ID mappings from numeric doc values
(#19897)
add 19dec509431 Reject the join distribution types that answer the join
wrong (#19862)
add 9c8b2901d3e Require consecutive readiness checks in the
integration-test base (#20015)
add 7835c7dc378 Support MODE on STRING, BOOLEAN and TIMESTAMP inputs
(#19991)
add b58776bca94 Return the input's logical type from ANY_VALUE (#19990)
add 3381db70831 Support is_leaf_return_final_result hint for aggregation
without GROUP BY (#20063)
add eeebb520118 Preserve wide Parquet INT96 timestamps (#20060)
add 3a83ea5f587 Fix NPE when sorting a no-dictionary sorted column that is
an aggregation key (#20002)
add ee7f50ac767 Retry task deletion when Helix fails to remove the job
context (#20009)
add 52d9c8027b8 Add scoped buffer reads for compressed forward indexes
(#20069)
add fcd05cca42d Fix compiled aggregation inputs after timestamp index
rewrites (#20035)
add e94cf06c161 Update DOAP metadata for Apache Pinot 1.6.0 (#20078)
add c03190564e6 Remove expired plan maker constant aliases (#20079)
add 6e0a0a1b35a Support ARRAY_AGG without a data type (#19988)
add 0de3f5e3ffc Cover ANY_VALUE block reads across all stored types
(#19905)
add 3f6fef49e05 Send v1 DataTable responses to Netty and gRPC as pages
instead of one byte[] (#19943)
add ad97f19f7d6 Bump dnsjava:dnsjava from 3.6.5 to 3.7.0 (#20087)
add c6c5aee5e47 Bump org.jacoco:jacoco-maven-plugin from 0.8.15 to 0.8.16
(#20086)
add 7341df07c8d Preserve live instance fields during controller updates
(#20064)
add 577dc1fd937 Bump com.google.errorprone:error_prone_annotations from
2.50.0 to 2.51.0 (#20048)
add e58b6fc2e6a Fix MSE leaf final results for array and raw sketch
aggregations (#20074)
add c91d0e8378d Fix the shared test query server answering every query
with an error (#20096)
add 20cac709644 Add per-table hybrid retention strategy with cluster-level
fallback (#20057)
add 481d4768b72 Revert "Add per-table hybrid retention strategy with
cluster-level fallback (…" (#20101)
add f526b645ce8 Bump software.amazon.awssdk:bom from 2.57.3 to 2.57.4
(#20103)
add 863356920f2 Bump org.roaringbitmap:RoaringBitmap from 1.6.25 to 1.6.27
(#20102)
add e12b8435c75 Add per-table SHADOW enforcement mode for storage and
query quotas (#19838)
add 1b552624f52 Add a broker hook to prepare each MSE stage before
dispatch (#20104)
add 3117dbb262a Avoid repeated force commits during rebalance (#20097)
add 96fe7141147 Merge final results of array, raw sketch and other
aggregation functions (#20098)
add eca9ea7afde Reuse the shared hybrid cluster for tenant rebalance tests
(#20084)
add 66eb74d69d1 Add requiresFetch flag to a segment to for prefetch
planning (#20068)
add d226fb5b358 Reduce instance selector memory by grouping segments with
the same candidates (#20077)
add df57b73726a Store sealed segment metadata in a mapped sidecar (#19998)
add bfdb305e148 Skip mailbox config scans for non-server membership
changes (#20107)
add e9be2ee203a Bump follow-redirects in
/pinot-controller/src/main/resources (#20110)
add 2b0da67bc43 Extract hybrid table retention check into overridable
shouldEnableHybridTableRetentionStrategy method (#20088)
add 9fe9d4d8f04 Spy a plain RetentionManager in hybrid retention override
tests so subclass suites can reuse them (#20119)
add 250cbe0b654 Bump com.github.jnr:jnr-ffi from 2.3.4 to 2.3.5 (#20132)
add ad1f044fe02 Bump joda-time:joda-time from 2.15.0 to 2.15.1 (#20131)
add 0ea960fb6a9 Bump org.apache.zookeeper:zookeeper from 3.9.6 to 3.9.7
(#20127)
add 300d124b1aa Bump org.apache.maven.plugins:maven-enforcer-plugin from
3.6.3 to 3.7.0 (#20126)
add 68b836d6f2a Bump wire.version from 7.1.0 to 7.1.2 (#20124)
add 4d76004ccad Bump com.google.cloud:libraries-bom from 26.90.0 to
26.91.0 (#20123)
add 63cd8e006d1 Let segment directories serve mapped segment metadata
(#20114)
add 677a903191f Bump org.roaringbitmap:RoaringBitmap from 1.6.27 to 1.6.29
(#20130)
add 5d81ce915d1 Bump com.github.luben:zstd-jni from 1.5.7-24 to 1.5.7-25
(#20125)
add 3ac4b6e6c7b Add option to require an existing server data directory
(#20115)
add 791cdbd86c7 Bump software.amazon.awssdk:bom from 2.57.4 to 2.57.7
(#20129)
add 3fc2fd8c126 Bump org.apache.maven.enforcer:enforcer-api from 3.6.3 to
3.7.0 (#20128)
add 25d05ce4a90 Bump org.mongodb:bson from 5.13.0 to 5.14.0 (#20122)
add 9bacd8af0f9 Handle literal ORDER BY keys in single-stage queries
(#20134)
add 772f7723380 [test] Stabilize table rebalance disk utilization
prechecks (#20135)
add 4f8a1b419a4 Bump com.github.jnr:jnr-ffi from 2.3.5 to 2.3.6 (#20139)
add 2c5ff02a3e4 Bump software.amazon.awssdk:bom from 2.57.7 to 2.57.8
(#20140)
add 361346ce7b8 Bump org.webjars:swagger-ui from 5.34.0 to 5.34.1 (#20142)
add 16321377e94 Bump com.github.jnr:jnr-constants from 0.11.2 to 0.11.3
(#20141)
add da086263777 Bump
org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_25 (#20143)
add 9f812e9a0e2 Bump com.github.jnr:jnr-posix from 3.2.5 to 3.2.6 (#20144)
add 660962fbdc1 Include column name in realtime indexing error logs
(#20138)
add 03a7c5da4ed [VARIANT 1/8] Add VARIANT logical type foundation
(pinot-spi)
add cea4ccb6e93 [VARIANT 2/8] Add Variant codec, scalar functions and
VARIANT column type (pinot-common)
add 0e2c3909448 [VARIANT 3/8] VARIANT segment storage and
schema/table-config validation
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (9a360a81bd3)
\
N -- N -- N refs/heads/xiangfu0/variant-split/03-segment-storage
(0e2c3909448)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.github/workflows/build-all-pinot-docker-image.yml | 6 +-
.../workflows/build-pinot-base-docker-image.yml | 4 +-
.github/workflows/build-pinot-docker-image.yml | 2 +-
.github/workflows/build-superset-docker-image.yml | 2 +-
.../nightly-build-all-pinot-docker-image.yml | 6 +-
...htly-build-multi-arch-superset-docker-image.yml | 4 +-
.github/workflows/pr-flow-checks.yml | 2 +-
.github/workflows/pr-flow.yml | 4 +-
.mvn/extensions.xml | 2 +-
doap_Pinot.rdf | 6 +-
pinot-bom/pom.xml | 4 +-
.../broker/broker/helix/BaseBrokerStarter.java | 21 +-
.../HelixExternalViewBasedQueryQuotaManager.java | 25 +
.../pinot/broker/queryquota/QueryQuotaEntity.java | 10 +
.../pinot/broker/queryquota/QueryQuotaManager.java | 11 +
.../requesthandler/BaseBrokerRequestHandler.java | 86 +-
.../BaseSingleStageBrokerRequestHandler.java | 30 +-
.../requesthandler/GrpcBrokerRequestHandler.java | 4 +-
.../requesthandler/InferAnyValueTypeProvider.java | 70 ++
.../MultiStageBrokerRequestHandler.java | 61 +-
.../SingleConnectionBrokerRequestHandler.java | 77 +-
.../AdaptiveServerSelector.java | 3 +-
.../PriorityPoolInstanceSelector.java | 182 +--
.../instanceselector/BalancedInstanceSelector.java | 91 +-
.../instanceselector/BaseInstanceSelector.java | 556 +++------
.../routing/instanceselector/CandidateGroup.java | 80 ++
.../routing/instanceselector/GroupRouting.java | 64 ++
.../routing/instanceselector/InstanceSelector.java | 49 +-
.../MultiStageReplicaGroupSelector.java | 305 ++---
.../routing/instanceselector/NewSegmentState.java | 49 -
.../ReplicaGroupInstanceSelector.java | 330 ++----
.../routing/instanceselector/SegmentGroups.java | 75 ++
.../instanceselector/SegmentGroupsBuilder.java | 546 +++++++++
.../routing/instanceselector/SegmentStates.java | 126 ++-
.../instanceselector/SelectionCollector.java | 170 +++
.../StrictReplicaGroupInstanceSelector.java | 45 +-
.../instanceselector/TableReplicaHealth.java | 2 +-
.../routing/manager/BaseBrokerRoutingManager.java | 75 +-
...elixExternalViewBasedQueryQuotaManagerTest.java | 148 +++
.../BaseSingleStageBrokerRequestHandlerTest.java | 94 +-
.../BrokerQpsQuotaEnforcementModeTest.java | 219 ++++
.../InferAnyValueTypeProviderTest.java | 69 ++
.../MultiStageBrokerRequestHandlerTest.java | 259 ++++-
...ionBrokerRequestHandlerFailureDetectorTest.java | 454 ++++++++
.../PriorityPoolInstanceSelectorTest.java | 111 +-
.../CompactAssignmentInstanceSelectorTest.java | 5 +-
.../instanceselector/InstanceSelectorTest.java | 504 +++++----
.../MultiStageReplicaGroupSelectorTest.java | 20 +-
.../MultiStageSelectionReferenceModelTest.java | 415 +++++++
.../instanceselector/ReplicaGroupSelectorTest.java | 170 ++-
.../SegmentStatesReferenceModelTest.java | 1180 ++++++++++++++++++++
.../instanceselector/SegmentStatesTestUtils.java | 147 +++
.../SelectionReferenceModelTest.java | 598 ++++++++++
.../instanceselector/SelectionResultTestUtils.java | 69 ++
.../routing/manager/BrokerRoutingManagerTest.java | 72 +-
.../assignment/PartitionAssignmentUtils.java | 117 ++
.../apache/pinot/common/datatable/DataTable.java | 11 +
.../pinot/common/datatable/DataTableImplV4.java | 89 +-
...BaseExponentialBackoffRetryFailureDetector.java | 132 ++-
.../failuredetector/ConnectionFailureDetector.java | 9 +-
.../common/failuredetector/FailureDetector.java | 23 +
.../pinot/common/metadata/ZKMetadataProvider.java | 10 +-
.../pinot/common/metrics/ControllerMeter.java | 5 +-
.../pinot/common/request/InstanceRequest.java | 106 +-
.../apache/pinot/common/utils/ArrayListUtils.java | 94 +-
.../org/apache/pinot/common/utils/DataSchema.java | 98 +-
.../apache/pinot/common/utils/ExceptionUtils.java | 45 +
.../pinot/common/utils/TarCompressionUtils.java | 33 +-
.../common/utils/config/QueryOptionsUtils.java | 5 +
.../utils/helix/ImmutableSortedArrayMap.java | 10 +
.../apache/pinot/sql/parsers/CalciteSqlParser.java | 11 +
.../parsers/rewriter/OrderByLiteralRemover.java | 45 +
.../sql/parsers/rewriter/QueryRewriterFactory.java | 3 +-
.../assignment/PartitionAssignmentUtilsTest.java | 65 ++
.../ConnectionFailureDetectorTest.java | 284 ++++-
.../pinot/common/utils/ArrayListUtilsTest.java | 117 --
.../pinot/common/utils/ExceptionUtilsTest.java | 129 +++
.../common/utils/TarCompressionUtilsTest.java | 21 +
.../utils/config/TableConfigSerDeUtilsTest.java | 6 +-
.../utils/helix/ImmutableSortedArrayMapTest.java | 3 +
.../pinot/sql/parsers/CalciteSqlParserTest.java | 14 +
.../rewriter/OrderByLiteralRemoverTest.java | 75 ++
.../parsers/rewriter/QueryRewriterFactoryTest.java | 55 +-
pinot-common/src/thrift/request.thrift | 4 +
.../exception/ControllerApplicationException.java | 23 +-
.../resources/PinotIngestionRestletResource.java | 27 +-
.../resources/WebApplicationExceptionMapper.java | 19 +-
.../api/upload/SegmentValidationUtils.java | 7 +
.../helix/core/PinotHelixResourceManager.java | 76 +-
.../assignment/segment/BaseSegmentAssignment.java | 174 ++-
.../BaseStrictRealtimeSegmentAssignment.java | 146 ---
.../MultiTierStrictRealtimeSegmentAssignment.java | 131 ---
.../segment/OfflineSegmentAssignment.java | 136 ++-
.../segment/RealtimeSegmentAssignment.java | 193 +++-
.../core/assignment/segment/SegmentAssignment.java | 18 +-
.../segment/SegmentAssignmentFactory.java | 26 +-
.../assignment/segment/SegmentAssignmentUtils.java | 11 +
.../SingleTierStrictRealtimeSegmentAssignment.java | 75 --
.../core/assignment/segment/TableAssignment.java | 59 +
.../core/cleanup/StaleInstancesCleanupTask.java | 9 +-
.../core/minion/PinotHelixTaskResourceManager.java | 36 +-
.../realtime/PinotLLCRealtimeSegmentManager.java | 44 +-
.../helix/core/rebalance/TableRebalancer.java | 891 +++++++++------
.../helix/core/retention/RetentionManager.java | 15 +-
.../RealtimeSegmentValidationManager.java | 15 +-
.../controller/validation/StorageQuotaChecker.java | 28 +-
.../src/main/resources/package-lock.json | 6 +-
...PinotIngestionRestletResourceStatelessTest.java | 86 +-
.../ControllerApplicationExceptionTest.java | 36 +-
.../PinotIngestionRestletResourceTest.java | 140 +++
.../WebApplicationExceptionMapperTest.java | 98 ++
.../api/upload/SegmentValidationUtilsTest.java | 29 +
...otHelixResourceManagerConfigValidationTest.java | 107 ++
.../PinotHelixResourceManagerStatelessTest.java | 17 +
...NonReplicaGroupTieredSegmentAssignmentTest.java | 49 +-
...artitionPinnedOfflineSegmentAssignmentTest.java | 353 ++++++
...rtitionPinnedRealtimeSegmentAssignmentTest.java | 704 ++++++++++++
...altimeNonReplicaGroupSegmentAssignmentTest.java | 82 +-
...NonReplicaGroupTieredSegmentAssignmentTest.java | 42 +-
.../RealtimeReplicaGroupSegmentAssignmentTest.java | 158 +--
.../segment/SegmentAssignmentTestUtils.java | 10 +
.../StrictRealtimeSegmentAssignmentTest.java | 527 ---------
.../AllServersSegmentAssignmentStrategyTest.java | 37 +-
.../BalancedNumSegmentAssignmentStrategyTest.java | 40 +-
.../ReplicaGroupSegmentAssignmentStrategyTest.java | 148 +--
...nReplicaGroupSegmentAssignmentStrategyTest.java | 43 +-
.../RoundRobinSegmentAssignmentStrategyTest.java | 28 +-
.../cleanup/StaleInstancesCleanupTaskTest.java | 29 +
.../minion/PinotHelixTaskResourceManagerTest.java | 54 +
.../PinotLLCRealtimeSegmentManagerTest.java | 2 +-
.../TableRebalancerClusterStatelessTest.java | 43 +
.../rebalance/TableRebalancerForceCommitTest.java | 352 ++++++
.../helix/core/rebalance/TableRebalancerTest.java | 339 ++++--
.../helix/core/retention/RetentionManagerTest.java | 55 +-
.../RealtimeSegmentValidationManagerTest.java | 49 +
.../validation/StorageQuotaCheckerTest.java | 118 ++
.../apache/pinot/core/common/ObjectSerDeUtils.java | 53 +-
.../common/datatable/DataTableBuilderUtils.java | 52 +-
.../core/data/manager/BaseTableDataManager.java | 3 +-
.../apache/pinot/core/data/table/IndexedTable.java | 6 +-
.../pinot/core/data/table/IntermediateRecord.java | 7 +-
.../apache/pinot/core/data/table/TableResizer.java | 40 +-
.../blocks/results/AggregationResultsBlock.java | 63 +-
.../SelectionPartiallyOrderedByDescOperation.java | 9 +-
.../SelectionPartiallyOrderedByLinearOperator.java | 14 +-
.../operator/streaming/StreamingResponseUtils.java | 20 +-
.../core/plan/maker/InstancePlanMakerImplV2.java | 307 +++--
.../aggregation/function/AggregationFunction.java | 32 +-
.../function/AggregationFunctionUtils.java | 26 +-
.../function/AnyValueAggregationFunction.java | 268 +++--
...ValueIntegerTupleSketchAggregationFunction.java | 6 +
.../function/BaseAggregationFunction.java | 2 +-
.../BaseSingleInputAggregationFunction.java | 3 +-
...stinctCountRawCPCSketchAggregationFunction.java | 17 +
.../DistinctCountRawHLLAggregationFunction.java | 15 +-
...DistinctCountRawHLLPlusAggregationFunction.java | 15 +-
...inctCountRawThetaSketchAggregationFunction.java | 17 +
.../DistinctCountRawULLAggregationFunction.java | 15 +-
.../FrequentLongsSketchAggregationFunction.java | 10 +
.../FrequentStringsSketchAggregationFunction.java | 10 +
.../function/HistogramAggregationFunction.java | 18 +-
.../function/IdSetAggregationFunction.java | 9 +
.../IntegerTupleSketchAggregationFunction.java | 11 +
.../function/ModeAggregationFunction.java | 201 +++-
.../PercentileRawEstAggregationFunction.java | 16 +-
.../PercentileRawKLLAggregationFunction.java | 14 +
.../PercentileRawKLLMVAggregationFunction.java | 14 +
.../PercentileRawTDigestAggregationFunction.java | 15 +-
.../function/StUnionAggregationFunction.java | 7 +
.../function/SumPrecisionAggregationFunction.java | 24 +-
.../function/array/ArrayAggBytesFunction.java | 4 +-
.../array/ArrayAggDistinctBytesFunction.java | 5 +-
.../array/BaseArrayAggBigDecimalFunction.java | 12 +-
.../function/array/BaseArrayAggBytesFunction.java | 3 +-
.../function/array/BaseArrayAggDoubleFunction.java | 9 +-
.../function/array/BaseArrayAggFloatFunction.java | 9 +-
.../function/array/BaseArrayAggFunction.java | 22 +-
.../function/array/BaseArrayAggIntFunction.java | 9 +-
.../function/array/BaseArrayAggLongFunction.java | 9 +-
.../function/array/BaseArrayAggStringFunction.java | 13 +-
.../function/array/ListAggFunction.java | 15 +-
.../array/SumArrayDoubleAggregationFunction.java | 17 +-
.../array/SumArrayLongAggregationFunction.java | 17 +-
.../function/funnel/BitmapMergeStrategy.java | 5 +-
.../funnel/FunnelCountAggregationFunction.java | 16 +-
.../FunnelCountAggregationFunctionFactory.java | 5 +-
.../aggregation/function/funnel/MergeStrategy.java | 3 +-
.../function/funnel/PartitionedMergeStrategy.java | 24 +-
.../function/funnel/SetMergeStrategy.java | 5 +-
.../function/funnel/ThetaSketchMergeStrategy.java | 5 +-
.../window/FunnelBaseAggregationFunction.java | 2 +-
...unnelEventsFunctionEvalAggregationFunction.java | 63 +-
.../window/FunnelMatchStepAggregationFunction.java | 18 +-
...FunnelStepDurationStatsAggregationFunction.java | 36 +-
.../AggregateFunctionRewriteOptimizer.java | 111 +-
.../pinot/core/query/prefetch/FetchPlanner.java | 10 +-
.../query/pruner/BloomFilterSegmentPruner.java | 5 +-
.../query/reduce/AggregationDataTableReducer.java | 13 +-
.../core/query/reduce/GroupByDataTableReducer.java | 19 +-
.../query/reduce/SelectionDataTableReducer.java | 2 +
.../core/query/request/context/QueryContext.java | 128 ++-
.../context/utils/QueryContextConverterUtils.java | 11 +-
.../query/scheduler/BinaryWorkloadScheduler.java | 15 +-
.../core/query/scheduler/PriorityScheduler.java | 11 +-
.../pinot/core/query/scheduler/QueryScheduler.java | 119 +-
.../query/scheduler/SchedulerQueryContext.java | 7 +-
.../core/query/scheduler/WorkloadScheduler.java | 9 +-
.../query/scheduler/fcfs/FCFSQueryScheduler.java | 7 +-
.../core/transport/InstanceRequestHandler.java | 53 +-
.../apache/pinot/core/transport/QueryRouter.java | 149 ++-
.../pinot/core/transport/ServerChannels.java | 47 +-
.../pinot/core/common/ObjectSerDeUtilsTest.java | 17 +
.../core/common/datatable/DataTableSerDeTest.java | 23 +-
.../data/manager/BaseTableDataManagerTest.java | 55 +
.../table/UnboundedConcurrentIndexedTableTest.java | 2 +-
.../streaming/StreamingResponseUtilsTest.java | 72 ++
.../plan/maker/InstancePlanMakerPrefetchTest.java | 283 +++++
.../plan/maker/QueryOverrideWithHintsTest.java | 349 +++++-
.../function/AggregationFunctionFactoryTest.java | 6 +
.../AggregationFunctionNullContractTest.java | 12 +-
.../function/AnyValueAggregationFunctionTest.java | 176 ++-
...tinctCountCPCSketchAggregationFunctionTest.java | 65 +-
.../DistinctCountULLAggregationFunctionTest.java | 15 +
.../aggregation/function/FinalResultMergeTest.java | 176 +++
.../IntegerTupleSketchAggregationFunctionTest.java | 65 +-
.../aggregation/function/ListAggFunctionTest.java | 34 +
.../function/ModeAggregationFunctionTest.java | 181 +++
.../function/StUnionAggregationFunctionTest.java | 14 +
.../SumPrecisionAggregationFunctionTest.java | 19 +
.../function/ValueAggregationNullHandlingTest.java | 20 +-
.../function/funnel/FunnelFinalResultTest.java | 64 +-
.../function/funnel/FunnelNullHandlingTest.java | 16 +-
.../AggregateFunctionRewriteOptimizerTest.java | 117 ++
.../query/pruner/BloomFilterSegmentPrunerTest.java | 50 +
.../scheduler/QuerySchedulerResponseApiTest.java | 98 ++
.../core/transport/InstanceRequestHandlerTest.java | 56 +
.../pinot/core/transport/QueryRoutingTest.java | 353 +++++-
.../pinot/core/transport/QueryServerTestUtils.java | 89 ++
.../pinot/core/transport/ServerChannelsTest.java | 39 +-
.../apache/pinot/queries/AnyValueQueriesTest.java | 230 ++++
.../pinot/queries/ArrayAggMvQueriesTest.java | 19 +
.../apache/pinot/queries/ArrayAggQueriesTest.java | 70 +-
.../pinot/queries/ArrayAggStarTreeQueriesTest.java | 42 +-
.../org/apache/pinot/queries/BaseQueriesTest.java | 2 +-
.../DistinctCountThetaSketchQueriesTest.java | 7 +
...nnerSegmentSelectionSingleValueQueriesTest.java | 176 +++
...erSegmentAggregationSingleValueQueriesTest.java | 16 +
.../org/apache/pinot/queries/ModeQueriesTest.java | 210 +++-
.../apache/pinot/queries/TimestampQueriesTest.java | 70 ++
.../tests/BaseClusterIntegrationTest.java | 20 +-
pinot-integration-tests/pom.xml | 1 -
.../tests/BaseClusterIntegrationTestSet.java | 46 +-
.../tests/QueryQuotaClusterIntegrationTest.java | 59 +
.../SharedHybridClusterIntegrationTestSuite.java | 18 +-
.../tests/TableRebalanceIntegrationTest.java | 7 +-
.../tests/TenantRebalanceIntegrationTest.java | 72 +-
.../tests/suites/SharedHybridSuite.java | 4 +-
.../AbstractAggregationFunctionBenchmark.java | 8 +-
.../apache/pinot/plugin/filesystem/GcsPinotFS.java | 4 +-
.../filesystem/GcsPinotFSNullSafetyTest.java | 22 +
.../parquet/ParquetAvroRecordExtractor.java | 12 +-
.../parquet/ParquetAvroRecordReader.java | 20 +-
.../parquet/ParquetNativeRecordExtractor.java | 9 +-
.../parquet/ParquetNativeRecordReader.java | 3 +-
.../parquet/ParquetRecordReaderConfig.java | 1 +
.../plugin/inputformat/parquet/ParquetUtils.java | 42 +-
.../parquet/ParquetAvroRecordExtractorTest.java | 27 +-
.../parquet/ParquetNativeRecordExtractorTest.java | 85 +-
.../parquet/ParquetReaderLogicalTypesTest.java | 73 ++
.../inputformat/parquet/ParquetTestUtils.java | 14 +
.../pinot/calcite/rel/hint/PinotHintOptions.java | 8 +
.../PinotAggregateExchangeNodeInsertRule.java | 9 +
.../PinotAggregateTypeArgumentRewriteRule.java | 188 +++-
.../PinotApproximateAggregateRewriteRule.java | 22 +-
.../rel/rules/PinotJoinExchangeNodeInsertRule.java | 134 ++-
.../pinot/calcite/rel/rules/PinotRuleUtils.java | 21 +
.../org/apache/pinot/query/QueryEnvironment.java | 25 +-
.../v2/opt/rules/AggregatePushdownRule.java | 9 +
.../rules/PinotJoinExchangeNodeInsertRuleTest.java | 143 +++
.../query/ApproximateAggregateRewriteTest.java | 32 +
.../pinot/query/QueryPlannerRuleOptionsTest.java | 18 +-
.../src/test/resources/queries/AggregatePlans.json | 203 ++++
.../apache/pinot/query/mailbox/MailboxService.java | 18 +-
.../query/mailbox/channel/ChannelManager.java | 73 +-
.../query/runtime/operator/AggregateOperator.java | 8 +-
.../pinot/query/runtime/operator/LeafOperator.java | 45 +-
.../operator/MultistageAggregationExecutor.java | 37 +-
.../operator/MultistageGroupByExecutor.java | 12 +-
.../query/runtime/operator/utils/SortUtils.java | 4 +-
.../query/runtime/operator/utils/TypeUtils.java | 44 -
.../plan/server/ServerPlanRequestVisitor.java | 40 +-
.../query/service/dispatch/DispatchClient.java | 21 +
.../query/service/dispatch/QueryDispatcher.java | 89 +-
.../query/mailbox/channel/ChannelManagerTest.java | 111 +-
.../runtime/operator/AggregateOperatorTest.java | 106 +-
.../query/runtime/operator/LeafOperatorTest.java | 67 ++
.../SerializedIntermediateResultMergeTest.java | 2 +-
.../service/dispatch/QueryDispatcherTest.java | 142 +++
.../test/resources/queries/NonEquiOuterJoins.json | 58 +
.../src/test/resources/queries/QueryHints.json | 89 ++
.../src/test/resources/queries/UDFAggregates.json | 202 +++-
.../dedup/BasePartitionDedupMetadataManager.java | 2 +-
.../indexsegment/immutable/EmptyIndexSegment.java | 8 +-
.../immutable/ImmutableSegmentImpl.java | 33 +-
.../immutable/ImmutableSegmentLoader.java | 34 +-
.../indexsegment/mutable/MutableSegmentImpl.java | 65 +-
.../loader/DefaultSegmentDirectoryLoader.java | 4 +
.../loader/TierBasedSegmentDirectoryLoader.java | 4 +
.../MultiColumnRealtimeLuceneTextIndex.java | 2 +-
.../invertedindex/RealtimeLuceneTextIndex.java | 2 +-
.../segment/creator/RecordProcessingException.java | 16 +-
.../RecordReaderSegmentCreationDataSource.java | 2 +-
.../impl/SegmentIndexCreationDriverImpl.java | 3 +-
.../impl/inv/json/OffHeapJsonIndexCreator.java | 25 +-
.../creator/impl/text/LuceneTextIndexCreator.java | 112 +-
.../segment/index/loader/IndexLoadingConfig.java | 22 +-
.../segment/index/loader/SegmentPreProcessor.java | 5 +-
.../defaultcolumn/BaseDefaultColumnHandler.java | 57 +-
.../forward/BaseChunkForwardIndexReader.java | 10 +-
.../forward/VarByteChunkForwardIndexReaderV4.java | 15 +-
.../index/readers/text/LuceneTextIndexReader.java | 164 +--
.../text/MultiColumnLuceneTextIndexReader.java | 4 +-
.../readers/PinotSegmentColumnReaderImpl.java | 9 +-
.../segment/readers/PinotSegmentRecordReader.java | 11 +-
.../segment/store/SegmentLocalFSDirectory.java | 71 +-
.../local/segment/store/TextIndexUtils.java | 68 ++
.../PartitionIdVirtualColumnProvider.java | 13 +-
.../upsert/BasePartitionUpsertMetadataManager.java | 82 +-
.../segment/local/utils/TableConfigUtils.java | 5 +-
...apPartitionDedupMetadataManagerWithTTLTest.java | 2 +
.../local/dedup/DedupWithTimestampColumnTest.java | 1 +
.../immutable/EmptyIndexSegmentTest.java | 9 +-
.../immutable/ImmutableSegmentImplTest.java | 34 +-
.../converter/RealtimeSegmentConverterTest.java | 142 +++
.../creator/RecordProcessingExceptionTest.java | 143 +++
.../segment/local/segment/index/JsonIndexTest.java | 97 ++
.../segment/index/SegmentMetadataImplTest.java | 610 ++++++++++
.../index/creator/LuceneTextIndexCreatorTest.java | 287 ++++-
.../index/loader/IndexLoadingConfigTest.java | 26 +-
.../local/segment/index/loader/LoaderTest.java | 59 +-
.../index/loader/SegmentPreProcessorTest.java | 67 ++
.../TextIndexRollbackRecoveryTest.java | 2 +-
.../OpenStructPerKeyIndexReloadTest.java | 16 +
.../text/LuceneTextIndexCompatibleTest.java | 174 ++-
.../readers/text/LuceneTextIndexCountTest.java | 22 +-
.../readers/PinotSegmentColumnReaderImplTest.java | 33 +
.../segment/store/FilePerIndexDirectoryTest.java | 22 +-
.../segment/store/SegmentLocalFSDirectoryTest.java | 25 +
.../store/SingleFileIndexDirectoryTest.java | 22 +-
.../v2/builder/MultipleTreesBuilderCloseTest.java | 4 +-
.../BasePartitionUpsertMetadataManagerTest.java | 96 +-
...ertMetadataManagerForConsistentDeletesTest.java | 1 +
...rrentMapPartitionUpsertMetadataManagerTest.java | 3 +
.../segment/local/utils/TableConfigUtilsTest.java | 27 +
.../pinot/segment/spi/AggregationFunctionType.java | 16 +-
.../apache/pinot/segment/spi/ColumnMetadata.java | 28 +
.../org/apache/pinot/segment/spi/IndexSegment.java | 8 +
.../apache/pinot/segment/spi/SegmentMetadata.java | 1 +
.../pinot/segment/spi/index/TextIndexConfig.java | 6 +-
.../spi/index/metadata/ColumnMetadataImpl.java | 18 +
.../spi/index/metadata/MappedColumnMetadata.java | 405 +++++++
.../index/metadata/MappedSegmentMetadataStore.java | 970 ++++++++++++++++
.../spi/index/metadata/SegmentMetadataImpl.java | 372 +++++-
.../spi/loader/SegmentDirectoryLoaderContext.java | 17 +-
.../pinot/segment/spi/memory/PinotDataBuffer.java | 12 +
.../pinot/segment/spi/store/SegmentDirectory.java | 15 +
.../resources/SegmentCompressionStatsReader.java | 9 +-
.../pinot/server/api/resources/TablesResource.java | 31 +-
.../server/starter/helix/BaseServerStarter.java | 10 +-
.../starter/helix/HelixInstanceDataManager.java | 30 +-
.../helix/HelixInstanceDataManagerConfig.java | 4 +
.../ServerGrpcChannelBackoffResetHandler.java | 168 ---
.../helix/ServerMailboxChannelRefreshHandler.java | 302 +++++
.../warmup/PageCacheWarmupServerQueryExecutor.java | 5 +-
.../pinot/server/api/TablesResourceTest.java | 22 +-
.../SegmentCompressionStatsReaderTest.java | 13 +
.../helix/HelixInstanceDataManagerConfigTest.java | 17 +
.../helix/HelixInstanceDataManagerTest.java | 68 +-
.../ServerGrpcChannelBackoffResetHandlerTest.java | 253 -----
.../ServerMailboxChannelRefreshHandlerTest.java | 543 +++++++++
.../PageCacheWarmupServerQueryExecutorTest.java | 7 +-
.../apache/pinot/spi/config/table/QuotaConfig.java | 32 +-
.../spi/config/table/QuotaEnforcementMode.java | 42 +
.../apache/pinot/spi/utils/CommonConstants.java | 38 +-
.../pinot/spi/utils/StoredValueComparator.java | 59 +
.../pinot/spi/config/table/QuotaConfigTest.java | 59 +
...nfigTest.java => QuotaEnforcementModeTest.java} | 24 +-
.../src/main/resources/conf/pinot-server.conf | 4 +
pom.xml | 42 +-
389 files changed, 24030 insertions(+), 5886 deletions(-)
create mode 100644
pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/InferAnyValueTypeProvider.java
create mode 100644
pinot-broker/src/main/java/org/apache/pinot/broker/routing/instanceselector/CandidateGroup.java
create mode 100644
pinot-broker/src/main/java/org/apache/pinot/broker/routing/instanceselector/GroupRouting.java
delete mode 100644
pinot-broker/src/main/java/org/apache/pinot/broker/routing/instanceselector/NewSegmentState.java
create mode 100644
pinot-broker/src/main/java/org/apache/pinot/broker/routing/instanceselector/SegmentGroups.java
create mode 100644
pinot-broker/src/main/java/org/apache/pinot/broker/routing/instanceselector/SegmentGroupsBuilder.java
create mode 100644
pinot-broker/src/main/java/org/apache/pinot/broker/routing/instanceselector/SelectionCollector.java
create mode 100644
pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/BrokerQpsQuotaEnforcementModeTest.java
create mode 100644
pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/InferAnyValueTypeProviderTest.java
create mode 100644
pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/SingleConnectionBrokerRequestHandlerFailureDetectorTest.java
create mode 100644
pinot-broker/src/test/java/org/apache/pinot/broker/routing/instanceselector/MultiStageSelectionReferenceModelTest.java
create mode 100644
pinot-broker/src/test/java/org/apache/pinot/broker/routing/instanceselector/SegmentStatesReferenceModelTest.java
create mode 100644
pinot-broker/src/test/java/org/apache/pinot/broker/routing/instanceselector/SegmentStatesTestUtils.java
create mode 100644
pinot-broker/src/test/java/org/apache/pinot/broker/routing/instanceselector/SelectionReferenceModelTest.java
create mode 100644
pinot-broker/src/test/java/org/apache/pinot/broker/routing/instanceselector/SelectionResultTestUtils.java
create mode 100644
pinot-common/src/main/java/org/apache/pinot/common/assignment/PartitionAssignmentUtils.java
create mode 100644
pinot-common/src/main/java/org/apache/pinot/sql/parsers/rewriter/OrderByLiteralRemover.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/assignment/PartitionAssignmentUtilsTest.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/utils/ExceptionUtilsTest.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/sql/parsers/rewriter/OrderByLiteralRemoverTest.java
delete mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/assignment/segment/BaseStrictRealtimeSegmentAssignment.java
delete mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/assignment/segment/MultiTierStrictRealtimeSegmentAssignment.java
delete mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/assignment/segment/SingleTierStrictRealtimeSegmentAssignment.java
create mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/assignment/segment/TableAssignment.java
create mode 100644
pinot-controller/src/test/java/org/apache/pinot/controller/api/resources/PinotIngestionRestletResourceTest.java
create mode 100644
pinot-controller/src/test/java/org/apache/pinot/controller/api/resources/WebApplicationExceptionMapperTest.java
create mode 100644
pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/assignment/segment/PartitionPinnedOfflineSegmentAssignmentTest.java
create mode 100644
pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/assignment/segment/PartitionPinnedRealtimeSegmentAssignmentTest.java
delete mode 100644
pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/assignment/segment/StrictRealtimeSegmentAssignmentTest.java
create mode 100644
pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/rebalance/TableRebalancerForceCommitTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/operator/streaming/StreamingResponseUtilsTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/plan/maker/InstancePlanMakerPrefetchTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/function/FinalResultMergeTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/query/scheduler/QuerySchedulerResponseApiTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/transport/QueryServerTestUtils.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/queries/AnyValueQueriesTest.java
create mode 100644
pinot-query-planner/src/test/java/org/apache/pinot/calcite/rel/rules/PinotJoinExchangeNodeInsertRuleTest.java
copy
pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/admin/PinotAdminException.java
=>
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/RecordProcessingException.java
(70%)
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/RecordProcessingExceptionTest.java
create mode 100644
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/MappedColumnMetadata.java
create mode 100644
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/MappedSegmentMetadataStore.java
delete mode 100644
pinot-server/src/main/java/org/apache/pinot/server/starter/helix/ServerGrpcChannelBackoffResetHandler.java
create mode 100644
pinot-server/src/main/java/org/apache/pinot/server/starter/helix/ServerMailboxChannelRefreshHandler.java
delete mode 100644
pinot-server/src/test/java/org/apache/pinot/server/starter/helix/ServerGrpcChannelBackoffResetHandlerTest.java
create mode 100644
pinot-server/src/test/java/org/apache/pinot/server/starter/helix/ServerMailboxChannelRefreshHandlerTest.java
create mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/config/table/QuotaEnforcementMode.java
create mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/utils/StoredValueComparator.java
copy
pinot-spi/src/test/java/org/apache/pinot/spi/config/table/{StarTreeIndexConfigTest.java
=> QuotaEnforcementModeTest.java} (55%)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
