This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a change to branch xiangfu0/codex1/sorted-exchange-leaf
in repository https://gitbox.apache.org/repos/asf/pinot.git
omit 88ef14a0fd9 Align shared window rule test setup with the core
dependency
omit a8f370c96dc [MSE] Preserve complete window input during receiver
sorting
omit ad2c8c428a5 [MSE] Port sorted leaf streaming to explicit merge
exchanges
omit 0d93c7c9920 Share sorted selection setup and trim duplicate coverage
omit bf80d6736be Check the DESC null-bearing case against fixed rows
instead of MinMax
omit e2ac1fdca50 Keep frontier pruning for segments flagged non-null.
omit 2a70ea4773d Address second-round review on the streaming selection
ORDER BY combine
omit 37f2896ff31 Defer tied cursors under a single-column ORDER BY
omit f9022e736c6 Address review feedback on the streaming selection ORDER
BY combine
omit 0f16e34fd8b Add streaming selection ORDER BY combine for physically
sorted segments
omit 755721f7fd1 Retain complete input for global window receiver sorts
omit 496b2db88fa [MSE] Port ordered leaf streaming to the merge plan
contract
omit b95589c513a [MSE] Preserve merge test imports after restacking
omit 017b2661c71 Simplify ordered mailbox test fixtures
omit 38a69731ab5 Check cancellation while validating merged ordering
omit 92fe29c64ad Stream ORDER BY over confirmed sorted mailbox producers
omit b047dd479ea Align window input sort plan expectations with complete
input
omit fde1708cf3d Retain complete input for global window receiver sorts
omit 7a4465c289c [MSE] Express ordered streams with a k-way merge exchange
omit dcf319f474a [build] Include Zstd API annotations for compiler
validation
omit 6dff6515d08 [MSE] Use current planner test APIs
omit ddc8a053bef [MSE] Simplify sorted mailbox merging
omit 50913f03c9b [MSE] Speed up four-sender sorted window merge
omit e9623577c18 [MSE] Address sender-sorted window review feedback
omit 2413e6fca8a [MSE] Minimize sender-sorted merge changes
omit 35b15632a5f [MSE] Avoid eager sort buffer allocation
omit 93ec7813f74 [MSE] Limit sender-sorted merge to global windows
omit 6266c48f92b [MSE] Optimize sorted mailbox merge
omit a5237b30667 [MSE] Merge sender-sorted streams for ordered windows
add d5542e22727 Make an OPEN_STRUCT key behave like an ordinary column:
apply its index settings on reload, and keep its type across tiers (#19698)
add 1b94d4bc698 Bump axios from 1.18.0 to 1.20.0 in
/pinot-controller/src/main/resources (#19722)
add a080910ca48 Build TimeSegmentPruner's interval tree from flat arrays
and skip no-op rebuilds (#19643)
add 816375d50c7 Take upsert/dedup primary key columns from the metadata
manager context (#19726)
add 747f9470257 Log and count protected upsert revert failures (#19505)
add d6a37880dfe Bump org.apache.maven:maven-core from 3.9.16 to 3.10.0
(#19734)
add 17961e1bdb7 Bump org.apache.maven:apache-maven from 3.9.16 to 3.10.0
(#19733)
add 3f80ac76d0c Bump org.codehaus.woodstox:stax2-api from 4.3.0 to 4.3.1
(#19732)
add 8cbbc1adc40 Bump com.gradle:develocity-maven-extension from 2.5.0 to
2.6.0 (#19690)
add 4e27013f964 Make _numPendingOperations protected in
BasePartitionUpsertMetadataManager (#19744)
add b861ac5503f Bump com.fasterxml.woodstox:woodstox-core from 7.2.2 to
7.3.0 (#19741)
add 6bf036907f2 Bump software.amazon.awssdk:bom from 2.55.6 to 2.55.7
(#19742)
add c95ff8e51c5 Bump net.alchim31.maven:scala-maven-plugin from 4.9.10 to
4.10.0 (#19743)
add e149b276bed Bump com.puppycrawl.tools:checkstyle from 14.1.0 to 14.3.0
(#19731)
add 931c67a0f25 Introduce reason codes for segment completion requests
(#19251)
add 4f495df35c8 Fix wrong results for multi-key GROUP BY with transform
expressions (#19663)
add 7a00b2318be [core] Fix ANY_VALUE bytes intermediate round trip (#19667)
add a7cfea0b824 Upgrade Commons Lang to 3.21.0 and discard trailing empty
split fields (#19747)
add f5d9be62448 Build bitmap and sorted doc id sets only through their
factories (#19746)
add 36b526bf159 Bump org.apache.thrift:libthrift from 0.24.0 to 0.25.0
(#19750)
add 624dcc2c666 Bump joda-time:joda-time from 2.14.4 to 2.15.0 (#19751)
add af3a63278ed Bump software.amazon.awssdk:bom from 2.55.7 to 2.55.10
(#19752)
add 60fcf580835 Bump org.checkerframework:checker-qual from 4.2.3 to 4.3.0
(#19753)
add 22102756dae Index CASE THEN results by array instead of a boxed map
(#19685)
add b8e2ef27671 Build METADATA push metadata from the local segment
instead of re-downloading the staged tar (#19490)
add 55a760d0d64 Stop MSE query planning when the planning thread is
interrupted (#19709)
add 6a2ab1221db Reject input file segment name templates that reference no
match group (#19755)
add 18d18ac2afa Split upsert keys-not-replaced metric by table type
(#19748)
add 98bda679fd3 Add planning budget tests for large IN lists (#19745)
add ade41d16899 Log lead-controller VIP lookup at debug, not info (#19758)
add 85db332cfca Reject consuming during download for upsert tables that
revert metadata in PROTECTED mode (#19710)
add 27e8839ade5 Bump software.amazon.awssdk:bom from 2.55.10 to 2.55.11
(#19763)
add c9af1d06ae2 Bump source-map-js in /pinot-controller/src/main/resources
(#19765)
add 21596bbffad Configure OPEN_STRUCT indexes the way every other column
is configured (#19736)
add 902d3864bed Fix wrong results from MinMax selection order-by combine
under null handling (#19724)
add b51535ed34a Bump org.xerial.snappy:snappy-java from 1.1.10.8 to
1.1.10.9 (#19764)
add eb86bc77f5c Upgrade Guava to 33.7.2 and libraries-bom to 26.90.0
(#19772)
add 274601b020e Add numRemoteAccesses and remoteAccessBytes query stats
(#19756)
add 005c5ab64b6 Fix Unknown nodes in multi-stage EXPLAIN when asking
servers (#19766)
add 73645dafec9 Fix missing aggregate FILTER in multi-stage EXPLAIN when
asking servers (#19767)
add c58c13feca4 Fix missing GROUPING SETS in multi-stage EXPLAIN when
asking servers (#19769)
add 3de4f315e04 Fix CAST type in multi-stage EXPLAIN when asking servers
(#19768)
add 08f4ab17e67 Make IDSET and IN_ID_SET work in the multi-stage engine
(#19757)
add 9c3f0e6d208 Prepare partition pruning predicates once per query
(#19539)
add 43ecf19f925 Report whether upsert snapshot files can be compared
across replicas (#19728)
add 79486707b92 Bump org.xerial.snappy:snappy-java from 1.1.10.9 to
1.1.10.10 (#19780)
add bb6b8b9d906 Bump org.webjars:swagger-ui from 5.32.15 to 5.33.1 (#19779)
add a75f2e54c14 Build compatibility verifier checkouts with their own
Maven wrapper (#19777)
add 5b6c7a2983c Fix missing UNNEST input columns in multi-stage EXPLAIN
when asking servers (#19771)
add b587d0a8c1b Bump com.github.luben:zstd-jni from 1.5.7-20 to 1.5.7-21
(#19778)
add 657d7805e7a Support range index on ingestion-aggregated no-dictionary
columns (#19111)
add 4d93fce3399 Fix SIGSEGV when upsert reads a concurrently destroyed
segment (#19596)
add 651ff41a3a9 Upgrade Pulsar to 5.0.0 (#19787)
add 1ea817199aa Support IdSet literals of millions of ids (#19759)
add 3a36f139597 Optimize bitmap unions in query aggregation, star-tree
build, and JSON indexes (#19587)
add 859f7f9f4a1 [MSE] Merge sender-sorted streams for ordered windows
add faf5338b438 [MSE] Optimize sorted mailbox merge
add e721d638cde [MSE] Limit sender-sorted merge to global windows
add e14e5068453 [MSE] Avoid eager sort buffer allocation
add aeefe49f4dd [MSE] Minimize sender-sorted merge changes
add b34c4a05b35 [MSE] Address sender-sorted window review feedback
add d8bfb8b6d3e [MSE] Speed up four-sender sorted window merge
add afbd893dcaa [MSE] Simplify sorted mailbox merging
add 77fd9592548 [MSE] Use current planner test APIs
add e69fc0fe32f [build] Include Zstd API annotations for compiler
validation
add faa77b1385e [MSE] Express ordered streams with a k-way merge exchange
add d78f73c22da Retain complete input for global window receiver sorts
add 2df4abdf0e0 Align window input sort plan expectations with complete
input
add d36ecd023fb [test] Verify homogeneous snapshot builds disable k-way
merge
add 6f63db504b9 Address ordered-window merge review feedback
add 45ab1ab1123 Complete ordered-window review fixes
add 72706083aa0 Keep complete window input when fallback stages use older
servers
add c7c883a1e2e Stream ORDER BY over confirmed sorted mailbox producers
add 5e88c778ae8 Check cancellation while validating merged ordering
add 64cd1673c36 Simplify ordered mailbox test fixtures
add c80711dabf4 [MSE] Preserve merge test imports after restacking
add cb075ec0d20 [MSE] Port ordered leaf streaming to the merge plan
contract
add e8714812aad Retain complete input for global window receiver sorts
add 3ff15f7a07f [ci] Enable existing checks for stacked PR #19711
add 777d7cbbee0 Adapt ordered exchange pushdown to the shared merge
contract
add 267b78f26ef Check sliced merge output after completed senders
add 09147b32c0d Add streaming selection ORDER BY combine for physically
sorted segments
add f84d1ae49be Address review feedback on the streaming selection ORDER
BY combine
add 6b4247ae088 Defer tied cursors under a single-column ORDER BY
add 912f42be116 Address second-round review on the streaming selection
ORDER BY combine
add 001ff89cb67 Keep frontier pruning for segments flagged non-null.
add f658bf2e04f Check the DESC null-bearing case against fixed rows
instead of MinMax
add f01e5589150 Share sorted selection setup and trim duplicate coverage
add c5b1a760833 [MSE] Port sorted leaf streaming to explicit merge
exchanges
add 0cc2cf9712f [MSE] Preserve complete window input during receiver
sorting
add 4c8ebb30a6f Align shared window rule test setup with the core
dependency
add 75d733f1717 [ci] Enable existing checks for stacked PR #19712
add 417e6eedb98 Import the ordered window fixture mock helper
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (88ef14a0fd9)
\
N -- N -- N refs/heads/xiangfu0/codex1/sorted-exchange-leaf
(417e6eedb98)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.github/workflows/pinot_compatibility_checks.yml | 2 +
.github/workflows/pinot_integration_tests.yml | 2 +
.../pinot_java11_client_compatibility.yml | 2 +
.github/workflows/pinot_quickstart_tests.yml | 2 +
.github/workflows/pinot_unit_tests.yml | 2 +
.github/workflows/pinot_vuln_check.yml | 2 +
.mvn/extensions.xml | 2 +-
.mvn/wrapper/maven-wrapper.properties | 2 +-
compatibility-verifier/checkoutAndBuild.sh | 22 +-
.../broker/broker/helix/BaseBrokerStarter.java | 9 +
.../helix/MultiClusterHelixBrokerStarter.java | 1 +
.../requesthandler/KWayMergeSupportPredicate.java | 70 +--
.../MultiStageBrokerRequestHandler.java | 23 +-
.../routing/manager/BaseBrokerRoutingManager.java | 29 +-
.../segmentpruner/SegmentPrunerFactory.java | 27 +-
.../SinglePartitionColumnSegmentPruner.java | 186 ++++++
.../routing/segmentpruner/TimeSegmentPruner.java | 38 +-
.../segmentpruner/interval/IntervalTree.java | 240 ++++----
.../KWayMergeSupportPredicateTest.java | 76 +--
.../MultiStageBrokerRequestHandlerTest.java | 19 +-
.../routing/intervalst/IntervalTreeTest.java | 60 ++
.../routing/manager/BrokerRoutingManagerTest.java | 33 ++
.../routing/segmentpruner/SegmentPrunerTest.java | 120 ++++
.../SinglePartitionColumnSegmentPrunerTest.java | 632 +++++++++++++++++++++
.../apache/pinot/common/datatable/DataTable.java | 8 +-
.../common/function/TransformFunctionType.java | 5 +-
.../common/function/scalar/StringFunctions.java | 72 +--
.../apache/pinot/common/metrics/ServerMeter.java | 3 +
.../function/ByteArrayPartitionFunction.java | 24 +
.../partition/function/FnvPartitionFunction.java | 28 +
.../function/HashCodePartitionFunction.java | 24 +
.../function/ModuloPartitionFunction.java | 24 +
.../function/Murmur3PartitionFunction.java | 29 +
.../function/MurmurPartitionFunction.java | 26 +
.../protocols/SegmentCompletionProtocol.java | 96 +++-
.../pinot/common/response/BrokerResponse.java | 8 +
.../response/broker/BrokerResponseNative.java | 45 +-
.../response/broker/BrokerResponseNativeV2.java | 31 +-
.../response/broker/CursorResponseNative.java | 10 +-
.../response/mapper/TimeSeriesResponseMapper.java | 4 +
.../common/utils/MutableRoaringBitmapUnion.java | 182 ++++++
.../pinot/common/utils/RoaringBitmapUnion.java | 240 ++++++++
.../pinot/common/utils/RoaringBitmapUtils.java | 8 +
.../common/utils/config/QueryOptionsUtils.java | 12 +-
.../common/utils/helix/ClusterVersionTracker.java | 189 ++++++
.../apache/pinot/sql/parsers/CalciteSqlParser.java | 15 +-
.../sql/parsers/parser/GrowingCharStream.java | 86 +++
pinot-common/src/main/proto/plan.proto | 11 +-
.../function/scalar/StringFunctionsTest.java | 89 ++-
.../partition/function/PartitionFunctionTest.java | 123 +++-
.../protocols/SegmentCompletionProtocolTest.java | 89 ++-
.../utils/MutableRoaringBitmapUnionTest.java | 299 ++++++++++
.../pinot/common/utils/RoaringBitmapUnionTest.java | 399 +++++++++++++
.../common/utils/config/QueryOptionsUtilsTest.java | 20 +-
.../utils/helix/ClusterVersionTrackerTest.java | 222 ++++++++
.../pinot/sql/parsers/CalciteSqlParserTest.java | 57 +-
.../sql/parsers/parser/GrowingCharStreamTest.java | 138 +++++
.../resources/LLCSegmentCompletionHandlers.java | 23 +-
.../helix/core/minion/ClusterInfoAccessor.java | 2 +-
.../src/main/resources/package-lock.json | 16 +-
pinot-controller/src/main/resources/package.json | 2 +-
.../LLCSegmentCompletionHandlersTest.java | 92 +++
.../helix/core/realtime/SegmentCompletionTest.java | 38 ++
.../apache/pinot/core/common/BlockDocIdSet.java | 3 +-
.../realtime/RealtimeSegmentDataManager.java | 36 +-
.../pinot/core/function/scalar/IdSetFunctions.java | 169 ++++++
.../core/operator/InstanceResponseOperator.java | 10 +
...xValueBasedSelectionOrderByCombineOperator.java | 49 +-
.../core/operator/docidsets/BitmapDocIdSet.java | 15 +-
.../pinot/core/operator/docidsets/NotDocIdSet.java | 2 +-
.../docidsets/RangelessBitmapDocIdSet.java | 6 +-
.../core/operator/docidsets/SortedDocIdSet.java | 2 +-
.../operator/filter/BaseColumnFilterOperator.java | 15 +-
.../operator/filter/BitmapBasedFilterOperator.java | 2 +-
.../filter/InvertedIndexFilterOperator.java | 9 +-
.../core/operator/filter/MapFilterOperator.java | 17 +-
.../transform/function/CaseTransformFunction.java | 146 ++---
.../function/InIdSetTransformFunction.java | 48 +-
.../transform/function/TransformFunction.java | 12 +
.../function/TransformFunctionFactory.java | 2 +-
.../function/AnyValueAggregationFunction.java | 11 +-
.../DistinctCountBitmapAggregationFunction.java | 78 ++-
.../query/reduce/ExecutionStatsAggregator.java | 8 +
.../core/query/request/context/QueryContext.java | 8 +-
.../core/query/utils/OrderByComparatorFactory.java | 17 +-
.../core/query/utils/idset/BloomFilterIdSet.java | 29 +-
.../pinot/core/query/utils/idset/IdSets.java | 53 +-
.../utils/idset/Roaring64NavigableMapIdSet.java | 13 +-
.../pinot/core/util/MemoizedClassAssociation.java | 42 --
.../realtime/RealtimeSegmentDataManagerTest.java | 113 +++-
.../pinot/core/function/FunctionRegistryTest.java | 4 +-
.../core/function/scalar/IdSetFunctionsTest.java | 219 +++++++
.../operator/InstanceResponseOperatorTest.java | 36 ++
.../combine/SelectionCombineOperatorTest.java | 148 ++++-
.../combine/SortedGroupByCombineOperatorsTest.java | 89 +++
.../dociditerators/SortedDocIdIteratorTest.java | 10 +-
.../core/operator/docidsets/AndDocIdSetTest.java | 4 +-
.../filter/MapFilterOperatorOpenStructTest.java | 22 +
.../function/InIdSetTransformFunctionTest.java | 98 ++++
.../function/AnyValueAggregationFunctionTest.java | 59 ++
.../function/DistinctCountBitmapLazyUnionTest.java | 221 +++++++
.../query/reduce/ExecutionStatsAggregatorTest.java | 24 +
.../query/utils/OrderByComparatorFactoryTest.java | 38 ++
.../pinot/core/query/utils/idset/IdSetTest.java | 76 +++
.../pinot/core/startree/v2/BaseStarTreeV2Test.java | 11 +-
.../v2/DistinctCountBitmapStarTreeV2Test.java | 18 +-
.../DistinctCountBitmapWithMVStarTreeV2Test.java | 17 +-
.../v2/DistinctCountHLLOnHeapStarTreeV2Test.java | 28 +-
.../org/apache/pinot/queries/IdSetQueriesTest.java | 157 ++++-
.../pinot/queries/NullQueriesFluentTest.java | 137 +++++
.../CustomDataQueryClusterIntegrationTest.java | 6 +
.../tests/custom/IdSetIntegrationTest.java | 236 ++++++++
.../custom/WindowKWayMergeIntegrationTest.java | 176 ++++++
.../resources/udf-test-results/all-functions.yaml | 2 +-
.../pinot/perf/BenchmarkTimeSegmentPruner.java | 163 ++++++
...> BenchmarkDistinctCountBitmapAggregation.java} | 82 ++-
.../BaseMultipleSegmentsConversionExecutor.java | 67 +--
.../tasks/BaseSingleSegmentConversionExecutor.java | 20 +-
.../plugin/minion/tasks/BaseTaskExecutor.java | 19 -
...BaseMultipleSegmentsConversionExecutorTest.java | 177 +++++-
.../BaseSingleSegmentConversionExecutorTest.java | 42 +-
.../pinot-stream-ingestion/pinot-pulsar/pom.xml | 17 +-
.../plugin/stream/pulsar/PulsarMessageId.java | 138 +++++
.../PulsarPartitionLevelConnectionHandler.java | 32 +-
.../pulsar/PulsarPartitionLevelConsumer.java | 3 +-
.../pinot/plugin/stream/pulsar/PulsarUtils.java | 10 +-
.../plugin/stream/pulsar/PulsarConsumerTest.java | 6 +-
.../stream/pulsar/PulsarMessageBatchTest.java | 5 +
.../plugin/stream/pulsar/PulsarMessageIdTest.java | 64 +++
.../plugin/stream/pulsar/PulsarUtilsTest.java | 17 +
.../rel/logical/PinotLogicalSortExchange.java | 4 +
.../rules/PinotWindowExchangeNodeInsertRule.java | 30 +-
.../org/apache/pinot/query/QueryEnvironment.java | 6 +-
.../planner/explain/ExplainNodeSimplifier.java | 6 +
.../explain/PhysicalExplainPlanVisitor.java | 11 +
.../query/planner/explain/PlanNodeMerger.java | 41 +-
.../query/planner/explain/PlanNodeSorter.java | 6 +
.../planner/logical/EquivalentStagesFinder.java | 29 +-
.../planner/logical/EquivalentStagesReplacer.java | 11 +
.../query/planner/logical/LogicalPlanner.java | 20 +
.../query/planner/logical/PlanFragmenter.java | 15 +-
.../planner/logical/PlanNodeToRelConverter.java | 105 +++-
.../planner/logical/RelToPlanNodeConverter.java | 27 +-
.../query/planner/logical/RexExpressionUtils.java | 48 +-
.../query/planner/logical/SubPlanFragmenter.java | 6 +
.../planner/physical/DispatchablePlanVisitor.java | 11 +-
.../planner/physical/PinotDispatchPlanner.java | 6 +-
...eceiveNode.java => BaseMailboxReceiveNode.java} | 71 +--
.../plannode/DefaultPostOrderTraversalVisitor.java | 6 +
.../pinot/query/planner/plannode/ExchangeNode.java | 57 +-
.../planner/plannode/KWayMergeExchangeNode.java | 75 ---
.../planner/plannode/MailboxMergeReceiveNode.java | 50 +-
.../query/planner/plannode/MailboxReceiveNode.java | 74 +--
.../query/planner/plannode/MailboxSendNode.java | 4 +-
.../query/planner/plannode/PlanNodeVisitor.java | 12 +
.../pinot/query/planner/plannode/SortNode.java | 12 +
.../query/planner/serde/PlanNodeDeserializer.java | 3 +-
.../query/planner/serde/PlanNodeSerializer.java | 30 +-
.../validation/ArrayToMvValidationVisitor.java | 7 +
.../pinot/query/LargeInListPlanningBudgetTest.java | 198 +++++++
.../apache/pinot/query/QueryCompilationTest.java | 47 +-
.../pinot/query/QueryPlanningCancellationTest.java | 179 ++++++
.../query/planner/logical/LogicalPlannerTest.java | 61 ++
.../query/planner/logical/PlanFragmenterTest.java | 19 +-
.../logical/PlanNodeToRelConverterTest.java | 136 +++++
.../planner/logical/RexExpressionUtilsTest.java | 35 ++
.../query/planner/logical/StagesTestBase.java | 84 ++-
.../query/planner/serde/PlanNodeSerDeTest.java | 13 +-
.../src/test/resources/queries/JoinPlans.json | 14 +
.../resources/queries/PhysicalOptimizerPlans.json | 15 +
.../resources/queries/WindowFunctionPlans.json | 248 ++++----
.../query/runtime/InStageStatsTreeBuilder.java | 12 +
.../query/runtime/MultiStageStatsTreeBuilder.java | 10 +-
.../pinot/query/runtime/SendStatsPredicate.java | 115 +---
.../operator/BaseMailboxReceiveOperator.java | 4 +-
.../pinot/query/runtime/operator/LeafOperator.java | 12 +-
.../pinot/query/runtime/operator/SortOperator.java | 3 +
.../SortedMailboxMergeReceiveOperator.java | 15 +-
.../utils/BlockingMultiStreamConsumer.java | 8 +-
.../query/runtime/plan/PlanNodeToOpChain.java | 17 +-
.../plan/pipeline/PipelineBreakerVisitor.java | 7 +
.../plan/server/ServerPlanRequestVisitor.java | 19 +-
.../TimeSeriesExchangeReceiveOperator.java | 4 +-
.../query/service/dispatch/DispatchClient.java | 18 +-
.../query/service/dispatch/QueryDispatcher.java | 35 +-
.../pinot/query/service/server/QueryServer.java | 36 +-
.../query/runtime/SendStatsPredicateTest.java | 80 +++
.../query/runtime/operator/LeafOperatorTest.java | 33 ++
.../query/runtime/operator/SortOperatorTest.java | 21 +
.../SortedMailboxMergeReceiveOperatorTest.java | 89 +--
.../operator/WindowAggregateOperatorTest.java | 46 --
.../plan/OpChainConverterDispatcherTest.java | 2 +-
.../plan/server/ServerPlanRequestVisitorTest.java | 7 +-
.../query/service/dispatch/DispatchClientTest.java | 138 +++++
.../query/service/server/QueryServerTest.java | 55 ++
.../src/test/resources/queries/IdSet.json | 360 ++++++++++++
pinot-segment-local/pom.xml | 7 -
.../DistinctCountBitmapValueAggregator.java | 60 +-
.../segment/local/aggregator/ValueAggregator.java | 5 +
.../immutable/ImmutableSegmentImpl.java | 49 +-
.../indexsegment/mutable/MutableSegmentImpl.java | 14 +-
.../stats/CompactedNoDictColumnStatistics.java | 2 +-
.../stats/MutableNoDictColumnStatistics.java | 69 ++-
.../realtime/impl/json/MutableJsonIndexImpl.java | 67 ++-
.../impl/openstruct/OpenStructColumnSplitter.java | 45 +-
.../segment/index/loader/IndexLoadingConfig.java | 45 +-
.../segment/index/loader/SegmentPreProcessor.java | 53 +-
.../loader/invertedindex/RangeIndexHandler.java | 69 ++-
.../openstruct/ImmutableOpenStructDataSource.java | 52 +-
.../index/openstruct/OpenStructIndexType.java | 8 +-
.../readers/json/ImmutableJsonIndexReader.java | 59 +-
.../startree/v2/builder/BaseSingleTreeBuilder.java | 17 +
.../v2/builder/OnHeapSingleTreeBuilder.java | 24 +
.../upsert/BasePartitionUpsertMetadataManager.java | 108 +++-
...oncurrentMapPartitionUpsertMetadataManager.java | 84 ++-
...nUpsertMetadataManagerForConsistentDeletes.java | 94 +--
.../upsert/PartitionUpsertMetadataManager.java | 19 +
.../segment/local/utils/SegmentPushUtils.java | 307 ++++++----
.../segment/local/utils/TableConfigUtils.java | 32 ++
.../DistinctCountBitmapValueAggregatorTest.java | 161 ++++++
.../immutable/ImmutableSegmentImplTest.java | 90 ++-
.../mutable/MutableSegmentDedupTest.java | 44 ++
.../mutable/MutableSegmentImplUpsertTest.java | 55 ++
.../io/codec/CompressionCodecCorruptInputTest.java | 29 +-
.../converter/RealtimeSegmentConverterTest.java | 84 +++
.../stats/MutableNoDictColumnStatisticsTest.java | 147 +++++
.../RangeIndexHandlerNullMinMaxTest.java | 243 ++++++++
.../ImmutableOpenStructDataSourceTest.java | 48 +-
.../index/openstruct/OpenStructIndexTypeTest.java | 3 -
.../OpenStructPerKeyIndexReloadTest.java | 397 +++++++++++++
.../OpenStructSparseDenseParityTest.java | 11 +-
.../v2/builder/NullAwareStarTreeBuilderTest.java | 99 +++-
.../BasePartitionUpsertMetadataManagerTest.java | 153 +++++
...ertMetadataManagerForConsistentDeletesTest.java | 153 +++++
...rrentMapPartitionUpsertMetadataManagerTest.java | 240 ++++++++
.../utils/IndexCombinationValidationTest.java | 117 ++++
.../segment/local/utils/SegmentPushUtilsTest.java | 112 ++++
.../segment/local/utils/TableConfigUtilsTest.java | 88 +++
.../org/apache/pinot/segment/spi/FetchContext.java | 23 +
.../org/apache/pinot/segment/spi/V1Constants.java | 7 +
.../segment/spi/creator/IndexCreationContext.java | 31 +-
.../creator/name/FixedSegmentNameGenerator.java | 1 -
.../name/InputFileSegmentNameGenerator.java | 35 +-
.../name/NormalizedDateSegmentNameGenerator.java | 1 -
.../creator/name/SimpleSegmentNameGenerator.java | 1 -
.../segment/spi/index/FieldIndexConfigsUtil.java | 60 +-
.../spi/index/metadata/ColumnMetadataImpl.java | 39 +-
.../segment/spi/partition/PartitionFunction.java | 11 +
.../name/InputFileSegmentNameGeneratorTest.java | 12 +
.../metadata/ColumnMetadataSparseKeysTest.java | 29 +
.../pinot/server/api/resources/TablesResource.java | 16 +
.../server/starter/helix/BaseServerStarter.java | 8 +-
.../pinot/server/api/TablesResourceTest.java | 15 +
.../spi/config/table/OpenStructIndexConfig.java | 30 +-
.../pinot/spi/config/table/RoutingConfig.java | 19 +-
.../apache/pinot/spi/utils/CommonConstants.java | 32 +-
pom.xml | 120 ++--
257 files changed, 12970 insertions(+), 2064 deletions(-)
create mode 100644
pinot-broker/src/test/java/org/apache/pinot/broker/routing/segmentpruner/SinglePartitionColumnSegmentPrunerTest.java
create mode 100644
pinot-common/src/main/java/org/apache/pinot/common/utils/MutableRoaringBitmapUnion.java
create mode 100644
pinot-common/src/main/java/org/apache/pinot/common/utils/RoaringBitmapUnion.java
create mode 100644
pinot-common/src/main/java/org/apache/pinot/common/utils/helix/ClusterVersionTracker.java
create mode 100644
pinot-common/src/main/java/org/apache/pinot/sql/parsers/parser/GrowingCharStream.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/utils/MutableRoaringBitmapUnionTest.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/utils/RoaringBitmapUnionTest.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/utils/helix/ClusterVersionTrackerTest.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/sql/parsers/parser/GrowingCharStreamTest.java
create mode 100644
pinot-controller/src/test/java/org/apache/pinot/controller/api/resources/LLCSegmentCompletionHandlersTest.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/function/scalar/IdSetFunctions.java
delete mode 100644
pinot-core/src/main/java/org/apache/pinot/core/util/MemoizedClassAssociation.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/function/scalar/IdSetFunctionsTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/InIdSetTransformFunctionTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/function/DistinctCountBitmapLazyUnionTest.java
copy
pinot-common/src/main/java/org/apache/pinot/common/metrics/BrokerQueryPhase.java
=>
pinot-core/src/test/java/org/apache/pinot/core/startree/v2/DistinctCountHLLOnHeapStarTreeV2Test.java
(57%)
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/IdSetIntegrationTest.java
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/WindowKWayMergeIntegrationTest.java
create mode 100644
pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkTimeSegmentPruner.java
copy
pinot-perf/src/main/java/org/apache/pinot/perf/aggregation/{BenchmarkSumAggregation.java
=> BenchmarkDistinctCountBitmapAggregation.java} (56%)
create mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-pulsar/src/main/java/org/apache/pinot/plugin/stream/pulsar/PulsarMessageId.java
create mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-pulsar/src/test/java/org/apache/pinot/plugin/stream/pulsar/PulsarMessageIdTest.java
copy
pinot-query-planner/src/main/java/org/apache/pinot/query/planner/plannode/{MailboxReceiveNode.java
=> BaseMailboxReceiveNode.java} (55%)
delete mode 100644
pinot-query-planner/src/main/java/org/apache/pinot/query/planner/plannode/KWayMergeExchangeNode.java
create mode 100644
pinot-query-planner/src/test/java/org/apache/pinot/query/LargeInListPlanningBudgetTest.java
create mode 100644
pinot-query-planner/src/test/java/org/apache/pinot/query/QueryPlanningCancellationTest.java
create mode 100644
pinot-query-planner/src/test/java/org/apache/pinot/query/planner/logical/LogicalPlannerTest.java
create mode 100644
pinot-query-planner/src/test/java/org/apache/pinot/query/planner/logical/PlanNodeToRelConverterTest.java
create mode 100644
pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/SendStatsPredicateTest.java
create mode 100644
pinot-query-runtime/src/test/java/org/apache/pinot/query/service/dispatch/DispatchClientTest.java
create mode 100644 pinot-query-runtime/src/test/resources/queries/IdSet.json
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/aggregator/DistinctCountBitmapValueAggregatorTest.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/RangeIndexHandlerNullMinMaxTest.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/openstruct/OpenStructPerKeyIndexReloadTest.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
