This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a change to branch
xiangfu0/data-3221-7-slim-metadata-lazy-schema
in repository https://gitbox.apache.org/repos/asf/pinot.git
discard 64382186301 Invalidate the derived schema when the loader registers
virtual columns
discard 069d076a859 Use per-segment schema state in lazy metadata tests
discard 1199bcaa7e7 Slim ColumnMetadataImpl to 72 bytes and derive the
per-segment Schema lazily
discard 0f17b6301fb Serialize v1/v2 index buffer access for lazy
materialization and cover destroy vs in-flight
discard d8f0e3cef25 Retain compact index configs for segment-specific loads
discard 43aee25f62a Reuse resolved index configs for lazy column
materialization
discard 3b630ad8a92 Materialize immutable-segment columns lazily behind an
opt-in instance config (default off)
add 845c851eab2 Add V7 raw forward index format for codec pipelines
(#19307)
add 2354206647a Propagate failed IdealState group commits after queue
drains (#19679)
add 341b903121a Extract previous record locations size condition rather
than counting (#19683)
add c85b9c26843 Disallow JUnit test APIs in tests and drop the unused
JUnit artifacts (#19675)
add 2014abbdcea Copy the column set in the server table metadata endpoint
instead of narrowing a segment's own columns (#19665)
add 911ac3619c9 Fix Docker broker query log appender reference (#19626)
add ef7b1b7a733 perf: avoid materializing skipped Fory JSON strings
(#19672)
add 66b7f21344a Preserve query error codes during mailbox cancellation
(#19682)
add f0b4da30c8c Bump org.slf4j:slf4j-bom from 2.0.19 to 2.0.20 (#19694)
add f7ffb3a1e9a Bump org.mockito:mockito-bom from 5.23.0 to 5.24.0 (#19691)
add 94867aa8b98 Bump org.jline:jline from 4.4.5 to 4.4.6 (#19689)
add afe264086c3 Bump software.amazon.awssdk:bom from 2.55.2 to 2.55.5
(#19693)
add af39d1a6c1f Fix leaked read lock and unknown-n-gram crash in
RealtimeNgramFilteringIndex (#19612)
add b2cc6dd2b58 Bump com.google.cloud:libraries-bom from 26.88.1 to
26.89.0 (#19695)
add c825dc4730f Bump com.github.luben:zstd-jni from 1.5.7-18 to 1.5.7-20
(#19688)
add 2fbc0284a8b OPEN_STRUCT fixes: nested paths, multi-value keys,
declared types, sparse blob reads, and whole-column projection (#19608)
add 866bb59cb9e Report V7 compression stats and reject per-key OPEN_STRUCT
codecSpec (#19308)
add 9f6f5ca10df Fix duplicate QueryErrorCode id 429 so quota rejections
decode as TOO_MANY_REQUESTS (#19676)
add a3af29e92b8 Trim listener access protocols and skipIndexes values
(#19648)
add 2cd9a2fea0f feat(controller): label browser SQL queries (#19697)
add 2d6288e64a3 Bump org.mongodb:bson from 5.12.0 to 5.13.0 (#19705)
add c8a1add2e8f Bump at.yawk.lz4:lz4-java from 1.11.3 to 1.12.0 (#19704)
add 04cca1a58dc Bump software.amazon.awssdk:bom from 2.55.5 to 2.55.6
(#19703)
add 0f1c98d9e14 Add codec pipeline integration tests (#19309)
add 9e5a41fd6a5 Reject per-key OPEN_STRUCT settings that materialized keys
silently ignore (#19687)
add 41b8c20a8b6 Normalize empty filter results to EmptyDocIdSet when they
are built (#19684)
add cd504afe901 Add JDBC support for MAP, arrays and extended scalar types
add f5614058a96 Bump com.h2database:h2 from 2.5.250 to 2.5.252 (#19719)
add a42e748472a Bump brace-expansion in
/pinot-controller/src/main/resources (#19721)
add cbe92c2ccce Bump org.apache.fory:fory-json from 1.7.5 to 1.7.6 (#19717)
add fcbe3e4a73b Seal large IN lists from Calcite during multi-stage
planning (#19674)
add 99401dec621 Bump moment in /pinot-controller/src/main/resources
(#19720)
add d5542e22727 Make an OPEN_STRUCT key behave like an ordinary column:
apply its index settings on reload, and keep its type across tiers (#19698)
add 1b94d4bc698 Bump axios from 1.18.0 to 1.20.0 in
/pinot-controller/src/main/resources (#19722)
add a080910ca48 Build TimeSegmentPruner's interval tree from flat arrays
and skip no-op rebuilds (#19643)
add 816375d50c7 Take upsert/dedup primary key columns from the metadata
manager context (#19726)
add 747f9470257 Log and count protected upsert revert failures (#19505)
add d6a37880dfe Bump org.apache.maven:maven-core from 3.9.16 to 3.10.0
(#19734)
add 17961e1bdb7 Bump org.apache.maven:apache-maven from 3.9.16 to 3.10.0
(#19733)
add 3f80ac76d0c Bump org.codehaus.woodstox:stax2-api from 4.3.0 to 4.3.1
(#19732)
add 8cbbc1adc40 Bump com.gradle:develocity-maven-extension from 2.5.0 to
2.6.0 (#19690)
add 4e27013f964 Make _numPendingOperations protected in
BasePartitionUpsertMetadataManager (#19744)
add b861ac5503f Bump com.fasterxml.woodstox:woodstox-core from 7.2.2 to
7.3.0 (#19741)
add 6bf036907f2 Bump software.amazon.awssdk:bom from 2.55.6 to 2.55.7
(#19742)
add c95ff8e51c5 Bump net.alchim31.maven:scala-maven-plugin from 4.9.10 to
4.10.0 (#19743)
add e149b276bed Bump com.puppycrawl.tools:checkstyle from 14.1.0 to 14.3.0
(#19731)
add 931c67a0f25 Introduce reason codes for segment completion requests
(#19251)
add 4f495df35c8 Fix wrong results for multi-key GROUP BY with transform
expressions (#19663)
add 7a00b2318be [core] Fix ANY_VALUE bytes intermediate round trip (#19667)
add a7cfea0b824 Upgrade Commons Lang to 3.21.0 and discard trailing empty
split fields (#19747)
add f5d9be62448 Build bitmap and sorted doc id sets only through their
factories (#19746)
add 36b526bf159 Bump org.apache.thrift:libthrift from 0.24.0 to 0.25.0
(#19750)
add 624dcc2c666 Bump joda-time:joda-time from 2.14.4 to 2.15.0 (#19751)
add af3a63278ed Bump software.amazon.awssdk:bom from 2.55.7 to 2.55.10
(#19752)
add 60fcf580835 Bump org.checkerframework:checker-qual from 4.2.3 to 4.3.0
(#19753)
add 22102756dae Index CASE THEN results by array instead of a boxed map
(#19685)
add b8e2ef27671 Build METADATA push metadata from the local segment
instead of re-downloading the staged tar (#19490)
add 55a760d0d64 Stop MSE query planning when the planning thread is
interrupted (#19709)
add 6a2ab1221db Reject input file segment name templates that reference no
match group (#19755)
add 18d18ac2afa Split upsert keys-not-replaced metric by table type
(#19748)
add 98bda679fd3 Add planning budget tests for large IN lists (#19745)
add ade41d16899 Log lead-controller VIP lookup at debug, not info (#19758)
add 85db332cfca Reject consuming during download for upsert tables that
revert metadata in PROTECTED mode (#19710)
add 27e8839ade5 Bump software.amazon.awssdk:bom from 2.55.10 to 2.55.11
(#19763)
add c9af1d06ae2 Bump source-map-js in /pinot-controller/src/main/resources
(#19765)
add 21596bbffad Configure OPEN_STRUCT indexes the way every other column
is configured (#19736)
add 902d3864bed Fix wrong results from MinMax selection order-by combine
under null handling (#19724)
add b51535ed34a Bump org.xerial.snappy:snappy-java from 1.1.10.8 to
1.1.10.9 (#19764)
add eb86bc77f5c Upgrade Guava to 33.7.2 and libraries-bom to 26.90.0
(#19772)
add 274601b020e Add numRemoteAccesses and remoteAccessBytes query stats
(#19756)
add 005c5ab64b6 Fix Unknown nodes in multi-stage EXPLAIN when asking
servers (#19766)
add 73645dafec9 Fix missing aggregate FILTER in multi-stage EXPLAIN when
asking servers (#19767)
add c58c13feca4 Fix missing GROUPING SETS in multi-stage EXPLAIN when
asking servers (#19769)
add 3de4f315e04 Fix CAST type in multi-stage EXPLAIN when asking servers
(#19768)
add 08f4ab17e67 Make IDSET and IN_ID_SET work in the multi-stage engine
(#19757)
add 9c3f0e6d208 Prepare partition pruning predicates once per query
(#19539)
add 43ecf19f925 Report whether upsert snapshot files can be compared
across replicas (#19728)
add 79486707b92 Bump org.xerial.snappy:snappy-java from 1.1.10.9 to
1.1.10.10 (#19780)
add bb6b8b9d906 Bump org.webjars:swagger-ui from 5.32.15 to 5.33.1 (#19779)
add a75f2e54c14 Build compatibility verifier checkouts with their own
Maven wrapper (#19777)
add 5b6c7a2983c Fix missing UNNEST input columns in multi-stage EXPLAIN
when asking servers (#19771)
add b587d0a8c1b Bump com.github.luben:zstd-jni from 1.5.7-20 to 1.5.7-21
(#19778)
add 657d7805e7a Support range index on ingestion-aggregated no-dictionary
columns (#19111)
add 4d93fce3399 Fix SIGSEGV when upsert reads a concurrently destroyed
segment (#19596)
add 651ff41a3a9 Upgrade Pulsar to 5.0.0 (#19787)
add 1ea817199aa Support IdSet literals of millions of ids (#19759)
add 3a36f139597 Optimize bitmap unions in query aggregation, star-tree
build, and JSON indexes (#19587)
add c8e5a00705a Materialize immutable-segment columns lazily behind an
opt-in instance config (default off)
add 88f4bcf1282 Reuse resolved index configs for lazy column
materialization
add 93f193c7199 Retain compact index configs for segment-specific loads
add 6041cd6972b Serialize v1/v2 index buffer access for lazy
materialization and cover destroy vs in-flight
add 6a9541b967f Keep rebased source imports in project order
add 11352519dc4 Keep lazy materialization compatible with the segment read
lifecycle
add 7677d53bb1d Use current OpenStruct config constructors in materializer
tests
add 73bf3edd64e Keep nested and sparse settings distinct during config
sharing
add f046b64ef42 Slim ColumnMetadataImpl to 72 bytes and derive the
per-segment Schema lazily
add 9c602b52a81 Use per-segment schema state in lazy metadata tests
add 40d323de9d7 Invalidate the derived schema when the loader registers
virtual columns
add e105b6ba7c3 Preserve sparse manifests through slim metadata and lazy
loading
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (64382186301)
\
N -- N -- N
refs/heads/xiangfu0/data-3221-7-slim-metadata-lazy-schema (e105b6ba7c3)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.mvn/extensions.xml | 2 +-
.mvn/wrapper/maven-wrapper.properties | 2 +-
compatibility-verifier/checkoutAndBuild.sh | 22 +-
config/checkstyle.xml | 5 +-
.../images/pinot/etc/conf/pinot-broker-log4j2.xml | 2 +-
kb/skills/review-testing.md | 3 +-
.../broker/broker/helix/BaseBrokerStarter.java | 1 +
.../helix/MultiClusterHelixBrokerStarter.java | 1 +
.../MultiStageBrokerRequestHandler.java | 13 +-
.../routing/manager/BaseBrokerRoutingManager.java | 29 +-
.../segmentpruner/SegmentPrunerFactory.java | 27 +-
.../SinglePartitionColumnSegmentPruner.java | 186 ++++
.../routing/segmentpruner/TimeSegmentPruner.java | 38 +-
.../segmentpruner/interval/IntervalTree.java | 240 +++--
.../MultiStageBrokerRequestHandlerTest.java | 19 +-
.../routing/intervalst/IntervalTreeTest.java | 60 ++
.../routing/manager/BrokerRoutingManagerTest.java | 33 +
.../routing/segmentpruner/SegmentPrunerTest.java | 120 +++
.../SinglePartitionColumnSegmentPrunerTest.java | 632 +++++++++++++
.../org/apache/pinot/client/PinotResultSet.java | 281 ++----
.../pinot/client/base/AbstractBaseResultSet.java | 342 +++++++-
.../pinot/client/grpc/PinotGrpcResultSet.java | 293 +++----
.../apache/pinot/client/utils/BigDecimalUtils.java | 20 +-
.../apache/pinot/client/utils/DateTimeUtils.java | 16 +-
.../org/apache/pinot/client/utils/DriverUtils.java | 20 +
.../apache/pinot/client/PinotResultSetTest.java | 164 +++-
.../pinot/client/grpc/PinotGrpcResultSetTest.java | 260 ++++++
.../pinot/client/utils/BigDecimalUtilsTest.java | 35 +-
pinot-common/pom.xml | 16 +-
.../apache/pinot/common/datatable/DataTable.java | 8 +-
.../common/function/ForyJsonPathExtractor.java | 15 +-
.../common/function/TransformFunctionType.java | 5 +-
.../common/function/scalar/JsonFunctions.java | 6 +-
.../common/function/scalar/StringFunctions.java | 72 +-
.../apache/pinot/common/metrics/ServerMeter.java | 3 +
.../function/ByteArrayPartitionFunction.java | 24 +
.../partition/function/FnvPartitionFunction.java | 28 +
.../function/HashCodePartitionFunction.java | 24 +
.../function/ModuloPartitionFunction.java | 24 +
.../function/Murmur3PartitionFunction.java | 29 +
.../function/MurmurPartitionFunction.java | 26 +
.../protocols/SegmentCompletionProtocol.java | 96 +-
.../pinot/common/response/BrokerResponse.java | 8 +
.../response/broker/BrokerResponseNative.java | 45 +-
.../response/broker/BrokerResponseNativeV2.java | 31 +-
.../response/broker/CursorResponseNative.java | 10 +-
.../response/encoder/JsonResponseEncoder.java | 95 +-
.../response/mapper/TimeSeriesResponseMapper.java | 4 +
.../ColumnCompressionStatsContribution.java | 3 +-
.../resources/ColumnCompressionStatsInfo.java | 3 +-
.../org/apache/pinot/common/utils/DataSchema.java | 24 +-
.../common/utils/MutableRoaringBitmapUnion.java | 182 ++++
.../pinot/common/utils/RoaringBitmapUnion.java | 240 +++++
.../pinot/common/utils/RoaringBitmapUtils.java | 8 +
.../common/utils/config/QueryOptionsUtils.java | 20 +-
.../common/utils/helix/IdealStateGroupCommit.java | 4 +-
.../apache/pinot/sql/parsers/CalciteSqlParser.java | 15 +-
.../sql/parsers/parser/GrowingCharStream.java | 86 ++
.../function/ForyJsonLinkageFallbackTest.java | 37 +
.../common/function/ForyJsonPathFunctionsTest.java | 27 +
.../function/scalar/StringFunctionsTest.java | 89 +-
.../partition/function/PartitionFunctionTest.java | 123 ++-
.../protocols/SegmentCompletionProtocolTest.java | 89 +-
.../utils/MutableRoaringBitmapUnionTest.java | 299 +++++++
.../pinot/common/utils/NativeTestNGSuiteTest.java | 230 -----
.../pinot/common/utils/OrderedTestNGSuiteTest.java | 86 ++
.../pinot/common/utils/RoaringBitmapUnionTest.java | 399 +++++++++
.../common/utils/config/QueryOptionsUtilsTest.java | 54 ++
.../pinot/sql/parsers/CalciteSqlParserTest.java | 57 +-
.../sql/parsers/parser/GrowingCharStreamTest.java | 138 +++
pinot-connectors/pinot-flink-connector/pom.xml | 1 +
pinot-controller/pom.xml | 1 +
.../resources/LLCSegmentCompletionHandlers.java | 23 +-
.../helix/core/minion/ClusterInfoAccessor.java | 2 +-
.../src/main/resources/app/pages/Query.tsx | 3 +-
.../src/main/resources/package-lock.json | 30 +-
pinot-controller/src/main/resources/package.json | 4 +-
.../LLCSegmentCompletionHandlersTest.java | 92 ++
.../helix/IdealStateGroupCommitTest.java | 252 +++---
.../helix/core/realtime/SegmentCompletionTest.java | 38 +
.../controller/util/ListenerConfigUtilTest.java | 38 +
.../apache/pinot/core/common/BlockDocIdSet.java | 20 +-
.../realtime/RealtimeSegmentDataManager.java | 36 +-
.../pinot/core/function/scalar/IdSetFunctions.java | 169 ++++
.../core/operator/InstanceResponseOperator.java | 10 +
.../core/operator/blocks/ProjectionBlock.java | 167 +++-
...xValueBasedSelectionOrderByCombineOperator.java | 49 +-
.../pinot/core/operator/docidsets/AndDocIdSet.java | 5 -
.../core/operator/docidsets/BitmapDocIdSet.java | 40 +-
.../core/operator/docidsets/EmptyDocIdSet.java | 19 +-
.../pinot/core/operator/docidsets/NotDocIdSet.java | 2 +-
.../pinot/core/operator/docidsets/OrDocIdSet.java | 5 -
.../docidsets/RangelessBitmapDocIdSet.java | 24 +-
.../docidsets/ShortCircuitingDocIdSet.java | 50 --
.../core/operator/docidsets/SortedDocIdSet.java | 15 +-
.../docvalsets/OpenStructDocumentBlockValSet.java | 159 ++++
.../core/operator/filter/AndFilterOperator.java | 14 +-
.../operator/filter/BaseColumnFilterOperator.java | 19 +-
.../core/operator/filter/BaseFilterOperator.java | 12 +-
.../operator/filter/BitmapBasedFilterOperator.java | 6 +-
.../operator/filter/CombinedFilterOperator.java | 7 +-
.../core/operator/filter/EmptyFilterOperator.java | 2 +-
.../filter/ExactVectorScanFilterOperator.java | 2 +-
.../operator/filter/ExpressionFilterOperator.java | 2 +-
.../filter/H3InclusionIndexFilterOperator.java | 7 +-
.../operator/filter/H3IndexFilterOperator.java | 11 +-
.../filter/InvertedIndexFilterOperator.java | 19 +-
.../operator/filter/JsonMatchFilterOperator.java | 2 +-
.../core/operator/filter/MapFilterOperator.java | 23 +-
.../core/operator/filter/NotFilterOperator.java | 2 +-
.../core/operator/filter/OrFilterOperator.java | 14 +-
.../filter/RangeIndexBasedFilterOperator.java | 13 +-
.../filter/SortedIndexBasedFilterOperator.java | 8 +-
.../operator/filter/TextMatchFilterOperator.java | 5 +-
.../filter/VectorRadiusFilterOperator.java | 2 +-
.../filter/VectorSimilarityFilterOperator.java | 2 +-
.../transform/function/CaseTransformFunction.java | 146 ++--
.../function/InIdSetTransformFunction.java | 48 +-
.../transform/function/ItemTransformFunction.java | 68 ++
.../JsonExtractScalarTransformFunction.java | 4 +-
.../transform/function/TransformFunction.java | 12 +
.../function/TransformFunctionFactory.java | 2 +-
.../function/AnyValueAggregationFunction.java | 11 +-
.../DistinctCountBitmapAggregationFunction.java | 78 +-
.../query/reduce/ExecutionStatsAggregator.java | 8 +
.../core/query/request/context/QueryContext.java | 8 +-
.../core/query/utils/OrderByComparatorFactory.java | 17 +-
.../core/query/utils/idset/BloomFilterIdSet.java | 29 +-
.../pinot/core/query/utils/idset/IdSets.java | 53 +-
.../utils/idset/Roaring64NavigableMapIdSet.java | 13 +-
.../startree/operator/StarTreeFilterOperator.java | 2 +-
.../apache/pinot/core/util/ListenerConfigUtil.java | 5 +-
.../pinot/core/util/MemoizedClassAssociation.java | 42 -
.../realtime/RealtimeSegmentDataManagerTest.java | 113 ++-
.../pinot/core/function/FunctionRegistryTest.java | 4 +-
.../core/function/scalar/IdSetFunctionsTest.java | 219 +++++
.../operator/InstanceResponseOperatorTest.java | 36 +
.../blocks/ProjectionBlockOpenStructTest.java | 163 +++-
.../combine/SelectionCombineOperatorTest.java | 148 +++-
.../combine/SortedGroupByCombineOperatorsTest.java | 89 ++
.../dociditerators/SortedDocIdIteratorTest.java | 10 +-
.../core/operator/docidsets/AndDocIdSetTest.java | 4 +-
.../operator/filter/AndFilterOperatorTest.java | 2 +-
.../filter/MapFilterOperatorOpenStructTest.java | 52 +-
.../core/operator/filter/OrFilterOperatorTest.java | 4 +-
.../core/operator/filter/TestFilterOperator.java | 4 +-
.../function/InIdSetTransformFunctionTest.java | 98 +++
.../ItemTransformFunctionMultiValueTest.java | 143 +++
.../ItemTransformFunctionScalarTypeTest.java | 140 +++
.../function/AnyValueAggregationFunctionTest.java | 59 ++
.../function/DistinctCountBitmapLazyUnionTest.java | 221 +++++
.../query/reduce/ExecutionStatsAggregatorTest.java | 24 +
.../query/utils/OrderByComparatorFactoryTest.java | 38 +
.../pinot/core/query/utils/idset/IdSetTest.java | 76 ++
.../pinot/core/startree/v2/BaseStarTreeV2Test.java | 11 +-
.../v2/DistinctCountBitmapStarTreeV2Test.java | 18 +-
.../DistinctCountBitmapWithMVStarTreeV2Test.java | 17 +-
.../v2/DistinctCountHLLOnHeapStarTreeV2Test.java | 28 +-
.../org/apache/pinot/queries/IdSetQueriesTest.java | 157 +++-
.../pinot/queries/NullQueriesFluentTest.java | 137 +++
pinot-integration-test-base/pom.xml | 1 +
pinot-integration-tests/pom.xml | 12 +-
.../tests/custom/CodecPipelineIntegrationTest.java | 973 +++++++++++++++++++++
.../CodecPipelineRealtimeIntegrationTest.java | 432 +++++++++
.../tests/custom/IdSetIntegrationTest.java | 236 +++++
.../resources/udf-test-results/all-functions.yaml | 2 +-
.../pinot/perf/BenchmarkTimeSegmentPruner.java | 163 ++++
.../pinot/perf/BenchmarkV7ForwardIndexWriter.java | 194 ++++
...> BenchmarkDistinctCountBitmapAggregation.java} | 82 +-
.../BaseMultipleSegmentsConversionExecutor.java | 67 +-
.../tasks/BaseSingleSegmentConversionExecutor.java | 20 +-
.../plugin/minion/tasks/BaseTaskExecutor.java | 19 -
...BaseMultipleSegmentsConversionExecutorTest.java | 177 +++-
.../BaseSingleSegmentConversionExecutorTest.java | 42 +-
.../pinot-stream-ingestion/pinot-kafka-3.0/pom.xml | 1 +
.../pinot-stream-ingestion/pinot-pulsar/pom.xml | 17 +-
.../plugin/stream/pulsar/PulsarMessageId.java | 138 +++
.../PulsarPartitionLevelConnectionHandler.java | 32 +-
.../pulsar/PulsarPartitionLevelConsumer.java | 3 +-
.../pinot/plugin/stream/pulsar/PulsarUtils.java | 10 +-
.../plugin/stream/pulsar/PulsarConsumerTest.java | 6 +-
.../stream/pulsar/PulsarMessageBatchTest.java | 5 +
.../plugin/stream/pulsar/PulsarMessageIdTest.java | 64 ++
.../plugin/stream/pulsar/PulsarUtilsTest.java | 17 +
.../pinot/calcite/rel/rules/PinotRuleUtils.java | 25 +
.../calcite/rex/PinotSealedSearchOperator.java | 117 +++
.../org/apache/pinot/calcite/rex/SearchSealer.java | 401 +++++++++
.../pinot/calcite/sql/fun/PinotInListOperator.java | 93 ++
.../calcite/sql2rel/PinotConvertletTable.java | 140 ++-
.../org/apache/pinot/query/QueryEnvironment.java | 17 +-
.../apache/pinot/query/context/PlannerContext.java | 15 +
.../query/planner/logical/LogicalPlanner.java | 20 +
.../planner/logical/PlanNodeToRelConverter.java | 77 +-
.../query/planner/logical/RexExpressionUtils.java | 151 +++-
.../apache/pinot/calcite/rex/SearchSealerTest.java | 315 +++++++
.../pinot/query/LargeInListPlanningBudgetTest.java | 198 +++++
.../pinot/query/QueryPlanningCancellationTest.java | 179 ++++
.../pinot/query/SealedInListPlanningTest.java | 555 ++++++++++++
.../query/planner/logical/LogicalPlannerTest.java | 61 ++
.../logical/PlanNodeToRelConverterTest.java | 136 +++
.../planner/logical/RexExpressionUtilsTest.java | 184 ++++
.../query/planner/logical/StagesTestBase.java | 84 +-
.../src/test/resources/queries/JoinPlans.json | 14 +
.../resources/queries/PhysicalOptimizerPlans.json | 15 +
.../pinot/query/mailbox/GrpcSendingMailbox.java | 14 +-
.../query/mailbox/InMemorySendingMailbox.java | 13 +-
.../pinot/query/runtime/operator/LeafOperator.java | 12 +-
.../TimeSeriesExchangeReceiveOperator.java | 4 +-
.../query/service/dispatch/DispatchClient.java | 18 +-
.../query/service/dispatch/QueryDispatcher.java | 35 +-
.../pinot/query/service/server/QueryServer.java | 26 +-
.../query/mailbox/GrpcSendingMailboxTest.java | 42 +
.../query/mailbox/InMemorySendingMailboxTest.java | 38 +
.../query/runtime/operator/LeafOperatorTest.java | 33 +
.../query/service/dispatch/DispatchClientTest.java | 138 +++
.../query/service/server/QueryServerTest.java | 55 ++
.../src/test/resources/queries/IdSet.json | 360 ++++++++
.../src/test/resources/queries/LargeInLists.json | 602 +++++++++++++
.../DistinctCountBitmapValueAggregator.java | 60 +-
.../segment/local/aggregator/ValueAggregator.java | 5 +
.../immutable/ImmutableSegmentImpl.java | 88 +-
.../indexsegment/mutable/MutableSegmentImpl.java | 14 +-
.../impl/FixedByteChunkForwardIndexWriter.java | 7 +-
.../impl/FixedByteChunkForwardIndexWriterV7.java | 419 +++++++++
.../local/io/writer/impl/FixedByteChunkWriter.java | 47 +
.../stats/CompactedNoDictColumnStatistics.java | 2 +-
.../stats/MutableNoDictColumnStatistics.java | 69 +-
.../invertedindex/RealtimeNgramFilteringIndex.java | 27 +-
.../realtime/impl/json/MutableJsonIndexImpl.java | 67 +-
...ompressionStatsTrackingForwardIndexCreator.java | 3 +-
.../fwd/SingleValueFixedByteRawIndexCreator.java | 53 +-
.../impl/openstruct/OpenStructColumnSplitter.java | 332 +++++--
.../index/forward/CompressionStatsMetadata.java | 6 +-
.../index/forward/ForwardIndexCreatorFactory.java | 22 +-
.../index/forward/ForwardIndexReaderFactory.java | 30 +-
.../segment/index/forward/ForwardIndexType.java | 53 +-
.../segment/index/loader/ForwardIndexHandler.java | 95 +-
.../segment/index/loader/IndexLoadingConfig.java | 45 +-
.../segment/index/loader/SegmentPreProcessor.java | 53 +-
.../loader/invertedindex/RangeIndexHandler.java | 69 +-
.../segment/index/map/SimpleColumnMetadata.java | 12 +-
.../openstruct/ImmutableOpenStructDataSource.java | 77 +-
.../segment/index/openstruct/MutableKeyColumn.java | 93 +-
.../index/openstruct/MutableOpenStructIndex.java | 167 +++-
.../index/openstruct/OpenStructIndexType.java | 136 ++-
.../openstruct/OpenStructSupportedIndexes.java | 3 +-
.../index/openstruct/SparseKeyDataSource.java | 258 +++++-
.../FixedByteChunkSVForwardIndexReaderV7.java | 489 +++++++++++
.../readers/json/ImmutableJsonIndexReader.java | 59 +-
.../startree/v2/builder/BaseSingleTreeBuilder.java | 17 +
.../v2/builder/OnHeapSingleTreeBuilder.java | 24 +
.../upsert/BasePartitionUpsertMetadataManager.java | 114 ++-
...oncurrentMapPartitionUpsertMetadataManager.java | 84 +-
...nUpsertMetadataManagerForConsistentDeletes.java | 94 +-
.../upsert/PartitionUpsertMetadataManager.java | 19 +
.../segment/local/utils/SegmentPushUtils.java | 307 ++++---
.../segment/local/utils/TableConfigUtils.java | 32 +
.../DistinctCountBitmapValueAggregatorTest.java | 161 ++++
.../immutable/ColumnMaterializerTest.java | 5 +-
.../immutable/ImmutableSegmentImplTest.java | 108 ++-
.../mutable/MutableSegmentDedupTest.java | 44 +
.../mutable/MutableSegmentImplUpsertTest.java | 55 ++
.../io/codec/CompressionCodecCorruptInputTest.java | 29 +-
.../converter/RealtimeSegmentConverterTest.java | 84 ++
.../stats/MutableNoDictColumnStatisticsTest.java | 147 ++++
.../RealtimeNgramFilteringIndexTest.java | 44 +
.../openstruct/OpenStructColumnSplitterTest.java | 39 +
.../OpenStructDeclaredChildSpecTest.java | 156 ++++
.../openstruct/OpenStructMultiValueKeyTest.java | 215 +++++
.../impl/openstruct/OpenStructNestedKeyTest.java | 238 +++++
.../forward/CompressionStatsMetadataTest.java | 6 +
.../forward/ForwardIndexCreatorFactoryTest.java | 459 +++++++++-
.../index/forward/ForwardIndexTypeTest.java | 35 +
.../ForwardIndexHandlerCompressionStatsTest.java | 147 +++-
.../index/loader/ForwardIndexHandlerTest.java | 202 ++++-
.../RangeIndexHandlerNullMinMaxTest.java | 243 +++++
.../ImmutableOpenStructDataSourceTest.java | 48 +-
.../MutableOpenStructDataSourceTest.java | 29 +-
.../OpenStructConsumingSealedParityTest.java | 286 ++++++
.../index/openstruct/OpenStructIndexTypeTest.java | 209 +++++
.../OpenStructPerKeyIndexReloadTest.java | 397 +++++++++
.../OpenStructSparseDenseParityTest.java | 102 ++-
.../index/openstruct/SparseKeyDataSourceTest.java | 41 +-
.../v2/builder/NullAwareStarTreeBuilderTest.java | 99 ++-
.../BasePartitionUpsertMetadataManagerTest.java | 153 ++++
...ertMetadataManagerForConsistentDeletesTest.java | 153 ++++
...rrentMapPartitionUpsertMetadataManagerTest.java | 240 +++++
.../utils/IndexCombinationValidationTest.java | 117 +++
.../segment/local/utils/SegmentPushUtilsTest.java | 112 +++
.../segment/local/utils/TableConfigUtilsTest.java | 208 +++++
.../org/apache/pinot/segment/spi/FetchContext.java | 23 +
.../org/apache/pinot/segment/spi/V1Constants.java | 14 +
.../segment/spi/creator/IndexCreationContext.java | 31 +-
.../creator/name/FixedSegmentNameGenerator.java | 1 -
.../name/InputFileSegmentNameGenerator.java | 35 +-
.../name/NormalizedDateSegmentNameGenerator.java | 1 -
.../creator/name/SimpleSegmentNameGenerator.java | 1 -
.../segment/spi/index/FieldIndexConfigsUtil.java | 60 +-
.../segment/spi/index/ForwardIndexConfig.java | 6 +
.../spi/index/metadata/ColumnMetadataImpl.java | 98 ++-
.../segment/spi/partition/PartitionFunction.java | 11 +
.../name/InputFileSegmentNameGeneratorTest.java | 12 +
.../spi/index/metadata/ColumnMetadataImplTest.java | 18 +-
.../metadata/ColumnMetadataSparseKeysTest.java | 29 +
.../resources/SegmentCompressionStatsReader.java | 2 +-
.../pinot/server/api/resources/TablesResource.java | 16 +
.../pinot/server/api/TablesResourceTest.java | 83 ++
.../SegmentCompressionStatsReaderTest.java | 77 ++
.../apache/pinot/spi/config/table/FieldConfig.java | 34 +-
.../spi/config/table/OpenStructIndexConfig.java | 86 +-
.../pinot/spi/config/table/RoutingConfig.java | 19 +-
.../pinot/spi/data/OpenStructKeyFlattener.java | 187 ++++
.../pinot/spi/data/OpenStructTypeInference.java | 39 +
.../apache/pinot/spi/exception/QueryErrorCode.java | 2 +-
.../apache/pinot/spi/utils/CommonConstants.java | 45 +
.../config/table/OpenStructIndexConfigTest.java | 34 +-
.../pinot/spi/data/OpenStructKeyFlattenerTest.java | 195 +++++
.../pinot/spi/exception/QueryErrorCodeTest.java | 56 ++
pom.xml | 210 ++---
319 files changed, 23430 insertions(+), 2692 deletions(-)
create mode 100644
pinot-broker/src/test/java/org/apache/pinot/broker/routing/segmentpruner/SinglePartitionColumnSegmentPrunerTest.java
copy
pinot-query-runtime/src/main/java/org/apache/pinot/query/service/dispatch/CancelObserver.java
=>
pinot-clients/pinot-jdbc-client/src/main/java/org/apache/pinot/client/utils/BigDecimalUtils.java
(65%)
create mode 100644
pinot-clients/pinot-jdbc-client/src/test/java/org/apache/pinot/client/grpc/PinotGrpcResultSetTest.java
copy
pinot-spi/src/test/java/org/apache/pinot/spi/utils/builder/TableConfigBuilderTest.java
=>
pinot-clients/pinot-jdbc-client/src/test/java/org/apache/pinot/client/utils/BigDecimalUtilsTest.java
(52%)
create mode 100644
pinot-common/src/main/java/org/apache/pinot/common/utils/MutableRoaringBitmapUnion.java
create mode 100644
pinot-common/src/main/java/org/apache/pinot/common/utils/RoaringBitmapUnion.java
create mode 100644
pinot-common/src/main/java/org/apache/pinot/sql/parsers/parser/GrowingCharStream.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/utils/MutableRoaringBitmapUnionTest.java
delete mode 100644
pinot-common/src/test/java/org/apache/pinot/common/utils/NativeTestNGSuiteTest.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/utils/OrderedTestNGSuiteTest.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/utils/RoaringBitmapUnionTest.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/sql/parsers/parser/GrowingCharStreamTest.java
create mode 100644
pinot-controller/src/test/java/org/apache/pinot/controller/api/resources/LLCSegmentCompletionHandlersTest.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/function/scalar/IdSetFunctions.java
delete mode 100644
pinot-core/src/main/java/org/apache/pinot/core/operator/docidsets/ShortCircuitingDocIdSet.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/operator/docvalsets/OpenStructDocumentBlockValSet.java
delete mode 100644
pinot-core/src/main/java/org/apache/pinot/core/util/MemoizedClassAssociation.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/function/scalar/IdSetFunctionsTest.java
rename pinot-core/src/{main =>
test}/java/org/apache/pinot/core/operator/filter/TestFilterOperator.java (97%)
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/InIdSetTransformFunctionTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ItemTransformFunctionMultiValueTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ItemTransformFunctionScalarTypeTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/function/DistinctCountBitmapLazyUnionTest.java
copy
pinot-common/src/main/java/org/apache/pinot/common/metrics/BrokerQueryPhase.java
=>
pinot-core/src/test/java/org/apache/pinot/core/startree/v2/DistinctCountHLLOnHeapStarTreeV2Test.java
(57%)
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/CodecPipelineIntegrationTest.java
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/CodecPipelineRealtimeIntegrationTest.java
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/IdSetIntegrationTest.java
create mode 100644
pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkTimeSegmentPruner.java
create mode 100644
pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkV7ForwardIndexWriter.java
copy
pinot-perf/src/main/java/org/apache/pinot/perf/aggregation/{BenchmarkSumAggregation.java
=> BenchmarkDistinctCountBitmapAggregation.java} (56%)
create mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-pulsar/src/main/java/org/apache/pinot/plugin/stream/pulsar/PulsarMessageId.java
create mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-pulsar/src/test/java/org/apache/pinot/plugin/stream/pulsar/PulsarMessageIdTest.java
create mode 100644
pinot-query-planner/src/main/java/org/apache/pinot/calcite/rex/PinotSealedSearchOperator.java
create mode 100644
pinot-query-planner/src/main/java/org/apache/pinot/calcite/rex/SearchSealer.java
create mode 100644
pinot-query-planner/src/main/java/org/apache/pinot/calcite/sql/fun/PinotInListOperator.java
create mode 100644
pinot-query-planner/src/test/java/org/apache/pinot/calcite/rex/SearchSealerTest.java
create mode 100644
pinot-query-planner/src/test/java/org/apache/pinot/query/LargeInListPlanningBudgetTest.java
create mode 100644
pinot-query-planner/src/test/java/org/apache/pinot/query/QueryPlanningCancellationTest.java
create mode 100644
pinot-query-planner/src/test/java/org/apache/pinot/query/SealedInListPlanningTest.java
create mode 100644
pinot-query-planner/src/test/java/org/apache/pinot/query/planner/logical/LogicalPlannerTest.java
create mode 100644
pinot-query-planner/src/test/java/org/apache/pinot/query/planner/logical/PlanNodeToRelConverterTest.java
create mode 100644
pinot-query-runtime/src/test/java/org/apache/pinot/query/service/dispatch/DispatchClientTest.java
create mode 100644 pinot-query-runtime/src/test/resources/queries/IdSet.json
create mode 100644
pinot-query-runtime/src/test/resources/queries/LargeInLists.json
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/FixedByteChunkForwardIndexWriterV7.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/FixedByteChunkWriter.java
create mode 100644
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedByteChunkSVForwardIndexReaderV7.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/aggregator/DistinctCountBitmapValueAggregatorTest.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/openstruct/OpenStructDeclaredChildSpecTest.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/openstruct/OpenStructMultiValueKeyTest.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/openstruct/OpenStructNestedKeyTest.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/RangeIndexHandlerNullMinMaxTest.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/openstruct/OpenStructPerKeyIndexReloadTest.java
create mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/data/OpenStructKeyFlattener.java
create mode 100644
pinot-spi/src/test/java/org/apache/pinot/spi/data/OpenStructKeyFlattenerTest.java
create mode 100644
pinot-spi/src/test/java/org/apache/pinot/spi/exception/QueryErrorCodeTest.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
