This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a change to branch xiangfu0/codex2/table-size-retention
in repository https://gitbox.apache.org/repos/asf/pinot.git
omit 258073becc9 Restore controller job limits after metadata expiry tests
omit 21bd1f36b2d Protect all retained lineage members during size retention
omit d20f817f264 Stop size retention at first lineage-owned segment
omit c32ac7fa3ae Fix size retention gauge lifecycle and reuse lineage
snapshot
omit 2e170fcecfd Protect live data during size-based retention
omit 496029f48c0 Add size-based table retention
add 902d3864bed Fix wrong results from MinMax selection order-by combine
under null handling (#19724)
add b51535ed34a Bump org.xerial.snappy:snappy-java from 1.1.10.8 to
1.1.10.9 (#19764)
add eb86bc77f5c Upgrade Guava to 33.7.2 and libraries-bom to 26.90.0
(#19772)
add 274601b020e Add numRemoteAccesses and remoteAccessBytes query stats
(#19756)
add 005c5ab64b6 Fix Unknown nodes in multi-stage EXPLAIN when asking
servers (#19766)
add 73645dafec9 Fix missing aggregate FILTER in multi-stage EXPLAIN when
asking servers (#19767)
add c58c13feca4 Fix missing GROUPING SETS in multi-stage EXPLAIN when
asking servers (#19769)
add 3de4f315e04 Fix CAST type in multi-stage EXPLAIN when asking servers
(#19768)
add 08f4ab17e67 Make IDSET and IN_ID_SET work in the multi-stage engine
(#19757)
add 9c3f0e6d208 Prepare partition pruning predicates once per query
(#19539)
add 43ecf19f925 Report whether upsert snapshot files can be compared
across replicas (#19728)
add 79486707b92 Bump org.xerial.snappy:snappy-java from 1.1.10.9 to
1.1.10.10 (#19780)
add bb6b8b9d906 Bump org.webjars:swagger-ui from 5.32.15 to 5.33.1 (#19779)
add a75f2e54c14 Build compatibility verifier checkouts with their own
Maven wrapper (#19777)
add 5b6c7a2983c Fix missing UNNEST input columns in multi-stage EXPLAIN
when asking servers (#19771)
add b587d0a8c1b Bump com.github.luben:zstd-jni from 1.5.7-20 to 1.5.7-21
(#19778)
add 657d7805e7a Support range index on ingestion-aggregated no-dictionary
columns (#19111)
add 4d93fce3399 Fix SIGSEGV when upsert reads a concurrently destroyed
segment (#19596)
add 651ff41a3a9 Upgrade Pulsar to 5.0.0 (#19787)
add 1ea817199aa Support IdSet literals of millions of ids (#19759)
add 3a36f139597 Optimize bitmap unions in query aggregation, star-tree
build, and JSON indexes (#19587)
add 800b084ea8e Add size-based table retention
add 83a20548d97 Protect live data during size-based retention
add d86521028d4 Fix size retention gauge lifecycle and reuse lineage
snapshot
add 1d5e7d5a606 Stop size retention at first lineage-owned segment
add 914fb0b984e Protect all retained lineage members during size retention
add 5b357d58c0e Restore controller job limits after metadata expiry tests
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (258073becc9)
\
N -- N -- N refs/heads/xiangfu0/codex2/table-size-retention
(5b357d58c0e)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
compatibility-verifier/checkoutAndBuild.sh | 22 +-
.../broker/broker/helix/BaseBrokerStarter.java | 1 +
.../helix/MultiClusterHelixBrokerStarter.java | 1 +
.../MultiStageBrokerRequestHandler.java | 5 +-
.../routing/manager/BaseBrokerRoutingManager.java | 29 +-
.../segmentpruner/SegmentPrunerFactory.java | 27 +-
.../SinglePartitionColumnSegmentPruner.java | 186 ++++++
.../MultiStageBrokerRequestHandlerTest.java | 19 +-
.../routing/manager/BrokerRoutingManagerTest.java | 33 ++
.../SinglePartitionColumnSegmentPrunerTest.java | 632 +++++++++++++++++++++
.../apache/pinot/common/datatable/DataTable.java | 8 +-
.../common/function/TransformFunctionType.java | 5 +-
.../function/ByteArrayPartitionFunction.java | 24 +
.../partition/function/FnvPartitionFunction.java | 28 +
.../function/HashCodePartitionFunction.java | 24 +
.../function/ModuloPartitionFunction.java | 24 +
.../function/Murmur3PartitionFunction.java | 29 +
.../function/MurmurPartitionFunction.java | 26 +
.../pinot/common/response/BrokerResponse.java | 8 +
.../response/broker/BrokerResponseNative.java | 45 +-
.../response/broker/BrokerResponseNativeV2.java | 31 +-
.../response/broker/CursorResponseNative.java | 10 +-
.../response/mapper/TimeSeriesResponseMapper.java | 4 +
.../common/utils/MutableRoaringBitmapUnion.java | 182 ++++++
.../pinot/common/utils/RoaringBitmapUnion.java | 240 ++++++++
.../pinot/common/utils/RoaringBitmapUtils.java | 8 +
.../common/utils/config/QueryOptionsUtils.java | 8 +
.../apache/pinot/sql/parsers/CalciteSqlParser.java | 15 +-
.../sql/parsers/parser/GrowingCharStream.java | 86 +++
.../partition/function/PartitionFunctionTest.java | 123 +++-
.../utils/MutableRoaringBitmapUnionTest.java | 299 ++++++++++
.../pinot/common/utils/RoaringBitmapUnionTest.java | 399 +++++++++++++
.../common/utils/config/QueryOptionsUtilsTest.java | 10 +
.../pinot/sql/parsers/CalciteSqlParserTest.java | 57 +-
.../sql/parsers/parser/GrowingCharStreamTest.java | 138 +++++
.../pinot/core/function/scalar/IdSetFunctions.java | 169 ++++++
.../core/operator/InstanceResponseOperator.java | 10 +
...xValueBasedSelectionOrderByCombineOperator.java | 49 +-
.../filter/InvertedIndexFilterOperator.java | 9 +-
.../function/InIdSetTransformFunction.java | 48 +-
.../transform/function/TransformFunction.java | 12 +
.../function/TransformFunctionFactory.java | 2 +-
.../DistinctCountBitmapAggregationFunction.java | 78 ++-
.../query/reduce/ExecutionStatsAggregator.java | 8 +
.../core/query/request/context/QueryContext.java | 8 +-
.../core/query/utils/idset/BloomFilterIdSet.java | 29 +-
.../pinot/core/query/utils/idset/IdSets.java | 53 +-
.../utils/idset/Roaring64NavigableMapIdSet.java | 13 +-
.../pinot/core/util/MemoizedClassAssociation.java | 42 --
.../pinot/core/function/FunctionRegistryTest.java | 4 +-
.../core/function/scalar/IdSetFunctionsTest.java | 219 +++++++
.../operator/InstanceResponseOperatorTest.java | 36 ++
.../combine/SelectionCombineOperatorTest.java | 148 ++++-
.../function/InIdSetTransformFunctionTest.java | 98 ++++
.../function/DistinctCountBitmapLazyUnionTest.java | 221 +++++++
.../query/reduce/ExecutionStatsAggregatorTest.java | 24 +
.../pinot/core/query/utils/idset/IdSetTest.java | 76 +++
.../pinot/core/startree/v2/BaseStarTreeV2Test.java | 11 +-
.../v2/DistinctCountBitmapStarTreeV2Test.java | 18 +-
.../DistinctCountBitmapWithMVStarTreeV2Test.java | 17 +-
.../v2/DistinctCountHLLOnHeapStarTreeV2Test.java | 28 +-
.../org/apache/pinot/queries/IdSetQueriesTest.java | 157 ++++-
.../pinot/queries/NullQueriesFluentTest.java | 137 +++++
.../tests/custom/IdSetIntegrationTest.java | 236 ++++++++
.../resources/udf-test-results/all-functions.yaml | 2 +-
...> BenchmarkDistinctCountBitmapAggregation.java} | 82 ++-
.../pinot-stream-ingestion/pinot-pulsar/pom.xml | 17 +-
.../plugin/stream/pulsar/PulsarMessageId.java | 138 +++++
.../PulsarPartitionLevelConnectionHandler.java | 32 +-
.../pulsar/PulsarPartitionLevelConsumer.java | 3 +-
.../pinot/plugin/stream/pulsar/PulsarUtils.java | 10 +-
.../plugin/stream/pulsar/PulsarConsumerTest.java | 6 +-
.../stream/pulsar/PulsarMessageBatchTest.java | 5 +
.../plugin/stream/pulsar/PulsarMessageIdTest.java | 64 +++
.../plugin/stream/pulsar/PulsarUtilsTest.java | 17 +
.../planner/logical/PlanNodeToRelConverter.java | 77 ++-
.../query/planner/logical/RexExpressionUtils.java | 48 +-
.../logical/PlanNodeToRelConverterTest.java | 136 +++++
.../planner/logical/RexExpressionUtilsTest.java | 35 ++
.../src/test/resources/queries/JoinPlans.json | 14 +
.../resources/queries/PhysicalOptimizerPlans.json | 15 +
.../pinot/query/runtime/operator/LeafOperator.java | 12 +-
.../TimeSeriesExchangeReceiveOperator.java | 4 +-
.../query/service/dispatch/DispatchClient.java | 18 +-
.../query/service/dispatch/QueryDispatcher.java | 35 +-
.../pinot/query/service/server/QueryServer.java | 26 +-
.../query/runtime/operator/LeafOperatorTest.java | 33 ++
.../query/service/dispatch/DispatchClientTest.java | 138 +++++
.../query/service/server/QueryServerTest.java | 55 ++
.../src/test/resources/queries/IdSet.json | 360 ++++++++++++
.../DistinctCountBitmapValueAggregator.java | 60 +-
.../segment/local/aggregator/ValueAggregator.java | 5 +
.../immutable/ImmutableSegmentImpl.java | 44 ++
.../stats/CompactedNoDictColumnStatistics.java | 2 +-
.../stats/MutableNoDictColumnStatistics.java | 69 ++-
.../realtime/impl/json/MutableJsonIndexImpl.java | 67 ++-
.../loader/invertedindex/RangeIndexHandler.java | 69 ++-
.../readers/json/ImmutableJsonIndexReader.java | 59 +-
.../startree/v2/builder/BaseSingleTreeBuilder.java | 17 +
.../v2/builder/OnHeapSingleTreeBuilder.java | 24 +
.../upsert/BasePartitionUpsertMetadataManager.java | 81 +++
...oncurrentMapPartitionUpsertMetadataManager.java | 73 ++-
...nUpsertMetadataManagerForConsistentDeletes.java | 81 ++-
.../upsert/PartitionUpsertMetadataManager.java | 14 +
.../DistinctCountBitmapValueAggregatorTest.java | 161 ++++++
.../immutable/ImmutableSegmentImplTest.java | 90 ++-
.../io/codec/CompressionCodecCorruptInputTest.java | 29 +-
.../converter/RealtimeSegmentConverterTest.java | 84 +++
.../stats/MutableNoDictColumnStatisticsTest.java | 147 +++++
.../RangeIndexHandlerNullMinMaxTest.java | 243 ++++++++
.../v2/builder/NullAwareStarTreeBuilderTest.java | 99 +++-
.../BasePartitionUpsertMetadataManagerTest.java | 153 +++++
...ertMetadataManagerForConsistentDeletesTest.java | 75 +++
...rrentMapPartitionUpsertMetadataManagerTest.java | 82 +++
.../utils/IndexCombinationValidationTest.java | 117 ++++
.../org/apache/pinot/segment/spi/FetchContext.java | 23 +
.../segment/spi/creator/IndexCreationContext.java | 31 +-
.../segment/spi/partition/PartitionFunction.java | 11 +
.../pinot/server/api/resources/TablesResource.java | 16 +
.../pinot/server/api/TablesResourceTest.java | 15 +
.../pinot/spi/config/table/RoutingConfig.java | 19 +-
.../apache/pinot/spi/utils/CommonConstants.java | 22 +
pom.xml | 100 ++--
123 files changed, 7799 insertions(+), 511 deletions(-)
create mode 100644
pinot-broker/src/test/java/org/apache/pinot/broker/routing/segmentpruner/SinglePartitionColumnSegmentPrunerTest.java
create mode 100644
pinot-common/src/main/java/org/apache/pinot/common/utils/MutableRoaringBitmapUnion.java
create mode 100644
pinot-common/src/main/java/org/apache/pinot/common/utils/RoaringBitmapUnion.java
create mode 100644
pinot-common/src/main/java/org/apache/pinot/sql/parsers/parser/GrowingCharStream.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/utils/MutableRoaringBitmapUnionTest.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/utils/RoaringBitmapUnionTest.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/sql/parsers/parser/GrowingCharStreamTest.java
create mode 100644
pinot-core/src/main/java/org/apache/pinot/core/function/scalar/IdSetFunctions.java
delete mode 100644
pinot-core/src/main/java/org/apache/pinot/core/util/MemoizedClassAssociation.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/function/scalar/IdSetFunctionsTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/InIdSetTransformFunctionTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/function/DistinctCountBitmapLazyUnionTest.java
copy
pinot-common/src/main/java/org/apache/pinot/common/metrics/BrokerQueryPhase.java
=>
pinot-core/src/test/java/org/apache/pinot/core/startree/v2/DistinctCountHLLOnHeapStarTreeV2Test.java
(57%)
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/IdSetIntegrationTest.java
copy
pinot-perf/src/main/java/org/apache/pinot/perf/aggregation/{BenchmarkSumAggregation.java
=> BenchmarkDistinctCountBitmapAggregation.java} (56%)
create mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-pulsar/src/main/java/org/apache/pinot/plugin/stream/pulsar/PulsarMessageId.java
create mode 100644
pinot-plugins/pinot-stream-ingestion/pinot-pulsar/src/test/java/org/apache/pinot/plugin/stream/pulsar/PulsarMessageIdTest.java
create mode 100644
pinot-query-planner/src/test/java/org/apache/pinot/query/planner/logical/PlanNodeToRelConverterTest.java
create mode 100644
pinot-query-runtime/src/test/java/org/apache/pinot/query/service/dispatch/DispatchClientTest.java
create mode 100644 pinot-query-runtime/src/test/resources/queries/IdSet.json
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/aggregator/DistinctCountBitmapValueAggregatorTest.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/RangeIndexHandlerNullMinMaxTest.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
