This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a change to branch
xiangfu0/codex/codec-stack/07-integration-docs
in repository https://gitbox.apache.org/repos/asf/pinot.git
omit c2e79cd65b6 Add codec pipeline integration tests and design doc
omit 664784acc8e Harden codec reload and compression stats
omit 015f098872a Support codecSpec on segment reload and enable codecSpec
end to end
omit 9f52fa61871 Validate V7 frames lazily and harden the review follow-ups
omit 27a0d4e5fa0 Add V7 raw forward index format for codec pipelines
add 1f17ab0ede1 Reopen the mutable HNSW searcher on a background thread
instead of on the query path (#19467)
add 5e05b9cf062 Make segment-replace timeouts and endReplaceSegments
convergence wait configurable (#19278)
add e83b59f0d8e Release MSE operator buffers on error and cancel, not only
on end of stream (#19462)
add a8476f988cf Stop a lingering receiving mailbox from pinning the
receive operator (#19465)
add cab13822b26 [MSE] Add ExplainableOperator so alternative leaf-stage
engines appear in EXPLAIN (#19447)
add 6aaa0405566 Add broker startup pre-connect for broker-to-server
channels (SSE) (#19407)
add 6c773871d9d Bump org.jline:jline from 4.4.1 to 4.4.2 (#19498)
add 32b9d6bf7a5 Bump org.slf4j:slf4j-bom from 2.0.18 to 2.0.19 (#19497)
add ec0507c764d Bump com.mchange:c3p0 from 0.14.1 to 0.14.2 (#19496)
add a88fa8750a2 Bump software.amazon.awssdk:bom from 2.54.9 to 2.54.12
(#19495)
add 16227abd341 Bump org.checkerframework:checker-qual from 4.2.2 to 4.2.3
(#19494)
add b9ee686b945 Bump io.grpc:grpc-bom from 1.83.1 to 1.84.0 (#19491)
add 7712e67bc9c Bump parquet.version from 1.18.0 to 1.18.1 (#19492)
add ff7aefbef30 Bump org.roaringbitmap:RoaringBitmap from 1.6.20 to 1.6.21
(#19493)
add 7f024b6b623 Release the consumer semaphore only after the segment's
upsert metadata is removed on offload (#19444)
add 49cf1be58b6 Bump com.google.cloud:libraries-bom from 26.87.0 to
26.88.0 (#19506)
add 51f24d8a476 Bump com.squareup.okio:okio-bom from 3.18.1 to 3.18.2
(#19507)
add 452f5a686e8 Bump com.h2database:h2 from 2.4.240 to 2.5.250 (#19508)
add 9d91a7bffb1 Bump software.amazon.awssdk:bom from 2.54.12 to 2.54.13
(#19509)
add 0e540de467a Use plain component names for the MultiValueLimit record
(#19482)
add 1f5b06c6001 Add free OpenRouter PR flow diagrams (#19502)
add 6fb1075c4de Sample resource usage while the DISTINCT cross-segment
merge grows (#19416)
add c6116095f7d Log a query on a single line (#19453)
add 568df83b4f4 Add broker startup warmup for the query serve path (SSE)
(#19500)
add d666cb5e1c0 Bump actions/upload-artifact from 4.6.2 to 7.0.1 (#19516)
add 74659c76ff6 Bump actions/checkout from 4.4.0 to 7.0.1 (#19517)
add ceb1fabf4ef Bump actions/setup-python from 5.6.0 to 7.0.0 (#19518)
add f727b44f6ef Bump js-yaml from 4.3.1 to 4.3.2 in
/pinot-controller/src/main/resources (#19522)
add 940284ded36 Add an opt-in controller-copy METADATA push for same-name
segment refreshes (#19488)
add 63b3b1f1b3c Bump org.jetbrains.kotlin:kotlin-bom from 2.4.10 to 2.4.20
(#19524)
add 26f47121863 Broker startup pre-connect: remove the straggler-grace
window (#19519)
add f2800272e44 docs: add RELEASE_POLICY.md for versioning and upgrades
#11642 (#19451)
add 86107957b03 Star-tree offheap build: read dims from sort buffer
instead of re-fetching from segment (#19430)
add 30f5d208a1e Prune values across the query executor instead of on the
calling thread (#19513)
add 718f4100551 Bump org.scala-lang.modules:scala-xml_2.13 from 2.4.0 to
2.5.0 (#19527)
add 372c535f053 Bump org.jline:jline from 4.4.2 to 4.4.3 (#19528)
add b65be1d599c Allocate ColumnMetadataImpl index sizes lazily and skip
the index_map lookup without an index dir (#19480)
add 26bf075d16c Skip missing consuming segment check for paused realtime
tables (#19310)
add 5771d6acea6 Defer TTL watermark bump until after segment
add/preload/replace (upsert & dedup) (#19512)
add 9ee1c681be8 Fix conflicting agent instructions and stale workflow
guidance (#19541)
add a6ea7b0debd Remove obsolete benchmarks and timing-only unit tests
(#19538)
add 66fca21494e Reuse fixed-row buffers for DataTable result construction
(#19533)
add 4290c50ff6b Bump org.mongodb:bson from 5.11.0 to 5.11.1 (#19545)
add 3b8759fe2c6 Bump com.google.cloud:libraries-bom from 26.88.0 to
26.88.1 (#19546)
add b5035a6cea8 Bump software.amazon.awssdk:bom from 2.54.13 to 2.54.16
(#19548)
add d7b8ce3b387 Bump org.apache.fory:fory-json from 1.7.1 to 1.7.2 (#19549)
add 4087449ec96 Bump org.roaringbitmap:RoaringBitmap from 1.6.21 to 1.6.23
(#19550)
add 4df89218b38 Bump io.netty:netty-bom from 4.1.137.Final to
4.1.138.Final (#19551)
add ea6b22dcfe4 Bump bouncycastle.version from 1.85 to 1.86 (#19547)
add 7fd2f1b157f PageCache warmup on Server Restarts and Segment Refresh
(#16033)
add 47a5f8671f6 Return HTTP 400 for DDL table validator rejections (#19544)
add 810d7985b7e Add Jupyter JupySQL tutorial for querying Pinot (#19374)
add 92360684523 Make filter count and bitmap shortcuts three-valued under
null handling (#19440)
add 6ac0a577da5 Reject QUALIFY in the single-stage engine instead of
silently ignoring it (#19553)
add 8228534299a Reuse compiled constant string casts during planning
(#19514)
add a3e25756e17 Read each broker's instance config once per refresh in
ExternalViewReader (#19567)
add 719e486ac1d Bump org.codehaus.mojo:animal-sniffer-annotations from
1.27 to 1.28 (#19559)
add d49d91f4252 Bump software.amazon.awssdk:bom from 2.54.16 to 2.54.17
(#19560)
add f74a76d5909 Bump org.codehaus.mojo:buildnumber-maven-plugin from 3.3.0
to 3.3.1 (#19561)
add ac708ce2b27 Bump org.codehaus.mojo:versions-maven-plugin from 2.21.0
to 2.22.0 (#19562)
add 52095c8eb6a Bump org.codehaus.mojo:javacc-maven-plugin from 3.2.0 to
3.2.1 (#19563)
add d3a4b6e527a Bump org.apache.zookeeper:zookeeper from 3.9.5 to 3.9.6
(#19564)
add 12f9ebb6027 Bump org.codehaus.mojo:exec-maven-plugin from 3.6.3 to
3.6.4 (#19565)
add a979a6a0d8e Bump org.codehaus.mojo:build-helper-maven-plugin from
3.6.1 to 3.6.2 (#19566)
add ee615670b3f Reduce per-segment allocations in broker routing (#19530)
add 4b06328c0cb Fix binary compatibility CI baseline selection (#19569)
add 051049b9be3 Report BrokerResourceMissing instead of a generic internal
error when MSE routing is absent (#19558)
add 81f1a03abf8 Bump org.apache.fory:fory-json from 1.7.2 to 1.7.3 (#19573)
add 2a07d3ec173 Bump at.yawk.lz4:lz4-java from 1.11.2 to 1.11.3 (#19574)
add 8e454baabaa fix(readers): don't leak column readers or skip segment
destroy on close() failure (#19572)
add 3b760adb71a Optimise the OPEN_STRUCT per-row consumption path, and
follow-up fixes from #18643 review (#19093)
add 22470be7320 Add a cluster config to rewrite exact DISTINCTCOUNT and
PERCENTILE into approximations (#19510)
add 42cd1d67b6f Tolerate concurrent deletes when sizing consumer and
segment directories (#19580)
add c1a26e91c1e Restore the default when an adaptive routing metric export
cluster config is removed (#19578)
add b319ffd5785 Share FieldSpecs and repeated values across segment
metadata loads (#19473)
add 257ec17d6fa Fix final group-by schema independently of server response
order (#19543)
add ec52ed81f6f Intern the instance id in ServerInstance (#19583)
add ce2dc49cb0f Add SQL query option policies: legacy OPTION() cluster
config and per-request sqlOptionsMode (#19570)
add 06d6079e49f Stop the single-stage engine from silently ignoring the
HAVING clause (#19554)
add bf0793af430 Fix protobuf descriptor file temp directory leak (#19378)
add 3764aea534e Drop star-tree indexes left unreadable by a dimension
encoding change (#19349)
add 15b838a5df8 Fall back to last-known-good protobuf descriptor when the
remote fetch fails (#19434)
add ef75ed97148 Remove unused 5-arg getValidDocIdFromServerMatchingCrc
overload (#19585)
add be6eb519c65 Bump org.webjars:swagger-ui from 5.32.14 to 5.32.15
(#19588)
add e19541a477f Serialize cluster config listener registration with change
processing (#19590)
add 6d522d9a77d Do not build a discarded result when an accumulator
reaches its threshold (#19449)
add e1762ad216d Use dedicated weak interners for metadata column names and
defaults (#19584)
add 6f68898b74b Mark the "query received" log record so it can be routed
independently (#19454)
add 7436330fa0d Add V7 raw forward index format for codec pipelines
add 8c8ec39984b Validate V7 frames lazily and harden the review follow-ups
add 3d9bd063d57 Resolve configured codec specs once per reload and add a
V7 write benchmark
add 7f446948e16 Support codecSpec on segment reload and enable codecSpec
end to end
add 9e2503a4f87 Harden codec reload and compression stats
add a76317d2a07 Add codec pipeline integration tests and design doc
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (c2e79cd65b6)
\
N -- N -- N
refs/heads/xiangfu0/codex/codec-stack/07-integration-docs (a76317d2a07)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.claude/agents/code-reviewer.md | 2 +-
.claude/skills/bench-compare/SKILL.md | 2 +-
.claude/skills/flaky-analyze/SKILL.md | 2 +-
.claude/skills/precommit/SKILL.md | 2 +-
.claude/skills/quickstart/SKILL.md | 2 +-
.claude/skills/review-architecture/SKILL.md | 2 +-
.claude/skills/review-concurrency-state/SKILL.md | 2 +-
.claude/skills/review-config-backcompat/SKILL.md | 2 +-
.claude/skills/review-correctness-nulls/SKILL.md | 2 +-
.claude/skills/review-naming-api/SKILL.md | 2 +-
.claude/skills/review-performance/SKILL.md | 2 +-
.claude/skills/review-process-scope/SKILL.md | 2 +-
.claude/skills/review-testing/SKILL.md | 2 +-
.claude/skills/run-test/SKILL.md | 2 +-
.github/scripts/pr_flow/README.md | 121 ++++
.github/scripts/pr_flow/main.py | 524 +++++++++++++++++
.github/scripts/pr_flow/model.py | 470 +++++++++++++++
.github/scripts/pr_flow/test_main.py | 572 ++++++++++++++++++
.github/scripts/pr_flow/test_model.py | 476 +++++++++++++++
.github/workflows/build-all-pinot-docker-image.yml | 8 +-
.../workflows/build-pinot-base-docker-image.yml | 4 +-
.github/workflows/build-pinot-docker-image.yml | 2 +-
.github/workflows/build-superset-docker-image.yml | 2 +-
.../nightly-build-all-pinot-docker-image.yml | 8 +-
...htly-build-multi-arch-superset-docker-image.yml | 2 +-
.github/workflows/pinot_compatibility_checks.yml | 36 +-
.github/workflows/pinot_compatibility_tests.yml | 4 +-
.github/workflows/pinot_integration_tests.yml | 4 +-
.../pinot_java11_client_compatibility.yml | 2 +-
...ulti_stage_query_engine_compatibility_tests.yml | 4 +-
.github/workflows/pinot_quickstart_tests.yml | 2 +-
.github/workflows/pinot_unit_tests.yml | 6 +-
.github/workflows/pinot_vuln_check.yml | 2 +-
.github/workflows/pr-flow-checks.yml | 55 ++
.../workflows/pr-flow-signal.yml | 26 +-
.github/workflows/pr-flow.yml | 134 +++++
AGENTS.md | 91 +--
CLAUDE.md | 91 +--
README.md | 1 +
RELEASE_POLICY.md | 126 ++++
contrib/jupyter-jupysql/README.md | 76 +++
contrib/jupyter-jupysql/pinot_jupysql_eda.ipynb | 215 +++++++
.../jupyter-jupysql/requirements.txt | 13 +-
kb/agents/code-reviewer.md | 47 +-
kb/claude.md | 2 +-
kb/code-review-principles.md | 106 ++--
kb/plugin-modules.md | 69 +++
kb/skills/README.md | 151 +----
kb/skills/bench-compare.md | 86 ++-
kb/skills/flaky-analyze.md | 32 +-
kb/skills/precommit.md | 63 +-
kb/skills/quickstart.md | 42 +-
kb/skills/review-architecture.md | 17 +-
kb/skills/review-concurrency-state.md | 24 +-
kb/skills/review-config-backcompat.md | 12 +-
kb/skills/review-correctness-nulls.md | 20 +-
kb/skills/review-naming-api.md | 13 +-
kb/skills/review-performance.md | 18 +-
kb/skills/review-process-scope.md | 19 +-
kb/skills/review-testing.md | 64 +--
kb/skills/run-test.md | 26 +-
.../broker/api/resources/PinotClientRequest.java | 3 +-
.../broker/broker/helix/BaseBrokerStarter.java | 282 ++++++++-
.../apache/pinot/broker/grpc/BrokerGrpcServer.java | 3 +-
.../apache/pinot/broker/querylog/QueryLogger.java | 38 +-
.../ApproximateFunctionOverrideProvider.java | 161 ++++++
.../requesthandler/BaseBrokerRequestHandler.java | 23 +-
.../BaseSingleStageBrokerRequestHandler.java | 112 ++--
.../requesthandler/BrokerRequestHandler.java | 37 ++
.../BrokerRequestHandlerDelegate.java | 20 +-
.../broker/requesthandler/BrokerWarmupConfig.java | 60 ++
.../MultiStageBrokerRequestHandler.java | 18 +
.../broker/requesthandler/ServerPreConnector.java | 180 ++++++
.../SingleConnectionBrokerRequestHandler.java | 582 +++++++++++++++++++
.../instanceselector/BalancedInstanceSelector.java | 16 +-
.../ReplicaGroupInstanceSelector.java | 15 +-
.../routing/manager/BaseBrokerRoutingManager.java | 28 +-
.../manager/MultiClusterRoutingManager.java | 12 +
.../broker/broker/helix/BrokerWarmupGateTest.java | 50 ++
.../pinot/broker/querylog/QueryLoggerTest.java | 165 +++++-
.../ApproximateFunctionOverrideProviderTest.java | 134 +++++
.../BaseSingleStageBrokerRequestHandlerTest.java | 61 ++
.../broker/requesthandler/BrokerWarmupTest.java | 454 +++++++++++++++
.../broker/requesthandler/QueryOverrideTest.java | 56 +-
.../broker/requesthandler/QueryValidationTest.java | 8 +-
.../requesthandler/ServerPreConnectorTest.java | 458 +++++++++++++++
.../instanceselector/InstanceSelectorTest.java | 62 ++
.../routing/manager/BrokerRoutingManagerTest.java | 114 +++-
.../manager/MultiClusterRoutingManagerTest.java | 14 +
.../apache/pinot/client/ExternalViewReader.java | 20 +-
.../client/admin/PageCacheWarmupAdminClient.java | 91 +++
.../pinot/client/admin/PinotAdminClient.java | 11 +
.../pinot/client/ExternalViewReaderTest.java | 103 +++-
.../config/DefaultClusterConfigChangeHandler.java | 47 +-
.../apache/pinot/common/metrics/BrokerGauge.java | 11 +
.../apache/pinot/common/metrics/BrokerMeter.java | 5 +
.../apache/pinot/common/metrics/BrokerMetrics.java | 7 +
.../apache/pinot/common/metrics/BrokerTimer.java | 12 +
.../pinot/common/metrics/ControllerMeter.java | 3 +
.../apache/pinot/common/metrics/ServerMeter.java | 4 +
.../pinot/common/pagecache/WarmupQueryUtils.java | 497 ++++++++++++++++
.../pinot/common/response/BrokerResponse.java | 13 +
.../response/broker/BrokerResponseNative.java | 19 +-
.../response/broker/BrokerResponseNativeV2.java | 18 +-
.../response/broker/CursorResponseNative.java | 5 +-
.../org/apache/pinot/common/utils/FileUtils.java | 22 +
.../utils/config/QueryOptionConfigListener.java | 78 +++
.../common/utils/config/QueryOptionsUtils.java | 55 +-
.../utils/config/QueryWorkloadConfigUtils.java | 58 +-
.../common/utils/config/TableConfigSerDeUtils.java | 13 +-
.../pinot/common/utils/helix/HelixHelper.java | 52 ++
.../apache/pinot/common/utils/http/HttpClient.java | 10 +-
.../pinot/common/utils/request/RequestUtils.java | 32 +-
.../apache/pinot/sql/parsers/CalciteSqlParser.java | 134 ++++-
.../sql/parsers/parser/TableNameExtractor.java | 5 +
...nAggregationGroupByToDistinctQueryRewriter.java | 7 +
.../common/pagecache/WarmupQueryUtilsTest.java | 318 ++++++++++
.../pinot/common/utils/FALFInternerTest.java | 71 +--
.../config/QueryOptionConfigListenerTest.java | 71 +++
.../pinot/sql/parsers/CalciteSqlParserTest.java | 37 ++
.../pinot/sql/parsers/SqlOptionsModeTest.java | 136 +++++
.../sql/parsers/parser/TableNameExtractorTest.java | 14 +
.../apache/pinot/controller/ControllerConf.java | 40 ++
.../pinot/controller/api/resources/Constants.java | 1 +
.../resources/PageCacheWarmupRestletResource.java | 290 ++++++++++
.../api/resources/PinotDdlRestletResource.java | 3 +-
.../api/resources/PinotQueryResource.java | 54 +-
.../controller/helix/SegmentStatusChecker.java | 16 +-
.../helix/core/PinotHelixResourceManager.java | 31 +-
.../realtime/MissingConsumingSegmentFinder.java | 12 +
.../util/PageCacheWarmupControllerExecutor.java | 378 ++++++++++++
.../src/main/resources/package-lock.json | 6 +-
.../pinot/controller/ControllerConfTest.java | 24 +
.../api/PinotDdlRestletResourceTest.java | 39 ++
.../api/resources/PinotQueryResourceTest.java | 40 ++
.../pinot/controller/helix/ControllerTest.java | 1 +
.../controller/helix/SegmentStatusCheckerTest.java | 86 +++
.../helix/core/retention/RetentionManagerTest.java | 79 ---
.../PageCacheWarmupControllerExecutorTest.java | 331 +++++++++++
.../apache/pinot/core/common/MinionConstants.java | 4 +
.../common/datatable/BaseDataTableBuilder.java | 9 +-
.../core/common/datatable/DataTableBuilder.java | 1 +
.../realtime/RealtimeSegmentDataManager.java | 32 +-
.../core/operator/combine/BaseCombineOperator.java | 8 +-
.../core/operator/filter/AndFilterOperator.java | 80 ++-
.../operator/filter/BaseColumnFilterOperator.java | 66 ++-
.../core/operator/filter/BaseFilterOperator.java | 65 ++-
.../operator/filter/BitmapBasedFilterOperator.java | 45 +-
.../core/operator/filter/BitmapCollection.java | 127 +++-
.../operator/filter/ExpressionFilterOperator.java | 25 +-
.../core/operator/filter/FilterOperatorUtils.java | 43 +-
.../filter/InvertedIndexFilterOperator.java | 24 +-
.../core/operator/filter/MapFilterOperator.java | 10 +
.../core/operator/filter/NotFilterOperator.java | 33 +-
.../core/operator/filter/OrFilterOperator.java | 77 ++-
.../filter/RangeIndexBasedFilterOperator.java | 35 +-
.../filter/SortedIndexBasedFilterOperator.java | 19 +-
.../streaming/BaseStreamingCombineOperator.java | 9 +-
...istinctCountSmartSketchAggregationFunction.java | 77 ++-
.../DistinctCountSmartHLLAggregationFunction.java | 19 +-
...stinctCountSmartHLLPlusAggregationFunction.java | 19 +-
.../DistinctCountSmartULLAggregationFunction.java | 19 +-
.../PercentileSmartTDigestAggregationFunction.java | 69 ++-
.../distinct/table/BigDecimalDistinctTable.java | 6 +
.../query/distinct/table/BytesDistinctTable.java | 6 +
.../query/distinct/table/DoubleDistinctTable.java | 6 +
.../query/distinct/table/FloatDistinctTable.java | 6 +
.../query/distinct/table/IntDistinctTable.java | 6 +
.../query/distinct/table/LongDistinctTable.java | 6 +
.../distinct/table/MultiColumnDistinctTable.java | 7 +
.../query/distinct/table/StringDistinctTable.java | 6 +
.../query/pruner/BloomFilterSegmentPruner.java | 99 +---
.../core/query/pruner/ValueBasedSegmentPruner.java | 90 ++-
.../query/reduce/AggregationDataTableReducer.java | 26 +-
.../core/query/reduce/GroupByDataTableReducer.java | 3 +
.../apache/pinot/core/routing/RoutingManager.java | 11 +
.../apache/pinot/core/startree/StarTreeUtils.java | 46 +-
.../apache/pinot/core/transport/QueryRouter.java | 26 +
.../pinot/core/transport/ServerChannels.java | 136 ++++-
.../pinot/core/transport/ServerInstance.java | 10 +-
.../routing/stats/ServerRoutingStatsManager.java | 14 +-
.../core/common/datatable/DataTableSerDeTest.java | 66 +++
.../realtime/RealtimeSegmentDataManagerTest.java | 111 ++++
.../operator/filter/AndFilterOperatorTest.java | 111 ++++
.../core/operator/filter/BitmapCollectionTest.java | 91 +++
.../filter/InvertedIndexFilterOperatorTest.java | 64 ++-
.../operator/filter/NotFilterOperatorTest.java | 81 ++-
.../core/operator/filter/OrFilterOperatorTest.java | 68 +++
.../function/AnyValueAggregationFunctionTest.java | 5 +-
...stinctCountSmartHLLAggregationFunctionTest.java | 126 ++++
...centileSmartTDigestAggregationFunctionTest.java | 76 +++
.../table/DistinctTableMergeAccountingTest.java | 292 ++++++++++
.../query/pruner/BloomFilterSegmentPrunerTest.java | 136 +++++
.../query/pruner/ColumnValueSegmentPrunerTest.java | 189 ++++++
.../core/query/reduce/GroupByFinalSchemaTest.java | 90 +++
.../pinot/core/startree/v2/BaseStarTreeV2Test.java | 4 +-
.../pinot/core/transport/ServerChannelsTest.java | 95 +++
.../pinot/core/transport/ServerInstanceTest.java | 12 +
.../stats/ServerRoutingStatsManagerTest.java | 41 +-
.../apache/pinot/queries/HavingQueriesTest.java | 186 ++++++
.../queries/NullHandlingEnabledQueriesTest.java | 210 +++++++
.../queries/StarTreeNullHandlingQueriesTest.java | 210 +++++++
...ApproximateFunctionOverrideIntegrationTest.java | 202 +++++++
.../BrokerServerPreConnectIntegrationTest.java | 142 +++++
.../tests/BrokerStartupWarmupIntegrationTest.java | 142 +++++
.../tests/NullHandlingIntegrationTest.java | 6 +-
.../tests/PageCacheWarmupIntegrationTest.java | 247 ++++++++
.../integration/tests/TlsIntegrationTest.java | 29 +
.../java/org/apache/pinot/minion/MinionConf.java | 6 +
.../org/apache/pinot/minion/MinionConfTest.java | 14 +
pinot-perf/README.md | 19 +
pinot-perf/pom.xml | 20 -
.../pinot/perf/BenchmarkConstantCastPlanning.java | 145 +++++
.../perf/BenchmarkFixedIntArrayOffHeapIdMap.java | 59 --
.../perf/BenchmarkGroovyExpressionEvaluation.java | 190 ------
.../org/apache/pinot/perf/BenchmarkJsonKeyMap.java | 172 ------
.../perf/BenchmarkOffHeapDictionaryMemory.java | 133 -----
.../apache/pinot/perf/BenchmarkQueryEngine.java | 152 -----
.../pinot/perf/BenchmarkRoaringBitmapCreation.java | 217 -------
.../pinot/perf/BenchmarkRoaringBitmapMapping.java | 328 -----------
.../pinot/perf/BenchmarkScalarCastPlanning.java | 151 +++++
.../pinot/perf/BenchmarkV7ForwardIndexWriter.java | 194 +++++++
.../pinot/perf/BenchmarkVectorFilterWorkloads.java | 169 ++++++
.../pinot/perf/ForwardIndexWriterBenchmark.java | 120 ----
.../java/org/apache/pinot/perf/LazyDataList.java | 163 ------
.../org/apache/pinot/perf/RawIndexBenchmark.java | 293 ----------
.../pinot/perf/StringDictionaryPerfTest.java | 189 ------
.../pinot/perf/aggregation/BenchmarkSumQuery.java | 121 ----
.../pinot/plugin/filesystem/HadoopPinotFSTest.java | 16 +-
.../protobuf/ProtoBufMessageDecoder.java | 38 +-
.../inputformat/protobuf/ProtoBufRecordReader.java | 7 +-
.../plugin/inputformat/protobuf/ProtoBufUtils.java | 158 ++++-
.../protobuf/ProtoBufMessageDecoderTest.java | 25 +
.../protobuf/ProtoBufTempFileLeakTest.java | 185 ++++++
.../protobuf/ProtoBufUtilsDescriptorCacheTest.java | 312 ++++++++++
.../BaseMultipleSegmentsConversionExecutor.java | 3 +-
.../tasks/BaseSingleSegmentConversionExecutor.java | 104 +++-
.../plugin/minion/tasks/BaseTaskExecutor.java | 45 +-
.../pinot/plugin/minion/tasks/MinionTaskUtils.java | 25 +-
.../minion/tasks/SegmentConversionUtils.java | 44 +-
.../BaseSingleSegmentConversionExecutorTest.java | 281 ++++++++-
.../PinotApproximateAggregateRewriteRule.java | 191 ++++++
.../calcite/rel/rules/PinotQueryRuleSets.java | 4 +
.../apache/pinot/calcite/rex/PinotRexExecutor.java | 130 +++++
.../org/apache/pinot/query/QueryEnvironment.java | 31 +
.../apache/pinot/query/context/PlannerContext.java | 14 +
.../apache/pinot/query/routing/WorkerManager.java | 15 +-
.../pinot/calcite/rex/PinotRexExecutorTest.java | 532 +++++++++++++++++
.../query/ApproximateAggregateRewriteTest.java | 162 ++++++
.../pinot/query/ConstantCastPlanningTest.java | 186 ++++++
.../pinot/query/context/PlannerContextTest.java | 2 +-
.../pinot/query/routing/WorkerManagerTest.java | 178 +++++-
.../resources/queries/WindowFunctionPlans.json | 15 +
.../apache/pinot/query/runtime/QueryRunner.java | 8 +-
.../runtime/executor/OpChainSchedulerService.java | 21 +-
.../query/runtime/operator/AggregateOperator.java | 33 +-
.../query/runtime/operator/AsofJoinOperator.java | 7 +-
.../query/runtime/operator/BaseJoinOperator.java | 6 +-
.../operator/BaseMailboxReceiveOperator.java | 21 +-
.../runtime/operator/ExplainableOperator.java | 56 ++
.../query/runtime/operator/HashJoinOperator.java | 7 +-
.../pinot/query/runtime/operator/LeafOperator.java | 12 +-
.../query/runtime/operator/MultiStageOperator.java | 60 ++
.../runtime/operator/NonEquiJoinOperator.java | 13 +-
.../query/runtime/operator/RepeatOperator.java | 26 +
.../pinot/query/runtime/operator/SortOperator.java | 37 +-
.../operator/SortedMailboxReceiveOperator.java | 27 +-
.../runtime/operator/set/BinarySetOperator.java | 19 +-
.../query/runtime/operator/set/UnionOperator.java | 18 +-
.../plan/pipeline/PipelineBreakerOperator.java | 10 +
.../operator/MailboxReceiveOperatorTest.java | 41 ++
.../operator/OperatorBufferReleaseTest.java | 640 +++++++++++++++++++++
.../operator/SortedMailboxReceiveOperatorTest.java | 34 ++
.../operator/set/SetOperatorBufferReleaseTest.java | 228 ++++++++
.../test/resources/queries/WindowFunctions.json | 44 +-
.../local/customobject/CpcSketchAccumulator.java | 24 +-
.../customobject/CustomObjectAccumulator.java | 8 +-
.../local/customobject/ThetaSketchAccumulator.java | 38 +-
.../customobject/TupleIntSketchAccumulator.java | 38 +-
.../dedup/BasePartitionDedupMetadataManager.java | 47 +-
...ConcurrentMapPartitionDedupMetadataManager.java | 5 +-
.../immutable/ImmutableSegmentImpl.java | 8 +
.../indexsegment/mutable/MutableSegmentImpl.java | 22 +-
.../local/io/reader/impl/FixedBitIntReader.java | 3 +-
.../realtime/impl/vector/MutableVectorIndex.java | 307 +++++++---
.../creator/impl/SegmentColumnarIndexCreator.java | 25 +-
.../impl/openstruct/OpenStructColumnSplitter.java | 57 +-
.../stats/NoDictColumnStatisticsCollector.java | 18 +
.../segment/index/loader/ForwardIndexHandler.java | 16 +-
.../segment/index/loader/SegmentPreProcessor.java | 32 +-
.../openstruct/ImmutableOpenStructDataSource.java | 123 ++++
.../segment/index/openstruct/MutableKeyColumn.java | 23 +-
.../index/openstruct/MutableOpenStructIndex.java | 86 ++-
.../index/openstruct/OpenStructIndexType.java | 33 ++
.../segment/readers/PinotSegmentRecordReader.java | 49 +-
.../segment/store/SegmentLocalFSDirectory.java | 8 +-
.../local/startree/StarTreeBuilderUtils.java | 34 ++
.../startree/v2/builder/BaseSingleTreeBuilder.java | 20 +
.../v2/builder/OffHeapSingleTreeBuilder.java | 108 ++--
.../startree/v2/store/StarTreeLoaderUtils.java | 19 +-
.../upsert/BasePartitionUpsertMetadataManager.java | 54 +-
...oncurrentMapPartitionUpsertMetadataManager.java | 9 +-
...nUpsertMetadataManagerForConsistentDeletes.java | 10 +-
.../customobject/CpcSketchAccumulatorTest.java | 32 ++
.../customobject/ThetaSketchAccumulatorTest.java | 33 ++
.../TupleIntSketchAccumulatorTest.java | 47 ++
...apPartitionDedupMetadataManagerWithTTLTest.java | 31 +
.../mutable/MutableSegmentImplOpenStructTest.java | 106 ++++
.../impl/vector/MutableVectorIndexTest.java | 395 ++++++++++---
.../openstruct/OpenStructColumnSplitterTest.java | 15 +-
.../segment/index/SegmentMetadataImplTest.java | 234 +++++++-
.../forward/ForwardIndexCreatorFactoryTest.java | 30 +-
.../index/loader/SegmentPreProcessorTest.java | 272 +++++++++
.../ImmutableOpenStructDataSourceTest.java | 389 ++++++++++++-
.../openstruct/MutableOpenStructIndexTest.java | 155 ++++-
.../index/openstruct/OpenStructIndexTypeTest.java | 30 +
.../index/vector/VectorSearchBenchmark.java | 389 -------------
.../readers/PinotSegmentRecordReaderCloseTest.java | 90 +++
.../v2/builder/OffHeapSingleTreeBuilderTest.java | 254 ++++++++
...rrentMapPartitionUpsertMetadataManagerTest.java | 33 ++
.../pinot/segment/spi/AggregationFunctionType.java | 23 +-
.../spi/datasource/OpenStructDataSource.java | 26 +
.../index/creator/VectorIndexConfigValidator.java | 30 +-
.../spi/index/metadata/ColumnMetadataImpl.java | 135 ++++-
.../spi/index/metadata/SegmentMetadataImpl.java | 15 +-
.../creator/VectorIndexConfigValidatorTest.java | 46 ++
.../spi/index/metadata/ColumnMetadataImplTest.java | 528 ++++++++++++++++-
.../pinot/server/api/resources/TablesResource.java | 77 +++
.../pinot/server/starter/ServerInstance.java | 8 +
.../server/starter/helix/BaseServerStarter.java | 18 +-
.../server/warmup/PageCacheWarmupQueryUtils.java | 99 ++++
.../warmup/PageCacheWarmupServerQueryExecutor.java | 334 +++++++++++
.../PageCacheWarmupServerQueryExecutorTest.java | 166 ++++++
.../spi/config/table/PageCacheWarmupConfig.java | 112 ++++
.../spi/config/table/PageCacheWarmupRequest.java | 65 +++
.../apache/pinot/spi/config/table/TableConfig.java | 21 +-
.../apache/pinot/spi/exception/QueryErrorCode.java | 6 +
.../ColumnNameInterner.java} | 24 +-
.../apache/pinot/spi/utils/CommonConstants.java | 118 +++-
.../utils/ProtoBufDescriptorFallbackListener.java | 87 +++
.../utils/builder/ControllerRequestURLBuilder.java | 4 +
.../spi/utils/builder/TableConfigBuilder.java | 9 +-
.../ProtoBufDescriptorFallbackListenerTest.java | 75 +++
.../pinot/tools/admin/PinotAdministrator.java | 2 +
.../GeneratePageCacheWarmupQueriesCommand.java | 335 +++++++++++
.../pinot/tools/pagecache/WarmupQuerySource.java | 214 +++++++
.../pagecache/QueryLogWarmupQuerySourceTest.java | 91 +++
pom.xml | 52 +-
348 files changed, 23773 insertions(+), 4581 deletions(-)
create mode 100644 .github/scripts/pr_flow/README.md
create mode 100644 .github/scripts/pr_flow/main.py
create mode 100644 .github/scripts/pr_flow/model.py
create mode 100644 .github/scripts/pr_flow/test_main.py
create mode 100644 .github/scripts/pr_flow/test_model.py
create mode 100644 .github/workflows/pr-flow-checks.yml
copy docker/images/pinot-superset/Dockerfile =>
.github/workflows/pr-flow-signal.yml (66%)
create mode 100644 .github/workflows/pr-flow.yml
create mode 100644 RELEASE_POLICY.md
create mode 100644 contrib/jupyter-jupysql/README.md
create mode 100644 contrib/jupyter-jupysql/pinot_jupysql_eda.ipynb
copy docker/images/pinot-superset/.dockerignore =>
contrib/jupyter-jupysql/requirements.txt (86%)
create mode 100644 kb/plugin-modules.md
create mode 100644
pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/ApproximateFunctionOverrideProvider.java
create mode 100644
pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BrokerWarmupConfig.java
create mode 100644
pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/ServerPreConnector.java
create mode 100644
pinot-broker/src/test/java/org/apache/pinot/broker/broker/helix/BrokerWarmupGateTest.java
create mode 100644
pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/ApproximateFunctionOverrideProviderTest.java
create mode 100644
pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/BrokerWarmupTest.java
create mode 100644
pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/ServerPreConnectorTest.java
create mode 100644
pinot-clients/pinot-java-client/src/main/java/org/apache/pinot/client/admin/PageCacheWarmupAdminClient.java
create mode 100644
pinot-common/src/main/java/org/apache/pinot/common/pagecache/WarmupQueryUtils.java
create mode 100644
pinot-common/src/main/java/org/apache/pinot/common/utils/config/QueryOptionConfigListener.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/pagecache/WarmupQueryUtilsTest.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/common/utils/config/QueryOptionConfigListenerTest.java
create mode 100644
pinot-common/src/test/java/org/apache/pinot/sql/parsers/SqlOptionsModeTest.java
create mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PageCacheWarmupRestletResource.java
create mode 100644
pinot-controller/src/main/java/org/apache/pinot/controller/util/PageCacheWarmupControllerExecutor.java
create mode 100644
pinot-controller/src/test/java/org/apache/pinot/controller/util/PageCacheWarmupControllerExecutorTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/query/distinct/table/DistinctTableMergeAccountingTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/core/query/reduce/GroupByFinalSchemaTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/queries/HavingQueriesTest.java
create mode 100644
pinot-core/src/test/java/org/apache/pinot/queries/StarTreeNullHandlingQueriesTest.java
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/ApproximateFunctionOverrideIntegrationTest.java
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BrokerServerPreConnectIntegrationTest.java
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BrokerStartupWarmupIntegrationTest.java
create mode 100644
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/PageCacheWarmupIntegrationTest.java
create mode 100644
pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkConstantCastPlanning.java
delete mode 100644
pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkGroovyExpressionEvaluation.java
delete mode 100644
pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkJsonKeyMap.java
delete mode 100644
pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkOffHeapDictionaryMemory.java
delete mode 100644
pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkQueryEngine.java
delete mode 100644
pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkRoaringBitmapCreation.java
delete mode 100644
pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkRoaringBitmapMapping.java
create mode 100644
pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkScalarCastPlanning.java
create mode 100644
pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkV7ForwardIndexWriter.java
delete mode 100644
pinot-perf/src/main/java/org/apache/pinot/perf/ForwardIndexWriterBenchmark.java
delete mode 100644
pinot-perf/src/main/java/org/apache/pinot/perf/LazyDataList.java
delete mode 100644
pinot-perf/src/main/java/org/apache/pinot/perf/RawIndexBenchmark.java
delete mode 100644
pinot-perf/src/main/java/org/apache/pinot/perf/StringDictionaryPerfTest.java
delete mode 100644
pinot-perf/src/main/java/org/apache/pinot/perf/aggregation/BenchmarkSumQuery.java
create mode 100644
pinot-plugins/pinot-input-format/pinot-protobuf/src/test/java/org/apache/pinot/plugin/inputformat/protobuf/ProtoBufTempFileLeakTest.java
create mode 100644
pinot-plugins/pinot-input-format/pinot-protobuf/src/test/java/org/apache/pinot/plugin/inputformat/protobuf/ProtoBufUtilsDescriptorCacheTest.java
create mode 100644
pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotApproximateAggregateRewriteRule.java
create mode 100644
pinot-query-planner/src/main/java/org/apache/pinot/calcite/rex/PinotRexExecutor.java
create mode 100644
pinot-query-planner/src/test/java/org/apache/pinot/calcite/rex/PinotRexExecutorTest.java
create mode 100644
pinot-query-planner/src/test/java/org/apache/pinot/query/ApproximateAggregateRewriteTest.java
create mode 100644
pinot-query-planner/src/test/java/org/apache/pinot/query/ConstantCastPlanningTest.java
create mode 100644
pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/ExplainableOperator.java
create mode 100644
pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/OperatorBufferReleaseTest.java
create mode 100644
pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/set/SetOperatorBufferReleaseTest.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/indexsegment/mutable/MutableSegmentImplOpenStructTest.java
delete mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/vector/VectorSearchBenchmark.java
create mode 100644
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/readers/PinotSegmentRecordReaderCloseTest.java
create mode 100644
pinot-server/src/main/java/org/apache/pinot/server/warmup/PageCacheWarmupQueryUtils.java
create mode 100644
pinot-server/src/main/java/org/apache/pinot/server/warmup/PageCacheWarmupServerQueryExecutor.java
create mode 100644
pinot-server/src/test/java/org/apache/pinot/server/warmup/PageCacheWarmupServerQueryExecutorTest.java
create mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/config/table/PageCacheWarmupConfig.java
create mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/config/table/PageCacheWarmupRequest.java
copy
pinot-spi/src/main/java/org/apache/pinot/spi/{config/table/TableCustomConfig.java
=> utils/ColumnNameInterner.java} (59%)
create mode 100644
pinot-spi/src/main/java/org/apache/pinot/spi/utils/ProtoBufDescriptorFallbackListener.java
create mode 100644
pinot-spi/src/test/java/org/apache/pinot/spi/utils/ProtoBufDescriptorFallbackListenerTest.java
create mode 100644
pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/GeneratePageCacheWarmupQueriesCommand.java
create mode 100644
pinot-tools/src/main/java/org/apache/pinot/tools/pagecache/WarmupQuerySource.java
create mode 100644
pinot-tools/src/test/java/org/apache/pinot/tools/pagecache/QueryLogWarmupQuerySourceTest.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
