This is an automated email from the ASF dual-hosted git repository.
mrhhsg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new c04e7e17c2a [refactor](be) Move adaptive block size predictor to core
(#65995)
c04e7e17c2a is described below
commit c04e7e17c2afdd6cc4ac7a7994cd8f4bbaa7df28
Author: Jerry Hu <[email protected]>
AuthorDate: Fri Jul 31 10:36:31 2026 +0800
[refactor](be) Move adaptive block size predictor to core (#65995)
### What problem does this PR solve?
Issue Number: None
Related PR: #62835
Problem Summary:
`AdaptiveBlockSizePredictor` is a block-level helper shared by segment
scans and external file scanners. Keeping it under `storage/segment`
forces external scan code to depend on the storage layer even though the
predictor only uses `Block` statistics and EWMA calculations.
This PR moves the predictor implementation and its focused unit test to
`core/block`, updates all consumers to use the new include path, and
removes the unused segment-specific `ColumnMetadata` declaration. It
also makes `predict_next_rows()` const because it does not modify
predictor state. The prediction behavior is unchanged.
The branch has been rebuilt on the current `master`, preserving the
later external-scan rollback from #65998 rather than restoring the old
`update(rows, bytes)` path.
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [x] Unit Test
- ASAN focused harness compiled the unchanged production and test
sources directly: `AdaptiveBlockSizePredictorTest.*` (25 tests passed)
- The standard `./run-be-ut.sh --run
--filter='AdaptiveBlockSizePredictorTest.*' -j 48` runner did not
complete because this worktree's third-party installation is missing the
unrelated `liblance_c.a`; CI will provide the full monolithic BE
validation
- [x] Manual test (add detailed scripts or steps below)
- Targeted ASAN_UT compilation passed for the predictor implementation,
its focused test, and all three production consumers (`SegmentIterator`,
`FileScanner`, and `FileScannerV2`)
- `./build-support/clang-format.sh`
- `./build-support/check-format.sh`
- changed-line clang-tidy passed; the stock local invocation
additionally encounters a pre-existing unmatched `NOLINTEND` in
`be/src/core/types.h`
- `git diff --check origin/master...HEAD`
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [x] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [x] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
.../block}/adaptive_block_size_predictor.cpp | 4 +--
.../block}/adaptive_block_size_predictor.h | 20 +++-----------
be/src/exec/scan/file_scanner.h | 2 +-
be/src/exec/scan/file_scanner_v2.h | 2 +-
be/src/storage/segment/segment_iterator.cpp | 1 -
be/src/storage/segment/segment_iterator.h | 2 +-
.../block}/adaptive_block_size_predictor_test.cpp | 32 ++++++++++------------
7 files changed, 23 insertions(+), 40 deletions(-)
diff --git a/be/src/storage/segment/adaptive_block_size_predictor.cpp
b/be/src/core/block/adaptive_block_size_predictor.cpp
similarity index 95%
rename from be/src/storage/segment/adaptive_block_size_predictor.cpp
rename to be/src/core/block/adaptive_block_size_predictor.cpp
index d8cc700f579..3d7fb6e5a8b 100644
--- a/be/src/storage/segment/adaptive_block_size_predictor.cpp
+++ b/be/src/core/block/adaptive_block_size_predictor.cpp
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-#include "storage/segment/adaptive_block_size_predictor.h"
+#include "core/block/adaptive_block_size_predictor.h"
#include <algorithm>
#include <cstddef>
@@ -46,7 +46,7 @@ void AdaptiveBlockSizePredictor::update(const Block& block) {
}
}
-size_t AdaptiveBlockSizePredictor::predict_next_rows() {
+size_t AdaptiveBlockSizePredictor::predict_next_rows() const {
if (_block_size_bytes == 0) {
return _block_size_rows;
}
diff --git a/be/src/storage/segment/adaptive_block_size_predictor.h
b/be/src/core/block/adaptive_block_size_predictor.h
similarity index 87%
rename from be/src/storage/segment/adaptive_block_size_predictor.h
rename to be/src/core/block/adaptive_block_size_predictor.h
index e03f18c2a53..4ee85117517 100644
--- a/be/src/storage/segment/adaptive_block_size_predictor.h
+++ b/be/src/core/block/adaptive_block_size_predictor.h
@@ -17,12 +17,7 @@
#pragma once
-#include <stddef.h>
-#include <stdint.h>
-
-#include <vector>
-
-#include "storage/olap_common.h"
+#include <cstddef>
namespace doris {
@@ -41,12 +36,6 @@ public:
static constexpr size_t kDefaultProbeRows = 4096;
static constexpr size_t kDefaultBlockSizeRows = 65535;
- // Per-column metadata for computing segment-level hints.
- struct ColumnMetadata {
- ColumnId column_id;
- uint64_t raw_bytes; // total raw data bytes for this column in the
segment
- };
-
// |preferred_block_size_bytes|: target total bytes of each output block
chunk.
// |metadata_hint_bytes_per_row|: pre-computed conservative estimate from
metadata (e.g.
// segment footer or file statistics). 0.0 means no hint available.
@@ -65,7 +54,7 @@ public:
// Never exceeds |block_size_rows|; never returns less than 1.
// Uses pre-computed metadata hint for first-call estimate when no history
exists.
// Does NOT modify internal state (_has_history is only flipped by
update()).
- size_t predict_next_rows();
+ size_t predict_next_rows() const;
bool has_history() const { return _has_history; }
@@ -84,9 +73,8 @@ private:
// Whether at least one update() has been called (i.e. we have real
measured history).
bool _has_history = false;
- // Cached conservative metadata estimate computed on the first
predict_next_rows() call.
- // Reused on subsequent first-round predictions (before _has_history is
set) to avoid
- // re-traversing the segment footer on every call.
+ // Conservative metadata estimate provided by the caller at construction.
+ // Reused for predictions before _has_history is set.
double _metadata_hint_bytes_per_row = 0.0;
#ifdef BE_TEST
diff --git a/be/src/exec/scan/file_scanner.h b/be/src/exec/scan/file_scanner.h
index f575cdae892..38b39d58db7 100644
--- a/be/src/exec/scan/file_scanner.h
+++ b/be/src/exec/scan/file_scanner.h
@@ -30,6 +30,7 @@
#include "common/factory_creator.h"
#include "common/global_types.h"
#include "common/status.h"
+#include "core/block/adaptive_block_size_predictor.h"
#include "core/block/block.h"
#include "exec/operator/file_scan_operator.h"
#include "exec/scan/file_scan_io_context.h"
@@ -43,7 +44,6 @@
#include "runtime/runtime_profile.h"
#include "storage/olap_common.h"
#include "storage/olap_scan_common.h"
-#include "storage/segment/adaptive_block_size_predictor.h"
#include "storage/segment/condition_cache.h"
namespace doris {
diff --git a/be/src/exec/scan/file_scanner_v2.h
b/be/src/exec/scan/file_scanner_v2.h
index fc217506eea..92edbc1a181 100644
--- a/be/src/exec/scan/file_scanner_v2.h
+++ b/be/src/exec/scan/file_scanner_v2.h
@@ -26,6 +26,7 @@
#include "common/factory_creator.h"
#include "common/status.h"
+#include "core/block/adaptive_block_size_predictor.h"
#include "core/block/block.h"
#include "exec/operator/file_scan_operator.h"
#include "exec/scan/scanner.h"
@@ -37,7 +38,6 @@
#include "gen_cpp/PlanNodes_types.h"
#include "io/io_common.h"
#include "runtime/runtime_profile.h"
-#include "storage/segment/adaptive_block_size_predictor.h"
namespace doris {
diff --git a/be/src/storage/segment/segment_iterator.cpp
b/be/src/storage/segment/segment_iterator.cpp
index bc04ffa2aca..84dd82fc932 100644
--- a/be/src/storage/segment/segment_iterator.cpp
+++ b/be/src/storage/segment/segment_iterator.cpp
@@ -382,7 +382,6 @@ std::unique_ptr<AdaptiveBlockSizePredictor>
SegmentIterator::_make_block_size_pr
// Collect per-column raw byte metadata from the segment footer for the
columns
// this iterator will actually output (defined by _schema, which is built
from
// _opts.return_columns).
- std::vector<AdaptiveBlockSizePredictor::ColumnMetadata> col_metadata;
uint32_t seg_rows = _segment->num_rows();
uint64_t total_raw_bytes = 0;
double metadata_hint_bytes_per_row = 0.0;
diff --git a/be/src/storage/segment/segment_iterator.h
b/be/src/storage/segment/segment_iterator.h
index 99e40f2a5d1..b67361d53ff 100644
--- a/be/src/storage/segment/segment_iterator.h
+++ b/be/src/storage/segment/segment_iterator.h
@@ -32,6 +32,7 @@
#include <vector>
#include "common/status.h"
+#include "core/block/adaptive_block_size_predictor.h"
#include "core/block/block.h"
#include "core/block/column_with_type_and_name.h"
#include "core/block/columns_with_type_and_name.h"
@@ -52,7 +53,6 @@
#include "storage/predicate/column_predicate.h"
#include "storage/row_cursor.h"
#include "storage/schema.h"
-#include "storage/segment/adaptive_block_size_predictor.h"
#include "storage/segment/common.h"
#include "storage/segment/segment.h"
#include "util/slice.h"
diff --git a/be/test/storage/segment/adaptive_block_size_predictor_test.cpp
b/be/test/core/block/adaptive_block_size_predictor_test.cpp
similarity index 94%
rename from be/test/storage/segment/adaptive_block_size_predictor_test.cpp
rename to be/test/core/block/adaptive_block_size_predictor_test.cpp
index 60b6f37b8ce..163ddcbdf8e 100644
--- a/be/test/storage/segment/adaptive_block_size_predictor_test.cpp
+++ b/be/test/core/block/adaptive_block_size_predictor_test.cpp
@@ -15,13 +15,16 @@
// specific language governing permissions and limitations
// under the License.
-#include "storage/segment/adaptive_block_size_predictor.h"
+#include "core/block/adaptive_block_size_predictor.h"
#include <gmock/gmock.h>
#include <gtest/gtest.h>
+#include <algorithm>
+#include <cstdint>
#include <memory>
-#include <vector>
+#include <string>
+#include <utility>
#include "common/config.h"
#include "core/block/block.h"
@@ -29,7 +32,6 @@
#include "core/column/column_vector.h"
#include "core/data_type/data_type_number.h"
#include "core/data_type/data_type_string.h"
-#include "storage/olap_common.h"
namespace doris {
@@ -81,7 +83,6 @@ TEST_F(AdaptiveBlockSizePredictorTest,
NoHistoryReturnsMaxRows) {
// After one update the first sample is stored directly (no EWMA blending).
Block blk = make_int32_block(100);
- std::vector<ColumnId> cols = {0};
pred.update(blk);
EXPECT_TRUE(pred.has_history_for_test());
@@ -95,8 +96,6 @@ TEST_F(AdaptiveBlockSizePredictorTest,
NoHistoryReturnsMaxRows) {
TEST_F(AdaptiveBlockSizePredictorTest, EwmaConvergence) {
AdaptiveBlockSizePredictor pred(kBlockBytes, 0.0);
- std::vector<ColumnId> cols = {0};
-
// Compute expected bytes-per-row from an actual block so the test does not
// hard-code internal column memory layout assumptions.
Block probe = make_string_block(100, 100);
@@ -120,7 +119,6 @@ TEST_F(AdaptiveBlockSizePredictorTest,
ZeroRowsBlockIgnored) {
// update() with an empty block must be a no-op.
Block blk = make_int32_block(0);
- std::vector<ColumnId> cols = {0};
pred.update(blk);
EXPECT_FALSE(pred.has_history_for_test());
@@ -139,7 +137,6 @@ TEST_F(AdaptiveBlockSizePredictorTest,
DisabledWhenBlockSizeIsZero) {
AdaptiveBlockSizePredictor pred(0, 0.0);
Block blk = make_int32_block(1000);
- std::vector<ColumnId> cols = {0};
pred.update(blk);
// update() still records history even when budget == 0.
@@ -162,7 +159,6 @@ TEST_F(AdaptiveBlockSizePredictorTest,
PredictReturnsBlockSizeRowsWhenDisabled)
// Even after update, still returns block_size_rows because
block_size_bytes == 0.
Block blk = make_int32_block(100);
- std::vector<ColumnId> cols = {0};
pred.update(blk);
EXPECT_EQ(pred.predict_next_rows(), pred.block_size_rows_for_test());
}
@@ -191,7 +187,7 @@ TEST_F(AdaptiveBlockSizePredictorTest,
PredictNoHistoryMetadataHint) {
size_t result = pred.predict_next_rows();
- size_t expected = static_cast<size_t>(static_cast<double>(kBlockBytes) /
hint_bpr);
+ auto expected = static_cast<size_t>(static_cast<double>(kBlockBytes) /
hint_bpr);
// No history: probe_rows clamps the result.
expected = std::min(expected, pred.probe_rows_for_test());
EXPECT_EQ(result, expected);
@@ -237,7 +233,7 @@ TEST_F(AdaptiveBlockSizePredictorTest,
PredictWithHistoryNoClamping) {
pred.set_has_history_for_test(true, 100.0);
size_t result = pred.predict_next_rows();
- EXPECT_EQ(result, 81u);
+ EXPECT_EQ(result, 81U);
}
// ── Test: predicted > block_size_rows → clamped to block_size_rows
─────────────
@@ -257,7 +253,7 @@ TEST_F(AdaptiveBlockSizePredictorTest, PredictClampedToOne)
{
// bytes_per_row so large that predicted rounds to 0.
pred.set_has_history_for_test(true, static_cast<double>(kBlockBytes) *
10.0);
- EXPECT_EQ(pred.predict_next_rows(), 1u);
+ EXPECT_EQ(pred.predict_next_rows(), 1U);
}
// ── Test: metadata hint with multiple columns ───────────────────────────────
@@ -268,7 +264,7 @@ TEST_F(AdaptiveBlockSizePredictorTest,
PredictNoHistoryMultiColumnMetadata) {
AdaptiveBlockSizePredictor pred(kBlockBytes, hint_bpr);
size_t result = pred.predict_next_rows();
- size_t expected = static_cast<size_t>(static_cast<double>(kBlockBytes) /
hint_bpr);
+ auto expected = static_cast<size_t>(static_cast<double>(kBlockBytes) /
hint_bpr);
// No history: probe_rows clamps the result.
expected = std::min(expected, pred.probe_rows_for_test());
EXPECT_EQ(result, expected);
@@ -301,14 +297,14 @@ TEST_F(AdaptiveBlockSizePredictorTest,
PredictUsesCustomProbeRowsWithHint) {
TEST_F(AdaptiveBlockSizePredictorTest, PredictProbeRowsZeroFallsBackToOne) {
AdaptiveBlockSizePredictor pred(kBlockBytes, 0.0, 0);
- EXPECT_EQ(pred.probe_rows_for_test(), 0u);
- EXPECT_EQ(pred.predict_next_rows(), 1u);
+ EXPECT_EQ(pred.probe_rows_for_test(), 0U);
+ EXPECT_EQ(pred.predict_next_rows(), 1U);
}
TEST_F(AdaptiveBlockSizePredictorTest, PredictProbeRowsOneWorks) {
AdaptiveBlockSizePredictor pred(kBlockBytes, 0.0, 1);
- EXPECT_EQ(pred.predict_next_rows(), 1u);
+ EXPECT_EQ(pred.predict_next_rows(), 1U);
}
// ── batch_size tests ────────────────────────────────────────────────────────
@@ -318,7 +314,7 @@ TEST_F(AdaptiveBlockSizePredictorTest,
DefaultBlockSizeRows) {
EXPECT_EQ(pred.block_size_rows_for_test(),
AdaptiveBlockSizePredictor::default_block_size_rows_for_test());
- EXPECT_EQ(pred.block_size_rows_for_test(), 65535u);
+ EXPECT_EQ(pred.block_size_rows_for_test(), 65535U);
}
TEST_F(AdaptiveBlockSizePredictorTest, CustomBlockSizeRows) {
@@ -351,7 +347,7 @@ TEST_F(AdaptiveBlockSizePredictorTest,
BlockSizeRowsDoesNotAffectSmallPrediction
// 100 bytes/row → predicted = 8192/100 = 81 < custom_rows.
pred.set_has_history_for_test(true, 100.0);
- EXPECT_EQ(pred.predict_next_rows(), 81u);
+ EXPECT_EQ(pred.predict_next_rows(), 81U);
}
} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
