This is an automated email from the ASF dual-hosted git repository.
Yukang-Lian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new a94b7a0e629 [improvement](compaction) Disable MOW version-count
promotion by default (#67178)
a94b7a0e629 is described below
commit a94b7a0e62918f70006aa9aa95fec25df2c7c017
Author: Jamie <[email protected]>
AuthorDate: Mon Oct 26 17:03:54 2026 +0800
[improvement](compaction) Disable MOW version-count promotion by default
(#67178)
### What problem does this PR solve?
Issue Number: None
Related PR: #49383
Problem Summary:
Size-based cumulative compaction currently promotes a merge-on-write
output rowset to the base layer once its version span exceeds
`compaction_promotion_version_count`, even when the output rowset is
only a few hundred KB. For high-frequency tiny loads, this moves small
rowsets into the base layer prematurely and causes frequent base
compactions.
PR #49383 makes cumulative compaction aggregate historical delete
bitmaps into the output rowset version and remove the pre-rowset delete
bitmap versions. With that optimization enabled, raw rowset version span
no longer needs to trigger promotion by default.
This PR changes the default `compaction_promotion_version_count` from
1,000 to the maximum `int64` value. This effectively disables
version-count promotion by default in both local and cloud modes, while
retaining the existing logic and configuration as an operational
fallback. Size-based and delete-version promotion remain unchanged.
### Release note
MOW cumulative compaction no longer promotes output rowsets by version
count under the default configuration. The existing configuration
remains available as a fallback.
---
be/src/common/config.cpp | 8 ++++----
be/src/common/config.h | 6 +++---
.../cloud_cumulative_compaction_policy_test.cpp | 13 ++++++++++++
.../cumulative_compaction_policy_test.cpp | 23 ++++++++++++++++++++++
4 files changed, 43 insertions(+), 7 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 4c9957dfdfa..a98196fa334 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -534,10 +534,10 @@ DEFINE_mDouble(compaction_promotion_ratio, "0.05");
DEFINE_mInt64(compaction_promotion_min_size_mbytes, "128");
// When output rowset of cumulative compaction total version count
(end_version - start_version)
-// exceed this config count, the rowset will be moved to base compaction
-// NOTE: this config will work for unique key merge-on-write table only, to
reduce version count
-// related cost on delete bitmap more effectively.
-DEFINE_mInt64(compaction_promotion_version_count, "1000");
+// exceed this config count, the rowset will be moved to base compaction.
+// NOTE: this config only works for unique key merge-on-write tables. The
default maximum value
+// disables version-count-based promotion while retaining the option to enable
it when needed.
+DEFINE_mInt64(compaction_promotion_version_count, "9223372036854775807");
// The lower bound size to do cumulative compaction. When total disk size of
candidate rowsets is less than
// this size, size_based policy may not do to cumulative compaction. The unit
is m byte.
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 62becc90f28..b2f352d1fb5 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -615,9 +615,9 @@ DECLARE_mDouble(compaction_promotion_ratio);
DECLARE_mInt64(compaction_promotion_min_size_mbytes);
// When output rowset of cumulative compaction total version count
(end_version - start_version)
-// exceed this config count, the rowset will be moved to base compaction
-// NOTE: this config will work for unique key merge-on-write table only, to
reduce version count
-// related cost on delete bitmap more effectively.
+// exceed this config count, the rowset will be moved to base compaction.
+// NOTE: this config only works for unique key merge-on-write tables. The
default maximum value
+// disables version-count-based promotion while retaining the option to enable
it when needed.
DECLARE_mInt64(compaction_promotion_version_count);
// The lower bound size to do cumulative compaction. When total disk size of
candidate rowsets is less than
diff --git a/be/test/cloud/cloud_cumulative_compaction_policy_test.cpp
b/be/test/cloud/cloud_cumulative_compaction_policy_test.cpp
index 01904b3391a..86128a2ac62 100644
--- a/be/test/cloud/cloud_cumulative_compaction_policy_test.cpp
+++ b/be/test/cloud/cloud_cumulative_compaction_policy_test.cpp
@@ -214,6 +214,19 @@ TEST_F(TestCloudSizeBasedCumulativeCompactionPolicy,
new_cumulative_point) {
EXPECT_EQ(policy.new_cumulative_point(&_tablet, output_rowset, version,
2), 6);
}
+TEST_F(TestCloudSizeBasedCumulativeCompactionPolicy,
+
new_cumulative_point_does_not_promote_small_mow_rowset_with_default_version_limit)
{
+ CloudTablet tablet(_engine, _tablet_meta);
+ tablet._tablet_meta->_enable_unique_key_merge_on_write = true;
+ tablet._base_size = kGiB;
+
+ CloudSizeBasedCumulativeCompactionPolicy policy;
+ RowsetSharedPtr output_rowset = create_rowset(Version(2, 1003), 1, false,
kMiB);
+ Version last_delete_version {-1, -1};
+
+ EXPECT_EQ(2, policy.new_cumulative_point(&tablet, output_rowset,
last_delete_version, 2));
+}
+
TEST_F(TestCloudSizeBasedCumulativeCompactionPolicy,
pick_input_rowsets_notready_keeps_latest_versions) {
auto base_rowset = create_rowset(Version(0, 1), 1, false, kGiB);
diff --git a/be/test/storage/compaction/cumulative_compaction_policy_test.cpp
b/be/test/storage/compaction/cumulative_compaction_policy_test.cpp
index 33dce1a6d8d..a2f5ba3d593 100644
--- a/be/test/storage/compaction/cumulative_compaction_policy_test.cpp
+++ b/be/test/storage/compaction/cumulative_compaction_policy_test.cpp
@@ -1238,6 +1238,29 @@ TEST_F(TestSizeBasedCumulativeCompactionPolicy,
EXPECT_EQ(0, compaction_score);
}
+TEST_F(TestSizeBasedCumulativeCompactionPolicy,
+
update_cumulative_point_does_not_promote_small_mow_rowset_with_default_version_limit)
{
+ _tablet_meta->set_enable_unique_key_merge_on_write(true);
+ TabletSharedPtr tablet(
+ new Tablet(_engine, _tablet_meta, nullptr,
CUMULATIVE_SIZE_BASED_POLICY));
+ ASSERT_TRUE(tablet->init().ok());
+ tablet->set_cumulative_layer_point(1);
+ tablet->set_cumulative_promotion_size(64L * kMiB);
+
+ RowsetMetaSharedPtr output_meta(new RowsetMeta());
+ init_rs_meta(output_meta, 1, 1002);
+ output_meta->set_total_disk_size(kMiB);
+ RowsetSharedPtr output_rowset;
+ ASSERT_TRUE(RowsetFactory::create_rowset(nullptr, "", output_meta,
&output_rowset).ok());
+
+ std::vector<RowsetSharedPtr> input_rowsets;
+ Version last_delete_version {-1, -1};
+ tablet->_cumulative_compaction_policy->update_cumulative_point(
+ tablet.get(), input_rowsets, output_rowset, last_delete_version);
+
+ EXPECT_EQ(1, tablet->cumulative_layer_point());
+}
+
TEST_F(TestSizeBasedCumulativeCompactionPolicy,
pick_input_rowsets_large_head_single_overlapping_tail_selected) {
std::vector<RowsetMetaSharedPtr> rs_metas;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
