This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 0460f62ca5f branch-3.0: [Enhancement](Compaction) Make base compaction
use the same tablet selection strategy as cumulative compaction #51649 (#52389)
0460f62ca5f is described below
commit 0460f62ca5f0097e067d8fe86c4d4cf1acff2dbc
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Mon Jun 30 15:52:13 2025 +0800
branch-3.0: [Enhancement](Compaction) Make base compaction use the same
tablet selection strategy as cumulative compaction #51649 (#52389)
Cherry-picked from #51649
Co-authored-by: abmdocrt <[email protected]>
---
be/src/cloud/cloud_tablet_mgr.cpp | 24 +++++++++++++++++-------
be/src/cloud/config.cpp | 2 +-
be/src/cloud/config.h | 2 +-
3 files changed, 19 insertions(+), 9 deletions(-)
diff --git a/be/src/cloud/cloud_tablet_mgr.cpp
b/be/src/cloud/cloud_tablet_mgr.cpp
index cbf9a29ee90..25f50f7ef4b 100644
--- a/be/src/cloud/cloud_tablet_mgr.cpp
+++ b/be/src/cloud/cloud_tablet_mgr.cpp
@@ -31,6 +31,10 @@
namespace doris {
uint64_t g_tablet_report_inactive_duration_ms = 0;
+bvar::Adder<uint64_t> g_base_compaction_not_frozen_tablet_num(
+ "base_compaction_not_frozen_tablet_num");
+bvar::Adder<uint64_t> g_cumu_compaction_not_frozen_tablet_num(
+ "cumu_compaction_not_frozen_tablet_num");
namespace {
// port from
@@ -353,19 +357,25 @@ Status CloudTabletMgr::get_topn_tablets_to_compact(
using namespace std::chrono;
auto now =
duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count();
auto skip = [now, compaction_type](CloudTablet* t) {
+ int32_t max_version_config = t->max_version_config();
if (compaction_type == CompactionType::BASE_COMPACTION) {
- return now - t->last_base_compaction_success_time_ms <
config::base_compaction_freeze_interval_s * 1000 ||
- now - t->last_base_compaction_failure_time() <
config::min_compaction_failure_interval_ms;
+ bool is_recent_failure = now -
t->last_base_compaction_failure_time() <
config::min_compaction_failure_interval_ms;
+ bool is_frozen = (now - t->last_load_time_ms >
config::compaction_load_max_freeze_interval_s * 1000
+ && now - t->last_base_compaction_success_time_ms <
config::base_compaction_freeze_interval_s * 1000
+ && t->fetch_add_approximate_num_rowsets(0) <
max_version_config / 2);
+ g_base_compaction_not_frozen_tablet_num << !is_frozen;
+ return is_recent_failure || is_frozen;
}
+
// If tablet has too many rowsets but not be compacted for a long
time, compaction should be performed
// regardless of whether there is a load job recently.
-
- int32_t max_version_config = t->max_version_config();
- return now - t->last_cumu_compaction_failure_time() <
config::min_compaction_failure_interval_ms ||
- now - t->last_cumu_no_suitable_version_ms <
config::min_compaction_failure_interval_ms ||
- (now - t->last_load_time_ms >
config::cu_compaction_freeze_interval_s * 1000
+ bool is_recent_failure = now - t->last_cumu_compaction_failure_time()
< config::min_compaction_failure_interval_ms;
+ bool is_recent_no_suitable_version = now -
t->last_cumu_no_suitable_version_ms <
config::min_compaction_failure_interval_ms;
+ bool is_frozen = (now - t->last_load_time_ms >
config::compaction_load_max_freeze_interval_s * 1000
&& now - t->last_cumu_compaction_success_time_ms <
config::cumu_compaction_interval_s * 1000
&& t->fetch_add_approximate_num_rowsets(0) < max_version_config
/ 2);
+ g_cumu_compaction_not_frozen_tablet_num << !is_frozen;
+ return is_recent_failure || is_recent_no_suitable_version || is_frozen;
};
// We don't schedule tablets that are disabled for compaction
auto disable = [](CloudTablet* t) { return
t->tablet_meta()->tablet_schema()->disable_auto_compaction(); };
diff --git a/be/src/cloud/config.cpp b/be/src/cloud/config.cpp
index 800df55b3d6..d4ae356236b 100644
--- a/be/src/cloud/config.cpp
+++ b/be/src/cloud/config.cpp
@@ -43,7 +43,7 @@ DEFINE_mInt32(init_scanner_sync_rowsets_parallelism, "10");
DEFINE_mInt64(min_compaction_failure_interval_ms, "5000");
DEFINE_mInt64(base_compaction_freeze_interval_s, "7200");
-DEFINE_mInt64(cu_compaction_freeze_interval_s, "1200");
+DEFINE_mInt64(compaction_load_max_freeze_interval_s, "1200");
DEFINE_mInt64(cumu_compaction_interval_s, "1800");
DEFINE_mInt32(compaction_timeout_seconds, "86400");
diff --git a/be/src/cloud/config.h b/be/src/cloud/config.h
index 76e37163828..6f550ff25f8 100644
--- a/be/src/cloud/config.h
+++ b/be/src/cloud/config.h
@@ -77,7 +77,7 @@ DECLARE_mInt64(min_compaction_failure_interval_ms);
DECLARE_mBool(enable_new_tablet_do_compaction);
// For cloud read/write separate mode
DECLARE_mInt64(base_compaction_freeze_interval_s);
-DECLARE_mInt64(cu_compaction_freeze_interval_s);
+DECLARE_mInt64(compaction_load_max_freeze_interval_s);
DECLARE_mInt64(cumu_compaction_interval_s);
DECLARE_mInt32(compaction_timeout_seconds);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
