This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new d5f639d401e branch-3.0: [enhancement](compaction) replace hardcoded
compaction thresholds with config #49662 (#49681)
d5f639d401e is described below
commit d5f639d401efeae99f7678dccad0ecb2d969d810
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Mon Mar 31 19:52:15 2025 +0800
branch-3.0: [enhancement](compaction) replace hardcoded compaction
thresholds with config #49662 (#49681)
Cherry-picked from #49662
Co-authored-by: Luwei <[email protected]>
---
be/src/agent/task_worker_pool.cpp | 7 +++++--
be/src/cloud/config.cpp | 1 -
be/src/cloud/config.h | 1 -
be/src/common/config.cpp | 5 +++++
be/src/common/config.h | 5 +++++
be/src/olap/base_compaction.cpp | 2 +-
be/src/olap/olap_server.cpp | 3 ++-
be/src/olap/tablet_manager.cpp | 2 +-
8 files changed, 19 insertions(+), 7 deletions(-)
diff --git a/be/src/agent/task_worker_pool.cpp
b/be/src/agent/task_worker_pool.cpp
index 86ece5125ed..846cec00bd4 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -1885,14 +1885,17 @@ void
PublishVersionWorkerPool::publish_version_callback(const TAgentTaskRequest&
.error(status);
} else {
if (!config::disable_auto_compaction &&
-
!GlobalMemoryArbitrator::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE)) {
+ (!config::enable_compaction_pause_on_high_memory ||
+
!GlobalMemoryArbitrator::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE))) {
for (auto [tablet_id, _] : succ_tablets) {
TabletSharedPtr tablet =
_engine.tablet_manager()->get_tablet(tablet_id);
if (tablet != nullptr) {
if
(!tablet->tablet_meta()->tablet_schema()->disable_auto_compaction()) {
tablet->published_count.fetch_add(1);
int64_t published_count =
tablet->published_count.load();
- if
(tablet->exceed_version_limit(config::max_tablet_version_num * 2 / 3) &&
+ if (tablet->exceed_version_limit(
+ config::max_tablet_version_num *
+
config::load_trigger_compaction_version_percent / 100) &&
published_count % 20 == 0) {
auto st = _engine.submit_compaction_task(
tablet,
CompactionType::CUMULATIVE_COMPACTION, true, false);
diff --git a/be/src/cloud/config.cpp b/be/src/cloud/config.cpp
index 84df8c016e7..f66b29b38ca 100644
--- a/be/src/cloud/config.cpp
+++ b/be/src/cloud/config.cpp
@@ -46,7 +46,6 @@ DEFINE_mInt64(cumu_compaction_interval_s, "1800");
DEFINE_mInt32(compaction_timeout_seconds, "86400");
DEFINE_mInt32(lease_compaction_interval_seconds, "20");
-DEFINE_mInt64(base_compaction_interval_seconds_since_last_operation, "86400");
DEFINE_mBool(enable_parallel_cumu_compaction, "false");
DEFINE_mDouble(base_compaction_thread_num_factor, "0.25");
DEFINE_mDouble(cumu_compaction_thread_num_factor, "0.5");
diff --git a/be/src/cloud/config.h b/be/src/cloud/config.h
index 72151b9ce64..7453af2b597 100644
--- a/be/src/cloud/config.h
+++ b/be/src/cloud/config.h
@@ -79,7 +79,6 @@ DECLARE_mInt64(cumu_compaction_interval_s);
DECLARE_mInt32(compaction_timeout_seconds);
DECLARE_mInt32(lease_compaction_interval_seconds);
-DECLARE_mInt64(base_compaction_interval_seconds_since_last_operation);
DECLARE_mBool(enable_parallel_cumu_compaction);
DECLARE_mDouble(base_compaction_thread_num_factor);
DECLARE_mDouble(cumu_compaction_thread_num_factor);
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index b6ae5ccb5b3..f36bd991e09 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1473,6 +1473,11 @@
DEFINE_mBool(enable_prune_delete_sign_when_base_compaction, "true");
DEFINE_Bool(enable_root_path_of_hdfs_resource, "true");
+DEFINE_mInt32(tablet_sched_delay_time_ms, "5000");
+DEFINE_mInt32(load_trigger_compaction_version_percent, "66");
+DEFINE_mInt64(base_compaction_interval_seconds_since_last_operation, "86400");
+DEFINE_mBool(enable_compaction_pause_on_high_memory, "true");
+
// clang-format off
#ifdef BE_TEST
// test s3
diff --git a/be/src/common/config.h b/be/src/common/config.h
index b25c4849d2e..f927c0ae704 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1549,6 +1549,11 @@
DECLARE_mBool(enable_prune_delete_sign_when_base_compaction);
// and guarantee that the root_path works as expected.
DECLARE_Bool(enable_root_path_of_hdfs_resource);
+DECLARE_mInt32(tablet_sched_delay_time_ms);
+DECLARE_mInt32(load_trigger_compaction_version_percent);
+DECLARE_mInt64(base_compaction_interval_seconds_since_last_operation);
+DECLARE_mBool(enable_compaction_pause_on_high_memory);
+
#ifdef BE_TEST
// test s3
DECLARE_String(test_s3_resource);
diff --git a/be/src/olap/base_compaction.cpp b/be/src/olap/base_compaction.cpp
index 8b9cbd75ed3..548042bad3e 100644
--- a/be/src/olap/base_compaction.cpp
+++ b/be/src/olap/base_compaction.cpp
@@ -197,7 +197,7 @@ Status BaseCompaction::pick_rowsets_to_compact() {
// 3. the interval since last base compaction reaches the threshold
int64_t base_creation_time = _input_rowsets[0]->creation_time();
- int64_t interval_threshold = 86400;
+ int64_t interval_threshold =
config::base_compaction_interval_seconds_since_last_operation;
int64_t interval_since_last_base_compaction = time(nullptr) -
base_creation_time;
if (interval_since_last_base_compaction > interval_threshold) {
VLOG_NOTICE << "satisfy the base compaction policy. tablet=" <<
_tablet->tablet_id()
diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index dfb02153d22..13dc75862bd 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -653,7 +653,8 @@ void StorageEngine::_compaction_tasks_producer_callback() {
int64_t interval = config::generate_compaction_tasks_interval_ms;
do {
if (!config::disable_auto_compaction &&
-
!GlobalMemoryArbitrator::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE)) {
+ (!config::enable_compaction_pause_on_high_memory ||
+
!GlobalMemoryArbitrator::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE))) {
_adjust_compaction_thread_num();
bool check_score = false;
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index 3ab8dfe31a6..f74cf2bf1f6 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -771,7 +771,7 @@ std::vector<TabletSharedPtr>
TabletManager::find_best_tablets_to_compaction(
if (compaction_type == CompactionType::BASE_COMPACTION) {
last_failure_ms = tablet_ptr->last_base_compaction_failure_time();
}
- if (now_ms - last_failure_ms <= 5000) {
+ if (now_ms - last_failure_ms <= config::tablet_sched_delay_time_ms) {
VLOG_DEBUG << "Too often to check compaction, skip it. "
<< "compaction_type=" << compaction_type_str
<< ", last_failure_time_ms=" << last_failure_ms
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
