This is an automated email from the ASF dual-hosted git repository.
jianliangqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 13cb38053dd [fix] (compaction) fix time series compaction policy
(#38220)
13cb38053dd is described below
commit 13cb38053ddfd0a1f9e4769870661edaa97b5a5a
Author: Sun Chenyang <[email protected]>
AuthorDate: Mon Aug 5 11:29:36 2024 +0800
[fix] (compaction) fix time series compaction policy (#38220)
1. Time series policy does not rely on permits
2. fix compaction in cloud mode
3. fix compaction level
---
be/src/cloud/cloud_cumulative_compaction_policy.cpp | 4 ++++
be/src/cloud/config.cpp | 1 +
be/src/cloud/config.h | 1 +
be/src/olap/compaction.cpp | 9 ++++-----
be/src/olap/cumulative_compaction.cpp | 7 ++++---
be/src/olap/tablet.cpp | 7 ++++++-
6 files changed, 20 insertions(+), 9 deletions(-)
diff --git a/be/src/cloud/cloud_cumulative_compaction_policy.cpp
b/be/src/cloud/cloud_cumulative_compaction_policy.cpp
index fc56f971cad..b8c4ee20cb2 100644
--- a/be/src/cloud/cloud_cumulative_compaction_policy.cpp
+++ b/be/src/cloud/cloud_cumulative_compaction_policy.cpp
@@ -268,6 +268,10 @@ int32_t
CloudTimeSeriesCumulativeCompactionPolicy::pick_input_rowsets(
continue;
}
return transient_size;
+ } else if (
+ *compaction_score >=
+ config::compaction_max_rowset_count) { // If the number of
rowsets is too large: FDB_ERROR_CODE_TXN_TOO_LARGE
+ return transient_size;
}
}
diff --git a/be/src/cloud/config.cpp b/be/src/cloud/config.cpp
index e589025423e..82c466120e9 100644
--- a/be/src/cloud/config.cpp
+++ b/be/src/cloud/config.cpp
@@ -48,6 +48,7 @@ DEFINE_mDouble(cumu_compaction_thread_num_factor, "0.5");
DEFINE_mInt32(check_auto_compaction_interval_seconds, "5");
DEFINE_mInt32(max_base_compaction_task_num_per_disk, "2");
DEFINE_mBool(prioritize_query_perf_in_compaction, "false");
+DEFINE_mInt32(compaction_max_rowset_count, "10000");
DEFINE_mInt32(refresh_s3_info_interval_s, "60");
DEFINE_mInt32(vacuum_stale_rowsets_interval_s, "300");
diff --git a/be/src/cloud/config.h b/be/src/cloud/config.h
index bf041ba0fa6..02e7014801e 100644
--- a/be/src/cloud/config.h
+++ b/be/src/cloud/config.h
@@ -79,6 +79,7 @@ DECLARE_mDouble(cumu_compaction_thread_num_factor);
DECLARE_mInt32(check_auto_compaction_interval_seconds);
DECLARE_mInt32(max_base_compaction_task_num_per_disk);
DECLARE_mBool(prioritize_query_perf_in_compaction);
+DECLARE_mInt32(compaction_max_rowset_count);
// CloudStorageEngine config
DECLARE_mInt32(refresh_s3_info_interval_s);
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index 935c9991e97..8c109eec1c1 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -197,9 +197,6 @@ Status Compaction::merge_input_rowsets() {
_tablet->last_compaction_status = res;
if (!res.ok()) {
- LOG(WARNING) << "fail to do " << compaction_name() << ". res=" << res
- << ", tablet=" << _tablet->tablet_id()
- << ", output_version=" << _output_version;
return res;
}
@@ -1221,8 +1218,10 @@ Status
CloudCompactionMixin::construct_output_rowset_writer(RowsetWriterContext&
ctx.write_type = DataWriteType::TYPE_COMPACTION;
auto compaction_policy = _tablet->tablet_meta()->compaction_policy();
- ctx.compaction_level =
-
_engine.cumu_compaction_policy(compaction_policy)->new_compaction_level(_input_rowsets);
+ if (_tablet->tablet_meta()->time_series_compaction_level_threshold() >= 2)
{
+ ctx.compaction_level =
_engine.cumu_compaction_policy(compaction_policy)
+ ->new_compaction_level(_input_rowsets);
+ }
ctx.write_file_cache = compaction_type() ==
ReaderType::READER_CUMULATIVE_COMPACTION;
ctx.file_cache_ttl_sec = _tablet->ttl_seconds();
diff --git a/be/src/olap/cumulative_compaction.cpp
b/be/src/olap/cumulative_compaction.cpp
index 5b72ad0ce70..40f6579dd99 100644
--- a/be/src/olap/cumulative_compaction.cpp
+++ b/be/src/olap/cumulative_compaction.cpp
@@ -109,9 +109,10 @@ Status CumulativeCompaction::execute_compact() {
RETURN_IF_ERROR(CompactionMixin::execute_compact());
DCHECK_EQ(_state, CompactionState::SUCCESS);
-
-
tablet()->cumulative_compaction_policy()->update_compaction_level(tablet(),
_input_rowsets,
-
_output_rowset);
+ if (tablet()->tablet_meta()->time_series_compaction_level_threshold() >=
2) {
+
tablet()->cumulative_compaction_policy()->update_compaction_level(tablet(),
_input_rowsets,
+
_output_rowset);
+ }
tablet()->cumulative_compaction_policy()->update_cumulative_point(
tablet(), _input_rowsets, _output_rowset, _last_delete_version);
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 671fa1396ef..1a1d3be6bc9 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -1713,7 +1713,12 @@ Status Tablet::prepare_compaction_and_calculate_permits(
}
}
- permits = compaction->get_compaction_permits();
+ // Time series policy does not rely on permits, it uses goal size to
control memory
+ if (tablet->tablet_meta()->compaction_policy() ==
CUMULATIVE_TIME_SERIES_POLICY) {
+ permits = 0;
+ } else {
+ permits = compaction->get_compaction_permits();
+ }
return Status::OK();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
