This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 3e58cb1dc6d [fix] (compaction) fix time series compaction policy
#38220 (#38554)
3e58cb1dc6d is described below
commit 3e58cb1dc6dad11583a7f3f1201c09f6f30a9450
Author: qiye <[email protected]>
AuthorDate: Wed Jul 31 15:17:04 2024 +0800
[fix] (compaction) fix time series compaction policy #38220 (#38554)
## Proposed changes
bp #38220
---
be/src/cloud/cloud_cumulative_compaction_policy.cpp | 4 ++++
be/src/cloud/config.cpp | 1 +
be/src/cloud/config.h | 1 +
be/src/olap/compaction.cpp | 12 +++++++-----
be/src/olap/cumulative_compaction.cpp | 7 ++++---
be/src/olap/tablet.cpp | 7 ++++++-
6 files changed, 23 insertions(+), 9 deletions(-)
diff --git a/be/src/cloud/cloud_cumulative_compaction_policy.cpp
b/be/src/cloud/cloud_cumulative_compaction_policy.cpp
index fc56f971cad..b8c4ee20cb2 100644
--- a/be/src/cloud/cloud_cumulative_compaction_policy.cpp
+++ b/be/src/cloud/cloud_cumulative_compaction_policy.cpp
@@ -268,6 +268,10 @@ int32_t
CloudTimeSeriesCumulativeCompactionPolicy::pick_input_rowsets(
continue;
}
return transient_size;
+ } else if (
+ *compaction_score >=
+ config::compaction_max_rowset_count) { // If the number of
rowsets is too large: FDB_ERROR_CODE_TXN_TOO_LARGE
+ return transient_size;
}
}
diff --git a/be/src/cloud/config.cpp b/be/src/cloud/config.cpp
index 80522759b84..902502e0aac 100644
--- a/be/src/cloud/config.cpp
+++ b/be/src/cloud/config.cpp
@@ -48,6 +48,7 @@ DEFINE_mDouble(cumu_compaction_thread_num_factor, "0.5");
DEFINE_mInt32(check_auto_compaction_interval_seconds, "5");
DEFINE_mInt32(max_base_compaction_task_num_per_disk, "2");
DEFINE_mBool(prioritize_query_perf_in_compaction, "false");
+DEFINE_mInt32(compaction_max_rowset_count, "10000");
DEFINE_mInt32(refresh_s3_info_interval_s, "60");
DEFINE_mInt32(vacuum_stale_rowsets_interval_s, "300");
diff --git a/be/src/cloud/config.h b/be/src/cloud/config.h
index bf041ba0fa6..02e7014801e 100644
--- a/be/src/cloud/config.h
+++ b/be/src/cloud/config.h
@@ -79,6 +79,7 @@ DECLARE_mDouble(cumu_compaction_thread_num_factor);
DECLARE_mInt32(check_auto_compaction_interval_seconds);
DECLARE_mInt32(max_base_compaction_task_num_per_disk);
DECLARE_mBool(prioritize_query_perf_in_compaction);
+DECLARE_mInt32(compaction_max_rowset_count);
// CloudStorageEngine config
DECLARE_mInt32(refresh_s3_info_interval_s);
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index 5e25b39fe66..dc4dd419a17 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -197,9 +197,6 @@ Status Compaction::merge_input_rowsets() {
_tablet->last_compaction_status = res;
if (!res.ok()) {
- LOG(WARNING) << "fail to do " << compaction_name() << ". res=" << res
- << ", tablet=" << _tablet->tablet_id()
- << ", output_version=" << _output_version;
return res;
}
@@ -1185,6 +1182,9 @@ Status CloudCompactionMixin::execute_compact() {
int64_t permits = get_compaction_permits();
Status st = execute_compact_impl(permits);
if (!st.ok()) {
+ LOG(WARNING) << "failed to do " << compaction_name() << ". res=" << st
+ << ", tablet=" << _tablet->tablet_id()
+ << ", output_version=" << _output_version;
garbage_collection();
return st;
}
@@ -1223,8 +1223,10 @@ Status
CloudCompactionMixin::construct_output_rowset_writer(RowsetWriterContext&
ctx.write_type = DataWriteType::TYPE_COMPACTION;
auto compaction_policy = _tablet->tablet_meta()->compaction_policy();
- ctx.compaction_level =
-
_engine.cumu_compaction_policy(compaction_policy)->new_compaction_level(_input_rowsets);
+ if (_tablet->tablet_meta()->time_series_compaction_level_threshold() >= 2)
{
+ ctx.compaction_level =
_engine.cumu_compaction_policy(compaction_policy)
+ ->new_compaction_level(_input_rowsets);
+ }
ctx.write_file_cache = compaction_type() ==
ReaderType::READER_CUMULATIVE_COMPACTION;
ctx.file_cache_ttl_sec = _tablet->ttl_seconds();
diff --git a/be/src/olap/cumulative_compaction.cpp
b/be/src/olap/cumulative_compaction.cpp
index 2c7e654787a..9201c83b8ac 100644
--- a/be/src/olap/cumulative_compaction.cpp
+++ b/be/src/olap/cumulative_compaction.cpp
@@ -100,9 +100,10 @@ Status CumulativeCompaction::execute_compact() {
RETURN_IF_ERROR(CompactionMixin::execute_compact());
DCHECK_EQ(_state, CompactionState::SUCCESS);
-
-
tablet()->cumulative_compaction_policy()->update_compaction_level(tablet(),
_input_rowsets,
-
_output_rowset);
+ if (tablet()->tablet_meta()->time_series_compaction_level_threshold() >=
2) {
+
tablet()->cumulative_compaction_policy()->update_compaction_level(tablet(),
_input_rowsets,
+
_output_rowset);
+ }
tablet()->cumulative_compaction_policy()->update_cumulative_point(
tablet(), _input_rowsets, _output_rowset, _last_delete_version);
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 8b6ebc2c395..d8919fba417 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -1714,7 +1714,12 @@ Status Tablet::prepare_compaction_and_calculate_permits(
}
}
- permits = compaction->get_compaction_permits();
+ // Time series policy does not rely on permits, it uses goal size to
control memory
+ if (tablet->tablet_meta()->compaction_policy() ==
CUMULATIVE_TIME_SERIES_POLICY) {
+ permits = 0;
+ } else {
+ permits = compaction->get_compaction_permits();
+ }
return Status::OK();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
