This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 5e3caf3b17b8cd92706c21282d7c05d7bfb5350c Author: huanghaibin <[email protected]> AuthorDate: Tue Aug 29 17:40:37 2023 +0800 [improvement](compaction) add an option on delete stale rowset by judging _stale_rs_metas size when doing compaction (#23448) --- be/src/common/config.cpp | 3 +++ be/src/common/config.h | 3 +++ be/src/olap/compaction.cpp | 6 ++++++ be/src/olap/tablet.cpp | 3 +++ 4 files changed, 15 insertions(+) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index c865e53813..020e0de623 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -287,6 +287,9 @@ DEFINE_mInt32(default_num_rows_per_column_file_block, "1024"); DEFINE_mInt32(pending_data_expire_time_sec, "1800"); // inc_rowset snapshot rs sweep time interval DEFINE_mInt32(tablet_rowset_stale_sweep_time_sec, "300"); +// tablet stale rowset sweep by threshold size +DEFINE_Bool(tablet_rowset_stale_sweep_by_size, "false"); +DEFINE_mInt32(tablet_rowset_stale_sweep_threshold_size, "100"); // garbage sweep policy DEFINE_Int32(max_garbage_sweep_interval, "3600"); DEFINE_Int32(min_garbage_sweep_interval, "180"); diff --git a/be/src/common/config.h b/be/src/common/config.h index a6905c3f3f..0c0c0b5c25 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -332,6 +332,9 @@ DECLARE_mInt32(default_num_rows_per_column_file_block); DECLARE_mInt32(pending_data_expire_time_sec); // inc_rowset snapshot rs sweep time interval DECLARE_mInt32(tablet_rowset_stale_sweep_time_sec); +// tablet stale rowset sweep by threshold size +DECLARE_Bool(tablet_rowset_stale_sweep_by_size); +DECLARE_mInt32(tablet_rowset_stale_sweep_threshold_size); // garbage sweep policy DECLARE_Int32(max_garbage_sweep_interval); DECLARE_Int32(min_garbage_sweep_interval); diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index 32c15a3bde..8197706527 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -700,6 +700,12 @@ Status Compaction::modify_rowsets(const Merger::Statistics* stats) { RETURN_IF_ERROR(_tablet->modify_rowsets(output_rowsets, _input_rowsets, true)); } + if (config::tablet_rowset_stale_sweep_by_size && + _tablet->tablet_meta()->all_stale_rs_metas().size() >= + config::tablet_rowset_stale_sweep_threshold_size) { + _tablet->delete_expired_stale_rowset(); + } + int64_t cur_max_version = 0; { std::shared_lock rlock(_tablet->get_header_lock()); diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 444efd003b..d4bc57ab27 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -676,6 +676,9 @@ void Tablet::delete_expired_stale_rowset() { // Compute the end time to delete rowsets, when a expired rowset createtime less then this time, it will be deleted. double expired_stale_sweep_endtime = ::difftime(now, config::tablet_rowset_stale_sweep_time_sec); + if (config::tablet_rowset_stale_sweep_by_size) { + expired_stale_sweep_endtime = now; + } std::vector<int64_t> path_id_vec; // capture the path version to delete --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
