This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new daf4f0a7bf5 [fix](cloud-mow) Add enable config for procesing old
version delete bitmap when doing cu compaction (#43615)
daf4f0a7bf5 is described below
commit daf4f0a7bf51cd010b3c91067eeddb9b3753f5ee
Author: huanghaibin <[email protected]>
AuthorDate: Tue Nov 12 21:52:18 2024 +0800
[fix](cloud-mow) Add enable config for procesing old version delete bitmap
when doing cu compaction (#43615)
pr https://github.com/apache/doris/pull/40204 support deleting old
version delete bitmap when doing cu compaction, it a new function and
better to add a config to enable it or not.
pick pr:https://github.com/apache/doris/pull/42471
---
be/src/cloud/cloud_cumulative_compaction.cpp | 3 ++-
be/src/cloud/cloud_delete_bitmap_action.cpp | 2 ++
be/src/cloud/cloud_meta_mgr.cpp | 2 +-
be/src/common/config.cpp | 2 ++
be/src/common/config.h | 2 ++
be/src/olap/rowset/beta_rowset_writer.cpp | 3 +--
be/src/olap/tablet_meta.cpp | 3 +++
7 files changed, 13 insertions(+), 4 deletions(-)
diff --git a/be/src/cloud/cloud_cumulative_compaction.cpp
b/be/src/cloud/cloud_cumulative_compaction.cpp
index 4eb8aba2068..92e0f44dc5b 100644
--- a/be/src/cloud/cloud_cumulative_compaction.cpp
+++ b/be/src/cloud/cloud_cumulative_compaction.cpp
@@ -360,7 +360,8 @@ Status CloudCumulativeCompaction::modify_rowsets() {
stats.num_rows(),
stats.data_size());
}
}
- if (_tablet->keys_type() == KeysType::UNIQUE_KEYS &&
+ if (config::enable_delete_bitmap_merge_on_compaction &&
+ _tablet->keys_type() == KeysType::UNIQUE_KEYS &&
_tablet->enable_unique_key_merge_on_write() && _input_rowsets.size()
!= 1) {
RETURN_IF_ERROR(process_old_version_delete_bitmap());
}
diff --git a/be/src/cloud/cloud_delete_bitmap_action.cpp
b/be/src/cloud/cloud_delete_bitmap_action.cpp
index 60db5896dfa..672574a5aa8 100644
--- a/be/src/cloud/cloud_delete_bitmap_action.cpp
+++ b/be/src/cloud/cloud_delete_bitmap_action.cpp
@@ -95,6 +95,8 @@ Status
CloudDeleteBitmapAction::_handle_show_delete_bitmap_count(HttpRequest* re
auto count =
tablet->tablet_meta()->delete_bitmap().get_delete_bitmap_count();
auto cardinality = tablet->tablet_meta()->delete_bitmap().cardinality();
auto size = tablet->tablet_meta()->delete_bitmap().get_size();
+ LOG(INFO) << "show_delete_bitmap_count,tablet_id=" << tablet_id <<
",count=" << count
+ << ",cardinality=" << cardinality << ",size=" << size;
rapidjson::Document root;
root.SetObject();
diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp
index 0beeea42bd6..1ce2326c21f 100644
--- a/be/src/cloud/cloud_meta_mgr.cpp
+++ b/be/src/cloud/cloud_meta_mgr.cpp
@@ -1074,7 +1074,7 @@ Status CloudMetaMgr::update_delete_bitmap(const
CloudTablet& tablet, int64_t loc
Status CloudMetaMgr::cloud_update_delete_bitmap_without_lock(const
CloudTablet& tablet,
DeleteBitmap*
delete_bitmap) {
LOG(INFO) << "cloud_update_delete_bitmap_without_lock , tablet_id: " <<
tablet.tablet_id()
- << ",delete_bitmap size:" << delete_bitmap->delete_bitmap.size();
+ << ",delete_bitmap size:" <<
delete_bitmap->get_delete_bitmap_count();
UpdateDeleteBitmapRequest req;
UpdateDeleteBitmapResponse res;
req.set_cloud_unique_id(config::cloud_unique_id);
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 6c468e5abde..ef70fd9b88c 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1379,6 +1379,8 @@ DEFINE_mBool(enable_pipeline_task_leakage_detect,
"false");
DEFINE_Int32(query_cache_size, "512");
+DEFINE_mBool(enable_delete_bitmap_merge_on_compaction, "false");
+
// Enable validation to check the correctness of table size.
DEFINE_Bool(enable_table_size_correctness_check, "false");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index c763d1f8a06..726835ef85f 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1459,6 +1459,8 @@ DECLARE_mBool(enable_pipeline_task_leakage_detect);
// MB
DECLARE_Int32(query_cache_size);
+DECLARE_mBool(enable_delete_bitmap_merge_on_compaction);
+
// Enable validation to check the correctness of table size.
DECLARE_Bool(enable_table_size_correctness_check);
diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp
b/be/src/olap/rowset/beta_rowset_writer.cpp
index 97e3bcb1deb..435c080c6f0 100644
--- a/be/src/olap/rowset/beta_rowset_writer.cpp
+++ b/be/src/olap/rowset/beta_rowset_writer.cpp
@@ -336,8 +336,7 @@ Status
BaseBetaRowsetWriter::_generate_delete_bitmap(int32_t segment_id) {
LOG(INFO) << "[Memtable Flush] construct delete bitmap tablet: " <<
_context.tablet->tablet_id()
<< ", rowset_ids: " << _context.mow_context->rowset_ids.size()
<< ", cur max_version: " << _context.mow_context->max_version
- << ", transaction_id: " << _context.mow_context->txn_id << ",
delete_bitmap_count: "
- <<
_context.tablet->tablet_meta()->delete_bitmap().get_delete_bitmap_count()
+ << ", transaction_id: " << _context.mow_context->txn_id
<< ", cost: " << watch.get_elapse_time_us() << "(us), total
rows: " << total_rows;
return Status::OK();
}
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index b5e3accab86..9c60eeb3c65 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -1188,6 +1188,9 @@ void DeleteBitmap::add_to_remove_queue(
}
void DeleteBitmap::remove_stale_delete_bitmap_from_queue(const
std::vector<std::string>& vector) {
+ if (!config::enable_delete_bitmap_merge_on_compaction) {
+ return;
+ }
std::shared_lock l(stale_delete_bitmap_lock);
//<rowset_id, start_version, end_version>
std::vector<std::tuple<std::string, uint64_t, uint64_t>> to_delete;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
