This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 38838ace169 branch-3.1: [fix](mow) remove
enable_delete_bitmap_merge_on_compaction #51479 (#52262)
38838ace169 is described below
commit 38838ace1692f1a62dba6a0ab077aa9b3b324a1a
Author: meiyi <[email protected]>
AuthorDate: Thu Jun 26 14:12:11 2025 +0800
branch-3.1: [fix](mow) remove enable_delete_bitmap_merge_on_compaction
#51479 (#52262)
Cherry pick from #51479
---
be/src/cloud/cloud_cumulative_compaction.cpp | 71 ---------------------
be/src/cloud/cloud_cumulative_compaction.h | 2 -
be/src/cloud/cloud_meta_mgr.cpp | 18 ------
be/src/cloud/cloud_meta_mgr.h | 4 --
be/src/cloud/cloud_tablet.cpp | 3 -
be/src/common/config.cpp | 2 -
be/src/common/config.h | 2 -
be/src/olap/tablet_meta.cpp | 48 --------------
be/src/olap/tablet_meta.h | 10 ---
...compaction_remove_old_version_delete_bitmap.out | Bin 571 -> 0 bytes
.../test_mow_compaction_and_schema_change.out | Bin 720 -> 505 bytes
...paction_agg_and_remove_pre_delete_bitmap.groovy | 3 -
.../test_mow_compaction_and_schema_change.groovy | 11 +---
13 files changed, 1 insertion(+), 173 deletions(-)
diff --git a/be/src/cloud/cloud_cumulative_compaction.cpp
b/be/src/cloud/cloud_cumulative_compaction.cpp
index ec23f59a4b5..e7a5fb1ce2a 100644
--- a/be/src/cloud/cloud_cumulative_compaction.cpp
+++ b/be/src/cloud/cloud_cumulative_compaction.cpp
@@ -392,11 +392,6 @@ Status CloudCumulativeCompaction::modify_rowsets() {
stats.num_rows(),
stats.data_size());
}
}
- if (config::enable_delete_bitmap_merge_on_compaction &&
- _tablet->keys_type() == KeysType::UNIQUE_KEYS &&
- _tablet->enable_unique_key_merge_on_write() && _input_rowsets.size()
!= 1) {
- RETURN_IF_ERROR(process_old_version_delete_bitmap());
- }
// agg delete bitmap for pre rowsets
if (config::enable_agg_and_remove_pre_rowsets_delete_bitmap &&
_tablet->keys_type() == KeysType::UNIQUE_KEYS &&
@@ -443,72 +438,6 @@ Status CloudCumulativeCompaction::modify_rowsets() {
return Status::OK();
}
-Status CloudCumulativeCompaction::process_old_version_delete_bitmap() {
- // agg previously rowset old version delete bitmap
- std::vector<RowsetSharedPtr> pre_rowsets {};
- std::vector<std::string> pre_rowset_ids {};
- for (const auto& it : cloud_tablet()->rowset_map()) {
- if (it.first.second < _input_rowsets.front()->start_version()) {
- pre_rowsets.emplace_back(it.second);
- pre_rowset_ids.emplace_back(it.second->rowset_id().to_string());
- }
- }
- std::sort(pre_rowsets.begin(), pre_rowsets.end(), Rowset::comparator);
- if (!pre_rowsets.empty()) {
- auto pre_max_version = _output_rowset->version().second;
- DeleteBitmapPtr new_delete_bitmap =
-
std::make_shared<DeleteBitmap>(_tablet->tablet_meta()->tablet_id());
- std::vector<std::tuple<int64_t, DeleteBitmap::BitmapKey,
DeleteBitmap::BitmapKey>>
- to_remove_vec;
- for (auto& rowset : pre_rowsets) {
- if (rowset->rowset_meta()->total_disk_size() == 0) {
- continue;
- }
- for (uint32_t seg_id = 0; seg_id < rowset->num_segments();
++seg_id) {
- rowset->rowset_id().to_string();
- DeleteBitmap::BitmapKey start {rowset->rowset_id(), seg_id, 0};
- DeleteBitmap::BitmapKey end {rowset->rowset_id(), seg_id,
pre_max_version};
- auto d = _tablet->tablet_meta()->delete_bitmap().get_agg(
- {rowset->rowset_id(), seg_id, pre_max_version});
-
to_remove_vec.emplace_back(std::make_tuple(_tablet->tablet_id(), start, end));
- if (d->isEmpty()) {
- continue;
- }
- new_delete_bitmap->set(end, *d);
- }
- }
- if (!new_delete_bitmap->empty()) {
- // store agg delete bitmap
-
DBUG_EXECUTE_IF("CloudCumulativeCompaction.modify_rowsets.update_delete_bitmap_failed",
- {
- return Status::InternalError(
- "test fail to update delete bitmap for
tablet_id {}",
- cloud_tablet()->tablet_id());
- });
- std::map<std::string, int64_t> rowset_to_versions;
-
RETURN_IF_ERROR(_engine.meta_mgr().cloud_update_delete_bitmap_without_lock(
- *cloud_tablet(), new_delete_bitmap.get(),
rowset_to_versions));
-
- Version version(_input_rowsets.front()->start_version(),
- _input_rowsets.back()->end_version());
- for (auto it = new_delete_bitmap->delete_bitmap.begin();
- it != new_delete_bitmap->delete_bitmap.end(); it++) {
- _tablet->tablet_meta()->delete_bitmap().set(it->first,
it->second);
- }
-
_tablet->tablet_meta()->delete_bitmap().add_to_remove_queue(version.to_string(),
-
to_remove_vec);
- DBUG_EXECUTE_IF(
-
"CloudCumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets",
- {
static_cast<CloudTablet*>(_tablet.get())->delete_expired_stale_rowsets(); });
- }
- }
-
DBUG_EXECUTE_IF("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowset",
{
- LOG(INFO) << "delete_expired_stale_rowsets for tablet=" <<
_tablet->tablet_id();
- _engine.tablet_mgr().vacuum_stale_rowsets(CountDownLatch(1));
- });
- return Status::OK();
-}
-
Status CloudCumulativeCompaction::garbage_collection() {
RETURN_IF_ERROR(CloudCompactionMixin::garbage_collection());
cloud::TabletJobInfoPB job;
diff --git a/be/src/cloud/cloud_cumulative_compaction.h
b/be/src/cloud/cloud_cumulative_compaction.h
index ba3b768a24f..e139054bbc7 100644
--- a/be/src/cloud/cloud_cumulative_compaction.h
+++ b/be/src/cloud/cloud_cumulative_compaction.h
@@ -52,8 +52,6 @@ private:
void update_cumulative_point();
- Status process_old_version_delete_bitmap();
-
ReaderType compaction_type() const override { return
ReaderType::READER_CUMULATIVE_COMPACTION; }
int64_t _input_segments = 0;
diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp
index 62c68b5379f..ca179144310 100644
--- a/be/src/cloud/cloud_meta_mgr.cpp
+++ b/be/src/cloud/cloud_meta_mgr.cpp
@@ -1465,24 +1465,6 @@ void
CloudMetaMgr::remove_delete_bitmap_update_lock(int64_t table_id, int64_t lo
}
}
-Status CloudMetaMgr::remove_old_version_delete_bitmap(
- int64_t tablet_id,
- const std::vector<std::tuple<std::string, uint64_t, uint64_t>>&
to_delete) {
- LOG(INFO) << "remove_old_version_delete_bitmap , tablet_id: " << tablet_id;
- RemoveDeleteBitmapRequest req;
- RemoveDeleteBitmapResponse res;
- req.set_cloud_unique_id(config::cloud_unique_id);
- req.set_tablet_id(tablet_id);
- for (auto& value : to_delete) {
- req.add_rowset_ids(std::get<0>(value));
- req.add_begin_versions(std::get<1>(value));
- req.add_end_versions(std::get<2>(value));
- }
- auto st = retry_rpc("remove old delete bitmap", req, &res,
- &MetaService_Stub::remove_delete_bitmap);
- return st;
-}
-
void CloudMetaMgr::check_table_size_correctness(const RowsetMeta& rs_meta) {
if (!config::enable_table_size_correctness_check) {
return;
diff --git a/be/src/cloud/cloud_meta_mgr.h b/be/src/cloud/cloud_meta_mgr.h
index dac01eb4e5f..99f409ed6dd 100644
--- a/be/src/cloud/cloud_meta_mgr.h
+++ b/be/src/cloud/cloud_meta_mgr.h
@@ -118,10 +118,6 @@ public:
void remove_delete_bitmap_update_lock(int64_t table_id, int64_t lock_id,
int64_t initiator,
int64_t tablet_id);
- Status remove_old_version_delete_bitmap(
- int64_t tablet_id,
- const std::vector<std::tuple<std::string, uint64_t, uint64_t>>&
to_delete);
-
private:
bool sync_tablet_delete_bitmap_by_cache(CloudTablet* tablet, int64_t
old_max_version,
std::ranges::range auto&& rs_metas,
diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp
index 4b430cca13a..8151560599d 100644
--- a/be/src/cloud/cloud_tablet.cpp
+++ b/be/src/cloud/cloud_tablet.cpp
@@ -415,7 +415,6 @@ uint64_t CloudTablet::delete_expired_stale_rowsets() {
std::vector<std::pair<Version, std::vector<RowsetSharedPtr>>>
deleted_stale_rowsets;
int64_t expired_stale_sweep_endtime =
::time(nullptr) - config::tablet_rowset_stale_sweep_time_sec;
- std::vector<std::string> version_to_delete;
{
std::unique_lock wlock(_meta_lock);
@@ -456,14 +455,12 @@ uint64_t CloudTablet::delete_expired_stale_rowsets() {
_tablet_meta->delete_stale_rs_meta_by_version(v_ts->version());
}
Version version(start_version, end_version);
- version_to_delete.emplace_back(version.to_string());
if (!stale_rowsets.empty()) {
deleted_stale_rowsets.emplace_back(version,
std::move(stale_rowsets));
}
}
_reconstruct_version_tracker_if_necessary();
}
-
_tablet_meta->delete_bitmap().remove_stale_delete_bitmap_from_queue(version_to_delete);
recycle_cached_data(expired_rowsets);
if (config::enable_mow_verbose_log) {
LOG_INFO("finish delete_expired_stale_rowset for tablet={}",
tablet_id());
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 59da5fc908a..1768f315304 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1488,8 +1488,6 @@ DEFINE_mBool(enable_pipeline_task_leakage_detect,
"false");
DEFINE_Int32(query_cache_size, "512");
-DEFINE_mBool(enable_delete_bitmap_merge_on_compaction, "false");
-
// Enable validation to check the correctness of table size.
DEFINE_Bool(enable_table_size_correctness_check, "false");
DEFINE_Bool(force_regenerate_rowsetid_on_start_error, "false");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 1a5bb9aacea..98186d5726a 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1559,8 +1559,6 @@ DECLARE_mBool(enable_pipeline_task_leakage_detect);
DECLARE_Int32(query_cache_size);
DECLARE_Bool(force_regenerate_rowsetid_on_start_error);
-DECLARE_mBool(enable_delete_bitmap_merge_on_compaction);
-
// Enable validation to check the correctness of table size.
DECLARE_Bool(enable_table_size_correctness_check);
// Enable sleep 5s between delete cumulative compaction.
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index ba20216e0c3..db8bd9aaee9 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -1305,54 +1305,6 @@ void DeleteBitmap::merge(const DeleteBitmap& other) {
}
}
-void DeleteBitmap::add_to_remove_queue(
- const std::string& version_str,
- const std::vector<std::tuple<int64_t, DeleteBitmap::BitmapKey,
DeleteBitmap::BitmapKey>>&
- vector) {
- std::shared_lock l(stale_delete_bitmap_lock);
- _stale_delete_bitmap.emplace(version_str, vector);
-}
-
-void DeleteBitmap::remove_stale_delete_bitmap_from_queue(const
std::vector<std::string>& vector) {
- if (!config::enable_delete_bitmap_merge_on_compaction) {
- return;
- }
- std::shared_lock l(stale_delete_bitmap_lock);
- //<rowset_id, start_version, end_version>
- std::vector<std::tuple<std::string, uint64_t, uint64_t>> to_delete;
- int64_t tablet_id = -1;
- for (auto& version_str : vector) {
- auto it = _stale_delete_bitmap.find(version_str);
- if (it != _stale_delete_bitmap.end()) {
- auto delete_bitmap_vector = it->second;
- for (auto& delete_bitmap_tuple : it->second) {
- if (tablet_id < 0) {
- tablet_id = std::get<0>(delete_bitmap_tuple);
- }
- auto start_bmk = std::get<1>(delete_bitmap_tuple);
- auto end_bmk = std::get<2>(delete_bitmap_tuple);
- // the key range of to be removed is [start_bmk,end_bmk),
- // due to the different definitions of the right boundary,
- // so use end_bmk as right boundary when removing local delete
bitmap,
- // use (end_bmk - 1) as right boundary when removing ms delete
bitmap
- remove(start_bmk, end_bmk);
-
to_delete.emplace_back(std::make_tuple(std::get<0>(start_bmk).to_string(), 0,
- std::get<2>(end_bmk) -
1));
- }
- _stale_delete_bitmap.erase(version_str);
- }
- }
- if (tablet_id == -1 || to_delete.empty()) {
- return;
- }
- CloudStorageEngine& engine =
ExecEnv::GetInstance()->storage_engine().to_cloud();
- auto st = engine.meta_mgr().remove_old_version_delete_bitmap(tablet_id,
to_delete);
- if (!st.ok()) {
- LOG(WARNING) << "fail to remove_stale_delete_bitmap_from_queue for
tablet=" << tablet_id
- << ",st=" << st;
- }
-}
-
uint64_t DeleteBitmap::get_delete_bitmap_count() {
std::shared_lock l(lock);
uint64_t count = 0;
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index 9d765aa1c7f..4df84c56255 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -385,7 +385,6 @@ private:
class DeleteBitmap {
public:
mutable std::shared_mutex lock;
- mutable std::shared_mutex stale_delete_bitmap_lock;
using SegmentId = uint32_t;
using Version = uint64_t;
using BitmapKey = std::tuple<RowsetId, SegmentId, Version>;
@@ -552,11 +551,6 @@ public:
void remove_sentinel_marks();
- void add_to_remove_queue(const std::string& version_str,
- const std::vector<std::tuple<int64_t,
DeleteBitmap::BitmapKey,
-
DeleteBitmap::BitmapKey>>& vector);
- void remove_stale_delete_bitmap_from_queue(const std::vector<std::string>&
vector);
-
uint64_t get_delete_bitmap_count();
void traverse_rowset_and_version(
@@ -608,10 +602,6 @@ private:
int64_t _tablet_id;
mutable std::shared_mutex _rowset_cache_version_lock;
mutable std::map<RowsetId, std::map<SegmentId, Version>>
_rowset_cache_version;
- // <version, <tablet_id, BitmapKeyStart, BitmapKeyEnd>>
- std::map<std::string,
- std::vector<std::tuple<int64_t, DeleteBitmap::BitmapKey,
DeleteBitmap::BitmapKey>>>
- _stale_delete_bitmap;
};
static const std::string SEQUENCE_COL = "__DORIS_SEQUENCE_COL__";
diff --git
a/regression-test/data/compaction/test_cu_compaction_remove_old_version_delete_bitmap.out
b/regression-test/data/compaction/test_cu_compaction_remove_old_version_delete_bitmap.out
deleted file mode 100644
index 37dfa3b93a5..00000000000
Binary files
a/regression-test/data/compaction/test_cu_compaction_remove_old_version_delete_bitmap.out
and /dev/null differ
diff --git
a/regression-test/data/compaction/test_mow_compaction_and_schema_change.out
b/regression-test/data/compaction/test_mow_compaction_and_schema_change.out
index 43ba92316ae..e912c295a2b 100644
Binary files
a/regression-test/data/compaction/test_mow_compaction_and_schema_change.out and
b/regression-test/data/compaction/test_mow_compaction_and_schema_change.out
differ
diff --git
a/regression-test/suites/compaction/test_mow_compaction_agg_and_remove_pre_delete_bitmap.groovy
b/regression-test/suites/compaction/test_mow_compaction_agg_and_remove_pre_delete_bitmap.groovy
index 73942a55f80..27fbb97ac01 100644
---
a/regression-test/suites/compaction/test_mow_compaction_agg_and_remove_pre_delete_bitmap.groovy
+++
b/regression-test/suites/compaction/test_mow_compaction_agg_and_remove_pre_delete_bitmap.groovy
@@ -155,13 +155,11 @@
suite("test_mow_compaction_agg_and_remove_pre_delete_bitmap", "nonConcurrent") {
GetDebugPoint().clearDebugPointsForAllBEs()
get_be_param("tablet_rowset_stale_sweep_time_sec")
get_be_param("compaction_promotion_version_count")
- get_be_param("enable_delete_bitmap_merge_on_compaction")
get_be_param("enable_agg_and_remove_pre_rowsets_delete_bitmap")
try {
set_be_param("tablet_rowset_stale_sweep_time_sec", "0")
set_be_param("compaction_promotion_version_count", "5")
- set_be_param("enable_delete_bitmap_merge_on_compaction", "false") //
solution 1
set_be_param("enable_agg_and_remove_pre_rowsets_delete_bitmap",
"true") // solution 2
def testTable = "test_mow_compaction"
@@ -251,7 +249,6 @@
suite("test_mow_compaction_agg_and_remove_pre_delete_bitmap", "nonConcurrent") {
} finally {
reset_be_param("tablet_rowset_stale_sweep_time_sec")
reset_be_param("compaction_promotion_version_count")
- reset_be_param("enable_delete_bitmap_merge_on_compaction")
reset_be_param("enable_agg_and_remove_pre_rowsets_delete_bitmap")
GetDebugPoint().clearDebugPointsForAllBEs()
}
diff --git
a/regression-test/suites/compaction/test_mow_compaction_and_schema_change.groovy
b/regression-test/suites/compaction/test_mow_compaction_and_schema_change.groovy
index 942df3d9e24..8328980817e 100644
---
a/regression-test/suites/compaction/test_mow_compaction_and_schema_change.groovy
+++
b/regression-test/suites/compaction/test_mow_compaction_and_schema_change.groovy
@@ -174,24 +174,17 @@ suite("test_mow_compaction_and_schema_change",
"nonConcurrent") {
GetDebugPoint().clearDebugPointsForAllBEs()
get_be_param("tablet_rowset_stale_sweep_time_sec")
- get_be_param("enable_delete_bitmap_merge_on_compaction")
get_be_param("enable_agg_and_remove_pre_rowsets_delete_bitmap")
try {
set_be_param("tablet_rowset_stale_sweep_time_sec", "0")
- for (int method = 0; method < 3; method++) {
+ for (int method = 0; method < 2; method++) {
if (method == 0) {
// off
- set_be_param("enable_delete_bitmap_merge_on_compaction",
"false")
set_be_param("enable_agg_and_remove_pre_rowsets_delete_bitmap", "false")
} else if (method == 1) {
// solution2: no duplicated key problems
- set_be_param("enable_delete_bitmap_merge_on_compaction",
"false")
set_be_param("enable_agg_and_remove_pre_rowsets_delete_bitmap", "true")
- } else if (method == 2) {
- // solution1: has duplicated key problems
- set_be_param("enable_delete_bitmap_merge_on_compaction",
"true")
-
set_be_param("enable_agg_and_remove_pre_rowsets_delete_bitmap", "false")
}
testTable = "test_mow_compaction_and_schema_change_${method}"
@@ -231,7 +224,6 @@ suite("test_mow_compaction_and_schema_change",
"nonConcurrent") {
order_qt_sql2 "select * from ${testTable}"
GetDebugPoint().enableDebugPointForAllBEs("Tablet.delete_expired_stale_rowset.start_delete_unused_rowset")
-
GetDebugPoint().enableDebugPointForAllBEs("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets")
// solution 1
GetDebugPoint().enableDebugPointForAllBEs("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowset")
// solution 2
// 3.0 write some data
@@ -338,7 +330,6 @@ suite("test_mow_compaction_and_schema_change",
"nonConcurrent") {
}
} finally {
reset_be_param("tablet_rowset_stale_sweep_time_sec")
- reset_be_param("enable_delete_bitmap_merge_on_compaction")
reset_be_param("enable_agg_and_remove_pre_rowsets_delete_bitmap")
GetDebugPoint().clearDebugPointsForAllBEs()
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
