This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 1bd31dfa335 branch-4.0: [fix](mow) delete bitmap is not deleted if
commit compaction job failed #56758 (#56779)
1bd31dfa335 is described below
commit 1bd31dfa3358d49275817e0d74d9828045345e57
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sat Oct 11 16:41:44 2025 +0800
branch-4.0: [fix](mow) delete bitmap is not deleted if commit compaction
job failed #56758 (#56779)
Cherry-picked from #56758
Co-authored-by: meiyi <[email protected]>
---
be/src/cloud/cloud_meta_mgr.cpp | 3 +++
cloud/src/recycler/recycler.cpp | 21 ++++++++++++++++++++-
cloud/test/recycler_test.cpp | 22 ++++++++++++++++++----
3 files changed, 41 insertions(+), 5 deletions(-)
diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp
index cc2440e1b2e..1d8d54ddcda 100644
--- a/be/src/cloud/cloud_meta_mgr.cpp
+++ b/be/src/cloud/cloud_meta_mgr.cpp
@@ -1567,6 +1567,9 @@ Status CloudMetaMgr::prepare_tablet_job(const
TabletJobInfoPB& job, StartTabletJ
Status CloudMetaMgr::commit_tablet_job(const TabletJobInfoPB& job,
FinishTabletJobResponse* res) {
VLOG_DEBUG << "commit_tablet_job: " << job.ShortDebugString();
TEST_SYNC_POINT_RETURN_WITH_VALUE("CloudMetaMgr::commit_tablet_job",
Status::OK(), job, res);
+ DBUG_EXECUTE_IF("CloudMetaMgr::commit_tablet_job.fail", {
+ return Status::InternalError<false>("inject
CloudMetaMgr::commit_tablet_job.fail");
+ });
FinishTabletJobRequest req;
req.mutable_job()->CopyFrom(job);
diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp
index b610331059c..4b53439959e 100644
--- a/cloud/src/recycler/recycler.cpp
+++ b/cloud/src/recycler/recycler.cpp
@@ -3786,7 +3786,21 @@ int InstanceRecycler::recycle_tmp_rowsets() {
auto ret = txn_remove(txn_kv_.get(), dbm_start_key, dbm_end_key);
if (ret != 0) {
LOG(WARNING) << "failed to delete versioned delete bitmap kv,
instance_id="
- << instance_id_;
+ << instance_id_ << ", tablet_id=" << tablet_id
+ << ", rowset_id=" << rowset_id;
+ }
+ return ret;
+ };
+
+ auto delete_delete_bitmap_kvs = [&](int64_t tablet_id, const std::string&
rowset_id) {
+ auto delete_bitmap_start =
+ meta_delete_bitmap_key({instance_id_, tablet_id, rowset_id, 0,
0});
+ auto delete_bitmap_end =
+ meta_delete_bitmap_key({instance_id_, tablet_id, rowset_id,
INT64_MAX, INT64_MAX});
+ auto ret = txn_remove(txn_kv_.get(), delete_bitmap_start,
delete_bitmap_end);
+ if (ret != 0) {
+ LOG(WARNING) << "failed to delete delete bitmap kv, instance_id="
<< instance_id_
+ << ", tablet_id=" << tablet_id << ", rowset_id=" <<
rowset_id;
}
return ret;
};
@@ -3811,6 +3825,11 @@ int InstanceRecycler::recycle_tmp_rowsets() {
<< rs.ShortDebugString();
return;
}
+ if (delete_delete_bitmap_kvs(rs.tablet_id(),
rs.rowset_id_v2()) != 0) {
+ LOG(WARNING) << "failed to delete delete bitmap kv, rs="
+ << rs.ShortDebugString();
+ return;
+ }
}
if (txn_remove(txn_kv_.get(), tmp_rowset_keys_to_delete) != 0) {
LOG(WARNING) << "failed to tmp rowset kv, instance_id=" <<
instance_id_;
diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp
index f815aa7852b..bec871b7282 100644
--- a/cloud/test/recycler_test.cpp
+++ b/cloud/test/recycler_test.cpp
@@ -1173,12 +1173,19 @@ static int get_copy_file_num(TxnKv* txn_kv, const
std::string& stage_id, int64_t
return 0;
}
-static void check_delete_bitmap_keys_size(TxnKv* txn_kv, int64_t tablet_id,
int expected_size) {
+static void check_delete_bitmap_keys_size(TxnKv* txn_kv, int64_t tablet_id,
int expected_size,
+ int version = 2) {
std::unique_ptr<Transaction> txn;
ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK);
std::unique_ptr<RangeGetIterator> it;
- auto dbm_start_key = versioned::meta_delete_bitmap_key({instance_id,
tablet_id, ""});
- std::string dbm_end_key = versioned::meta_delete_bitmap_key({instance_id,
tablet_id + 1, ""});
+ std::string dbm_start_key, dbm_end_key;
+ if (version == 2) {
+ dbm_start_key = versioned::meta_delete_bitmap_key({instance_id,
tablet_id, ""});
+ dbm_end_key = versioned::meta_delete_bitmap_key({instance_id,
tablet_id + 1, ""});
+ } else if (version == 1) {
+ dbm_start_key = meta_delete_bitmap_key({instance_id, tablet_id, "", 0,
0});
+ dbm_end_key = meta_delete_bitmap_key({instance_id, tablet_id + 1, "",
0, 0});
+ }
ASSERT_EQ(txn->get(dbm_start_key, dbm_end_key, &it), TxnErrorCode::TXN_OK);
EXPECT_EQ(it->size(), expected_size);
}
@@ -1497,16 +1504,23 @@ TEST(RecyclerTest, recycle_tmp_rowsets) {
int64_t txn_id_base = 114115;
int64_t tablet_id_base = 10015;
int64_t index_id_base = 1000;
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK);
for (int i = 0; i < 50; ++i) {
int64_t txn_id = txn_id_base + i;
for (int j = 0; j < 1000; ++j) {
auto rowset = create_rowset("recycle_tmp_rowsets", tablet_id_base
+ j,
index_id_base + j, 5, schemas[i % 5],
txn_id);
create_tmp_rowset(txn_kv.get(), accessor.get(), rowset, i & 1,
false, i < 50);
+ if (i < 50) {
+ create_delete_bitmaps(txn.get(), tablet_id_base + j,
rowset.rowset_id_v2(), 0, 1);
+ }
}
}
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
for (int j = 0; j < 20; ++j) {
check_delete_bitmap_keys_size(txn_kv.get(), tablet_id_base + j, 50);
+ check_delete_bitmap_keys_size(txn_kv.get(), tablet_id_base + j, 100,
1);
}
auto start = std::chrono::steady_clock::now();
@@ -1521,7 +1535,6 @@ TEST(RecyclerTest, recycle_tmp_rowsets) {
ASSERT_EQ(0, accessor->list_directory("data/", &list_iter));
ASSERT_FALSE(list_iter->has_next());
// check all tmp rowset kv have been deleted
- std::unique_ptr<Transaction> txn;
ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK);
std::unique_ptr<RangeGetIterator> it;
auto begin_key = meta_rowset_tmp_key({instance_id, 0, 0});
@@ -1539,6 +1552,7 @@ TEST(RecyclerTest, recycle_tmp_rowsets) {
ASSERT_EQ(it->size(), 0);
for (int j = 0; j < 20; ++j) {
check_delete_bitmap_keys_size(txn_kv.get(), tablet_id_base + j, 0);
+ check_delete_bitmap_keys_size(txn_kv.get(), tablet_id_base + j, 0, 1);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
