This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new d7ada5bbccf branch-4.1: [improvement](cloud) Remove pre-rowset delete
bitmaps by key (#66598) (#67936)
d7ada5bbccf is described below
commit d7ada5bbccf853384d5674c7d2e45fe734f9f2e5
Author: meiyi <[email protected]>
AuthorDate: Tue Sep 15 12:00:37 2026 +0800
branch-4.1: [improvement](cloud) Remove pre-rowset delete bitmaps by key
(#66598) (#67936)
pick https://github.com/apache/doris/pull/66598
---
be/src/cloud/cloud_cumulative_compaction.cpp | 15 +-
be/src/cloud/cloud_meta_mgr.cpp | 27 +-
be/src/cloud/cloud_meta_mgr.h | 6 +-
be/src/cloud/cloud_tablet.cpp | 32 ++-
be/src/cloud/cloud_tablet.h | 15 +-
be/src/common/config.cpp | 6 +
be/src/common/config.h | 2 +
be/test/cloud/cloud_meta_mgr_test.cpp | 83 ++++++
be/test/cloud/cloud_tablet_test.cpp | 73 +++++
cloud/src/meta-service/meta_service.cpp | 390 ++++++++++++++++++++++---
cloud/test/meta_service_test.cpp | 413 ++++++++++++++++++++++++++-
gensrc/proto/cloud.proto | 19 ++
12 files changed, 1024 insertions(+), 57 deletions(-)
diff --git a/be/src/cloud/cloud_cumulative_compaction.cpp
b/be/src/cloud/cloud_cumulative_compaction.cpp
index cd61ea9c426..a032cfdb392 100644
--- a/be/src/cloud/cloud_cumulative_compaction.cpp
+++ b/be/src/cloud/cloud_cumulative_compaction.cpp
@@ -463,7 +463,8 @@ Status CloudCumulativeCompaction::modify_rowsets() {
}
}
// agg delete bitmap for pre rowsets
- if (config::enable_agg_and_remove_pre_rowsets_delete_bitmap &&
+ if (config::delete_bitmap_store_write_version != 2 &&
+ config::enable_agg_and_remove_pre_rowsets_delete_bitmap &&
_tablet->keys_type() == KeysType::UNIQUE_KEYS &&
_tablet->enable_unique_key_merge_on_write() && _input_rowsets.size()
!= 1) {
OlapStopWatch watch;
@@ -479,17 +480,23 @@ Status CloudCumulativeCompaction::modify_rowsets() {
std::sort(pre_rowsets.begin(), pre_rowsets.end(), Rowset::comparator);
auto pre_rowsets_delete_bitmap =
std::make_shared<DeleteBitmap>(_tablet->tablet_id());
std::map<std::string, int64_t> pre_rowset_to_versions;
+ std::unique_ptr<CloudTablet::PreRowsetDeleteBitmapStats>
pre_rowset_delete_bitmap_stats;
+ if (config::enable_remove_pre_rowsets_delete_bitmap_by_keys) {
+ pre_rowset_delete_bitmap_stats =
+
std::make_unique<CloudTablet::PreRowsetDeleteBitmapStats>();
+ }
cloud_tablet()->agg_delete_bitmap_for_compaction(
_output_rowset->start_version(),
_output_rowset->end_version(), pre_rowsets,
- pre_rowsets_delete_bitmap, pre_rowset_to_versions);
+ pre_rowsets_delete_bitmap, pre_rowset_to_versions,
+ pre_rowset_delete_bitmap_stats.get());
// update delete bitmap to ms
DBUG_EXECUTE_IF(
"CumulativeCompaction.modify_rowsets.cloud_update_delete_bitmap_without_lock.block",
DBUG_BLOCK);
auto status =
_engine.meta_mgr().cloud_update_delete_bitmap_without_lock(
*cloud_tablet(), pre_rowsets_delete_bitmap.get(),
pre_rowset_to_versions,
- cloud_tablet()->table_id(), _output_rowset->start_version(),
- _output_rowset->end_version());
+ pre_rowset_delete_bitmap_stats.get(),
cloud_tablet()->table_id(),
+ _output_rowset->start_version(),
_output_rowset->end_version());
if (!status.ok()) {
LOG(WARNING) << "failed to agg pre rowsets delete bitmap to ms.
tablet_id="
<< _tablet->tablet_id() << ", pre rowset num=" <<
pre_rowsets.size()
diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp
index 8b4ec76b333..378f3686f3a 100644
--- a/be/src/cloud/cloud_meta_mgr.cpp
+++ b/be/src/cloud/cloud_meta_mgr.cpp
@@ -2191,8 +2191,10 @@ Status CloudMetaMgr::update_delete_bitmap(const
CloudTablet& tablet, int64_t loc
Status CloudMetaMgr::cloud_update_delete_bitmap_without_lock(
const CloudTablet& tablet, DeleteBitmap* delete_bitmap,
- std::map<std::string, int64_t>& rowset_to_versions, int64_t table_id,
- int64_t pre_rowset_agg_start_version, int64_t
pre_rowset_agg_end_version) {
+ std::map<std::string, int64_t>& rowset_to_versions,
+ const CloudTablet::PreRowsetDeleteBitmapStats*
pre_rowset_delete_bitmap_stats,
+ int64_t table_id, int64_t pre_rowset_agg_start_version,
+ int64_t pre_rowset_agg_end_version) {
if (config::delete_bitmap_store_write_version == 2) {
VLOG_DEBUG << "no need to agg delete bitmap v1 in ms because use v2";
return Status::OK();
@@ -2208,6 +2210,10 @@ Status
CloudMetaMgr::cloud_update_delete_bitmap_without_lock(
// use a fake lock id to resolve compatibility issues
req.set_lock_id(-3);
req.set_without_lock(true);
+ req.set_enable_remove_agg_pre_rowsets_delete_bitmap_by_keys(
+ config::enable_remove_agg_pre_rowsets_delete_bitmap_by_keys);
+
req.set_enable_remove_pre_rowsets_delete_bitmap_by_keys(pre_rowset_delete_bitmap_stats
!=
+ nullptr);
for (auto& [key, bitmap] : delete_bitmap->delete_bitmap) {
req.add_rowset_ids(std::get<0>(key).to_string());
req.add_segment_ids(std::get<1>(key));
@@ -2231,6 +2237,23 @@ Status
CloudMetaMgr::cloud_update_delete_bitmap_without_lock(
req.set_pre_rowset_agg_start_version(pre_rowset_agg_start_version);
req.set_pre_rowset_agg_end_version(pre_rowset_agg_end_version);
}
+ if (pre_rowset_delete_bitmap_stats != nullptr) {
+ for (const auto& [rowset_id, delete_bitmap_stats] :
*pre_rowset_delete_bitmap_stats) {
+ if (delete_bitmap_stats.empty()) {
+ continue;
+ }
+ auto* rowset_stats_pb = req.add_pre_rowset_delete_bitmap_stats();
+ rowset_stats_pb->set_rowset_id(rowset_id);
+ for (const auto& [segment_id, version, delete_bitmap_size] :
delete_bitmap_stats) {
+ auto* delete_bitmap_stat_pb =
rowset_stats_pb->add_delete_bitmap_stats();
+ delete_bitmap_stat_pb->set_segment_id(segment_id);
+ delete_bitmap_stat_pb->set_version(version);
+
delete_bitmap_stat_pb->set_delete_bitmap_size(delete_bitmap_size);
+ }
+ }
+ }
+ TEST_SYNC_POINT_RETURN_WITH_VALUE(
+
"CloudMetaMgr::cloud_update_delete_bitmap_without_lock.before_rpc",
Status::OK(), &req);
return retry_rpc(MetaServiceRPC::UPDATE_DELETE_BITMAP, req, &res,
&MetaService_Stub::update_delete_bitmap,
{
diff --git a/be/src/cloud/cloud_meta_mgr.h b/be/src/cloud/cloud_meta_mgr.h
index 50b8353b7e6..31bec1c3628 100644
--- a/be/src/cloud/cloud_meta_mgr.h
+++ b/be/src/cloud/cloud_meta_mgr.h
@@ -155,8 +155,10 @@ public:
Status cloud_update_delete_bitmap_without_lock(
const CloudTablet& tablet, DeleteBitmap* delete_bitmap,
- std::map<std::string, int64_t>& rowset_to_versions, int64_t
table_id,
- int64_t pre_rowset_agg_start_version = 0, int64_t
pre_rowset_agg_end_version = 0);
+ std::map<std::string, int64_t>& rowset_to_versions,
+ const CloudTablet::PreRowsetDeleteBitmapStats*
pre_rowset_delete_bitmap_stats,
+ int64_t table_id, int64_t pre_rowset_agg_start_version = 0,
+ int64_t pre_rowset_agg_end_version = 0);
Status get_delete_bitmap_update_lock(const CloudTablet& tablet, int64_t
lock_id,
int64_t initiator);
diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp
index 271c81dd49c..3be758fefd1 100644
--- a/be/src/cloud/cloud_tablet.cpp
+++ b/be/src/cloud/cloud_tablet.cpp
@@ -1440,12 +1440,34 @@ Status CloudTablet::calc_delete_bitmap_for_compaction(
void CloudTablet::agg_delete_bitmap_for_compaction(
int64_t start_version, int64_t end_version, const
std::vector<RowsetSharedPtr>& pre_rowsets,
- DeleteBitmapPtr& new_delete_bitmap,
- std::map<std::string, int64_t>& pre_rowset_to_versions) {
- for (auto& rowset : pre_rowsets) {
+ DeleteBitmapPtr& new_delete_bitmap, std::map<std::string, int64_t>&
pre_rowset_to_versions,
+ PreRowsetDeleteBitmapStats* pre_rowset_delete_bitmap_stats) {
+ auto& delete_bitmap = tablet_meta()->delete_bitmap();
+ for (const auto& rowset : pre_rowsets) {
+ if (pre_rowset_delete_bitmap_stats != nullptr) {
+ auto& rowset_delete_bitmap_stats =
+
(*pre_rowset_delete_bitmap_stats)[rowset->rowset_id().to_string()];
+ std::shared_lock lock(delete_bitmap.lock);
+ const auto bitmap_start_version =
static_cast<DeleteBitmap::Version>(start_version);
+ const auto bitmap_end_version =
static_cast<DeleteBitmap::Version>(end_version);
+ for (uint32_t seg_id = 0; seg_id < rowset->num_segments();
++seg_id) {
+ DeleteBitmap::BitmapKey segment_start {rowset->rowset_id(),
seg_id,
+ bitmap_start_version};
+ for (auto it =
delete_bitmap.delete_bitmap.lower_bound(segment_start);
+ it != delete_bitmap.delete_bitmap.end(); ++it) {
+ const auto& [key, bitmap] = *it;
+ if (std::get<0>(key) != rowset->rowset_id() ||
std::get<1>(key) != seg_id ||
+ std::get<2>(key) >= bitmap_end_version) {
+ break;
+ }
+ rowset_delete_bitmap_stats.emplace_back(seg_id,
std::get<2>(key),
+
bitmap.getSizeInBytes());
+ }
+ }
+ }
for (uint32_t seg_id = 0; seg_id < rowset->num_segments(); ++seg_id) {
- auto d = tablet_meta()->delete_bitmap().get_agg_without_cache(
- {rowset->rowset_id(), seg_id, end_version}, start_version);
+ auto d = delete_bitmap.get_agg_without_cache({rowset->rowset_id(),
seg_id, end_version},
+ start_version);
if (d->isEmpty()) {
continue;
}
diff --git a/be/src/cloud/cloud_tablet.h b/be/src/cloud/cloud_tablet.h
index e731dc534b4..b08d39005c2 100644
--- a/be/src/cloud/cloud_tablet.h
+++ b/be/src/cloud/cloud_tablet.h
@@ -18,6 +18,7 @@
#pragma once
#include <memory>
+#include <tuple>
#include "storage/partial_update_info.h"
#include "storage/rowset/rowset.h"
@@ -76,6 +77,11 @@ struct RecycledRowsets {
class CloudTablet final : public BaseTablet {
public:
+ // rowset id -> [(segment id, version, serialized delete bitmap size)]
+ using PreRowsetDeleteBitmapStats = std::map<
+ std::string,
+ std::vector<std::tuple<DeleteBitmap::SegmentId,
DeleteBitmap::Version, size_t>>>;
+
CloudTablet(CloudStorageEngine& engine, TabletMetaSharedPtr tablet_meta);
~CloudTablet() override;
@@ -357,10 +363,11 @@ public:
// check that if the delete bitmap in delete bitmap cache has the same
cardinality with the expected_delete_bitmap's
Status check_delete_bitmap_cache(int64_t txn_id, DeleteBitmap*
expected_delete_bitmap) override;
- void agg_delete_bitmap_for_compaction(int64_t start_version, int64_t
end_version,
- const std::vector<RowsetSharedPtr>&
pre_rowsets,
- DeleteBitmapPtr& new_delete_bitmap,
- std::map<std::string, int64_t>&
pre_rowset_to_versions);
+ void agg_delete_bitmap_for_compaction(
+ int64_t start_version, int64_t end_version,
+ const std::vector<RowsetSharedPtr>& pre_rowsets, DeleteBitmapPtr&
new_delete_bitmap,
+ std::map<std::string, int64_t>& pre_rowset_to_versions,
+ PreRowsetDeleteBitmapStats* pre_rowset_delete_bitmap_stats);
bool need_remove_unused_rowsets();
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 0bb1fde8678..16d41bcc3e9 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1446,6 +1446,12 @@ DEFINE_mBool(enable_mow_get_agg_by_cache, "true");
DEFINE_mBool(enable_mow_get_agg_correctness_check_core, "false");
DEFINE_mBool(enable_agg_and_remove_pre_rowsets_delete_bitmap, "true");
DEFINE_mBool(enable_check_agg_and_remove_pre_rowsets_delete_bitmap, "false");
+// Remove pre-rowset delete bitmaps in [end_version, end_version] before
writing aggregated delete
+// bitmaps. True: point delete; false: range delete.
+DEFINE_mBool(enable_remove_agg_pre_rowsets_delete_bitmap_by_keys, "true");
+// Remove pre-rowset delete bitmaps in [start_version, end_version). True:
point delete; false:
+// range delete.
+DEFINE_mBool(enable_remove_pre_rowsets_delete_bitmap_by_keys, "true");
// The secure path with user files, used in the `local` table function.
DEFINE_String(user_files_secure_path, "${DORIS_HOME}");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index e9f64c2c429..65ea01ae452 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1508,6 +1508,8 @@ DECLARE_mBool(enable_mow_get_agg_by_cache);
DECLARE_mBool(enable_mow_get_agg_correctness_check_core);
DECLARE_mBool(enable_agg_and_remove_pre_rowsets_delete_bitmap);
DECLARE_mBool(enable_check_agg_and_remove_pre_rowsets_delete_bitmap);
+DECLARE_mBool(enable_remove_agg_pre_rowsets_delete_bitmap_by_keys);
+DECLARE_mBool(enable_remove_pre_rowsets_delete_bitmap_by_keys);
// The secure path with user files, used in the `local` table function.
DECLARE_String(user_files_secure_path);
diff --git a/be/test/cloud/cloud_meta_mgr_test.cpp
b/be/test/cloud/cloud_meta_mgr_test.cpp
index 9b10f80a218..ba27d082f97 100644
--- a/be/test/cloud/cloud_meta_mgr_test.cpp
+++ b/be/test/cloud/cloud_meta_mgr_test.cpp
@@ -29,12 +29,14 @@
#include "cloud/cloud_storage_engine.h"
#include "cloud/cloud_tablet.h"
+#include "cloud/config.h"
#include "cpp/sync_point.h"
#include "load/stream_load/stream_load_context.h"
#include "storage/olap_common.h"
#include "storage/rowset/rowset_factory.h"
#include "storage/rowset/rowset_meta.h"
#include "storage/tablet/tablet_meta.h"
+#include "util/defer_op.h"
#include "util/uid_util.h"
namespace doris {
@@ -97,6 +99,87 @@ TEST_F(CloudMetaMgrTest,
response_status_returns_undefined_without_any_code) {
EXPECT_EQ(get_response_code(status), MetaServiceCode::OK);
}
+TEST_F(CloudMetaMgrTest, PreRowsetDeleteBitmapStatsRequestEncoding) {
+ CloudStorageEngine engine(EngineOptions {});
+ CloudMetaMgr meta_mgr;
+ TabletMetaSharedPtr tablet_meta(
+ new TabletMeta(1001, 2, 15673, 15674, 4, 5, TTabletSchema(), 6,
{{7, 8}},
+ UniqueId(9, 10), TTabletType::TABLET_TYPE_DISK,
TCompressionType::LZ4F));
+ auto tablet = std::make_shared<CloudTablet>(engine,
std::make_shared<TabletMeta>(*tablet_meta));
+ DeleteBitmap delete_bitmap(tablet->tablet_id());
+ std::map<std::string, int64_t> rowset_to_versions;
+
+ int32_t old_write_version = config::delete_bitmap_store_write_version;
+ bool old_remove_agg_by_keys =
config::enable_remove_agg_pre_rowsets_delete_bitmap_by_keys;
+ config::delete_bitmap_store_write_version = 1;
+ auto* sp = SyncPoint::get_instance();
+ sp->clear_all_call_backs();
+ sp->enable_processing();
+ Defer cleanup {[&] {
+ config::delete_bitmap_store_write_version = old_write_version;
+ config::enable_remove_agg_pre_rowsets_delete_bitmap_by_keys =
old_remove_agg_by_keys;
+ sp->disable_processing();
+ sp->clear_all_call_backs();
+ }};
+
+ auto capture_request = [&](const CloudTablet::PreRowsetDeleteBitmapStats*
stats) {
+ bool called = false;
+ UpdateDeleteBitmapRequest captured_req;
+ SyncPoint::CallbackGuard guard;
+ sp->set_call_back(
+
"CloudMetaMgr::cloud_update_delete_bitmap_without_lock.before_rpc",
+ [&](auto&& args) {
+ auto* req =
try_any_cast<UpdateDeleteBitmapRequest*>(args[0]);
+ captured_req.CopyFrom(*req);
+ called = true;
+ auto* ret = try_any_cast<std::pair<Status,
bool>*>(args.back());
+ ret->first = Status::OK();
+ ret->second = true;
+ },
+ &guard);
+ auto status = meta_mgr.cloud_update_delete_bitmap_without_lock(
+ *tablet, &delete_bitmap, rowset_to_versions, stats,
tablet->table_id(), 1, 2);
+ EXPECT_TRUE(status.ok()) << status;
+ EXPECT_TRUE(called);
+ return captured_req;
+ };
+
+ config::enable_remove_agg_pre_rowsets_delete_bitmap_by_keys = false;
+ auto config_disabled_req = capture_request(nullptr);
+
EXPECT_FALSE(config_disabled_req.enable_remove_agg_pre_rowsets_delete_bitmap_by_keys());
+
EXPECT_FALSE(config_disabled_req.enable_remove_pre_rowsets_delete_bitmap_by_keys());
+ EXPECT_EQ(config_disabled_req.pre_rowset_delete_bitmap_stats_size(), 0);
+
+ CloudTablet::PreRowsetDeleteBitmapStats empty_stats;
+ empty_stats.emplace(
+ "rowset_without_delete_bitmap",
+ std::vector<std::tuple<DeleteBitmap::SegmentId,
DeleteBitmap::Version, size_t>> {});
+ empty_stats.emplace(
+ "second_rowset_without_delete_bitmap",
+ std::vector<std::tuple<DeleteBitmap::SegmentId,
DeleteBitmap::Version, size_t>> {});
+ config::enable_remove_agg_pre_rowsets_delete_bitmap_by_keys = true;
+ auto config_enabled_req = capture_request(&empty_stats);
+
EXPECT_TRUE(config_enabled_req.enable_remove_agg_pre_rowsets_delete_bitmap_by_keys());
+
EXPECT_TRUE(config_enabled_req.enable_remove_pre_rowsets_delete_bitmap_by_keys());
+ EXPECT_EQ(config_enabled_req.pre_rowset_delete_bitmap_stats_size(), 0);
+
+ using DeleteBitmapStat = std::tuple<DeleteBitmap::SegmentId,
DeleteBitmap::Version, size_t>;
+ CloudTablet::PreRowsetDeleteBitmapStats populated_stats;
+ populated_stats.emplace("rowset_with_delete_bitmap",
+ std::vector<DeleteBitmapStat> {{3, 7, 1024}, {8,
9, 2048}});
+ auto populated_stats_req = capture_request(&populated_stats);
+ ASSERT_EQ(populated_stats_req.pre_rowset_delete_bitmap_stats_size(), 1);
+ const auto& rowset_stats =
populated_stats_req.pre_rowset_delete_bitmap_stats(0);
+ EXPECT_EQ(rowset_stats.rowset_id(), "rowset_with_delete_bitmap");
+ ASSERT_EQ(rowset_stats.delete_bitmap_stats_size(), 2);
+ EXPECT_EQ(rowset_stats.delete_bitmap_stats(0).segment_id(), 3);
+ EXPECT_EQ(rowset_stats.delete_bitmap_stats(0).version(), 7);
+ EXPECT_EQ(rowset_stats.delete_bitmap_stats(0).delete_bitmap_size(), 1024);
+ EXPECT_EQ(rowset_stats.delete_bitmap_stats(1).segment_id(), 8);
+ EXPECT_EQ(rowset_stats.delete_bitmap_stats(1).version(), 9);
+ EXPECT_EQ(rowset_stats.delete_bitmap_stats(1).delete_bitmap_size(), 2048);
+}
+
static AbortTxnRequest get_abort_txn_request(CloudMetaMgr* meta_mgr, const
StreamLoadContext& ctx) {
auto* sp = SyncPoint::get_instance();
sp->clear_all_call_backs();
diff --git a/be/test/cloud/cloud_tablet_test.cpp
b/be/test/cloud/cloud_tablet_test.cpp
index 228e7421ee3..7df8ad0607b 100644
--- a/be/test/cloud/cloud_tablet_test.cpp
+++ b/be/test/cloud/cloud_tablet_test.cpp
@@ -75,6 +75,79 @@ protected:
CloudStorageEngine _engine;
};
+class CloudTabletDeleteBitmapTest : public CloudTabletWarmUpStateTest {};
+
+TEST_F(CloudTabletDeleteBitmapTest,
AggDeleteBitmapForCompactionReturnsPreRowsetStats) {
+ auto rowset1 = create_rowset(Version(1, 1), 2);
+ auto rowset2 = create_rowset(Version(2, 2));
+ auto rowset_without_delete_bitmap = create_rowset(Version(3, 3));
+ ASSERT_NE(rowset1, nullptr);
+ ASSERT_NE(rowset2, nullptr);
+ ASSERT_NE(rowset_without_delete_bitmap, nullptr);
+
+ roaring::Roaring before_range;
+ before_range.add(1);
+ roaring::Roaring at_start;
+ at_start.add(2);
+ at_start.add(3);
+ roaring::Roaring within_range;
+ within_range.add(4);
+ roaring::Roaring at_end;
+ at_end.add(5);
+ roaring::Roaring second_segment;
+ second_segment.add(6);
+ second_segment.add(7);
+ second_segment.add(8);
+ roaring::Roaring second_rowset;
+ second_rowset.add(9);
+
+ auto& delete_bitmap = _tablet->tablet_meta()->delete_bitmap();
+ delete_bitmap.set({rowset1->rowset_id(), 0, 4}, before_range);
+ delete_bitmap.set({rowset1->rowset_id(), 0, 5}, at_start);
+ delete_bitmap.set({rowset1->rowset_id(), 0, 6}, within_range);
+ delete_bitmap.set({rowset1->rowset_id(), 0, 7}, at_end);
+ delete_bitmap.set({rowset1->rowset_id(), 1, 6}, second_segment);
+ delete_bitmap.set({rowset2->rowset_id(), 0, 5}, second_rowset);
+
+ auto aggregated_delete_bitmap =
std::make_shared<DeleteBitmap>(_tablet->tablet_id());
+ std::map<std::string, int64_t> pre_rowset_to_versions;
+ CloudTablet::PreRowsetDeleteBitmapStats pre_rowset_delete_bitmap_stats;
+ _tablet->agg_delete_bitmap_for_compaction(
+ 5, 7, {rowset1, rowset2, rowset_without_delete_bitmap},
aggregated_delete_bitmap,
+ pre_rowset_to_versions, &pre_rowset_delete_bitmap_stats);
+
+ using DeleteBitmapStat = std::tuple<DeleteBitmap::SegmentId,
DeleteBitmap::Version, size_t>;
+
EXPECT_EQ(pre_rowset_delete_bitmap_stats.at(rowset1->rowset_id().to_string()),
+ (std::vector<DeleteBitmapStat> {
+ {0, 5, at_start.getSizeInBytes()},
+ {0, 6, within_range.getSizeInBytes()},
+ {1, 6, second_segment.getSizeInBytes()},
+ }));
+
EXPECT_EQ(pre_rowset_delete_bitmap_stats.at(rowset2->rowset_id().to_string()),
+ (std::vector<DeleteBitmapStat> {{0, 5,
second_rowset.getSizeInBytes()}}));
+ EXPECT_TRUE(
+
pre_rowset_delete_bitmap_stats.at(rowset_without_delete_bitmap->rowset_id().to_string())
+ .empty());
+
+ roaring::Roaring aggregated;
+ ASSERT_EQ(aggregated_delete_bitmap->get({rowset1->rowset_id(), 0, 7},
&aggregated), 0);
+ EXPECT_EQ(aggregated.cardinality(), 4);
+ ASSERT_EQ(aggregated_delete_bitmap->get({rowset1->rowset_id(), 1, 7},
&aggregated), 0);
+ EXPECT_EQ(aggregated.cardinality(), 3);
+ ASSERT_EQ(aggregated_delete_bitmap->get({rowset2->rowset_id(), 0, 7},
&aggregated), 0);
+ EXPECT_EQ(aggregated.cardinality(), 1);
+ EXPECT_EQ(pre_rowset_to_versions.at(rowset1->rowset_id().to_string()), 1);
+ EXPECT_EQ(pre_rowset_to_versions.at(rowset2->rowset_id().to_string()), 2);
+
+ auto aggregated_without_stats =
std::make_shared<DeleteBitmap>(_tablet->tablet_id());
+ std::map<std::string, int64_t> rowset_versions_without_stats;
+ _tablet->agg_delete_bitmap_for_compaction(
+ 5, 7, {rowset1, rowset2, rowset_without_delete_bitmap},
aggregated_without_stats,
+ rowset_versions_without_stats, nullptr);
+ EXPECT_EQ(aggregated_without_stats->delete_bitmap,
aggregated_delete_bitmap->delete_bitmap);
+ EXPECT_EQ(rowset_versions_without_stats, pre_rowset_to_versions);
+}
+
// Test get_rowset_warmup_state for non-existent rowset
TEST_F(CloudTabletWarmUpStateTest, TestGetRowsetWarmupStateNonExistent) {
auto rowset = create_rowset(Version(1, 1));
diff --git a/cloud/src/meta-service/meta_service.cpp
b/cloud/src/meta-service/meta_service.cpp
index 670eab8c92b..dd5dc3cd05e 100644
--- a/cloud/src/meta-service/meta_service.cpp
+++ b/cloud/src/meta-service/meta_service.cpp
@@ -3799,6 +3799,112 @@ struct UpdateDeleteBitmapTxnStats {
size_t total_txn_count = 0;
};
+static bool check_delete_bitmap_point_delete(MetaServiceCode& code,
std::string& msg,
+ std::unique_ptr<Transaction>& txn,
+ const std::string& key,
+ const UpdateDeleteBitmapRequest*
request,
+ size_t request_index, bool&
point_delete,
+ bool& delete_bitmap_exists,
+ std::optional<uint16_t>&
max_blob_sequence) {
+ point_delete =
request->enable_remove_agg_pre_rowsets_delete_bitmap_by_keys() &&
+ request->lock_id() ==
COMPACTION_WITHOUT_LOCK_DELETE_BITMAP_LOCK_ID;
+ if (!point_delete) {
+ return true;
+ }
+ std::string end_key {key};
+ encode_int64(INT64_MAX, &end_key);
+ RangeGetOptions opts;
+ opts.batch_limit = 1;
+ opts.reverse = true;
+ std::unique_ptr<RangeGetIterator> it;
+ auto err = txn->get(key, end_key, &it, opts);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::READ>(err);
+ msg = fmt::format(
+ "failed to get the last delete bitmap blob key, err={},
tablet_id={}, key={}", err,
+ request->tablet_id(), hex(key));
+ return false;
+ }
+
+ delete_bitmap_exists = it->has_next();
+ max_blob_sequence.reset();
+ if (!delete_bitmap_exists) {
+ return true;
+ }
+ auto max_key = it->next().first;
+ if (max_key.size() < key.size()) [[unlikely]] {
+ code = MetaServiceCode::UNDEFINED_ERR;
+ msg = fmt::format(
+ "delete bitmap range scan returned a shorter key,
tablet_id={}, key={}, "
+ "max_key={}",
+ request->tablet_id(), hex(key), hex(max_key));
+ LOG(WARNING) << msg;
+ return false;
+ }
+ if (max_key.size() == key.size()) {
+ if (max_key != key) [[unlikely]] {
+ code = MetaServiceCode::UNDEFINED_ERR;
+ msg = fmt::format(
+ "delete bitmap range scan returned an unexpected key,
tablet_id={}, key={}, "
+ "max_key={}",
+ request->tablet_id(), hex(key), hex(max_key));
+ LOG(WARNING) << msg;
+ return false;
+ }
+ return true;
+ }
+ uint8_t blob_version = 0;
+ uint16_t sequence = 0;
+ std::vector<std::tuple<std::variant<int64_t, std::string>, int, int>> out;
+ if (!decode_blob_key(max_key, nullptr, &blob_version, &sequence, &out))
[[unlikely]] {
+ code = MetaServiceCode::UNDEFINED_ERR;
+ msg = fmt::format("failed to decode delete bitmap blob key,
tablet_id={}, key={}",
+ request->tablet_id(), hex(max_key));
+ LOG(WARNING) << msg;
+ return false;
+ }
+
+ constexpr size_t delete_bitmap_key_field_count = 7;
+ if (out.size() != delete_bitmap_key_field_count) [[unlikely]] {
+ code = MetaServiceCode::UNDEFINED_ERR;
+ msg = fmt::format(
+ "invalid delete bitmap blob origin key field count,
tablet_id={}, key={}, "
+ "field_count={}",
+ request->tablet_id(), hex(max_key), out.size());
+ LOG(WARNING) << msg;
+ return false;
+ }
+
+ const auto* rowset_id = std::get_if<std::string>(&std::get<0>(out[4]));
+ const auto* version = std::get_if<int64_t>(&std::get<0>(out[5]));
+ const auto* segment_id = std::get_if<int64_t>(&std::get<0>(out[6]));
+ if (rowset_id == nullptr || version == nullptr || segment_id == nullptr)
[[unlikely]] {
+ code = MetaServiceCode::UNDEFINED_ERR;
+ msg = fmt::format("invalid delete bitmap blob origin key fields,
tablet_id={}, key={}",
+ request->tablet_id(), hex(max_key));
+ LOG(WARNING) << msg;
+ return false;
+ }
+
+ if (*rowset_id != request->rowset_ids(request_index) ||
+ *version != request->versions(request_index) ||
+ *segment_id != request->segment_ids(request_index)) [[unlikely]] {
+ code = MetaServiceCode::UNDEFINED_ERR;
+ msg = fmt::format(
+ "unexpected delete bitmap blob origin key, tablet_id={},
key={}, rowset_id={}, "
+ "version={}, segment_id={}, expected_rowset_id={}, "
+ "expected_version={}, expected_segment_id={}",
+ request->tablet_id(), hex(max_key), *rowset_id, *version,
*segment_id,
+ request->rowset_ids(request_index),
request->versions(request_index),
+ request->segment_ids(request_index));
+ LOG(WARNING) << msg;
+ return false;
+ }
+
+ max_blob_sequence = sequence;
+ return true;
+}
+
void _write_delete_bitmap_kvs(MetaServiceCode& code, std::string& msg,
std::stringstream& ss,
std::shared_ptr<TxnKv> txn_kv,
std::unique_ptr<Transaction>& txn,
std::string& use_version, std::set<std::string>&
non_exist_rowset_ids,
@@ -3934,12 +4040,37 @@ void _write_delete_bitmap_kvs(MetaServiceCode& code,
std::string& msg, std::stri
}
// remove first
if (request->lock_id() == COMPACTION_WITHOUT_LOCK_DELETE_BITMAP_LOCK_ID) {
- auto& start_key = key;
- std::string end_key {start_key};
- encode_int64(INT64_MAX, &end_key);
- txn->remove(start_key, end_key);
- LOG(INFO) << "xxx remove delete_bitmap_key=" << hex(start_key) << "
tablet_id=" << tablet_id
- << " lock_id=" << request->lock_id() << " initiator=" <<
request->initiator();
+ bool point_delete = false;
+ bool delete_bitmap_exists = false;
+ std::optional<uint16_t> max_blob_sequence;
+ if (!check_delete_bitmap_point_delete(code, msg, txn, key, request, i,
point_delete,
+ delete_bitmap_exists,
max_blob_sequence)) {
+ return;
+ }
+ if (point_delete) {
+ if (delete_bitmap_exists) {
+ if (max_blob_sequence.has_value()) {
+ for (size_t sequence = 0; sequence <= *max_blob_sequence;
++sequence) {
+ txn->remove(encode_blob_key(key, 0, sequence));
+ }
+ } else {
+ // Compatibility with delete bitmaps written directly by
the legacy txn->put.
+ txn->remove(key);
+ }
+ LOG(INFO) << "xxx point remove delete_bitmap_key=" << hex(key)
+ << " tablet_id=" << tablet_id << " lock_id=" <<
request->lock_id()
+ << " initiator=" << request->initiator()
+ << " delete_key_count=" <<
(max_blob_sequence.value_or(0) + 1);
+ }
+ } else {
+ auto& start_key = key;
+ std::string end_key {start_key};
+ encode_int64(INT64_MAX, &end_key);
+ txn->remove(start_key, end_key);
+ LOG(INFO) << "xxx range remove delete_bitmap_key=" <<
hex(start_key)
+ << " tablet_id=" << tablet_id << " lock_id=" <<
request->lock_id()
+ << " initiator=" << request->initiator();
+ }
}
// splitting large values (>90*1000) into multiple KVs
cloud::blob_put(txn.get(), key, val, 0);
@@ -3952,6 +4083,136 @@ void _write_delete_bitmap_kvs(MetaServiceCode& code,
std::string& msg, std::stri
<< " key_size: " << key.size() << " value_size: " << val.size();
}
+static bool commit_pre_rowset_delete_bitmap_removal(
+ MetaServiceCode& code, std::string& msg, std::stringstream& ss,
+ const std::shared_ptr<TxnKv>& txn_kv, std::unique_ptr<Transaction>&
txn, KVStats& stats,
+ UpdateDeleteBitmapTxnStats& txn_stats, int64_t tablet_id, const
std::string& rowset_id) {
+ auto txn_size = txn->approximate_bytes();
+ LOG(INFO) << "commit delete bitmap point deletes before transaction size
exceeds limit, "
+ "tablet_id="
+ << tablet_id << ", rowset=" << rowset_id << ", txn_size=" <<
txn_size;
+ auto err = txn->commit();
+ TEST_SYNC_POINT_CALLBACK("update_delete_bitmap:remove_pre_rowsets:commit",
txn_size);
+ txn_stats.total_txn_put_keys += txn->num_put_keys();
+ txn_stats.total_txn_put_bytes += txn->put_bytes();
+ txn_stats.total_txn_size += txn_size;
+ txn_stats.total_txn_count++;
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::COMMIT>(err);
+ ss << "failed to remove pre rowsets delete bitmap, err=" << err
+ << " tablet_id=" << tablet_id << " rowset_id=" << rowset_id << "
txn_size=" << txn_size;
+ msg = ss.str();
+ g_bvar_update_delete_bitmap_fail_counter << 1;
+ return false;
+ }
+ stats.get_bytes += txn->get_bytes();
+ stats.put_bytes += txn->put_bytes();
+ stats.del_bytes += txn->delete_bytes();
+ stats.get_counter += txn->num_get_keys();
+ stats.put_counter += txn->num_put_keys();
+ stats.del_counter += txn->num_del_keys();
+ txn_stats.current_key_count = 0;
+ txn_stats.current_value_count = 0;
+ err = txn_kv->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::CREATE>(err);
+ msg = "failed to init txn when removing pre rowsets delete bitmap";
+ return false;
+ }
+ return true;
+}
+
+static bool remove_pre_rowset_delete_bitmap(
+ MetaServiceCode& code, std::string& msg, std::stringstream& ss,
+ const std::shared_ptr<TxnKv>& txn_kv, std::unique_ptr<Transaction>&
txn, KVStats& stats,
+ const UpdateDeleteBitmapRequest* request, const std::string&
instance_id,
+ const std::set<std::string>& non_exist_rowset_ids,
UpdateDeleteBitmapTxnStats& txn_stats) {
+ if (!request->has_pre_rowset_agg_start_version() ||
+ !request->has_pre_rowset_agg_end_version() ||
+ request->pre_rowset_agg_start_version() >=
request->pre_rowset_agg_end_version()) {
+ return true;
+ }
+
+ auto tablet_id = request->tablet_id();
+ if (!request->enable_remove_pre_rowsets_delete_bitmap_by_keys()) {
+ std::string pre_rowset_id;
+ for (size_t i = 0; i < request->rowset_ids_size(); ++i) {
+ if (request->rowset_ids(i) == pre_rowset_id) {
+ continue;
+ }
+ if (non_exist_rowset_ids.contains(request->rowset_ids(i))) {
+ LOG(INFO) << "skip remove pre rowsets delete bitmap,
rowset_id="
+ << request->rowset_ids(i) << " tablet_id=" <<
tablet_id
+ << " because the rowset does not exist";
+ continue;
+ }
+ pre_rowset_id = request->rowset_ids(i);
+ auto delete_bitmap_start =
+ meta_delete_bitmap_key({instance_id, tablet_id,
request->rowset_ids(i),
+
request->pre_rowset_agg_start_version(), 0});
+ auto delete_bitmap_end =
+ meta_delete_bitmap_key({instance_id, tablet_id,
request->rowset_ids(i),
+
request->pre_rowset_agg_end_version(), 0});
+ txn->remove(delete_bitmap_start, delete_bitmap_end);
+ LOG(INFO) << "remove pre rowsets delete bitmap by range,
tablet_id=" << tablet_id
+ << ", rowset=" << request->rowset_ids(i)
+ << ", start_version=" <<
request->pre_rowset_agg_start_version()
+ << ", end_version=" <<
request->pre_rowset_agg_end_version()
+ << ", start_key=" << hex(delete_bitmap_start)
+ << ", end_key=" << hex(delete_bitmap_end);
+ }
+ return true;
+ }
+
+ for (const auto& rowset_stats : request->pre_rowset_delete_bitmap_stats())
{
+ if (non_exist_rowset_ids.contains(rowset_stats.rowset_id())) {
+ LOG(INFO) << "skip remove pre rowsets delete bitmap, rowset_id="
+ << rowset_stats.rowset_id() << " tablet_id=" << tablet_id
+ << " because the rowset does not exist";
+ continue;
+ }
+ uint64_t delete_key_count = 0;
+ for (const auto& delete_bitmap_stat :
rowset_stats.delete_bitmap_stats()) {
+ DCHECK(delete_bitmap_stat.version() >=
request->pre_rowset_agg_start_version() &&
+ delete_bitmap_stat.version() <
request->pre_rowset_agg_end_version());
+ auto delete_bitmap_key = meta_delete_bitmap_key(
+ {instance_id, tablet_id, rowset_stats.rowset_id(),
delete_bitmap_stat.version(),
+ delete_bitmap_stat.segment_id()});
+ auto remove_key = [&](std::string_view key) {
+ auto txn_size = txn->approximate_bytes();
+ if (txn_size > 0 &&
+ txn_size + key.size() * 3 >
static_cast<size_t>(config::max_txn_commit_byte) &&
+ !commit_pre_rowset_delete_bitmap_removal(code, msg, ss,
txn_kv, txn, stats,
+ txn_stats,
tablet_id,
+
rowset_stats.rowset_id())) {
+ return false;
+ }
+ txn->remove(key);
+ delete_key_count++;
+ return true;
+ };
+ if (!remove_key(delete_bitmap_key)) {
+ return false;
+ }
+ auto split_key_count =
+ delete_bitmap_stat.delete_bitmap_size() /
DEFAULT_BLOB_SPLIT_SIZE +
+ (delete_bitmap_stat.delete_bitmap_size() %
DEFAULT_BLOB_SPLIT_SIZE != 0);
+ for (size_t i = 0; i < split_key_count; ++i) {
+ if (!remove_key(encode_blob_key(delete_bitmap_key, 0, i))) {
+ return false;
+ }
+ }
+ }
+ if (delete_key_count > 0) {
+ LOG(INFO) << "remove pre rowsets delete bitmap by keys,
tablet_id=" << tablet_id
+ << ", rowset=" << rowset_stats.rowset_id()
+ << ", delete_bitmap_count=" <<
rowset_stats.delete_bitmap_stats_size()
+ << ", delete_key_count=" << delete_key_count;
+ }
+ }
+ return true;
+}
+
void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController*
controller,
const UpdateDeleteBitmapRequest*
request,
UpdateDeleteBitmapResponse*
response,
@@ -4173,35 +4434,9 @@ void
MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont
if (code != MetaServiceCode::OK) return;
}
- // remove pre rowset delete bitmap
- if (request->has_pre_rowset_agg_start_version() &&
request->has_pre_rowset_agg_end_version() &&
- request->pre_rowset_agg_start_version() <
request->pre_rowset_agg_end_version()) {
- std::string pre_rowset_id = "";
- for (size_t i = 0; i < request->rowset_ids_size(); ++i) {
- if (request->rowset_ids(i) == pre_rowset_id) {
- continue;
- }
- if (non_exist_rowset_ids.contains(request->rowset_ids(i))) {
- LOG(INFO) << "skip remove pre rowsets delete bitmap,
rowset_id="
- << request->rowset_ids(i) << " tablet_id=" <<
tablet_id
- << " because the rowset does not exist";
- continue;
- }
- pre_rowset_id = request->rowset_ids(i);
- auto delete_bitmap_start =
- meta_delete_bitmap_key({instance_id, tablet_id,
request->rowset_ids(i),
-
request->pre_rowset_agg_start_version(), 0});
- auto delete_bitmap_end =
- meta_delete_bitmap_key({instance_id, tablet_id,
request->rowset_ids(i),
-
request->pre_rowset_agg_end_version(), 0});
- txn->remove(delete_bitmap_start, delete_bitmap_end);
- LOG(INFO) << "remove pre rowsets delete bitmap, tablet_id=" <<
tablet_id
- << ", rowset=" << request->rowset_ids(i)
- << ", start_version=" <<
request->pre_rowset_agg_start_version()
- << ", end_version=" <<
request->pre_rowset_agg_end_version()
- << ", start_key=" << hex(delete_bitmap_start)
- << ", end_key=" << hex(delete_bitmap_end);
- }
+ if (!remove_pre_rowset_delete_bitmap(code, msg, ss, txn_kv_, txn, stats,
request, instance_id,
+ non_exist_rowset_ids, txn_stats)) {
+ return;
}
err = txn->commit();
txn_stats.total_txn_put_keys += txn->num_put_keys();
@@ -4338,6 +4573,9 @@ void
MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control
std::unique_ptr<RangeGetIterator> it;
int64_t last_ver = -1;
int64_t last_seg_id = -1;
+ std::string last_delete_bitmap_first_blob_key;
+ uint16_t next_blob_sequence = 0;
+ bool skip_last_delete_bitmap = false;
int64_t round = 0;
while (it == nullptr /* may be not init */ || it->more()) {
if (test) {
@@ -4359,6 +4597,31 @@ void
MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control
msg = ss.str();
return;
}
+ if (!skip_last_delete_bitmap &&
!last_delete_bitmap_first_blob_key.empty()) {
+ std::string first_blob_value;
+ err = txn->get(last_delete_bitmap_first_blob_key,
&first_blob_value);
+ if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) {
+ delete_bitmap_byte -=
+ response->segment_delete_bitmaps(
+
response->segment_delete_bitmaps_size() - 1)
+ .size();
+ delete_bitmap_num--;
+ response->mutable_rowset_ids()->RemoveLast();
+ response->mutable_segment_ids()->RemoveLast();
+ response->mutable_versions()->RemoveLast();
+
response->mutable_segment_delete_bitmaps()->RemoveLast();
+ skip_last_delete_bitmap = true;
+ LOG(WARNING)
+ << "skip incomplete delete bitmap whose
first blob key "
+ "disappeared after transaction retry"
+ << ", tablet_id=" << tablet_id
+ << ", rowset_id=" << rowset_ids[i] << ",
version=" << last_ver
+ << ", segment_id=" << last_seg_id;
+ } else if (err != TxnErrorCode::TXN_OK) {
+ retry++;
+ continue;
+ }
+ }
if (test) {
err = txn->get(start_key, end_key, &it, false, 2);
} else {
@@ -4384,7 +4647,15 @@ void
MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control
auto k1 = k;
k1.remove_prefix(1);
std::vector<std::tuple<std::variant<int64_t, std::string>,
int, int>> out;
- decode_key(&k1, &out);
+ auto decode_ret = decode_key(&k1, &out);
+ if (decode_ret != 0 || (out.size() != 7 && out.size() !=
8)) {
+ code = MetaServiceCode::KV_TXN_GET_ERR;
+ ss << "invalid delete bitmap key: " << hex(k)
+ << ", decode_ret=" << decode_ret << ",
field_count=" << out.size();
+ msg = ss.str();
+ g_bvar_get_delete_bitmap_fail_counter << 1;
+ return;
+ }
// 0x01 "meta" ${instance_id} "delete_bitmap" ${tablet_id}
// ${rowset_id0} ${version1} ${segment_id0} ->
DeleteBitmapPB
auto ver = std::get<int64_t>(std::get<0>(out[5]));
@@ -4393,15 +4664,58 @@ void
MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control
// FIXME: Don't expose the implementation details of
splitting large value.
// merge splitted large values (>90*1000)
if (ver != last_ver || seg_id != last_seg_id) {
+ auto sequence =
+ out.size() == 8
+ ? static_cast<uint16_t>(
+
std::get<int64_t>(std::get<0>(out[7])) & 0xffff)
+ : uint16_t {0};
+ last_ver = ver;
+ last_seg_id = seg_id;
+ last_delete_bitmap_first_blob_key.clear();
+ next_blob_sequence = static_cast<uint16_t>(sequence +
1);
+ // Key-based pre-rowset cleanup may leave an obsolete
tail if its size estimate is
+ // too small. The bitmap is already aggregated, so the
tail can be skipped and will
+ // be recycled with its rowset.
+ skip_last_delete_bitmap = sequence != 0;
+ if (skip_last_delete_bitmap) {
+ LOG(WARNING)
+ << "skip incomplete delete bitmap whose
first blob "
+ "sequence is not zero"
+ << ", tablet_id=" << tablet_id
+ << ", rowset_id=" << rowset_ids[i] << ",
version=" << ver
+ << ", segment_id=" << seg_id << ",
first_sequence=" << sequence;
+ continue;
+ }
+ if (out.size() == 8) {
+ last_delete_bitmap_first_blob_key.assign(k.data(),
k.size());
+ }
response->add_rowset_ids(rowset_ids[i]);
response->add_segment_ids(seg_id);
response->add_versions(ver);
response->add_segment_delete_bitmaps(std::string(v));
- last_ver = ver;
- last_seg_id = seg_id;
delete_bitmap_num++;
delete_bitmap_byte += v.length();
} else {
+ if (skip_last_delete_bitmap) {
+ continue;
+ }
+ auto sequence =
+ out.size() == 8
+ ? static_cast<uint16_t>(
+
std::get<int64_t>(std::get<0>(out[7])) & 0xffff)
+ : uint16_t {0};
+ if (out.size() != 8 || sequence != next_blob_sequence)
{
+ code = MetaServiceCode::KV_TXN_GET_ERR;
+ ss << "non-contiguous delete bitmap blob sequence,
tablet_id="
+ << tablet_id << ", rowset_id=" << rowset_ids[i]
+ << ", version=" << ver << ", segment_id=" <<
seg_id
+ << ", expected_sequence=" << next_blob_sequence
+ << ", actual_sequence=" << sequence;
+ msg = ss.str();
+ g_bvar_get_delete_bitmap_fail_counter << 1;
+ return;
+ }
+ next_blob_sequence++;
TEST_SYNC_POINT_CALLBACK("get_delete_bitmap_code",
&code);
if (code != MetaServiceCode::OK) {
ss << "test get get_delete_bitmap fail, code="
diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp
index 12e31b6e8a2..1896ac01a2a 100644
--- a/cloud/test/meta_service_test.cpp
+++ b/cloud/test/meta_service_test.cpp
@@ -40,6 +40,7 @@
#include "cpp/sync_point.h"
#include "meta-service/meta_service_helper.h"
#include "meta-store/blob_message.h"
+#include "meta-store/codec.h"
#include "meta-store/document_message.h"
#include "meta-store/keys.h"
#include "meta-store/mem_txn_kv.h"
@@ -6901,6 +6902,93 @@ TEST(MetaServiceTest, UpdateDeleteBitmap) {
testUpdateDeleteBitmap(1);
}
+TEST(MetaServiceTest, UpdateDeleteBitmapPointDelete) {
+ auto meta_service = get_meta_service();
+ auto txn_kv = std::dynamic_pointer_cast<MemTxnKv>(meta_service->txn_kv());
+ ASSERT_NE(txn_kv, nullptr);
+ extern std::string get_instance_id(const std::shared_ptr<ResourceManager>&
rc_mgr,
+ const std::string& cloud_unique_id);
+ auto instance_id = get_instance_id(meta_service->resource_mgr(),
"test_cloud_unique_id");
+ auto old_max_txn_commit_byte = config::max_txn_commit_byte;
+ auto sp = SyncPoint::get_instance();
+ DORIS_CLOUD_DEFER {
+ config::max_txn_commit_byte = old_max_txn_commit_byte;
+ sp->clear_all_call_backs();
+ sp->disable_processing();
+ };
+ int split_commit_count = 0;
+ sp->set_call_back("update_delete_bitmap:commit:err", [&](auto&&) {
split_commit_count++; });
+ sp->enable_processing();
+
+ constexpr int64_t tablet_id = 3330;
+ auto replace_delete_bitmap = [&](const std::string& rowset_id, const
std::string& value,
+ MetaServiceCode expected_code =
MetaServiceCode::OK) {
+ brpc::Controller cntl;
+ UpdateDeleteBitmapRequest req;
+ UpdateDeleteBitmapResponse res;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_table_id(1120);
+ req.set_partition_id(1230);
+ req.set_without_lock(true);
+ req.set_enable_remove_agg_pre_rowsets_delete_bitmap_by_keys(true);
+ req.set_lock_id(-3);
+ req.set_initiator(tablet_id);
+ req.set_tablet_id(tablet_id);
+ req.add_rowset_ids(rowset_id);
+ req.add_segment_ids(0);
+ req.add_versions(2);
+ req.add_segment_delete_bitmaps(value);
+ meta_service->update_delete_bitmap(
+ reinterpret_cast<google::protobuf::RpcController*>(&cntl),
&req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), expected_code);
+ };
+ auto put_old_delete_bitmap = [&](const std::string& key, size_t
blob_count) {
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK);
+ std::string old_value(DEFAULT_BLOB_SPLIT_SIZE * (blob_count - 1) + 1,
'a');
+ blob_put(txn.get(), key, old_value, 0);
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+ };
+ auto check_delete_bitmap = [&](const std::string& key, const std::string&
expected) {
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK);
+ ValueBuf value;
+ ASSERT_EQ(blob_get(txn.get(), key, &value), TxnErrorCode::TXN_OK);
+ EXPECT_EQ(value.value(), expected);
+ };
+
+ const std::string point_rowset_id = "point_delete_rowset";
+ auto point_key = meta_delete_bitmap_key({instance_id, tablet_id,
point_rowset_id, 2, 0});
+ put_old_delete_bitmap(point_key, 4);
+ auto delete_count = txn_kv->del_count_;
+ replace_delete_bitmap(point_rowset_id, "point_replacement");
+ EXPECT_EQ(txn_kv->del_count_ - delete_count, 4);
+ EXPECT_EQ(split_commit_count, 0);
+ check_delete_bitmap(point_key, "point_replacement");
+
+ const std::string large_rowset_id = "large_point_delete_rowset";
+ auto large_key = meta_delete_bitmap_key({instance_id, tablet_id,
large_rowset_id, 2, 0});
+ put_old_delete_bitmap(large_key, 11);
+ config::max_txn_commit_byte = 1500;
+ delete_count = txn_kv->del_count_;
+ replace_delete_bitmap(large_rowset_id, "large_point_replacement");
+ EXPECT_EQ(txn_kv->del_count_ - delete_count, 11);
+ EXPECT_EQ(split_commit_count, 0);
+ check_delete_bitmap(large_key, "large_point_replacement");
+
+ const std::string invalid_rowset_id = "invalid_point_delete_rowset";
+ auto invalid_key = meta_delete_bitmap_key({instance_id, tablet_id,
invalid_rowset_id, 2, 0});
+ auto invalid_origin_key = invalid_key;
+ encode_int64(0, &invalid_origin_key);
+ {
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK);
+ txn->put(encode_blob_key(invalid_origin_key, 0, 0), "invalid");
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+ }
+ replace_delete_bitmap(invalid_rowset_id, "replacement",
MetaServiceCode::UNDEFINED_ERR);
+}
+
TEST(MetaServiceTest, UpdateDeleteBitmapWithException) {
auto meta_service = get_meta_service();
brpc::Controller cntl;
@@ -6983,9 +7071,25 @@ TEST(MetaServiceTest, UpdateDeleteBitmapWithException) {
}
}
+static void put_delete_bitmap_test_rowset(MetaServiceProxy* meta_service,
+ const std::string& instance_id,
int64_t tablet_id,
+ int64_t version, const std::string&
rowset_id) {
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ std::string rowset_key = meta_rowset_key({instance_id, tablet_id,
version});
+ doris::RowsetMetaCloudPB rowset_meta;
+ rowset_meta.set_rowset_id(0);
+ rowset_meta.set_rowset_id_v2(rowset_id);
+ std::string rowset_value;
+ ASSERT_TRUE(rowset_meta.SerializeToString(&rowset_value));
+ txn->put(rowset_key, rowset_value);
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+}
+
void update_delete_bitmap_with_remove_pre(MetaServiceProxy* meta_service,
int64_t table_id,
int64_t tablet_id, bool inject =
false,
- bool rowset_non_exist = false) {
+ bool rowset_non_exist = false,
+ bool use_delete_bitmap_stats = true)
{
// create rowset, if `rowset_non_exist` enabled, only r4 exists
{
std::unique_ptr<Transaction> txn;
@@ -7098,6 +7202,8 @@ void
update_delete_bitmap_with_remove_pre(MetaServiceProxy* meta_service, int64_
update_delete_bitmap_req.set_initiator(tablet_id);
update_delete_bitmap_req.set_pre_rowset_agg_start_version(4);
update_delete_bitmap_req.set_pre_rowset_agg_end_version(6);
+
update_delete_bitmap_req.set_enable_remove_pre_rowsets_delete_bitmap_by_keys(
+ use_delete_bitmap_stats);
std::vector<std::tuple<std::string, int64_t, int64_t, int64_t>>
new_rowset_segment_version_vector = {/* r2-0 */ {"r2", 0, 6, 2},
/* r3-0 */ {"r3", 0, 6, 3},
@@ -7113,6 +7219,21 @@ void
update_delete_bitmap_with_remove_pre(MetaServiceProxy* meta_service, int64_
update_delete_bitmap_req.add_segment_delete_bitmaps(new_large_value);
update_delete_bitmap_req.add_pre_rowset_versions(rowset_version);
}
+ if (use_delete_bitmap_stats) {
+ for (const auto& rowset : rowset_vector) {
+ auto* rowset_stats =
update_delete_bitmap_req.add_pre_rowset_delete_bitmap_stats();
+ rowset_stats->set_rowset_id(rowset);
+ for (const auto& [delete_bitmap_rowset, segment, version] :
+ rowset_segment_version_vector) {
+ if (delete_bitmap_rowset == rowset && version >= 4 && version
< 6) {
+ auto* delete_bitmap_stat =
rowset_stats->add_delete_bitmap_stats();
+ delete_bitmap_stat->set_segment_id(segment);
+ delete_bitmap_stat->set_version(version);
+
delete_bitmap_stat->set_delete_bitmap_size(large_value.size());
+ }
+ }
+ }
+ }
meta_service->update_delete_bitmap(reinterpret_cast<google::protobuf::RpcController*>(&cntl),
&update_delete_bitmap_req,
&update_delete_bitmap_res,
nullptr);
@@ -7183,11 +7304,18 @@ TEST(MetaServiceTest,
UpdateDeleteBitmapWithRemovePreDeleteBitmap) {
SyncPoint::get_instance()->clear_all_call_backs();
};
- update_delete_bitmap_with_remove_pre(meta_service.get(), 200, 202);
+ update_delete_bitmap_with_remove_pre(meta_service.get(), 200, 202, false,
false, false);
int64_t max_txn_commit_byte = config::max_txn_commit_byte;
config::max_txn_commit_byte = 1000;
+ int remove_pre_rowsets_commit_count = 0;
+ sp->set_call_back("update_delete_bitmap:remove_pre_rowsets:commit",
+ [&](auto&&) { remove_pre_rowsets_commit_count++; });
+ sp->enable_processing();
update_delete_bitmap_with_remove_pre(meta_service.get(), 300, 302);
+ EXPECT_GT(remove_pre_rowsets_commit_count, 0);
+ sp->clear_all_call_backs();
+ sp->disable_processing();
sp->set_call_back("update_delete_bitmap:commit:err", [&](auto&& args) {
auto initiator = try_any_cast<int64_t>(args[0]);
@@ -7206,6 +7334,287 @@ TEST(MetaServiceTest,
UpdateDeleteBitmapWithRemovePreDeleteBitmap) {
update_delete_bitmap_with_remove_pre(meta_service.get(), 500, 502, false,
true);
}
+TEST(MetaServiceTest, EmptyPreRowsetStatsUsesKeyRemoval) {
+ auto meta_service = get_meta_service();
+ extern std::string get_instance_id(const std::shared_ptr<ResourceManager>&
rc_mgr,
+ const std::string& cloud_unique_id);
+ auto instance_id = get_instance_id(meta_service->resource_mgr(),
"test_cloud_unique_id");
+ constexpr int64_t tablet_id = 622;
+ const std::string rowset_id = "empty_pre_rowset_stats";
+ put_delete_bitmap_test_rowset(meta_service.get(), instance_id, tablet_id,
1, rowset_id);
+
+ auto old_delete_bitmap_key = meta_delete_bitmap_key({instance_id,
tablet_id, rowset_id, 1, 0});
+ const std::string old_delete_bitmap = "old_delete_bitmap";
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ blob_put(txn.get(), old_delete_bitmap_key, old_delete_bitmap, 0);
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+
+ brpc::Controller cntl;
+ UpdateDeleteBitmapRequest req;
+ UpdateDeleteBitmapResponse res;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_table_id(620);
+ req.set_partition_id(621);
+ req.set_tablet_id(tablet_id);
+ req.set_lock_id(-3);
+ req.set_without_lock(true);
+ req.set_initiator(tablet_id);
+ req.set_pre_rowset_agg_start_version(1);
+ req.set_pre_rowset_agg_end_version(2);
+ req.add_rowset_ids(rowset_id);
+ req.add_segment_ids(0);
+ req.add_versions(2);
+ req.add_segment_delete_bitmaps("aggregated_delete_bitmap");
+ req.add_pre_rowset_versions(1);
+ req.set_enable_remove_pre_rowsets_delete_bitmap_by_keys(true);
+
meta_service->update_delete_bitmap(reinterpret_cast<google::protobuf::RpcController*>(&cntl),
+ &req, &res, nullptr);
+
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ ValueBuf val;
+ ASSERT_EQ(blob_get(txn.get(), old_delete_bitmap_key, &val),
TxnErrorCode::TXN_OK);
+ EXPECT_EQ(val.value(), old_delete_bitmap);
+}
+
+TEST(MetaServiceTest,
UnderestimatedPreRowsetDeleteBitmapSizeLeavesSkippedTail) {
+ auto meta_service = get_meta_service();
+ extern std::string get_instance_id(const std::shared_ptr<ResourceManager>&
rc_mgr,
+ const std::string& cloud_unique_id);
+ auto instance_id = get_instance_id(meta_service->resource_mgr(),
"test_cloud_unique_id");
+ constexpr int64_t tablet_id = 632;
+ const std::string rowset_id = "underestimated_delete_bitmap";
+ put_delete_bitmap_test_rowset(meta_service.get(), instance_id, tablet_id,
1, rowset_id);
+
+ auto old_delete_bitmap_key = meta_delete_bitmap_key({instance_id,
tablet_id, rowset_id, 1, 0});
+ const std::string old_delete_bitmap(DEFAULT_BLOB_SPLIT_SIZE * 3 + 1, 'a');
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ blob_put(txn.get(), old_delete_bitmap_key, old_delete_bitmap, 0);
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+
+ const std::string aggregated_delete_bitmap = "aggregated_delete_bitmap";
+ brpc::Controller update_cntl;
+ UpdateDeleteBitmapRequest update_req;
+ UpdateDeleteBitmapResponse update_res;
+ update_req.set_cloud_unique_id("test_cloud_unique_id");
+ update_req.set_table_id(630);
+ update_req.set_partition_id(631);
+ update_req.set_tablet_id(tablet_id);
+ update_req.set_lock_id(-3);
+ update_req.set_without_lock(true);
+ update_req.set_initiator(tablet_id);
+ update_req.set_pre_rowset_agg_start_version(1);
+ update_req.set_pre_rowset_agg_end_version(2);
+ update_req.add_rowset_ids(rowset_id);
+ update_req.add_segment_ids(0);
+ update_req.add_versions(2);
+ update_req.add_segment_delete_bitmaps(aggregated_delete_bitmap);
+ update_req.add_pre_rowset_versions(1);
+ update_req.set_enable_remove_pre_rowsets_delete_bitmap_by_keys(true);
+ auto* rowset_stats = update_req.add_pre_rowset_delete_bitmap_stats();
+ rowset_stats->set_rowset_id(rowset_id);
+ auto* delete_bitmap_stat = rowset_stats->add_delete_bitmap_stats();
+ delete_bitmap_stat->set_segment_id(0);
+ delete_bitmap_stat->set_version(1);
+ delete_bitmap_stat->set_delete_bitmap_size(DEFAULT_BLOB_SPLIT_SIZE);
+ meta_service->update_delete_bitmap(
+ reinterpret_cast<google::protobuf::RpcController*>(&update_cntl),
&update_req,
+ &update_res, nullptr);
+ ASSERT_EQ(update_res.status().code(), MetaServiceCode::OK);
+
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ std::string value;
+ EXPECT_EQ(txn->get(encode_blob_key(old_delete_bitmap_key, 0, 0), &value),
+ TxnErrorCode::TXN_KEY_NOT_FOUND);
+ EXPECT_EQ(txn->get(encode_blob_key(old_delete_bitmap_key, 0, 1), &value),
TxnErrorCode::TXN_OK);
+
+ brpc::Controller get_cntl;
+ GetDeleteBitmapRequest get_req;
+ GetDeleteBitmapResponse get_res;
+ get_req.set_cloud_unique_id("test_cloud_unique_id");
+ get_req.set_tablet_id(tablet_id);
+ get_req.add_rowset_ids(rowset_id);
+ get_req.add_begin_versions(0);
+ get_req.add_end_versions(3);
+
meta_service->get_delete_bitmap(reinterpret_cast<google::protobuf::RpcController*>(&get_cntl),
+ &get_req, &get_res, nullptr);
+
+ ASSERT_EQ(get_res.status().code(), MetaServiceCode::OK);
+ ASSERT_EQ(get_res.rowset_ids_size(), 1);
+ ASSERT_EQ(get_res.segment_ids_size(), 1);
+ ASSERT_EQ(get_res.versions_size(), 1);
+ ASSERT_EQ(get_res.segment_delete_bitmaps_size(), 1);
+ EXPECT_EQ(get_res.rowset_ids(0), rowset_id);
+ EXPECT_EQ(get_res.segment_ids(0), 0);
+ EXPECT_EQ(get_res.versions(0), 2);
+ EXPECT_EQ(get_res.segment_delete_bitmaps(0), aggregated_delete_bitmap);
+}
+
+static void test_get_delete_bitmap_during_point_cleanup(size_t
delete_bitmap_size) {
+ auto meta_service = get_meta_service();
+ auto sp = SyncPoint::get_instance();
+ DORIS_CLOUD_DEFER {
+ sp->clear_all_call_backs();
+ sp->disable_processing();
+ };
+
+ extern std::string get_instance_id(const std::shared_ptr<ResourceManager>&
rc_mgr,
+ const std::string& cloud_unique_id);
+ auto instance_id = get_instance_id(meta_service->resource_mgr(),
"test_cloud_unique_id");
+ constexpr int64_t table_id = 640;
+ constexpr int64_t partition_id = 641;
+ constexpr int64_t tablet_id = 642;
+ const std::string rowset_id = "point_cleanup_during_paginated_read";
+ put_delete_bitmap_test_rowset(meta_service.get(), instance_id, tablet_id,
1, rowset_id);
+
+ auto old_delete_bitmap_key = meta_delete_bitmap_key({instance_id,
tablet_id, rowset_id, 1, 0});
+ std::string old_delete_bitmap(DEFAULT_BLOB_SPLIT_SIZE, 'a');
+ old_delete_bitmap.append(DEFAULT_BLOB_SPLIT_SIZE, 'b');
+ old_delete_bitmap.append(DEFAULT_BLOB_SPLIT_SIZE, 'c');
+ old_delete_bitmap.push_back('d');
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ blob_put(txn.get(), old_delete_bitmap_key, old_delete_bitmap, 0);
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+
+ const std::string aggregated_delete_bitmap = "aggregated_delete_bitmap";
+ UpdateDeleteBitmapRequest update_req;
+ update_req.set_cloud_unique_id("test_cloud_unique_id");
+ update_req.set_table_id(table_id);
+ update_req.set_partition_id(partition_id);
+ update_req.set_tablet_id(tablet_id);
+ update_req.set_lock_id(-3);
+ update_req.set_without_lock(true);
+ update_req.set_initiator(tablet_id);
+ update_req.set_pre_rowset_agg_start_version(1);
+ update_req.set_pre_rowset_agg_end_version(2);
+ update_req.add_rowset_ids(rowset_id);
+ update_req.add_segment_ids(0);
+ update_req.add_versions(2);
+ update_req.add_segment_delete_bitmaps(aggregated_delete_bitmap);
+ update_req.add_pre_rowset_versions(1);
+ update_req.set_enable_remove_pre_rowsets_delete_bitmap_by_keys(true);
+ auto* rowset_stats = update_req.add_pre_rowset_delete_bitmap_stats();
+ rowset_stats->set_rowset_id(rowset_id);
+ auto* delete_bitmap_stat = rowset_stats->add_delete_bitmap_stats();
+ delete_bitmap_stat->set_segment_id(0);
+ delete_bitmap_stat->set_version(1);
+ delete_bitmap_stat->set_delete_bitmap_size(delete_bitmap_size);
+
+ bool cleanup_done = false;
+ sp->set_call_back("get_delete_bitmap_test",
+ [&](auto&& args) { *try_any_cast<bool*>(args[0]) = true;
});
+ sp->set_call_back("get_delete_bitmap_err", [&](auto&& args) {
+ auto round = *try_any_cast<int64_t*>(args[0]);
+ if (round != 1 || cleanup_done) {
+ return;
+ }
+ cleanup_done = true;
+ brpc::Controller update_cntl;
+ UpdateDeleteBitmapResponse update_res;
+ meta_service->update_delete_bitmap(
+
reinterpret_cast<google::protobuf::RpcController*>(&update_cntl), &update_req,
+ &update_res, nullptr);
+ ASSERT_EQ(update_res.status().code(), MetaServiceCode::OK);
+ *try_any_cast<TxnErrorCode*>(args[1]) = TxnErrorCode::TXN_TOO_OLD;
+ });
+ sp->enable_processing();
+
+ brpc::Controller get_cntl;
+ GetDeleteBitmapRequest get_req;
+ GetDeleteBitmapResponse get_res;
+ get_req.set_cloud_unique_id("test_cloud_unique_id");
+ get_req.set_tablet_id(tablet_id);
+ get_req.add_rowset_ids(rowset_id);
+ get_req.add_begin_versions(0);
+ get_req.add_end_versions(3);
+
meta_service->get_delete_bitmap(reinterpret_cast<google::protobuf::RpcController*>(&get_cntl),
+ &get_req, &get_res, nullptr);
+
+ ASSERT_TRUE(cleanup_done);
+ ASSERT_EQ(get_res.status().code(), MetaServiceCode::OK);
+ ASSERT_EQ(get_res.rowset_ids_size(), 1);
+ ASSERT_EQ(get_res.segment_ids_size(), 1);
+ ASSERT_EQ(get_res.versions_size(), 1);
+ ASSERT_EQ(get_res.segment_delete_bitmaps_size(), 1);
+ EXPECT_EQ(get_res.rowset_ids(0), rowset_id);
+ EXPECT_EQ(get_res.segment_ids(0), 0);
+ EXPECT_EQ(get_res.versions(0), 2);
+ EXPECT_EQ(get_res.segment_delete_bitmaps(0), aggregated_delete_bitmap);
+}
+
+TEST(MetaServiceTest, GetDeleteBitmapSkipsPointCleanupTailAfterTxnTooOld) {
+ test_get_delete_bitmap_during_point_cleanup(DEFAULT_BLOB_SPLIT_SIZE * 3);
+}
+
+TEST(MetaServiceTest,
GetDeleteBitmapDropsPartialValueAfterAllBlobKeysAreRemoved) {
+ test_get_delete_bitmap_during_point_cleanup(DEFAULT_BLOB_SPLIT_SIZE * 3 +
1);
+}
+
+TEST(MetaServiceTest, RemovePreDeleteBitmapBatchesEachBlobKey) {
+ auto meta_service = get_meta_service();
+ auto sp = SyncPoint::get_instance();
+ int64_t old_max_txn_commit_byte = config::max_txn_commit_byte;
+ DORIS_CLOUD_DEFER {
+ config::max_txn_commit_byte = old_max_txn_commit_byte;
+ sp->clear_all_call_backs();
+ sp->disable_processing();
+ };
+
+ constexpr size_t max_txn_commit_byte = 300;
+ config::max_txn_commit_byte = max_txn_commit_byte;
+ std::vector<size_t> committed_txn_sizes;
+ sp->set_call_back("update_delete_bitmap:remove_pre_rowsets:commit",
[&](auto&& args) {
+ committed_txn_sizes.push_back(try_any_cast<size_t>(args[0]));
+ });
+ sp->enable_processing();
+
+ extern std::string get_instance_id(const std::shared_ptr<ResourceManager>&
rc_mgr,
+ const std::string& cloud_unique_id);
+ auto instance_id = get_instance_id(meta_service->resource_mgr(),
"test_cloud_unique_id");
+ constexpr int64_t tablet_id = 612;
+ const std::string rowset_id = "batch_blob_keys_rowset";
+ const std::string value(DEFAULT_BLOB_SPLIT_SIZE * 20, 'a');
+ auto delete_bitmap_key = meta_delete_bitmap_key({instance_id, tablet_id,
rowset_id, 1, 0});
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ blob_put(txn.get(), delete_bitmap_key, value, 0);
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+
+ brpc::Controller cntl;
+ UpdateDeleteBitmapRequest req;
+ UpdateDeleteBitmapResponse res;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_table_id(610);
+ req.set_partition_id(611);
+ req.set_tablet_id(tablet_id);
+ req.set_lock_id(-3);
+ req.set_without_lock(true);
+ req.set_initiator(tablet_id);
+ req.set_pre_rowset_agg_start_version(1);
+ req.set_pre_rowset_agg_end_version(2);
+ req.set_enable_remove_pre_rowsets_delete_bitmap_by_keys(true);
+ auto* rowset_stats = req.add_pre_rowset_delete_bitmap_stats();
+ rowset_stats->set_rowset_id(rowset_id);
+ auto* delete_bitmap_stat = rowset_stats->add_delete_bitmap_stats();
+ delete_bitmap_stat->set_segment_id(0);
+ delete_bitmap_stat->set_version(1);
+ delete_bitmap_stat->set_delete_bitmap_size(value.size());
+
meta_service->update_delete_bitmap(reinterpret_cast<google::protobuf::RpcController*>(&cntl),
+ &req, &res, nullptr);
+
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+ ASSERT_FALSE(committed_txn_sizes.empty());
+ for (auto txn_size : committed_txn_sizes) {
+ EXPECT_LE(txn_size, max_txn_commit_byte);
+ }
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ ValueBuf val;
+ EXPECT_EQ(blob_get(txn.get(), delete_bitmap_key, &val),
TxnErrorCode::TXN_KEY_NOT_FOUND);
+}
+
TEST(MetaServiceTest, GetDeleteBitmapWithIdx) {
auto meta_service = get_meta_service();
extern std::string get_instance_id(const std::shared_ptr<ResourceManager>&
rc_mgr,
diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto
index 3e1b80a8415..030029bedf2 100644
--- a/gensrc/proto/cloud.proto
+++ b/gensrc/proto/cloud.proto
@@ -1915,6 +1915,17 @@ message DeleteBitmapStoragePB {
optional PackedSliceLocationPB packed_slice_location = 3;
}
+message PreRowsetDeleteBitmapStatsPB {
+ message DeleteBitmapStatPB {
+ optional uint32 segment_id = 1;
+ optional int64 version = 2;
+ optional uint64 delete_bitmap_size = 3;
+ }
+
+ optional string rowset_id = 1;
+ repeated DeleteBitmapStatPB delete_bitmap_stats = 2;
+}
+
message UpdateDeleteBitmapRequest {
optional string cloud_unique_id = 1; // For auth
optional int64 table_id = 2;
@@ -1940,6 +1951,14 @@ message UpdateDeleteBitmapRequest {
optional int64 pre_rowset_agg_end_version = 17;
// when update delete_bitmap of pre rowsets, check the rowset exists
repeated int64 pre_rowset_versions = 18;
+ // remove pre-rowset delete bitmaps in [end_version, end_version] before
writing aggregated
+ // delete bitmaps; true: point delete; false or unset: range delete
+ optional bool enable_remove_agg_pre_rowsets_delete_bitmap_by_keys = 19;
+ // remove pre-rowset delete bitmaps in [start_version, end_version):
+ // true: point delete; false or unset: range delete
+ optional bool enable_remove_pre_rowsets_delete_bitmap_by_keys = 20;
+ // stats of pre-rowset delete bitmaps in [start_version, end_version) used
for point delete
+ repeated PreRowsetDeleteBitmapStatsPB pre_rowset_delete_bitmap_stats = 21;
// 50-51: used for storage v2
repeated string delta_rowset_ids = 50;
repeated DeleteBitmapStoragePB delete_bitmap_storages = 51;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
