This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 1dae1d30e2b branch-4.1: [fix](cloud) Split oversized delete bitmaps
across metadata transactions #70478 (#70653)
1dae1d30e2b is described below
commit 1dae1d30e2b66693a50cf05fe2312a7ba2e866fd
Author: bobhan1 <[email protected]>
AuthorDate: Wed Nov 18 09:24:01 2026 +0800
branch-4.1: [fix](cloud) Split oversized delete bitmaps across metadata
transactions #70478 (#70653)
### What problem does this PR solve?
Related PR: #70478
Backport commit `5e85976d4c58d5528cb49d3ef677d5e695c2a36d` to
`branch-4.1`.
A single locked delete bitmap can exceed the FoundationDB transaction
limit even though its value is split into 90 KB KVs. Split oversized
v1/v2 values across metadata transactions using the existing transaction
budget, and verify the committed prefix in the last data transaction
with non-snapshot reads. Missing or replaced prefix fragments fail with
`KV_TXN_CONFLICT`; unlocked aggregation retains atomic overwrites.
The conflict resolution preserves the 4.1 `void` writer interface and
existing pre-rowset aggregation behavior. All upstream large-bitmap
tests are included unchanged. There are no configuration, protocol, or
persistent-format changes.
### Testing
- ASAN Cloud UT compiled with `BUILD_TYPE=ASAN ENABLE_PCH=OFF
./run-cloud-ut.sh -j100`; the packaged repository runner executed
`meta_service_test` with `*DeleteBitmap*:*PreRowset*` against an
isolated FoundationDB 7.3.69 instance.
- **56 passed, 1 skipped**, with zero failures/errors. All **16 real FDB
cases passed**. MemTxnKv passed 15 new cases and skipped the range-clear
conflict case that its conflict detector cannot model. Coverage includes
large v1/v2 and dual writes, compaction/schema change, overlapping
cleanup/replacement, ambiguous commits, lock loss, shorter retries,
explicit sub-transactions, and unlocked atomic overwrites.
- The test runtime used the bundled FDB 7.3.69 external-client
directory; no ASAN failure was suppressed.
- `ENABLE_PCH=OFF ENABLE_INJECTION_POINT=OFF ./build.sh --cloud -j100`
passed for the production Release build with `MAKE_TEST=OFF`.
- clang-format 16 and `git diff --check upstream/branch-4.1...HEAD`
passed. FE/BE end-to-end tests were not run.
As in #70478, the pre-existing race involving delayed cleanup after an
RPC has returned OK remains outside this change.
### Release note
Fix Cloud Merge-on-Write writes failing when a single delete bitmap
exceeds the FoundationDB transaction limit.
### Check List (For Author)
- [x] Unit Test
- [x] Behavior changed: an oversized locked bitmap can span metadata
transactions while retaining the existing publish boundary.
- [x] No documentation change required.
---
cloud/src/meta-service/meta_service.cpp | 410 ++++++++++++-------
cloud/test/CMakeLists.txt | 1 +
cloud/test/meta_service_large_bitmap_test.cpp | 544 ++++++++++++++++++++++++++
3 files changed, 809 insertions(+), 146 deletions(-)
diff --git a/cloud/src/meta-service/meta_service.cpp
b/cloud/src/meta-service/meta_service.cpp
index dd5dc3cd05e..87610203a3d 100644
--- a/cloud/src/meta-service/meta_service.cpp
+++ b/cloud/src/meta-service/meta_service.cpp
@@ -3905,6 +3905,216 @@ static bool
check_delete_bitmap_point_delete(MetaServiceCode& code, std::string&
return true;
}
+static bool commit_and_renew_delete_bitmap_txn(MetaServiceCode& code,
std::string& msg,
+ std::stringstream& ss,
+ const std::shared_ptr<TxnKv>&
txn_kv,
+ std::unique_ptr<Transaction>&
txn, KVStats& stats,
+ UpdateDeleteBitmapTxnStats&
txn_stats,
+ const
UpdateDeleteBitmapRequest* request,
+ std::string& instance_id,
size_t request_index,
+ std::string& use_version,
std::string& log) {
+ auto table_id = request->table_id();
+ auto tablet_id = request->tablet_id();
+ LOG(INFO) << "fdb txn size more than " << config::max_txn_commit_byte
+ << ", current size: " << txn->approximate_bytes() << ",
tablet_id: " << tablet_id
+ << " lock_id=" << request->lock_id() << " initiator=" <<
request->initiator()
+ << ", need to commit";
+ TxnErrorCode err = txn->commit();
+ TEST_SYNC_POINT_CALLBACK("update_delete_bitmap:commit:err",
request->initiator(), request_index,
+ &err);
+ txn_stats.total_txn_put_keys += txn->num_put_keys();
+ txn_stats.total_txn_put_bytes += txn->put_bytes();
+ txn_stats.total_txn_size += txn->approximate_bytes();
+ txn_stats.total_txn_count++;
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::COMMIT>(err);
+ ss << "failed to update delete bitmap, err=" << err << " tablet_id="
<< tablet_id
+ << " lock_id=" << request->lock_id() << " initiator=" <<
request->initiator()
+ << " delete_bitmap_key=" << txn_stats.current_key_count
+ << " delete_bitmap_value=" << txn_stats.current_value_count
+ << " put_size=" << txn->put_bytes() << " num_put_keys=" <<
txn->num_put_keys()
+ << " txn_size=" << txn->approximate_bytes();
+ msg = ss.str();
+ g_bvar_update_delete_bitmap_fail_counter << 1;
+ return false;
+ }
+ stats.get_bytes += txn->get_bytes();
+ stats.put_bytes += txn->put_bytes();
+ stats.del_bytes += txn->delete_bytes();
+ stats.get_counter += txn->num_get_keys();
+ stats.put_counter += txn->num_put_keys();
+ stats.del_counter += txn->num_del_keys();
+ txn_stats.current_key_count = 0;
+ txn_stats.current_value_count = 0;
+ err = txn_kv->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::CREATE>(err);
+ msg = "failed to init txn";
+ return false;
+ }
+ if (!request->without_lock()) {
+ std::string lock_key =
meta_delete_bitmap_update_lock_key({instance_id, table_id, -1});
+ DeleteBitmapUpdateLockPB lock_info;
+ if (!check_delete_bitmap_lock(code, msg, ss, txn, instance_id,
table_id, request->lock_id(),
+ request->initiator(), lock_key,
lock_info, use_version,
+ log)) {
+ LOG(WARNING) << "failed to check delete bitmap lock, table_id=" <<
table_id
+ << " request lock_id=" << request->lock_id()
+ << " request initiator=" << request->initiator() << "
msg " << msg;
+ return false;
+ }
+ }
+ return true;
+}
+
+static bool check_delete_bitmap_rowset(MetaServiceCode& code, std::string& msg,
+ std::stringstream& ss, Transaction* txn,
+ const UpdateDeleteBitmapRequest*
request,
+ const std::string& instance_id, size_t
request_index,
+ bool is_versioned_read,
ResourceManager* resource_mgr,
+ std::set<std::string>&
non_exist_rowset_ids,
+ bool& should_write) {
+ should_write = true;
+ if (!request->without_lock() || !request->has_pre_rowset_agg_end_version()
||
+ request->pre_rowset_agg_end_version() <= 0) {
+ return true;
+ }
+ auto tablet_id = request->tablet_id();
+ // check the rowset exists
+ if (non_exist_rowset_ids.contains(request->rowset_ids(request_index))) {
+ LOG(INFO) << "skip update delete bitmap, rowset_id=" <<
request->rowset_ids(request_index)
+ << " version=" << request->pre_rowset_versions(request_index)
+ << " tablet_id=" << tablet_id << " because the rowset does
not exist";
+ should_write = false;
+ return true;
+ }
+
+ TxnErrorCode err = TxnErrorCode::TXN_OK;
+ doris::RowsetMetaCloudPB rs;
+ if (!is_versioned_read) {
+ auto rowset_key = meta_rowset_key(
+ {instance_id, tablet_id,
request->pre_rowset_versions(request_index)});
+ std::string rowset_val;
+ err = txn->get(rowset_key, &rowset_val);
+ if (err != TxnErrorCode::TXN_OK && TxnErrorCode::TXN_KEY_NOT_FOUND !=
err) {
+ ss << "failed to get rowset, instance_id=" << instance_id << "
tablet_id=" << tablet_id
+ << " version=" << request->pre_rowset_versions(request_index)
<< " err=" << err;
+ msg = ss.str();
+ code = cast_as<ErrCategory::READ>(err);
+ return false;
+ }
+ if (err == TxnErrorCode::TXN_OK &&
+ !rs.ParseFromArray(rowset_val.data(), rowset_val.size())) {
+ code = MetaServiceCode::PROTOBUF_PARSE_ERR;
+ ss << "malformed rowset meta, unable to deserialize, tablet_id="
<< tablet_id
+ << " key=" << hex(rowset_key);
+ msg = ss.str();
+ return false;
+ }
+ } else {
+ CloneChainReader reader(instance_id, resource_mgr);
+ err = reader.get_rowset_meta(txn, tablet_id,
request->pre_rowset_versions(request_index),
+ &rs);
+ if (err != TxnErrorCode::TXN_OK && TxnErrorCode::TXN_KEY_NOT_FOUND !=
err) {
+ ss << "failed to get versioned rowset, err=" << err << "
tablet_id=" << tablet_id
+ << " version=" << request->pre_rowset_versions(request_index);
+ msg = ss.str();
+ code = cast_as<ErrCategory::READ>(err);
+ return false;
+ }
+ }
+ if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) {
+ non_exist_rowset_ids.emplace(request->rowset_ids(request_index));
+ LOG(INFO) << "skip update delete bitmap, rowset_id=" <<
request->rowset_ids(request_index)
+ << " version=" << request->pre_rowset_versions(request_index)
+ << " tablet_id=" << tablet_id << " because the rowset is not
exist";
+ should_write = false;
+ return true;
+ }
+ if (rs.rowset_id_v2() != request->rowset_ids(request_index)) {
+ LOG(INFO) << "skip update delete bitmap, rowset_id=" <<
request->rowset_ids(request_index)
+ << " version=" << request->pre_rowset_versions(request_index)
+ << " tablet_id=" << tablet_id << " because the rowset is not
exist";
+ non_exist_rowset_ids.emplace(request->rowset_ids(request_index));
+ should_write = false;
+ return true;
+ }
+ return true;
+}
+
+static bool remove_delete_bitmap_before_update(MetaServiceCode& code,
std::string& msg,
+ std::unique_ptr<Transaction>&
txn,
+ const std::string& key,
+ const
UpdateDeleteBitmapRequest* request,
+ size_t request_index) {
+ if (request->lock_id() != COMPACTION_WITHOUT_LOCK_DELETE_BITMAP_LOCK_ID) {
+ return true;
+ }
+ auto tablet_id = request->tablet_id();
+ bool point_delete = false;
+ bool delete_bitmap_exists = false;
+ std::optional<uint16_t> max_blob_sequence;
+ if (!check_delete_bitmap_point_delete(code, msg, txn, key, request,
request_index, point_delete,
+ delete_bitmap_exists,
max_blob_sequence)) {
+ return false;
+ }
+ if (point_delete) {
+ if (delete_bitmap_exists) {
+ if (max_blob_sequence.has_value()) {
+ for (size_t sequence = 0; sequence <= *max_blob_sequence;
++sequence) {
+ txn->remove(encode_blob_key(key, 0, sequence));
+ }
+ } else {
+ // Compatibility with delete bitmaps written directly by the
legacy txn->put.
+ txn->remove(key);
+ }
+ LOG(INFO) << "xxx point remove delete_bitmap_key=" << hex(key)
+ << " tablet_id=" << tablet_id << " lock_id=" <<
request->lock_id()
+ << " initiator=" << request->initiator()
+ << " delete_key_count=" <<
(max_blob_sequence.value_or(0) + 1);
+ }
+ } else {
+ auto& start_key = key;
+ std::string end_key {start_key};
+ encode_int64(INT64_MAX, &end_key);
+ txn->remove(start_key, end_key);
+ LOG(INFO) << "xxx range remove delete_bitmap_key=" << hex(start_key)
+ << " tablet_id=" << tablet_id << " lock_id=" <<
request->lock_id()
+ << " initiator=" << request->initiator();
+ }
+ return true;
+}
+
+static bool check_delete_bitmap_prefix(MetaServiceCode& code, std::string&
msg, Transaction* txn,
+ const std::string& key,
std::string_view prefix) {
+ auto end_key = encode_blob_key(key, 0, prefix.size() /
DEFAULT_BLOB_SPLIT_SIZE);
+ // Read the committed prefix through the last data transaction, without
+ // snapshot reads, so pending cleanup also conflicts with its final commit.
+ auto it = txn->full_range_get(key, end_key);
+ size_t offset = 0;
+ TxnErrorCode err = TxnErrorCode::TXN_OK;
+ for (auto kv = it->next(); kv.has_value(); kv = it->next()) {
+ auto [fragment_key, fragment] = *kv;
+ if (fragment_key != encode_blob_key(key, 0, offset /
DEFAULT_BLOB_SPLIT_SIZE) ||
+ fragment != prefix.substr(offset, DEFAULT_BLOB_SPLIT_SIZE)) {
+ err = TxnErrorCode::TXN_CONFLICT;
+ break;
+ }
+ offset += fragment.size();
+ }
+ if (!it->is_valid()) {
+ err = it->error_code();
+ } else if (offset != prefix.size()) {
+ err = TxnErrorCode::TXN_CONFLICT;
+ }
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::READ>(err);
+ msg = fmt::format("failed to verify delete bitmap prefix, key={}
err={}", hex(key), err);
+ return false;
+ }
+ return true;
+}
+
void _write_delete_bitmap_kvs(MetaServiceCode& code, std::string& msg,
std::stringstream& ss,
std::shared_ptr<TxnKv> txn_kv,
std::unique_ptr<Transaction>& txn,
std::string& use_version, std::set<std::string>&
non_exist_rowset_ids,
@@ -3913,174 +4123,82 @@ void _write_delete_bitmap_kvs(MetaServiceCode& code,
std::string& msg, std::stri
std::string& key, const std::string& val,
UpdateDeleteBitmapTxnStats& txn_stats, bool
is_versioned_read,
ResourceManager* resource_mgr) {
- auto table_id = request->table_id();
- auto tablet_id = request->tablet_id();
bool without_lock = request->has_without_lock() ? request->without_lock()
: false;
- // Split into multiple fdb transactions, because the size of one fdb
- // transaction can't exceed 10MB.
- if (txn->approximate_bytes() + key.size() * 3 + val.size() >
config::max_txn_commit_byte) {
- LOG(INFO) << "fdb txn size more than " << config::max_txn_commit_byte
- << ", current size: " << txn->approximate_bytes() << ",
tablet_id: " << tablet_id
- << " lock_id=" << request->lock_id() << " initiator=" <<
request->initiator()
- << ", need to commit";
- TxnErrorCode err = txn->commit();
- TEST_SYNC_POINT_CALLBACK("update_delete_bitmap:commit:err",
request->initiator(), i, &err);
- txn_stats.total_txn_put_keys += txn->num_put_keys();
- txn_stats.total_txn_put_bytes += txn->put_bytes();
- txn_stats.total_txn_size += txn->approximate_bytes();
- txn_stats.total_txn_count++;
- if (err != TxnErrorCode::TXN_OK) {
- code = cast_as<ErrCategory::COMMIT>(err);
- ss << "failed to update delete bitmap, err=" << err << "
tablet_id=" << tablet_id
- << " lock_id=" << request->lock_id() << " initiator=" <<
request->initiator()
- << " delete_bitmap_key=" << txn_stats.current_key_count
- << " delete_bitmap_value=" << txn_stats.current_value_count
- << " put_size=" << txn->put_bytes() << " num_put_keys=" <<
txn->num_put_keys()
- << " txn_size=" << txn->approximate_bytes();
- msg = ss.str();
- g_bvar_update_delete_bitmap_fail_counter << 1;
- return;
- }
- stats.get_bytes += txn->get_bytes();
- stats.put_bytes += txn->put_bytes();
- stats.del_bytes += txn->delete_bytes();
- stats.get_counter += txn->num_get_keys();
- stats.put_counter += txn->num_put_keys();
- stats.del_counter += txn->num_del_keys();
- txn_stats.current_key_count = 0;
- txn_stats.current_value_count = 0;
- err = txn_kv->create_txn(&txn);
- if (err != TxnErrorCode::TXN_OK) {
- code = cast_as<ErrCategory::CREATE>(err);
- msg = "failed to init txn";
- return;
- }
- if (!without_lock) {
- std::string lock_key =
meta_delete_bitmap_update_lock_key({instance_id, table_id, -1});
- DeleteBitmapUpdateLockPB lock_info;
- if (!check_delete_bitmap_lock(code, msg, ss, txn, instance_id,
table_id,
- request->lock_id(),
request->initiator(), lock_key,
- lock_info, use_version, log)) {
- LOG(WARNING) << "failed to check delete bitmap lock,
table_id=" << table_id
- << " request lock_id=" << request->lock_id()
- << " request initiator=" << request->initiator()
<< " msg " << msg;
- return;
- }
- }
+ // Split between logical values when their accumulated size exceeds the
budget.
+ // For an oversized value, persist pending cleanup/intent first; its load
version
+ // or output rowset is published only after all fragments have been
written.
+ if (txn->approximate_bytes() + key.size() * 3 + val.size() >
config::max_txn_commit_byte &&
+ !commit_and_renew_delete_bitmap_txn(code, msg, ss, txn_kv, txn, stats,
txn_stats, request,
+ instance_id, i, use_version, log))
{
+ return;
}
#ifdef ENABLE_INJECTION_POINT
if (config::enable_update_delete_bitmap_kv_check) {
- if (!check_delete_bitmap_kv_exists(code, msg, txn, key, instance_id,
tablet_id,
+ if (!check_delete_bitmap_kv_exists(code, msg, txn, key, instance_id,
request->tablet_id(),
request->lock_id())) {
return;
}
}
#endif
- if (without_lock && request->has_pre_rowset_agg_end_version() &&
- request->pre_rowset_agg_end_version() > 0) {
- // check the rowset exists
- if (non_exist_rowset_ids.contains(request->rowset_ids(i))) {
- LOG(INFO) << "skip update delete bitmap, rowset_id=" <<
request->rowset_ids(i)
- << " version=" << request->pre_rowset_versions(i)
- << " tablet_id=" << tablet_id << " because the rowset
does not exist";
- return;
- }
-
- TxnErrorCode err = TxnErrorCode::TXN_OK;
- doris::RowsetMetaCloudPB rs;
- if (!is_versioned_read) {
- auto rowset_key =
- meta_rowset_key({instance_id, tablet_id,
request->pre_rowset_versions(i)});
- std::string rowset_val;
- err = txn->get(rowset_key, &rowset_val);
- if (err != TxnErrorCode::TXN_OK && TxnErrorCode::TXN_KEY_NOT_FOUND
!= err) {
- ss << "failed to get rowset, instance_id=" << instance_id
- << " tablet_id=" << tablet_id << " version=" <<
request->pre_rowset_versions(i)
- << " err=" << err;
- msg = ss.str();
- code = cast_as<ErrCategory::READ>(err);
- return;
+ bool should_write = false;
+ if (!check_delete_bitmap_rowset(code, msg, ss, txn.get(), request,
instance_id, i,
+ is_versioned_read, resource_mgr,
non_exist_rowset_ids,
+ should_write)) {
+ return;
+ }
+ if (!should_write) {
+ return;
+ }
+ if (!remove_delete_bitmap_before_update(code, msg, txn, key, request, i)) {
+ return;
+ }
+ // Unlocked aggregation overwrites visible values and must remain atomic.
+ if (!without_lock && val.size() > config::max_txn_commit_byte) {
+ // blob_put splits a value into KVs, but keeps them in one transaction.
+ // Preserve its key format while batching this single logical value
too.
+ bool counted_in_txn = false;
+ size_t committed_size = 0;
+ for (size_t offset = 0, sequence = 0; offset < val.size(); ++sequence)
{
+ auto fragment = std::string_view(val).substr(offset,
DEFAULT_BLOB_SPLIT_SIZE);
+ auto fragment_key = encode_blob_key(key, 0, sequence);
+ if (txn->approximate_bytes() + fragment_key.size() * 3 +
fragment.size() >
+ config::max_txn_commit_byte) {
+ if (!commit_and_renew_delete_bitmap_txn(code, msg, ss, txn_kv,
txn, stats,
+ txn_stats, request,
instance_id, i,
+ use_version, log)) {
+ return;
+ }
+ counted_in_txn = false;
+ committed_size = offset;
}
- if (err == TxnErrorCode::TXN_OK &&
- !rs.ParseFromArray(rowset_val.data(), rowset_val.size())) {
- code = MetaServiceCode::PROTOBUF_PARSE_ERR;
- ss << "malformed rowset meta, unable to deserialize,
tablet_id=" << tablet_id
- << " key=" << hex(rowset_key);
- msg = ss.str();
- return;
- }
- } else {
- CloneChainReader reader(instance_id, resource_mgr);
- err = reader.get_rowset_meta(txn.get(), tablet_id,
request->pre_rowset_versions(i),
- &rs);
- if (err != TxnErrorCode::TXN_OK && TxnErrorCode::TXN_KEY_NOT_FOUND
!= err) {
- ss << "failed to get versioned rowset, err=" << err << "
tablet_id=" << tablet_id
- << " version=" << request->pre_rowset_versions(i);
- msg = ss.str();
- code = cast_as<ErrCategory::READ>(err);
- return;
+ txn->put(fragment_key, fragment);
+ if (!counted_in_txn) {
+ ++txn_stats.current_key_count;
+ counted_in_txn = true;
}
+ txn_stats.current_value_count += fragment.size();
+ offset += fragment.size();
}
- if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) {
- non_exist_rowset_ids.emplace(request->rowset_ids(i));
- LOG(INFO) << "skip update delete bitmap, rowset_id=" <<
request->rowset_ids(i)
- << " version=" << request->pre_rowset_versions(i)
- << " tablet_id=" << tablet_id << " because the rowset is
not exist";
- return;
- }
- if (rs.rowset_id_v2() != request->rowset_ids(i)) {
- LOG(INFO) << "skip update delete bitmap, rowset_id=" <<
request->rowset_ids(i)
- << " version=" << request->pre_rowset_versions(i)
- << " tablet_id=" << tablet_id << " because the rowset is
not exist";
- non_exist_rowset_ids.emplace(request->rowset_ids(i));
- return;
- }
- }
- // remove first
- if (request->lock_id() == COMPACTION_WITHOUT_LOCK_DELETE_BITMAP_LOCK_ID) {
- bool point_delete = false;
- bool delete_bitmap_exists = false;
- std::optional<uint16_t> max_blob_sequence;
- if (!check_delete_bitmap_point_delete(code, msg, txn, key, request, i,
point_delete,
- delete_bitmap_exists,
max_blob_sequence)) {
- return;
- }
- if (point_delete) {
- if (delete_bitmap_exists) {
- if (max_blob_sequence.has_value()) {
- for (size_t sequence = 0; sequence <= *max_blob_sequence;
++sequence) {
- txn->remove(encode_blob_key(key, 0, sequence));
- }
- } else {
- // Compatibility with delete bitmaps written directly by
the legacy txn->put.
- txn->remove(key);
- }
- LOG(INFO) << "xxx point remove delete_bitmap_key=" << hex(key)
- << " tablet_id=" << tablet_id << " lock_id=" <<
request->lock_id()
- << " initiator=" << request->initiator()
- << " delete_key_count=" <<
(max_blob_sequence.value_or(0) + 1);
+ if (committed_size > 0) {
+ if (!check_delete_bitmap_prefix(code, msg, txn.get(), key,
+ std::string_view(val).substr(0,
committed_size))) {
+ return;
}
- } else {
- auto& start_key = key;
- std::string end_key {start_key};
- encode_int64(INT64_MAX, &end_key);
- txn->remove(start_key, end_key);
- LOG(INFO) << "xxx range remove delete_bitmap_key=" <<
hex(start_key)
- << " tablet_id=" << tablet_id << " lock_id=" <<
request->lock_id()
- << " initiator=" << request->initiator();
+ TEST_SYNC_POINT("update_delete_bitmap:check_prefix:after");
}
+ } else {
+ cloud::blob_put(txn.get(), key, val, 0);
+ ++txn_stats.current_key_count;
+ txn_stats.current_value_count += val.size();
}
- // splitting large values (>90*1000) into multiple KVs
- cloud::blob_put(txn.get(), key, val, 0);
- txn_stats.current_key_count++;
- txn_stats.current_value_count += val.size();
txn_stats.total_key_count++;
txn_stats.total_value_count += val.size();
VLOG_DEBUG << "xxx update delete bitmap put delete_bitmap_key=" << hex(key)
<< " lock_id=" << request->lock_id() << " initiator=" <<
request->initiator()
<< " key_size: " << key.size() << " value_size: " << val.size();
+ return;
}
static bool commit_pre_rowset_delete_bitmap_removal(
diff --git a/cloud/test/CMakeLists.txt b/cloud/test/CMakeLists.txt
index 057b0132356..06c8d527bce 100644
--- a/cloud/test/CMakeLists.txt
+++ b/cloud/test/CMakeLists.txt
@@ -53,6 +53,7 @@ add_executable(meta_reader_test meta_reader_test.cpp)
add_executable(meta_service_test
meta_service_test.cpp
+ meta_service_large_bitmap_test.cpp
meta_service_helper_test.cpp
meta_service_job_test.cpp
meta_service_http_test.cpp
diff --git a/cloud/test/meta_service_large_bitmap_test.cpp
b/cloud/test/meta_service_large_bitmap_test.cpp
new file mode 100644
index 00000000000..41a270d8343
--- /dev/null
+++ b/cloud/test/meta_service_large_bitmap_test.cpp
@@ -0,0 +1,544 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <brpc/controller.h>
+#include <butil/guid.h>
+#include <gen_cpp/cloud.pb.h>
+#include <gtest/gtest.h>
+
+#include <algorithm>
+#include <memory>
+#include <string>
+
+#include "common/config.h"
+#include "common/util.h"
+#include "cpp/sync_point.h"
+#include "meta-service/meta_service.h"
+#include "meta-service/meta_service_helper.h"
+#include "meta-store/blob_message.h"
+#include "meta-store/keys.h"
+#include "meta-store/mem_txn_kv.h"
+#include "meta-store/txn_kv.h"
+#include "mock_resource_manager.h"
+#include "rate-limiter/rate_limiter.h"
+#include "snapshot/snapshot_manager.h"
+
+namespace doris::cloud {
+namespace {
+
+// MemTxnKv normally accepts arbitrarily large transactions. Enforce the real
size
+// limits so one logical blob exceeding 10 MB cannot pass without being
batched.
+struct BitmapTxnLimits {
+ size_t largest_commit = 0;
+ size_t blob_commits = 0;
+};
+
+class BitmapSizeLimitedTransaction : public memkv::Transaction {
+public:
+ BitmapSizeLimitedTransaction(std::shared_ptr<MemTxnKv> kv,
BitmapTxnLimits& limits)
+ : memkv::Transaction(std::move(kv)), limits_(limits) {}
+
+ void put(std::string_view key, std::string_view value) override {
+ EXPECT_LE(value.size(), 100'000);
+ memkv::Transaction::put(key, value);
+ }
+
+ TxnErrorCode commit() override {
+ auto size = approximate_bytes();
+ limits_.largest_commit = std::max(limits_.largest_commit, size);
+ if (size > 10'000'000) {
+ return TxnErrorCode::TXN_BYTES_TOO_LARGE;
+ }
+ auto err = memkv::Transaction::commit();
+ if (err == TxnErrorCode::TXN_OK && put_bytes() > 100'000) {
+ ++limits_.blob_commits;
+ }
+ return err;
+ }
+
+private:
+ BitmapTxnLimits& limits_;
+};
+
+class BitmapSizeLimitedKv : public MemTxnKv {
+public:
+ TxnErrorCode create_txn(std::unique_ptr<Transaction>* txn) override {
+ auto result =
std::make_unique<BitmapSizeLimitedTransaction>(shared_from_this(), limits);
+ EXPECT_EQ(result->init(), 0);
+ *txn = std::move(result);
+ return TxnErrorCode::TXN_OK;
+ }
+
+ BitmapTxnLimits limits;
+};
+
+class LargeDeleteBitmapTest : public testing::TestWithParam<bool> {
+protected:
+ void SetUp() override {
+ old_limit_ = config::max_txn_commit_byte;
+ old_cluster_file_ = config::fdb_cluster_file_path;
+ config::max_txn_commit_byte = 7 * 1024 * 1024;
+ if (GetParam()) {
+ config::fdb_cluster_file_path = "fdb.cluster";
+ static auto fdb_kv = std::make_shared<FdbTxnKv>();
+ static const int init_result = fdb_kv->init();
+ ASSERT_EQ(init_result, 0);
+ kv_ = fdb_kv;
+ instance_id_ = "large_bitmap_" + butil::GenerateGUID();
+ auto* sp = SyncPoint::get_instance();
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_;
+ ret->second = true;
+ });
+ sp->enable_processing();
+ RecordProperty("storage_backend", "FoundationDB");
+ } else {
+ limited_kv_ = std::make_shared<BitmapSizeLimitedKv>();
+ kv_ = limited_kv_;
+ ASSERT_EQ(kv_->init(), 0);
+ RecordProperty("storage_backend", "MemTxnKv");
+ }
+ auto resource = std::make_shared<MockResourceManager>(kv_);
+ instance_id_ = get_instance_id(resource, "test_cloud_unique_id");
+ auto service =
+ std::make_unique<MetaServiceImpl>(kv_, resource,
std::make_shared<RateLimiter>(),
+
std::make_shared<SnapshotManager>(kv_));
+ service_ = std::make_unique<MetaServiceProxy>(std::move(service));
+ }
+
+ void TearDown() override {
+ config::max_txn_commit_byte = old_limit_;
+ config::fdb_cluster_file_path = old_cluster_file_;
+ SyncPoint::get_instance()->clear_all_call_backs();
+ SyncPoint::get_instance()->disable_processing();
+ if (kv_ != nullptr && limited_kv_ == nullptr) {
+ // Each FDB case owns a unique instance; never clear unrelated
keys.
+ std::unique_ptr<Transaction> txn;
+ auto err = kv_->create_txn(&txn);
+ EXPECT_EQ(err, TxnErrorCode::TXN_OK);
+ if (err == TxnErrorCode::TXN_OK) {
+ for (const auto& prefix :
+ {meta_key_prefix(instance_id_),
versioned::meta_key_prefix(instance_id_)}) {
+ txn->remove(prefix, prefix + '\xff');
+ }
+ EXPECT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+ }
+ }
+ }
+
+ void lock(int64_t lock_id = 1001) {
+ GetDeleteBitmapUpdateLockRequest req;
+ GetDeleteBitmapUpdateLockResponse res;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_table_id(100);
+ req.add_partition_ids(101);
+ req.set_lock_id(lock_id);
+ req.set_initiator(103);
+ req.set_expiration(60);
+ brpc::Controller cntl;
+ service_->get_delete_bitmap_update_lock(&cntl, &req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK) <<
res.status().msg();
+ }
+
+ UpdateDeleteBitmapRequest request(const std::string& value, int64_t
lock_id = 1001,
+ bool v2 = false) {
+ UpdateDeleteBitmapRequest req;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_table_id(100);
+ req.set_partition_id(101);
+ req.set_tablet_id(102);
+ req.set_lock_id(lock_id);
+ req.set_initiator(103);
+ req.set_store_version(v2 ? 2 : 1);
+ if (v2) {
+ req.add_delta_rowset_ids("new_rowset");
+ auto* storage = req.add_delete_bitmap_storages();
+ storage->set_store_in_fdb(true);
+ auto* bitmap = storage->mutable_delete_bitmap();
+ bitmap->add_rowset_ids("old_rowset");
+ bitmap->add_segment_ids(0);
+ bitmap->add_versions(2);
+ bitmap->add_segment_delete_bitmaps(value);
+ } else {
+ req.add_rowset_ids("old_rowset");
+ req.add_segment_ids(0);
+ req.add_versions(2);
+ req.add_segment_delete_bitmaps(value);
+ }
+ return req;
+ }
+
+ UpdateDeleteBitmapResponse update(const UpdateDeleteBitmapRequest& req) {
+ UpdateDeleteBitmapResponse res;
+ brpc::Controller cntl;
+ service_->update_delete_bitmap(&cntl, &req, &res, nullptr);
+ return res;
+ }
+
+ void check_value(const std::string& expected, bool v2 = false) {
+ GetDeleteBitmapRequest req;
+ GetDeleteBitmapResponse res;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_tablet_id(102);
+ req.set_store_version(v2 ? 2 : 1);
+ req.add_rowset_ids(v2 ? "new_rowset" : "old_rowset");
+ req.add_begin_versions(v2 ? 0 : 2);
+ req.add_end_versions(2);
+ brpc::Controller cntl;
+ service_->get_delete_bitmap(&cntl, &req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK) <<
res.status().msg();
+ if (v2) {
+ ASSERT_EQ(res.delete_bitmap_storages_size(), 1);
+
ASSERT_EQ(res.delete_bitmap_storages(0).delete_bitmap().segment_delete_bitmaps_size(),
+ 1);
+
EXPECT_EQ(res.delete_bitmap_storages(0).delete_bitmap().segment_delete_bitmaps(0),
+ expected);
+ } else {
+ ASSERT_EQ(res.segment_delete_bitmaps_size(), 1);
+ EXPECT_EQ(res.segment_delete_bitmaps(0), expected);
+ }
+ }
+
+ static std::string large_value(size_t size = 10'578'008) {
+ std::string value(size, '\0');
+ for (size_t offset = 0; offset < value.size(); ++offset) {
+ value[offset] = static_cast<char>((offset /
DEFAULT_BLOB_SPLIT_SIZE) % 251);
+ }
+ return value;
+ }
+
+ void write_and_check_large_value(size_t size, bool v2) {
+ RecordProperty("value_bytes", std::to_string(size));
+ lock();
+ auto value = large_value(size);
+ auto res = update(request(value, 1001, v2));
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK) <<
res.status().msg();
+ if (limited_kv_ != nullptr) {
+ auto expected_commits =
+ (size + config::max_txn_commit_byte - 1) /
config::max_txn_commit_byte;
+ EXPECT_GE(limited_kv_->limits.blob_commits, expected_commits);
+ EXPECT_LE(limited_kv_->limits.largest_commit,
config::max_txn_commit_byte);
+ }
+ check_value(value, v2);
+ }
+
+ void write_and_check_two_values(bool v2) {
+ constexpr size_t value_size = 6 * 1024 * 1024;
+ RecordProperty("value_bytes", std::to_string(value_size));
+ RecordProperty("logical_values", "2");
+ lock();
+ auto req = request(std::string(value_size, 'a'), 1001, v2);
+ const std::string second(value_size, 'b');
+ if (v2) {
+ req.add_delta_rowset_ids("new_rowset_2");
+ *req.add_delete_bitmap_storages() =
+ request(second, 1001, true).delete_bitmap_storages(0);
+ } else {
+ req.add_rowset_ids("old_rowset");
+ req.add_segment_ids(1);
+ req.add_versions(2);
+ req.add_segment_delete_bitmaps(second);
+ }
+
+ auto res = update(req);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK) <<
res.status().msg();
+ if (limited_kv_ != nullptr) {
+ EXPECT_EQ(limited_kv_->limits.blob_commits, 2);
+ EXPECT_LE(limited_kv_->limits.largest_commit,
config::max_txn_commit_byte);
+ }
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(kv_->create_txn(&txn), TxnErrorCode::TXN_OK);
+ for (int i = 0; i < 2; ++i) {
+ auto key = v2 ? versioned::meta_delete_bitmap_key(
+ {instance_id_, 102,
req.delta_rowset_ids(i)})
+ : meta_delete_bitmap_key({instance_id_, 102,
req.rowset_ids(i),
+ req.versions(i),
req.segment_ids(i)});
+ ValueBuf stored;
+ ASSERT_EQ(blob_get(txn.get(), key, &stored), TxnErrorCode::TXN_OK);
+ auto expected = v2 ?
req.delete_bitmap_storages(i).SerializeAsString()
+ : req.segment_delete_bitmaps(i);
+ EXPECT_EQ(stored.value(), expected);
+ }
+ }
+
+ void check_overlapping_retry(bool after_validation, bool different_value =
false) {
+ const bool old_retry = config::enable_txn_store_retry;
+ config::enable_txn_store_retry = false;
+ DORIS_CLOUD_DEFER {
+ config::enable_txn_store_retry = old_retry;
+ };
+ auto* sp = SyncPoint::get_instance();
+ for (size_t size : {8UL * 1024 * 1024, 32UL * 1024 * 1024}) {
+ for (bool v2 : {false, true}) {
+ SCOPED_TRACE(testing::Message() << "size=" << size << " v2="
<< v2);
+ lock();
+ const auto value = large_value(size);
+ auto req = request(value, 1001, v2);
+ bool delayed_handler = false;
+ bool injected = false;
+ size_t active_commits = 0;
+ size_t delayed_commits = 0;
+ auto run_delayed_handler = [&] {
+ injected = true;
+ delayed_handler = true;
+ auto delayed_req =
+ different_value ? request(std::string(size, 'r'),
1001, v2) : req;
+ // Reenter the RPC while the active handler is paused. Let
the
+ // delayed handler commit pending cleanup, optionally
rewrite a
+ // prefix, then stop it before it can finish the bitmap.
+ auto delayed_res = update(delayed_req);
+ EXPECT_EQ(delayed_res.status().code(),
MetaServiceCode::KV_TXN_CONFLICT)
+ << delayed_res.status().msg();
+ delayed_handler = false;
+ };
+ sp->set_call_back("update_delete_bitmap:commit:err",
[&](auto&& args) {
+ if (delayed_handler) {
+ if (++delayed_commits == (different_value ? 2 : 1)) {
+ *try_any_cast<TxnErrorCode*>(args[2]) =
TxnErrorCode::TXN_CONFLICT;
+ }
+ } else if (++active_commits == 2 && !after_validation) {
+ run_delayed_handler();
+ }
+ });
+ sp->set_call_back("update_delete_bitmap:check_prefix:after",
[&](auto&&) {
+ if (after_validation && !injected) {
+ run_delayed_handler();
+ }
+ });
+ sp->enable_processing();
+ auto res = update(req);
+ EXPECT_TRUE(injected);
+ EXPECT_EQ(res.status().code(),
MetaServiceCode::KV_TXN_CONFLICT)
+ << res.status().msg();
+ sp->clear_call_back("update_delete_bitmap:commit:err");
+ sp->clear_call_back("update_delete_bitmap:check_prefix:after");
+
+ res = update(req);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK) <<
res.status().msg();
+ check_value(value, v2);
+ }
+ }
+ }
+
+ int64_t old_limit_ = 0;
+ std::string old_cluster_file_;
+ std::shared_ptr<TxnKv> kv_;
+ std::shared_ptr<BitmapSizeLimitedKv> limited_kv_;
+ std::unique_ptr<MetaServiceProxy> service_;
+ std::string instance_id_;
+};
+
+TEST_P(LargeDeleteBitmapTest, SingleV1BitmapExceedsTransactionLimit) {
+ write_and_check_large_value(10'578'008, false);
+}
+
+TEST_P(LargeDeleteBitmapTest, SingleV2StorageExceedsTransactionLimit) {
+ write_and_check_large_value(10'578'008, true);
+}
+
+TEST_P(LargeDeleteBitmapTest, SingleV1BitmapSpansSeveralTransactions) {
+ write_and_check_large_value(32 * 1024 * 1024, false);
+}
+
+TEST_P(LargeDeleteBitmapTest, SingleV2StorageSpansSeveralTransactions) {
+ write_and_check_large_value(32 * 1024 * 1024, true);
+}
+
+TEST_P(LargeDeleteBitmapTest, TwoV1BitmapsExceedCombinedTransactionLimit) {
+ write_and_check_two_values(false);
+}
+
+TEST_P(LargeDeleteBitmapTest, TwoV2StoragesExceedCombinedTransactionLimit) {
+ write_and_check_two_values(true);
+}
+
+TEST_P(LargeDeleteBitmapTest, DualWriteExceedsTransactionLimit) {
+ lock();
+ auto value = large_value();
+ auto req = request(value);
+ auto v2 = request(value, 1001, true);
+ req.set_store_version(3);
+ req.add_delta_rowset_ids(v2.delta_rowset_ids(0));
+ *req.add_delete_bitmap_storages() = v2.delete_bitmap_storages(0);
+ auto res = update(req);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg();
+ check_value(value);
+ check_value(value, true);
+}
+
+TEST_P(LargeDeleteBitmapTest, CompactionBitmapExceedsTransactionLimit) {
+ lock(-1);
+ auto value = large_value();
+ auto res = update(request(value, -1));
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg();
+ check_value(value);
+}
+
+TEST_P(LargeDeleteBitmapTest, SchemaChangeBitmapExceedsTransactionLimit) {
+ lock(-2);
+ auto value = large_value();
+ auto res = update(request(value, -2));
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg();
+ check_value(value);
+}
+
+TEST_P(LargeDeleteBitmapTest, PendingCleanupBeforePrefixValidation) {
+ check_overlapping_retry(false);
+}
+
+TEST_P(LargeDeleteBitmapTest, PendingCleanupAfterPrefixValidation) {
+ if (!GetParam()) {
+ GTEST_SKIP() << "MemTxnKv does not detect read conflicts with covering
range clears";
+ }
+ check_overlapping_retry(true);
+}
+
+TEST_P(LargeDeleteBitmapTest, ChangedPrefixBeforePrefixValidation) {
+ check_overlapping_retry(false, true);
+}
+
+TEST_P(LargeDeleteBitmapTest, MaybeCommittedFragmentRetriesIdempotently) {
+ lock();
+ bool injected = false;
+ size_t commits = 0;
+ auto* sp = SyncPoint::get_instance();
+ sp->set_call_back("update_delete_bitmap:commit:err", [&](auto&& args) {
+ // The first commit persists pending intent; the second persists a
prefix
+ // of the large bitmap. Report an ambiguous result after the real
commit.
+ if (++commits == 2) {
+ *try_any_cast<TxnErrorCode*>(args[2]) =
TxnErrorCode::TXN_MAYBE_COMMITTED;
+ injected = true;
+ }
+ });
+ sp->enable_processing();
+ auto value = large_value();
+ auto res = update(request(value));
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg();
+ EXPECT_TRUE(injected);
+ check_value(value);
+}
+
+TEST_P(LargeDeleteBitmapTest, LockLossStopsFragmentsAndShorterRetryCleansTail)
{
+ lock();
+ bool removed_lock = false;
+ size_t commits = 0;
+ auto* sp = SyncPoint::get_instance();
+ sp->set_call_back("update_delete_bitmap:commit:err", [&](auto&&) {
+ if (++commits != 2) {
+ return;
+ }
+ removed_lock = true;
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(kv_->create_txn(&txn), TxnErrorCode::TXN_OK);
+ txn->remove(meta_delete_bitmap_update_lock_key({instance_id_, 100,
-1}));
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+ });
+ sp->enable_processing();
+ auto res = update(request(large_value()));
+ ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_EXPIRED) <<
res.status().msg();
+ ASSERT_TRUE(removed_lock);
+ sp->clear_call_back("update_delete_bitmap:commit:err");
+
+ lock();
+ const std::string shorter(DEFAULT_BLOB_SPLIT_SIZE + 1, 's');
+ res = update(request(shorter));
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg();
+ check_value(shorter);
+}
+
+TEST_P(LargeDeleteBitmapTest, ExplicitSubTransactionPreservesEarlierBitmaps) {
+ lock();
+ auto value = large_value();
+ auto req = request(value);
+ req.set_is_explicit_txn(true);
+ req.set_txn_id(1001);
+ auto res = update(req);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg();
+
+ const auto pending_key = meta_pending_delete_bitmap_key({instance_id_,
102});
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(kv_->create_txn(&txn), TxnErrorCode::TXN_OK);
+ std::string data;
+ ASSERT_EQ(txn->get(pending_key, &data), TxnErrorCode::TXN_OK);
+ PendingDeleteBitmapPB previous;
+ ASSERT_TRUE(previous.ParseFromString(data));
+ ASSERT_EQ(previous.delete_bitmap_keys_size(), 1);
+
+ req.set_txn_id(1002);
+ req.set_versions(0, 3);
+ res = update(req);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg();
+ check_value(value);
+ ASSERT_EQ(kv_->create_txn(&txn), TxnErrorCode::TXN_OK);
+ ASSERT_EQ(txn->get(pending_key, &data), TxnErrorCode::TXN_OK);
+ PendingDeleteBitmapPB current;
+ ASSERT_TRUE(current.ParseFromString(data));
+ EXPECT_EQ(current.delete_bitmap_keys_size(), 2);
+ EXPECT_EQ(current.delete_bitmap_keys(0), previous.delete_bitmap_keys(0));
+ ValueBuf next;
+ ASSERT_EQ(blob_get(txn.get(), meta_delete_bitmap_key({instance_id_, 102,
"old_rowset", 3, 0}),
+ &next),
+ TxnErrorCode::TXN_OK);
+ EXPECT_EQ(next.value(), value);
+}
+
+TEST_P(LargeDeleteBitmapTest, UnlockedAggregationKeepsAtomicBitmapOverwrite) {
+ auto old_key = meta_delete_bitmap_key({instance_id_, 102, "old_rowset", 1,
0});
+ auto current_key = meta_delete_bitmap_key({instance_id_, 102,
"old_rowset", 2, 0});
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(kv_->create_txn(&txn), TxnErrorCode::TXN_OK);
+ RowsetMetaCloudPB old_rowset;
+ old_rowset.set_rowset_id(0);
+ old_rowset.set_rowset_id_v2("old_rowset");
+ old_rowset.set_start_version(0);
+ old_rowset.set_end_version(0);
+ txn->put(meta_rowset_key({instance_id_, 102, 0}),
old_rowset.SerializeAsString());
+ RowsetMetaCloudPB output_rowset;
+ output_rowset.set_rowset_id(0);
+ output_rowset.set_rowset_id_v2("output_rowset");
+ output_rowset.set_start_version(1);
+ output_rowset.set_end_version(2);
+ txn->put(meta_rowset_key({instance_id_, 102, 2}),
output_rowset.SerializeAsString());
+ blob_put(txn.get(), old_key, "older_visible_bitmap", 0);
+ blob_put(txn.get(), current_key, "visible_bitmap", 0);
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+
+ auto req = request(large_value(), -3);
+ req.set_without_lock(true);
+ req.set_pre_rowset_agg_start_version(1);
+ req.set_pre_rowset_agg_end_version(2);
+ req.add_pre_rowset_versions(0);
+ auto res = update(req);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT) <<
res.status().msg();
+ if (limited_kv_ != nullptr) {
+ EXPECT_GT(limited_kv_->limits.largest_commit, 10'000'000);
+ }
+ ASSERT_EQ(kv_->create_txn(&txn), TxnErrorCode::TXN_OK);
+ ValueBuf current;
+ ASSERT_EQ(blob_get(txn.get(), current_key, ¤t),
TxnErrorCode::TXN_OK);
+ EXPECT_EQ(current.value(), "visible_bitmap");
+ ValueBuf old;
+ ASSERT_EQ(blob_get(txn.get(), old_key, &old), TxnErrorCode::TXN_OK);
+ EXPECT_EQ(old.value(), "older_visible_bitmap");
+}
+
+INSTANTIATE_TEST_SUITE_P(Mem, LargeDeleteBitmapTest, testing::Values(false));
+INSTANTIATE_TEST_SUITE_P(Fdb, LargeDeleteBitmapTest, testing::Values(true));
+
+} // namespace
+} // namespace doris::cloud
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]