This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 9957987d5bf [fix](cloud) Return existing rowset for concurrent commits
(#67637)
9957987d5bf is described below
commit 9957987d5bf43fa6a33aa59918435fed65e2912a
Author: Yixuan Wang <[email protected]>
AuthorDate: Tue Sep 8 17:07:30 2026 +0800
[fix](cloud) Return existing rowset for concurrent commits (#67637)
Problem Summary:
Different BEs may generate different rowsets for the same transaction
and tablet.
After rowset A is committed, the temporary rowset key points to A while
the recycle rowset key for B still exists. The recycle key check ran
before comparing rowset IDs, so commit_rowset incorrectly treated tmp(A)
and recycle(B) as an invalid conflict.
Compare rowset IDs first and only check the tmp and recycle key conflict
when they belong to the same rowset. For different rowsets, return
ALREADY_EXISTED with rowset A.
---
cloud/src/meta-service/meta_service.cpp | 40 ++++++++++++++---------------
cloud/test/meta_service_test.cpp | 45 +++++++++++++++++++++++++++++++++
2 files changed, 65 insertions(+), 20 deletions(-)
diff --git a/cloud/src/meta-service/meta_service.cpp
b/cloud/src/meta-service/meta_service.cpp
index c6aa71e38c8..8192f11323b 100644
--- a/cloud/src/meta-service/meta_service.cpp
+++ b/cloud/src/meta-service/meta_service.cpp
@@ -2758,26 +2758,6 @@ void
MetaServiceImpl::commit_rowset(::google::protobuf::RpcController* controlle
std::string existed_commit_val;
err = txn->get(tmp_rs_key, &existed_commit_val);
if (err == TxnErrorCode::TXN_OK) {
- if (config::enable_recycle_delete_rowset_key_check) {
- bool recycle_rs_key_exists = false;
- if (!check_recycle_rowset_key(txn.get(), recycle_rs_key,
rowset_meta,
- &recycle_rs_key_exists, code, msg)) {
- return;
- }
- if (recycle_rs_key_exists) {
- code = MetaServiceCode::INVALID_ARGUMENT;
- msg = fmt::format(
- "tmp rowset key and recycle rowset key are mutually
exclusive, "
- "tmp_rs_key={}, recycle_rs_key={}",
- hex(tmp_rs_key), hex(recycle_rs_key));
- LOG(INFO) << "skip commit rowset because tmp rowset key and
recycle rowset key are "
- "mutually exclusive, txn_id="
- << rowset_meta.txn_id() << ", tablet_id=" <<
tablet_id
- << ", rowset_id=" << rowset_id << ", tmp_rs_key=" <<
hex(tmp_rs_key)
- << ", recycle_rs_key=" << hex(recycle_rs_key) << ",
msg=" << msg;
- return;
- }
- }
auto existed_rowset_meta = response->mutable_existed_rowset_meta();
if (!existed_rowset_meta->ParseFromString(existed_commit_val)) {
code = MetaServiceCode::PROTOBUF_PARSE_ERR;
@@ -2785,6 +2765,26 @@ void
MetaServiceImpl::commit_rowset(::google::protobuf::RpcController* controlle
return;
}
if (existed_rowset_meta->rowset_id_v2() == rowset_meta.rowset_id_v2())
{
+ if (config::enable_recycle_delete_rowset_key_check) {
+ bool recycle_rs_key_exists = false;
+ if (!check_recycle_rowset_key(txn.get(), recycle_rs_key,
rowset_meta,
+ &recycle_rs_key_exists, code,
msg)) {
+ return;
+ }
+ if (recycle_rs_key_exists) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = fmt::format(
+ "tmp rowset key and recycle rowset key are
mutually exclusive, "
+ "tmp_rs_key={}, recycle_rs_key={}",
+ hex(tmp_rs_key), hex(recycle_rs_key));
+ LOG(INFO) << "skip commit rowset because tmp rowset key
and recycle rowset key "
+ "are mutually exclusive, txn_id="
+ << rowset_meta.txn_id() << ", tablet_id=" <<
tablet_id
+ << ", rowset_id=" << rowset_id << ",
tmp_rs_key=" << hex(tmp_rs_key)
+ << ", recycle_rs_key=" << hex(recycle_rs_key) <<
", msg=" << msg;
+ return;
+ }
+ }
// Same request, return OK
response->set_allocated_existed_rowset_meta(nullptr);
return;
diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp
index 4e569bc7ed6..924eb3897a6 100644
--- a/cloud/test/meta_service_test.cpp
+++ b/cloud/test/meta_service_test.cpp
@@ -11268,6 +11268,13 @@ TEST(MetaServiceTest,
CommitRowsetCheckTmpAndRecycleKeyExclusion) {
ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
};
+ auto remove_recycle_rowset = [&](const doris::RowsetMetaCloudPB& rowset) {
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn),
TxnErrorCode::TXN_OK);
+ txn->remove(recycle_rowset_key({instance_id, rowset.tablet_id(),
rowset.rowset_id_v2()}));
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+ };
+
{
int64_t table_id = 1;
int64_t partition_id = 1;
@@ -11316,6 +11323,44 @@ TEST(MetaServiceTest,
CommitRowsetCheckTmpAndRecycleKeyExclusion) {
ASSERT_TRUE(res.status().msg().find("recycle rowset key not found") !=
std::string::npos)
<< res.status().msg();
}
+
+ auto check_different_rowset_commit = [&](int64_t table_id, int64_t
tablet_id, int64_t db_id,
+ bool remove_recycle_key) {
+ std::string label = "test_commit_rowset_different_rowset";
+ create_tablet(meta_service.get(), table_id, 1, table_id, tablet_id);
+
+ int64_t txn_id = 0;
+ ASSERT_NO_FATAL_FAILURE(begin_txn(meta_service.get(), db_id, label,
table_id, txn_id));
+ CreateRowsetResponse res;
+ auto rowset_a = create_rowset(txn_id, tablet_id, table_id);
+ rowset_a.mutable_load_id()->set_hi(123);
+ rowset_a.mutable_load_id()->set_lo(456);
+ ASSERT_NO_FATAL_FAILURE(prepare_rowset(meta_service.get(), rowset_a,
res));
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << label;
+ res.Clear();
+
+ auto rowset_b = create_rowset(txn_id, tablet_id, table_id);
+ rowset_b.mutable_load_id()->set_hi(789);
+ rowset_b.mutable_load_id()->set_lo(101112);
+ ASSERT_NO_FATAL_FAILURE(prepare_rowset(meta_service.get(), rowset_b,
res));
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << label;
+ res.Clear();
+
+ ASSERT_NO_FATAL_FAILURE(commit_rowset(meta_service.get(), rowset_a,
res));
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << label;
+ res.Clear();
+
+ if (remove_recycle_key) {
+ ASSERT_NO_FATAL_FAILURE(remove_recycle_rowset(rowset_b));
+ }
+ ASSERT_NO_FATAL_FAILURE(commit_rowset(meta_service.get(), rowset_b,
res));
+ ASSERT_EQ(res.status().code(), MetaServiceCode::ALREADY_EXISTED) <<
res.status().msg();
+ ASSERT_TRUE(res.has_existed_rowset_meta());
+ ASSERT_EQ(res.existed_rowset_meta().rowset_id_v2(),
rowset_a.rowset_id_v2());
+ };
+
+ check_different_rowset_commit(3, 3, 100203, false);
+ check_different_rowset_commit(4, 4, 100204, true);
}
TEST(MetaServiceTest, AlterObjInfoTest) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
