This is an automated email from the ASF dual-hosted git repository.
luwei16 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 27e68bde457 [fix](cloud) Exclude stale rowsets from schema change
temporary tablet (#67351)
27e68bde457 is described below
commit 27e68bde457e15590fb5dfa2df82d3d5ed8f9379
Author: Luwei <[email protected]>
AuthorDate: Tue Sep 1 17:56:59 2026 +0800
[fix](cloud) Exclude stale rowsets from schema change temporary tablet
(#67351)
### What problem does this PR solve?
Issue Number: None
Related PR: None
Problem Summary: During merge-on-write cloud schema changes, the
delete-bitmap temporary tablet copied stale rowset metadata from the
real shadow tablet. Its version tracker could select a compacted stale
edge that was absent from the active rowset map, causing schema change
to fail with a rowset capture error under concurrent writes and
compaction. Clear only the copied stale rowset metadata before
constructing the temporary tablet so its tracker is built exclusively
from active schema-change output and incremental rowsets, without
altering the source tablet or shared delete-bitmap cache.
### Release note
Fix cloud merge-on-write schema changes that could fail during
concurrent writes and compaction.
### Check List (For Author)
- Test: Unit Test
- `./run-be-ut.sh --run --filter=CloudSchemaChangeJobTest.* -j 48`
- `./build.sh --be -j 48`
- Behavior changed: Yes. Cloud schema change temporary tablets no longer
inherit stale rowset metadata.
- Does this need documentation: No
---
be/src/cloud/cloud_schema_change_job.cpp | 3 ++
be/src/storage/tablet/tablet_meta.cpp | 6 ++-
be/src/storage/tablet/tablet_meta.h | 3 ++
be/test/cloud/cloud_schema_change_job_test.cpp | 63 +++++++++++++++++++++++++-
4 files changed, 73 insertions(+), 2 deletions(-)
diff --git a/be/src/cloud/cloud_schema_change_job.cpp
b/be/src/cloud/cloud_schema_change_job.cpp
index 97f4e0ec538..427da01054d 100644
--- a/be/src/cloud/cloud_schema_change_job.cpp
+++ b/be/src/cloud/cloud_schema_change_job.cpp
@@ -585,6 +585,9 @@ Status CloudSchemaChangeJob::_process_delete_bitmap(int64_t
alter_version,
.tag("alter_version", alter_version);
RETURN_IF_ERROR(_cloud_storage_engine.register_compaction_stop_token(_new_tablet,
initiator));
TabletMetaSharedPtr tmp_meta =
std::make_shared<TabletMeta>(*(_new_tablet->tablet_meta()));
+ // The temporary tablet must build its version graph only from active
rowsets. Stale
+ // rowsets copied from the real tablet are not present in its active
rowset map.
+ tmp_meta->clear_stale_rs_metas();
tmp_meta->delete_bitmap().delete_bitmap.clear();
// Keep only version [0-1] rowset, other rowsets will be added in
_output_rowsets
auto& rs_metas = tmp_meta->all_mutable_rs_metas();
diff --git a/be/src/storage/tablet/tablet_meta.cpp
b/be/src/storage/tablet/tablet_meta.cpp
index 201e914463e..1f6ee1012da 100644
--- a/be/src/storage/tablet/tablet_meta.cpp
+++ b/be/src/storage/tablet/tablet_meta.cpp
@@ -1184,12 +1184,16 @@ Status TabletMeta::set_partition_id(int64_t
partition_id) {
}
void TabletMeta::clear_stale_rowset() {
- _stale_rs_metas.clear();
+ clear_stale_rs_metas();
if (_enable_unique_key_merge_on_write) {
_delete_bitmap->clear_rowset_cache_version();
}
}
+void TabletMeta::clear_stale_rs_metas() {
+ _stale_rs_metas.clear();
+}
+
void TabletMeta::clear_rowsets() {
_rs_metas.clear();
if (_enable_unique_key_merge_on_write) {
diff --git a/be/src/storage/tablet/tablet_meta.h
b/be/src/storage/tablet/tablet_meta.h
index 9b3f4764677..0efce3d3f2e 100644
--- a/be/src/storage/tablet/tablet_meta.h
+++ b/be/src/storage/tablet/tablet_meta.h
@@ -223,6 +223,9 @@ public:
// used for after tablet cloned to clear stale rowset
void clear_stale_rowset();
+ // Clear stale rowset metadata without changing the delete bitmap cache.
+ void clear_stale_rs_metas();
+
void clear_rowsets();
// MUST hold EXCLUSIVE `_meta_lock` in belonged Tablet
diff --git a/be/test/cloud/cloud_schema_change_job_test.cpp
b/be/test/cloud/cloud_schema_change_job_test.cpp
index 9afce0bfb5c..2b7e7808367 100644
--- a/be/test/cloud/cloud_schema_change_job_test.cpp
+++ b/be/test/cloud/cloud_schema_change_job_test.cpp
@@ -153,9 +153,10 @@ public:
protected:
RowsetSharedPtr create_rowset(TabletSchemaSPtr schema, int64_t tablet_id,
int64_t start,
- int64_t end) {
+ int64_t end, int64_t rowset_id = 540081)
const {
RowsetMetaPB pb;
json2pb::JsonToProtoMessage(_json_rowset_meta, &pb);
+ pb.set_rowset_id(rowset_id);
pb.set_tablet_id(tablet_id);
pb.set_start_version(start);
pb.set_end_version(end);
@@ -173,6 +174,66 @@ protected:
std::shared_ptr<CloudClusterInfo> _cluster_info;
};
+// GTest assertion macros inflate cognitive complexity for this linear
scenario.
+// NOLINTNEXTLINE(readability-function-cognitive-complexity)
+TEST_F(CloudSchemaChangeJobTest,
DeleteBitmapTmpTabletDoesNotInheritStaleRowsets) {
+ constexpr int64_t new_tablet_id = 60002;
+
+ TabletMetaSharedPtr new_meta(new TabletMeta(
+ 1, 2, new_tablet_id, new_tablet_id + 100, 4, 5, TTabletSchema(),
6, {{7, 8}},
+ UniqueId(11, 12), TTabletType::TABLET_TYPE_DISK,
TCompressionType::LZ4F, -1, true));
+ auto placeholder = create_rowset(new_meta->tablet_schema(), new_tablet_id,
0, 1, 6000201);
+ auto compacted = create_rowset(new_meta->tablet_schema(), new_tablet_id,
2, 3, 6000202);
+ auto sc_output_2 = create_rowset(new_meta->tablet_schema(), new_tablet_id,
2, 2, 6000203);
+ auto sc_output_3 = create_rowset(new_meta->tablet_schema(), new_tablet_id,
3, 3, 6000204);
+ auto sc_output_4 = create_rowset(new_meta->tablet_schema(), new_tablet_id,
4, 4, 6000205);
+ ASSERT_NE(placeholder, nullptr);
+ ASSERT_NE(compacted, nullptr);
+ ASSERT_NE(sc_output_2, nullptr);
+ ASSERT_NE(sc_output_3, nullptr);
+ ASSERT_NE(sc_output_4, nullptr);
+
+ ASSERT_TRUE(new_meta->add_rs_meta(placeholder->rowset_meta()).ok());
+ ASSERT_TRUE(new_meta->add_rs_meta(compacted->rowset_meta()).ok());
+ new_meta->modify_rs_metas({}, {compacted->rowset_meta()});
+ ASSERT_EQ(new_meta->all_stale_rs_metas().size(), 1);
+
+ auto new_tablet = std::make_shared<CloudTablet>(_engine, new_meta);
+ auto* sp = SyncPoint::get_instance();
+ sp->clear_all_call_backs();
+ sp->enable_processing();
+ sp->set_call_back("CloudMetaMgr::prepare_tablet_job", [](auto&& outcome) {
+ auto* pairs = try_any_cast_ret<Status>(outcome);
+ pairs->second = true;
+ pairs->first = Status::OK();
+ auto* resp = try_any_cast<cloud::StartTabletJobResponse*>(outcome[1]);
+ resp->mutable_status()->set_code(cloud::MetaServiceCode::OK);
+ });
+
+ Status captured_status = Status::InternalError("temporary tablet was not
inspected");
+ RowsetIdUnorderedSet captured_rowset_ids;
+ sp->set_call_back("CloudMetaMgr::sync_tablet_rowsets", [&](auto&& outcome)
{
+ auto* tablet = try_any_cast<CloudTablet*>(outcome[0]);
+ std::shared_lock rlock(tablet->get_header_lock());
+ captured_status = tablet->get_all_rs_id_unlocked(4,
&captured_rowset_ids);
+ auto* pairs = try_any_cast_ret<Status>(outcome);
+ pairs->second = true;
+ pairs->first = Status::InternalError("stop after inspecting temporary
tablet");
+ });
+
+ CloudSchemaChangeJob sc_job(_engine, "test_tmp_tablet_stale_rowsets",
9999999999);
+ sc_job._new_tablet = new_tablet;
+ sc_job._output_rowsets = {sc_output_2, sc_output_3, sc_output_4};
+ auto status = sc_job._process_delete_bitmap(4, 5, 12345, "");
+ ASSERT_TRUE(_engine.unregister_compaction_stop_token(new_tablet,
false).ok());
+
+ ASSERT_FALSE(status.ok());
+ ASSERT_NE(status.to_string().find("stop after inspecting temporary
tablet"), std::string::npos);
+ ASSERT_TRUE(captured_status.ok()) << captured_status.to_string();
+ ASSERT_EQ(captured_rowset_ids.size(), 3);
+ ASSERT_EQ(new_meta->all_stale_rs_metas().size(), 1);
+}
+
TEST_F(CloudSchemaChangeJobTest, FillVersionHolesBeforeNewTabletRunning) {
int64_t base_tablet_id = 40001;
int64_t new_tablet_id = 40002;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
