This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new dceb8a7f8ce branch-4.0: [enhance](cloud)enable tablets meta url show
rs_meta for cloud #55167 (#56972)
dceb8a7f8ce is described below
commit dceb8a7f8ce33fc9cedcb39c016488c39b3337ac
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu Oct 16 10:04:22 2025 +0800
branch-4.0: [enhance](cloud)enable tablets meta url show rs_meta for cloud
#55167 (#56972)
Cherry-picked from #55167
Co-authored-by: koarz <[email protected]>
---
be/src/cloud/cloud_snapshot_mgr.cpp | 2 +-
be/src/http/action/meta_action.cpp | 2 +-
be/src/olap/base_tablet.cpp | 16 +++++++---------
be/src/olap/base_tablet.h | 5 +++--
be/src/olap/snapshot_manager.cpp | 2 +-
be/src/olap/tablet_meta.cpp | 10 +++++-----
be/src/olap/tablet_meta.h | 2 +-
be/src/olap/task/engine_storage_migration_task.cpp | 2 +-
be/src/tools/meta_tool.cpp | 2 +-
be/test/olap/index_builder_test.cpp | 6 +++---
be/test/olap/tablet_meta_test.cpp | 10 +++++-----
.../key_bounds/test_key_bounds_truncation_basic.groovy | 12 ++++++++++++
.../test_key_bounds_truncation_read_scenarios.groovy | 12 ++++++++++++
13 files changed, 53 insertions(+), 30 deletions(-)
diff --git a/be/src/cloud/cloud_snapshot_mgr.cpp
b/be/src/cloud/cloud_snapshot_mgr.cpp
index c943f6b55fb..a3f3c5dc213 100644
--- a/be/src/cloud/cloud_snapshot_mgr.cpp
+++ b/be/src/cloud/cloud_snapshot_mgr.cpp
@@ -77,7 +77,7 @@ Status CloudSnapshotMgr::make_snapshot(int64_t
target_tablet_id, StorageResource
// 1. deserialize tablet meta from memory
RETURN_IF_ERROR(tablet_meta.create_from_buffer((const
uint8_t*)slice->data, slice->size));
TabletMetaPB tablet_meta_pb;
- tablet_meta.to_meta_pb(&tablet_meta_pb);
+ tablet_meta.to_meta_pb(&tablet_meta_pb, false);
tablet_meta_pb.clear_rs_metas(); // copy the rs meta
if (tablet_meta.all_rs_metas().size() > 0) {
diff --git a/be/src/http/action/meta_action.cpp
b/be/src/http/action/meta_action.cpp
index 490750cfaea..e5d564a11f1 100644
--- a/be/src/http/action/meta_action.cpp
+++ b/be/src/http/action/meta_action.cpp
@@ -71,7 +71,7 @@ Status MetaAction::_handle_header(HttpRequest* req,
std::string* json_meta) {
std::string operation = req->param(OP);
if (operation == HEADER) {
TabletMeta tablet_meta;
- tablet->generate_tablet_meta_copy(tablet_meta);
+ tablet->generate_tablet_meta_copy(tablet_meta, true);
json2pb::Pb2JsonOptions json_options;
json_options.pretty_json = true;
json_options.bytes_to_base64 = enable_byte_to_base64;
diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp
index b919db8d92b..00a7db1d982 100644
--- a/be/src/olap/base_tablet.cpp
+++ b/be/src/olap/base_tablet.cpp
@@ -332,21 +332,19 @@ bool
BaseTablet::_reconstruct_version_tracker_if_necessary() {
// should use this method to get a copy of current tablet meta
// there are some rowset meta in local meta store and in in-memory tablet meta
// but not in tablet meta in local meta store
-void BaseTablet::generate_tablet_meta_copy(TabletMeta& new_tablet_meta) const {
- TabletMetaPB tablet_meta_pb;
- {
- std::shared_lock rdlock(_meta_lock);
- _tablet_meta->to_meta_pb(&tablet_meta_pb);
- }
- generate_tablet_meta_copy_unlocked(new_tablet_meta);
+void BaseTablet::generate_tablet_meta_copy(TabletMeta& new_tablet_meta,
+ bool cloud_get_rowset_meta) const {
+ std::shared_lock rdlock(_meta_lock);
+ generate_tablet_meta_copy_unlocked(new_tablet_meta, cloud_get_rowset_meta);
}
// this is a unlocked version of generate_tablet_meta_copy()
// some method already hold the _meta_lock before calling this,
// such as EngineCloneTask::_finish_clone -> tablet->revise_tablet_meta
-void BaseTablet::generate_tablet_meta_copy_unlocked(TabletMeta&
new_tablet_meta) const {
+void BaseTablet::generate_tablet_meta_copy_unlocked(TabletMeta&
new_tablet_meta,
+ bool
cloud_get_rowset_meta) const {
TabletMetaPB tablet_meta_pb;
- _tablet_meta->to_meta_pb(&tablet_meta_pb);
+ _tablet_meta->to_meta_pb(&tablet_meta_pb, cloud_get_rowset_meta);
new_tablet_meta.init_from_pb(tablet_meta_pb);
}
diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h
index bf1c600c0ce..3e1dbd82053 100644
--- a/be/src/olap/base_tablet.h
+++ b/be/src/olap/base_tablet.h
@@ -160,8 +160,9 @@ public:
Versions get_missed_versions(int64_t spec_version) const;
Versions get_missed_versions_unlocked(int64_t spec_version) const;
- void generate_tablet_meta_copy(TabletMeta& new_tablet_meta) const;
- void generate_tablet_meta_copy_unlocked(TabletMeta& new_tablet_meta) const;
+ void generate_tablet_meta_copy(TabletMeta& new_tablet_meta, bool
cloud_get_rowset_meta) const;
+ void generate_tablet_meta_copy_unlocked(TabletMeta& new_tablet_meta,
+ bool cloud_get_rowset_meta) const;
virtual int64_t max_version_unlocked() const { return
_tablet_meta->max_version().second; }
diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp
index 3a7634c62f2..dad9e22c057 100644
--- a/be/src/olap/snapshot_manager.cpp
+++ b/be/src/olap/snapshot_manager.cpp
@@ -588,7 +588,7 @@ Status SnapshotManager::_create_snapshot_files(const
TabletSharedPtr& ref_tablet
// copy the tablet meta to new_tablet_meta inside header lock
CHECK(res.ok()) << res;
- ref_tablet->generate_tablet_meta_copy_unlocked(*new_tablet_meta);
+ ref_tablet->generate_tablet_meta_copy_unlocked(*new_tablet_meta,
false);
// The delete bitmap update operation and the add_inc_rowset
operation is not atomic,
// so delete bitmap may contains some data generated by invisible
rowset, we should
// get rid of these useless bitmaps when doing snapshot.
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 60585a45359..bd23ed3f472 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -593,7 +593,7 @@ Status TabletMeta::save_as_json(const string& file_path) {
Status TabletMeta::save(const string& file_path) {
TabletMetaPB tablet_meta_pb;
- to_meta_pb(&tablet_meta_pb);
+ to_meta_pb(&tablet_meta_pb, false);
return TabletMeta::save(file_path, tablet_meta_pb);
}
@@ -645,7 +645,7 @@ Status TabletMeta::_save_meta(DataDir* data_dir) {
void TabletMeta::serialize(string* meta_binary) {
TabletMetaPB tablet_meta_pb;
- to_meta_pb(&tablet_meta_pb);
+ to_meta_pb(&tablet_meta_pb, false);
if (tablet_meta_pb.partition_id() <= 0) {
LOG(WARNING) << "invalid partition id " <<
tablet_meta_pb.partition_id() << " tablet "
<< tablet_meta_pb.tablet_id();
@@ -815,7 +815,7 @@ void TabletMeta::init_from_pb(const TabletMetaPB&
tablet_meta_pb) {
}
}
-void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb) {
+void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb, bool
cloud_get_rowset_meta) {
tablet_meta_pb->set_table_id(table_id());
tablet_meta_pb->set_index_id(index_id());
tablet_meta_pb->set_partition_id(partition_id());
@@ -847,7 +847,7 @@ void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb) {
}
// RowsetMetaPB is separated from TabletMetaPB
- if (!config::is_cloud_mode()) {
+ if (!config::is_cloud_mode() || cloud_get_rowset_meta) {
for (const auto& [_, rs] : _rs_metas) {
rs->to_rowset_pb(tablet_meta_pb->add_rs_metas());
}
@@ -910,7 +910,7 @@ void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb) {
void TabletMeta::to_json(string* json_string, json2pb::Pb2JsonOptions&
options) {
TabletMetaPB tablet_meta_pb;
- to_meta_pb(&tablet_meta_pb);
+ to_meta_pb(&tablet_meta_pb, false);
json2pb::ProtoMessageToJson(tablet_meta_pb, json_string, options);
}
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index a8f35520782..c86d9ad553a 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -144,7 +144,7 @@ public:
Status deserialize(std::string_view meta_binary);
void init_from_pb(const TabletMetaPB& tablet_meta_pb);
- void to_meta_pb(TabletMetaPB* tablet_meta_pb);
+ void to_meta_pb(TabletMetaPB* tablet_meta_pb, bool cloud_get_rowset_meta);
void to_json(std::string* json_string, json2pb::Pb2JsonOptions& options);
size_t tablet_columns_num() const { return _schema->num_columns(); }
diff --git a/be/src/olap/task/engine_storage_migration_task.cpp
b/be/src/olap/task/engine_storage_migration_task.cpp
index f3746be8755..e7935eae55f 100644
--- a/be/src/olap/task/engine_storage_migration_task.cpp
+++ b/be/src/olap/task/engine_storage_migration_task.cpp
@@ -346,7 +346,7 @@ Status EngineStorageMigrationTask::_migrate() {
void EngineStorageMigrationTask::_generate_new_header(
int32_t new_shard, const std::vector<RowsetSharedPtr>&
consistent_rowsets,
TabletMetaSharedPtr new_tablet_meta, int64_t end_version) {
- _tablet->generate_tablet_meta_copy_unlocked(*new_tablet_meta);
+ _tablet->generate_tablet_meta_copy_unlocked(*new_tablet_meta, false);
std::vector<RowsetMetaSharedPtr> rs_metas;
for (auto& rs : consistent_rowsets) {
diff --git a/be/src/tools/meta_tool.cpp b/be/src/tools/meta_tool.cpp
index 199214566fa..d0caf0d1dda 100644
--- a/be/src/tools/meta_tool.cpp
+++ b/be/src/tools/meta_tool.cpp
@@ -88,7 +88,7 @@ void show_meta() {
json2pb::Pb2JsonOptions json_options;
json_options.pretty_json = true;
doris::TabletMetaPB tablet_meta_pb;
- tablet_meta.to_meta_pb(&tablet_meta_pb);
+ tablet_meta.to_meta_pb(&tablet_meta_pb, false);
json2pb::ProtoMessageToJson(tablet_meta_pb, &json_meta, json_options);
std::cout << json_meta << std::endl;
}
diff --git a/be/test/olap/index_builder_test.cpp
b/be/test/olap/index_builder_test.cpp
index 2bc475d9a54..afe3a4b8e4d 100644
--- a/be/test/olap/index_builder_test.cpp
+++ b/be/test/olap/index_builder_test.cpp
@@ -626,7 +626,7 @@ TEST_F(IndexBuilderTest, AddIndexWhenOneExistsTestV1) {
// 4. Update schema in tablet
TabletMetaPB tablet_meta_pb;
- _tablet_meta->to_meta_pb(&tablet_meta_pb);
+ _tablet_meta->to_meta_pb(&tablet_meta_pb, false);
TabletSchemaPB v1_schema_pb;
v1_schema->to_schema_pb(&v1_schema_pb);
@@ -1425,7 +1425,7 @@ TEST_F(IndexBuilderTest,
AddNonExistentColumnIndexWhenOneExistsTestV1) {
// 4. Update schema in tablet
TabletMetaPB tablet_meta_pb;
- _tablet_meta->to_meta_pb(&tablet_meta_pb);
+ _tablet_meta->to_meta_pb(&tablet_meta_pb, false);
TabletSchemaPB v1_schema_pb;
v1_schema->to_schema_pb(&v1_schema_pb);
@@ -1830,7 +1830,7 @@ TEST_F(IndexBuilderTest, DropIndexV1FormatTest) {
// 4. Update schema in tablet
TabletMetaPB tablet_meta_pb;
- _tablet_meta->to_meta_pb(&tablet_meta_pb);
+ _tablet_meta->to_meta_pb(&tablet_meta_pb, false);
TabletSchemaPB v1_schema_pb;
v1_schema->to_schema_pb(&v1_schema_pb);
diff --git a/be/test/olap/tablet_meta_test.cpp
b/be/test/olap/tablet_meta_test.cpp
index 3aabdf9bafb..8fa3081731c 100644
--- a/be/test/olap/tablet_meta_test.cpp
+++ b/be/test/olap/tablet_meta_test.cpp
@@ -55,7 +55,7 @@ TEST(TabletMetaTest, SaveAsBufferAndParse) {
TTabletType::TABLET_TYPE_DISK,
TCompressionType::LZ4F);
TabletMetaPB tablet_meta_pb;
- src_tablet_meta.to_meta_pb(&tablet_meta_pb);
+ src_tablet_meta.to_meta_pb(&tablet_meta_pb, false);
FileHeader<TabletMetaPB> file_header("");
file_header.mutable_message()->CopyFrom(tablet_meta_pb);
@@ -85,7 +85,7 @@ TEST(TabletMetaTest, SerializeToMemoryWithSmallBuffer) {
TabletMeta src_tablet_meta(1, 2, 3, 3, 4, 5, TTabletSchema(), 6, {{7, 8}},
UniqueId(9, 10),
TTabletType::TABLET_TYPE_DISK,
TCompressionType::LZ4F);
TabletMetaPB tablet_meta_pb;
- src_tablet_meta.to_meta_pb(&tablet_meta_pb);
+ src_tablet_meta.to_meta_pb(&tablet_meta_pb, false);
FileHeader<TabletMetaPB> file_header("");
file_header.mutable_message()->CopyFrom(tablet_meta_pb);
@@ -99,7 +99,7 @@ TEST(TabletMetaTest, DeserializeFromMemoryWithOldHeader) {
TabletMeta src_tablet_meta(1, 2, 3, 3, 4, 5, TTabletSchema(), 6, {{7, 8}},
UniqueId(9, 10),
TTabletType::TABLET_TYPE_DISK,
TCompressionType::LZ4F);
TabletMetaPB tablet_meta_pb;
- src_tablet_meta.to_meta_pb(&tablet_meta_pb);
+ src_tablet_meta.to_meta_pb(&tablet_meta_pb, false);
std::string proto_string;
ASSERT_TRUE(tablet_meta_pb.SerializeToString(&proto_string));
@@ -132,7 +132,7 @@ TEST(TabletMetaTest,
DeserializeFromMemoryWithInvalidChecksum) {
TabletMeta src_tablet_meta(1, 2, 3, 3, 4, 5, TTabletSchema(), 6, {{7, 8}},
UniqueId(9, 10),
TTabletType::TABLET_TYPE_DISK,
TCompressionType::LZ4F);
TabletMetaPB tablet_meta_pb;
- src_tablet_meta.to_meta_pb(&tablet_meta_pb);
+ src_tablet_meta.to_meta_pb(&tablet_meta_pb, false);
FileHeader<TabletMetaPB> file_header("");
file_header.mutable_message()->CopyFrom(tablet_meta_pb);
@@ -152,7 +152,7 @@ TEST(TabletMetaTest,
DeserializeFromMemoryWithInvalidProtobuf) {
TabletMeta src_tablet_meta(1, 2, 3, 3, 4, 5, TTabletSchema(), 6, {{7, 8}},
UniqueId(9, 10),
TTabletType::TABLET_TYPE_DISK,
TCompressionType::LZ4F);
TabletMetaPB tablet_meta_pb;
- src_tablet_meta.to_meta_pb(&tablet_meta_pb);
+ src_tablet_meta.to_meta_pb(&tablet_meta_pb, false);
FileHeader<TabletMetaPB> file_header("");
file_header.mutable_message()->CopyFrom(tablet_meta_pb);
diff --git
a/regression-test/suites/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_basic.groovy
b/regression-test/suites/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_basic.groovy
index d9fad5970d0..e00615449ac 100644
---
a/regression-test/suites/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_basic.groovy
+++
b/regression-test/suites/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_basic.groovy
@@ -43,6 +43,18 @@ suite("test_key_bounds_truncation_basic", "nonConcurrent") {
return meta
}
}
+ if (cloudMode) {
+ for (int retryTimes = 0; retryTimes < 100; retryTimes++) {
+ Thread.sleep(1000)
+ jsonMeta = Http.GET(metaUrl, true, false)
+ for (def meta : jsonMeta.rs_metas) {
+ int end_version = meta.end_version
+ if (end_version == version) {
+ return meta
+ }
+ }
+ }
+ }
}
def truncateString = { String s, int l ->
diff --git
a/regression-test/suites/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_read_scenarios.groovy
b/regression-test/suites/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_read_scenarios.groovy
index 2620e302faa..d0acd394ba6 100644
---
a/regression-test/suites/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_read_scenarios.groovy
+++
b/regression-test/suites/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_read_scenarios.groovy
@@ -48,6 +48,18 @@ suite("test_key_bounds_truncation_read_scenarios",
"nonConcurrent") {
return meta
}
}
+ if (cloudMode) {
+ for (int retryTimes = 0; retryTimes < 100; retryTimes++) {
+ Thread.sleep(1000)
+ jsonMeta = Http.GET(metaUrl, true, false)
+ for (def meta : jsonMeta.rs_metas) {
+ int end_version = meta.end_version
+ if (end_version == version) {
+ return meta
+ }
+ }
+ }
+ }
}
def checkKeyBounds = { int version, int length, boolean turnedOn ->
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
