This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new a303308d7ed [fix](cloud) preserve rowset inverted index format in
recycler (#66793)
a303308d7ed is described below
commit a303308d7ed77027420c470dc314474463d38959
Author: hoshinojyunn <[email protected]>
AuthorDate: Tue Aug 18 10:07:55 2026 +0800
[fix](cloud) preserve rowset inverted index format in recycler (#66793)
Related PR: #66475
Problem Summary:
This PR separately submits the Meta Service changes from #66475. When
the shared schema format differs from the physical format used by an
existing rowset, recycler and checker must use the rowset format to
select the correct inverted-index files.
The change persists the physical inverted-index format in metadata and
makes recycler/checker prefer the rowset value, with the existing schema
and schema-KV fallback retained for old metadata.
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [x] Unit Test
- `./run-cloud-ut.sh --run
--filter=recycler_test:RecyclerTest.delete_tmp_rowset_data_with_rowset_idx_v3`
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [x] Yes. Recycler and checker select inverted-index files from the
persisted rowset format.
- Does this need documentation?
- [x] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
---
be/src/cloud/pb_convert.cpp | 24 +++++++++++++++
cloud/src/recycler/checker.cpp | 7 +++--
cloud/src/recycler/recycler.cpp | 23 ++++++++++-----
cloud/test/recycler_test.cpp | 65 +++++++++++++++++++++++++++++++++++++++++
gensrc/proto/olap_file.proto | 10 +++++++
5 files changed, 120 insertions(+), 9 deletions(-)
diff --git a/be/src/cloud/pb_convert.cpp b/be/src/cloud/pb_convert.cpp
index e0799ee1256..bcad8506d1a 100644
--- a/be/src/cloud/pb_convert.cpp
+++ b/be/src/cloud/pb_convert.cpp
@@ -102,6 +102,9 @@ void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out,
const RowsetMetaPB& in)
if (in.has___split_schema()) {
out->mutable___split_schema()->CopyFrom(in.__split_schema());
}
+ if (in.has_inverted_index_storage_format()) {
+
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
+ }
if (in.has_visible_ts_ms()) {
out->set_visible_ts_ms(in.visible_ts_ms());
}
@@ -198,6 +201,9 @@ void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out,
RowsetMetaPB&& in) {
if (in.has___split_schema()) {
out->mutable___split_schema()->Swap(in.mutable___split_schema());
}
+ if (in.has_inverted_index_storage_format()) {
+
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
+ }
if (in.has_visible_ts_ms()) {
out->set_visible_ts_ms(in.visible_ts_ms());
}
@@ -310,6 +316,9 @@ void cloud_rowset_meta_to_doris(RowsetMetaPB* out, const
RowsetMetaCloudPB& in)
if (in.has_reference_instance_id()) {
out->set_reference_instance_id(in.reference_instance_id());
}
+ if (in.has_inverted_index_storage_format()) {
+
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
+ }
auto* slice_locations = out->mutable_packed_slice_locations();
slice_locations->clear();
slice_locations->insert(in.packed_slice_locations().begin(),
in.packed_slice_locations().end());
@@ -405,6 +414,9 @@ void cloud_rowset_meta_to_doris(RowsetMetaPB* out,
RowsetMetaCloudPB&& in) {
if (in.has_reference_instance_id()) {
out->set_reference_instance_id(in.reference_instance_id());
}
+ if (in.has_inverted_index_storage_format()) {
+
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
+ }
auto* slice_locations = out->mutable_packed_slice_locations();
slice_locations->clear();
slice_locations->insert(in.packed_slice_locations().begin(),
in.packed_slice_locations().end());
@@ -790,6 +802,9 @@ void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out,
const TabletMetaPB& in)
if (in.has_encryption_algorithm()) {
out->set_encryption_algorithm(in.encryption_algorithm());
}
+ if (in.has_inverted_index_storage_format()) {
+
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
+ }
}
void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, TabletMetaPB&& in) {
@@ -875,6 +890,9 @@ void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out,
TabletMetaPB&& in) {
if (in.has_encryption_algorithm()) {
out->set_encryption_algorithm(in.encryption_algorithm());
}
+ if (in.has_inverted_index_storage_format()) {
+
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
+ }
}
TabletMetaPB cloud_tablet_meta_to_doris(const TabletMetaCloudPB& in) {
@@ -967,6 +985,9 @@ void cloud_tablet_meta_to_doris(TabletMetaPB* out, const
TabletMetaCloudPB& in)
if (in.has_encryption_algorithm()) {
out->set_encryption_algorithm(in.encryption_algorithm());
}
+ if (in.has_inverted_index_storage_format()) {
+
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
+ }
}
void cloud_tablet_meta_to_doris(TabletMetaPB* out, TabletMetaCloudPB&& in) {
@@ -1052,6 +1073,9 @@ void cloud_tablet_meta_to_doris(TabletMetaPB* out,
TabletMetaCloudPB&& in) {
if (in.has_encryption_algorithm()) {
out->set_encryption_algorithm(in.encryption_algorithm());
}
+ if (in.has_inverted_index_storage_format()) {
+
out->set_inverted_index_storage_format(in.inverted_index_storage_format());
+ }
}
} // namespace doris::cloud
diff --git a/cloud/src/recycler/checker.cpp b/cloud/src/recycler/checker.cpp
index 8930d327d6d..3369cc8dc97 100644
--- a/cloud/src/recycler/checker.cpp
+++ b/cloud/src/recycler/checker.cpp
@@ -772,10 +772,13 @@ int InstanceChecker::do_check() {
}
if (!index_ids.empty()) {
const auto& index_map = rs_meta.packed_slice_locations();
+ const auto index_format =
+ rs_meta.has_inverted_index_storage_format()
+ ? rs_meta.inverted_index_storage_format()
+ :
rs_meta.tablet_schema().inverted_index_storage_format();
for (int i = 0; i < rs_meta.num_segments(); ++i) {
std::vector<std::string> index_path_v;
- if (rs_meta.tablet_schema().inverted_index_storage_format() ==
- InvertedIndexStorageFormatPB::V1) {
+ if (index_format == InvertedIndexStorageFormatPB::V1) {
for (const auto& index_id : index_ids) {
LOG(INFO) << "check inverted index, tablet_id=" <<
rs_meta.tablet_id()
<< " rowset_id=" << rs_meta.rowset_id_v2()
<< " segment_id=" << i
diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp
index b2a9a2b815f..7c13d2d4bc2 100644
--- a/cloud/src/recycler/recycler.cpp
+++ b/cloud/src/recycler/recycler.cpp
@@ -3471,8 +3471,9 @@ int InstanceRecycler::delete_rowset_data(const
RowsetMetaCloudPB& rs_meta_pb) {
// Process inverted indexes
std::vector<std::pair<int64_t, std::string>> index_ids;
- // default format as v1.
- InvertedIndexStorageFormatPB index_format =
InvertedIndexStorageFormatPB::V1;
+ InvertedIndexStorageFormatPB index_format =
rs_meta_pb.has_inverted_index_storage_format()
+ ?
rs_meta_pb.inverted_index_storage_format()
+ :
InvertedIndexStorageFormatPB::V1;
bool delete_rowset_data_by_prefix = false;
if (rs_meta_pb.rowset_state() == RowsetStatePB::BEGIN_PARTIAL_UPDATE) {
// if rowset state is RowsetStatePB::BEGIN_PARTIAL_UPDATE, the number
of segments data
@@ -3484,7 +3485,8 @@ int InstanceRecycler::delete_rowset_data(const
RowsetMetaCloudPB& rs_meta_pb) {
index_ids.emplace_back(index.index_id(),
index.index_suffix_name());
}
}
- if (rs_meta_pb.tablet_schema().has_inverted_index_storage_format()) {
+ if (!rs_meta_pb.has_inverted_index_storage_format() &&
+ rs_meta_pb.tablet_schema().has_inverted_index_storage_format()) {
index_format =
rs_meta_pb.tablet_schema().inverted_index_storage_format();
}
} else if (!rs_meta_pb.has_index_id() || !rs_meta_pb.has_schema_version())
{
@@ -3498,7 +3500,9 @@ int InstanceRecycler::delete_rowset_data(const
RowsetMetaCloudPB& rs_meta_pb) {
TEST_SYNC_POINT_CALLBACK("InstanceRecycler::delete_rowset_data.tmp_rowset",
&inverted_index_get_ret);
if (inverted_index_get_ret == 0) {
- index_format = index_info.first;
+ if (!rs_meta_pb.has_inverted_index_storage_format()) {
+ index_format = index_info.first;
+ }
index_ids = index_info.second;
} else if (inverted_index_get_ret == 1) {
// 1. Schema kv not found means tablet has been recycled
@@ -4236,7 +4240,9 @@ int InstanceRecycler::delete_rowset_data(
// Process inverted indexes
std::vector<std::pair<int64_t, std::string>> index_ids;
// default format as v1.
- InvertedIndexStorageFormatPB index_format =
InvertedIndexStorageFormatPB::V1;
+ InvertedIndexStorageFormatPB index_format =
rs.has_inverted_index_storage_format()
+ ?
rs.inverted_index_storage_format()
+ :
InvertedIndexStorageFormatPB::V1;
int inverted_index_get_ret = 0;
if (rs.has_tablet_schema()) {
for (const auto& index : rs.tablet_schema().index()) {
@@ -4244,7 +4250,8 @@ int InstanceRecycler::delete_rowset_data(
index_ids.emplace_back(index.index_id(),
index.index_suffix_name());
}
}
- if (rs.tablet_schema().has_inverted_index_storage_format()) {
+ if (!rs.has_inverted_index_storage_format() &&
+ rs.tablet_schema().has_inverted_index_storage_format()) {
index_format =
rs.tablet_schema().inverted_index_storage_format();
}
} else {
@@ -4262,7 +4269,9 @@ int InstanceRecycler::delete_rowset_data(
TEST_SYNC_POINT_CALLBACK("InstanceRecycler::delete_rowset_data.tmp_rowset",
&inverted_index_get_ret);
if (inverted_index_get_ret == 0) {
- index_format = index_info.first;
+ if (!rs.has_inverted_index_storage_format()) {
+ index_format = index_info.first;
+ }
index_ids = index_info.second;
} else if (inverted_index_get_ret == 1) {
// 1. Schema kv not found means tablet has been recycled
diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp
index c1abd0a5e13..8c635e8d6e9 100644
--- a/cloud/test/recycler_test.cpp
+++ b/cloud/test/recycler_test.cpp
@@ -8066,6 +8066,71 @@ TEST(RecyclerTest, delete_tmp_rowset_data_with_idx_v2) {
}
}
+TEST(RecyclerTest, delete_tmp_rowset_data_with_rowset_idx_v3) {
+ auto txn_kv = std::make_shared<MemTxnKv>();
+ ASSERT_EQ(txn_kv->init(), 0);
+
+ InstanceInfoPB instance;
+ instance.set_instance_id(instance_id);
+ auto obj_info = instance.add_obj_info();
+ obj_info->set_id("delete_tmp_rowset_data_with_rowset_idx_v3");
+ obj_info->set_ak(config::test_s3_ak);
+ obj_info->set_sk(config::test_s3_sk);
+ obj_info->set_endpoint(config::test_s3_endpoint);
+ obj_info->set_region(config::test_s3_region);
+ obj_info->set_bucket(config::test_s3_bucket);
+ obj_info->set_prefix("delete_tmp_rowset_data_with_rowset_idx_v3");
+
+ doris::TabletSchemaCloudPB schema;
+ schema.set_schema_version(1);
+ schema.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V1);
+ auto index = schema.add_index();
+ index->set_index_id(1);
+ index->set_index_type(IndexType::INVERTED);
+
+ {
+ InstanceRecycler recycler(txn_kv, instance, thread_group,
+ std::make_shared<TxnLazyCommitter>(txn_kv));
+ ASSERT_EQ(recycler.init(), 0);
+ auto accessor = recycler.accessor_map_.begin()->second;
+ std::map<std::string, doris::RowsetMetaCloudPB> rowset_pbs;
+ doris::RowsetMetaCloudPB rowset;
+ rowset.set_rowset_id(0); // useless but required
+ rowset.set_rowset_id_v2("1");
+ rowset.set_num_segments(1);
+ rowset.set_tablet_id(10000);
+ rowset.set_index_id(10001);
+ rowset.set_resource_id("delete_tmp_rowset_data_with_rowset_idx_v3");
+ rowset.set_schema_version(schema.schema_version());
+ rowset.mutable_tablet_schema()->CopyFrom(schema);
+
rowset.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V3);
+ create_tmp_rowset(txn_kv.get(), accessor.get(), rowset, 1, true);
+ rowset_pbs.emplace(rowset.rowset_id_v2(), rowset);
+
+ std::unordered_set<std::string> list_files;
+ std::unique_ptr<ListIterator> iter;
+ EXPECT_EQ(accessor->list_all(&iter), 0);
+ EXPECT_TRUE(iter->has_next());
+ for (auto file = iter->next(); file.has_value(); file = iter->next()) {
+ list_files.insert(file->path);
+ }
+ EXPECT_EQ(list_files.size(), 2);
+ EXPECT_TRUE(list_files.contains("data/10000/1_0.dat"));
+ EXPECT_TRUE(list_files.contains("data/10000/1_0.idx"));
+
+ ASSERT_EQ(0,
+ recycler.delete_rowset_data(rowset_pbs,
RowsetRecyclingState::TMP_ROWSET, ctx));
+ list_files.clear();
+ iter.reset();
+ EXPECT_EQ(accessor->list_all(&iter), 0);
+ EXPECT_FALSE(iter->has_next());
+ for (auto file = iter->next(); file.has_value(); file = iter->next()) {
+ list_files.insert(file->path);
+ }
+ EXPECT_EQ(list_files.size(), 0);
+ }
+}
+
TEST(RecyclerTest, delete_tmp_rowset_without_resource_id) {
auto* sp = SyncPoint::get_instance();
DORIS_CLOUD_DEFER {
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index fe0ecc82af9..c672f33da5f 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -183,6 +183,9 @@ message RowsetMetaPB {
// For row binlog LSN allocation (FE auto-inc RPC)
optional int64 db_id = 1016;
optional int64 table_id = 1017;
+ // Actual inverted-index format used by this rowset's files. It is
independent
+ // from the shared (index_id, schema_version) schema key.
+ optional InvertedIndexStorageFormatPB inverted_index_storage_format = 1018;
}
message SchemaDictKeyList {
@@ -301,6 +304,9 @@ message RowsetMetaCloudPB {
optional int64 db_id = 115;
optional int64 table_id = 116;
+ // Actual inverted-index format used by this rowset's files. It is
independent
+ // from the shared (index_id, schema_version) schema key.
+ optional InvertedIndexStorageFormatPB inverted_index_storage_format = 117;
}
message SegmentStatisticsPB {
@@ -733,6 +739,8 @@ message TabletMetaPB {
optional string table_name = 1003;
optional int64 ttl_seconds = 1004;
optional int32 schema_version = 1005; // index_id, schema_version -> schema
+ // Immutable format of this tablet's inverted-index files.
+ optional InvertedIndexStorageFormatPB inverted_index_storage_format = 1006;
}
message TabletMetaCloudPB {
@@ -792,6 +800,8 @@ message TabletMetaCloudPB {
optional string table_name = 101;
optional int64 ttl_seconds = 102;
optional int32 schema_version = 103; // index_id, schema_version -> schema
+ // Immutable format of this tablet's inverted-index files.
+ optional InvertedIndexStorageFormatPB inverted_index_storage_format = 104;
}
message OLAPRawDeltaHeaderMessage {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
