This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 8af45166949 branch-4.0: [Bug](cloud restore) refresh index id in
rowset schema #57074 (#58660)
8af45166949 is described below
commit 8af4516694913e303831ba07eb9f879d591218f9
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Wed Dec 3 22:23:52 2025 +0800
branch-4.0: [Bug](cloud restore) refresh index id in rowset schema #57074
(#58660)
Cherry-picked from #57074
Co-authored-by: xy720 <[email protected]>
---
be/src/cloud/cloud_snapshot_mgr.cpp | 35 +++
be/src/cloud/cloud_snapshot_mgr.h | 3 +
be/src/olap/snapshot_manager.cpp | 44 +++
be/src/olap/snapshot_manager.h | 4 +
be/src/olap/tablet_schema.cpp | 12 +
be/src/olap/tablet_schema.h | 2 +
be/test/cloud/cloud_snapshot_mgr_test.cpp | 106 ++++++++
be/test/olap/snapshot_manager_test.cpp | 296 +++++++++++++++++++++
be/test/olap/tablet_schema_test.cpp | 92 ++++++-
.../test_backup_restore_reset_index_id.groovy | 161 +++++++++++
10 files changed, 754 insertions(+), 1 deletion(-)
diff --git a/be/src/cloud/cloud_snapshot_mgr.cpp
b/be/src/cloud/cloud_snapshot_mgr.cpp
index a3f3c5dc213..2d471d80975 100644
--- a/be/src/cloud/cloud_snapshot_mgr.cpp
+++ b/be/src/cloud/cloud_snapshot_mgr.cpp
@@ -145,6 +145,9 @@ Status CloudSnapshotMgr::convert_rowsets(
cooldown_meta_id->set_hi(0);
cooldown_meta_id->set_lo(0);
+ TabletSchemaSPtr target_tablet_schema = std::make_shared<TabletSchema>();
+ target_tablet_schema->copy_from(*target_tablet->tablet_schema());
+
TabletSchemaSPtr tablet_schema = std::make_shared<TabletSchema>();
tablet_schema->init_from_pb(in.schema());
@@ -155,10 +158,19 @@ Status CloudSnapshotMgr::convert_rowsets(
RETURN_IF_ERROR(_create_rowset_meta(new_rowset_meta_pb,
rowset_meta_pb, tablet_id,
target_tablet, storage_resource,
tablet_schema,
file_mapping, rowset_id_mapping));
+ if (new_rowset_meta_pb->has_tablet_schema() &&
+ new_rowset_meta_pb->tablet_schema().index_size() > 0) {
+
RETURN_IF_ERROR(_rename_index_ids(*new_rowset_meta_pb->mutable_tablet_schema(),
+ target_tablet_schema));
+ }
Version rowset_version = {rowset_meta_pb.start_version(),
rowset_meta_pb.end_version()};
rs_version_map[rowset_version] = new_rowset_meta_pb;
}
+ if (out->schema().index_size() > 0) {
+ RETURN_IF_ERROR(_rename_index_ids(*out->mutable_schema(),
target_tablet_schema));
+ }
+
if (!rowset_id_mapping.empty() && in.has_delete_bitmap()) {
const auto& old_del_bitmap_pb = in.delete_bitmap();
DeleteBitmapPB* new_del_bitmap_pb = out->mutable_delete_bitmap();
@@ -178,6 +190,7 @@ Status CloudSnapshotMgr::convert_rowsets(
new_del_bitmap_pb->set_rowset_ids(cast_set<int>(i),
it->second.to_string());
}
}
+
return Status::OK();
}
@@ -271,5 +284,27 @@ Status CloudSnapshotMgr::_create_rowset_meta(
return Status::OK();
}
+Status CloudSnapshotMgr::_rename_index_ids(TabletSchemaPB& schema_pb,
+ const TabletSchemaSPtr&
tablet_schema) const {
+ if (tablet_schema == nullptr) {
+ return Status::OK();
+ }
+
+ for (int i = 0; i < schema_pb.index_size(); ++i) {
+ TabletIndexPB* index_pb = schema_pb.mutable_index(i);
+ for (int32_t col_unique_id : index_pb->col_unique_id()) {
+ auto local_index = tablet_schema->get_index(col_unique_id,
index_pb->index_type(),
+
index_pb->index_suffix_name());
+ if (local_index) {
+ if (index_pb->index_id() != local_index->index_id()) {
+ index_pb->set_index_id(local_index->index_id());
+ }
+ break;
+ }
+ }
+ }
+ return Status::OK();
+}
+
#include "common/compile_check_end.h"
} // namespace doris
diff --git a/be/src/cloud/cloud_snapshot_mgr.h
b/be/src/cloud/cloud_snapshot_mgr.h
index e1888ac412e..e249e3e4865 100644
--- a/be/src/cloud/cloud_snapshot_mgr.h
+++ b/be/src/cloud/cloud_snapshot_mgr.h
@@ -90,6 +90,9 @@ private:
std::unordered_map<std::string, std::string>&
file_mapping,
std::unordered_map<RowsetId, RowsetId>&
rowset_id_mapping);
+ Status _rename_index_ids(TabletSchemaPB& schema_pb,
+ const TabletSchemaSPtr& tablet_schema) const;
+
private:
CloudStorageEngine& _engine;
std::atomic<uint64_t> _snapshot_base_id {0};
diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp
index 899a4bef265..684a93b150a 100644
--- a/be/src/olap/snapshot_manager.cpp
+++ b/be/src/olap/snapshot_manager.cpp
@@ -183,6 +183,13 @@ Result<std::vector<PendingRowsetGuard>>
SnapshotManager::convert_rowset_ids(
"clone dir not existed when convert rowsetids. clone_dir={}",
clone_dir));
}
+ TabletSharedPtr target_tablet =
_engine.tablet_manager()->get_tablet(tablet_id);
+ TabletSchemaSPtr target_tablet_schema = nullptr;
+ if (target_tablet != nullptr) {
+ target_tablet_schema = std::make_shared<TabletSchema>();
+ target_tablet_schema->copy_from(*target_tablet->tablet_schema());
+ }
+
// load original tablet meta
auto cloned_meta_file = fmt::format("{}/{}.hdr", clone_dir, tablet_id);
TabletMetaPB cloned_tablet_meta_pb;
@@ -241,6 +248,11 @@ Result<std::vector<PendingRowsetGuard>>
SnapshotManager::convert_rowset_ids(
rowset_meta->set_partition_id(partition_id);
}
+ if (rowset_meta->has_tablet_schema() &&
rowset_meta->tablet_schema().index_size() > 0) {
+ RETURN_IF_ERROR_RESULT(
+ _rename_index_ids(*rowset_meta->mutable_tablet_schema(),
target_tablet_schema));
+ }
+
Version rowset_version = {visible_rowset.start_version(),
visible_rowset.end_version()};
rs_version_map[rowset_version] = rowset_meta;
}
@@ -277,6 +289,16 @@ Result<std::vector<PendingRowsetGuard>>
SnapshotManager::convert_rowset_ids(
if (partition_id != -1) {
rowset_meta->set_partition_id(partition_id);
}
+
+ if (rowset_meta->has_tablet_schema() &&
rowset_meta->tablet_schema().index_size() > 0) {
+ RETURN_IF_ERROR_RESULT(
+ _rename_index_ids(*rowset_meta->mutable_tablet_schema(),
target_tablet_schema));
+ }
+ }
+
+ if (new_tablet_meta_pb.schema().index_size() > 0) {
+ RETURN_IF_ERROR_RESULT(
+ _rename_index_ids(*new_tablet_meta_pb.mutable_schema(),
target_tablet_schema));
}
if (!rowset_id_mapping.empty() &&
cloned_tablet_meta_pb.has_delete_bitmap()) {
@@ -346,6 +368,28 @@ Status SnapshotManager::_rename_rowset_id(const
RowsetMetaPB& rs_meta_pb,
return Status::OK();
}
+Status SnapshotManager::_rename_index_ids(TabletSchemaPB& schema_pb,
+ const TabletSchemaSPtr&
tablet_schema) const {
+ if (tablet_schema == nullptr) {
+ return Status::OK();
+ }
+
+ for (int i = 0; i < schema_pb.index_size(); ++i) {
+ TabletIndexPB* index_pb = schema_pb.mutable_index(i);
+ for (int32_t col_unique_id : index_pb->col_unique_id()) {
+ auto local_index = tablet_schema->get_index(col_unique_id,
index_pb->index_type(),
+
index_pb->index_suffix_name());
+ if (local_index) {
+ if (index_pb->index_id() != local_index->index_id()) {
+ index_pb->set_index_id(local_index->index_id());
+ }
+ break;
+ }
+ }
+ }
+ return Status::OK();
+}
+
// get snapshot path: curtime.seq.timeout
// eg: 20190819221234.3.86400
Status SnapshotManager::_calc_snapshot_id_path(const TabletSharedPtr& tablet,
int64_t timeout_s,
diff --git a/be/src/olap/snapshot_manager.h b/be/src/olap/snapshot_manager.h
index 668bb860e1b..01303dac4f2 100644
--- a/be/src/olap/snapshot_manager.h
+++ b/be/src/olap/snapshot_manager.h
@@ -30,6 +30,7 @@
namespace doris {
class RowsetMetaPB;
+class TabletSchemaPB;
class TSnapshotRequest;
struct RowsetId;
class StorageEngine;
@@ -133,6 +134,9 @@ private:
TabletSchemaSPtr tablet_schema, const RowsetId&
next_id,
RowsetMetaPB* new_rs_meta_pb);
+ Status _rename_index_ids(TabletSchemaPB& schema_pb,
+ const TabletSchemaSPtr& tablet_schema) const;
+
StorageEngine& _engine;
std::atomic<uint64_t> _snapshot_base_id {0};
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index b3331cdfb18..0408800d0d0 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -1673,6 +1673,18 @@ const TabletIndex*
TabletSchema::get_ngram_bf_index(int32_t col_unique_id) const
return nullptr;
}
+const TabletIndex* TabletSchema::get_index(int32_t col_unique_id, IndexType
index_type,
+ const std::string& suffix_path)
const {
+ IndexKey index_key(index_type, col_unique_id, suffix_path);
+ auto it = _col_id_suffix_to_index.find(index_key);
+ if (it != _col_id_suffix_to_index.end()) {
+ if (!it->second.empty() && it->second[0] < _indexes.size()) {
+ return _indexes[it->second[0]].get();
+ }
+ }
+ return nullptr;
+}
+
vectorized::Block TabletSchema::create_block(
const std::vector<uint32_t>& return_columns,
const std::unordered_set<uint32_t>* tablet_columns_need_convert_null)
const {
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index 429e102c9a2..9d76424acbe 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -545,6 +545,8 @@ public:
bool has_ngram_bf_index(int32_t col_unique_id) const;
const TabletIndex* get_ngram_bf_index(int32_t col_unique_id) const;
+ const TabletIndex* get_index(int32_t col_unique_id, IndexType index_type,
+ const std::string& suffix_path) const;
void update_indexes_from_thrift(const std::vector<doris::TOlapTableIndex>&
indexes);
// If schema version is not set, it should be -1
int32_t schema_version() const { return _schema_version; }
diff --git a/be/test/cloud/cloud_snapshot_mgr_test.cpp
b/be/test/cloud/cloud_snapshot_mgr_test.cpp
index 3ca72cd7eb8..5b097be04d4 100644
--- a/be/test/cloud/cloud_snapshot_mgr_test.cpp
+++ b/be/test/cloud/cloud_snapshot_mgr_test.cpp
@@ -61,6 +61,38 @@ TEST_F(CloudSnapshotMgrTest, TestConvertRowsets) {
input_meta_pb.set_schema_hash(123456);
*input_meta_pb.mutable_tablet_uid() = TabletUid::gen_uid().to_proto();
+ TabletSchemaPB* input_schema = input_meta_pb.mutable_schema();
+ input_schema->set_keys_type(KeysType::DUP_KEYS);
+ input_schema->set_num_short_key_columns(1);
+ input_schema->set_num_rows_per_row_block(1024);
+ input_schema->set_compress_kind(COMPRESS_LZ4);
+
+ ColumnPB* col1 = input_schema->add_column();
+ col1->set_unique_id(1);
+ col1->set_name("col1");
+ col1->set_type("INT");
+ col1->set_is_key(true);
+ col1->set_aggregation("NONE");
+
+ ColumnPB* col2 = input_schema->add_column();
+ col2->set_unique_id(2);
+ col2->set_name("col2");
+ col2->set_type("VARCHAR");
+ col2->set_is_key(false);
+ col2->set_aggregation("REPLACE");
+
+ doris::TabletIndexPB* index1 = input_schema->add_index();
+ index1->set_index_id(1001);
+ index1->set_index_name("test_index1");
+ index1->set_index_type(IndexType::BITMAP);
+ index1->add_col_unique_id(2);
+
+ doris::TabletIndexPB* index2 = input_schema->add_index();
+ index2->set_index_id(1002);
+ index2->set_index_name("test_index2");
+ index2->set_index_type(IndexType::INVERTED);
+ index2->add_col_unique_id(2);
+
RowsetMetaPB* rowset_meta = input_meta_pb.add_rs_metas();
RowsetId rowset_id;
rowset_id.init(10000);
@@ -78,8 +110,21 @@ TEST_F(CloudSnapshotMgrTest, TestConvertRowsets) {
rowset_meta->set_rowset_state(RowsetStatePB::VISIBLE);
rowset_meta->set_newest_write_timestamp(1678901234567890);
+ TabletSchemaPB* rowset_schema = rowset_meta->mutable_tablet_schema();
+ rowset_schema->CopyFrom(*input_schema);
+
auto tablet_meta = std::make_shared<TabletMeta>();
tablet_meta->init_from_pb(input_meta_pb);
+
+ TabletSchemaSPtr local_tablet_schema = std::make_shared<TabletSchema>();
+ local_tablet_schema->init_from_pb(input_meta_pb.schema());
+
+ // reset the index id
+ input_schema->mutable_index(0)->set_index_id(2001);
+ input_schema->mutable_index(1)->set_index_id(2002);
+ rowset_meta->mutable_tablet_schema()->mutable_index(0)->set_index_id(2001);
+ rowset_meta->mutable_tablet_schema()->mutable_index(1)->set_index_id(2002);
+
CloudTabletSPtr target_tablet = std::make_shared<CloudTablet>(*_engine,
tablet_meta);
StorageResource storage_resource {_fs};
std::unordered_map<std::string, std::string> file_mapping;
@@ -92,6 +137,16 @@ TEST_F(CloudSnapshotMgrTest, TestConvertRowsets) {
EXPECT_EQ(output_meta_pb.rs_metas(0).tablet_id(), 3000);
EXPECT_EQ(output_meta_pb.rs_metas(0).rowset_id(), 0);
EXPECT_NE(output_meta_pb.rs_metas(0).rowset_id_v2(),
rowset_id.to_string());
+ EXPECT_TRUE(output_meta_pb.has_schema());
+ EXPECT_EQ(output_meta_pb.schema().index_size(), 2);
+ EXPECT_EQ(output_meta_pb.schema().index(0).index_id(), 1001);
+ EXPECT_EQ(output_meta_pb.schema().index(0).index_name(), "test_index1");
+ EXPECT_EQ(output_meta_pb.schema().index(1).index_id(), 1002);
+ EXPECT_EQ(output_meta_pb.schema().index(1).index_name(), "test_index2");
+ EXPECT_TRUE(output_meta_pb.rs_metas(0).has_tablet_schema());
+ EXPECT_EQ(output_meta_pb.rs_metas(0).tablet_schema().index_size(), 2);
+ EXPECT_EQ(output_meta_pb.rs_metas(0).tablet_schema().index(0).index_id(),
1001);
+ EXPECT_EQ(output_meta_pb.rs_metas(0).tablet_schema().index(1).index_id(),
1002);
EXPECT_EQ(output_meta_pb.rs_metas(0).num_segments(), 3);
EXPECT_EQ(output_meta_pb.rs_metas(0).num_rows(), 100);
EXPECT_EQ(output_meta_pb.rs_metas(0).start_version(), 100);
@@ -105,4 +160,55 @@ TEST_F(CloudSnapshotMgrTest, TestConvertRowsets) {
EXPECT_TRUE(status.ok());
}
+TEST_F(CloudSnapshotMgrTest, TestRenameIndexIds) {
+ TabletSchemaPB source_schema_pb;
+ source_schema_pb.set_keys_type(KeysType::DUP_KEYS);
+ source_schema_pb.set_num_short_key_columns(1);
+ source_schema_pb.set_num_rows_per_row_block(1024);
+ source_schema_pb.set_compress_kind(COMPRESS_LZ4);
+
+ ColumnPB* col1 = source_schema_pb.add_column();
+ col1->set_unique_id(1);
+ col1->set_name("col1");
+ col1->set_type("INT");
+ col1->set_is_key(true);
+ col1->set_aggregation("NONE");
+
+ ColumnPB* col2 = source_schema_pb.add_column();
+ col2->set_unique_id(2);
+ col2->set_name("col2");
+ col2->set_type("VARCHAR");
+ col2->set_is_key(false);
+ col2->set_aggregation("REPLACE");
+
+ doris::TabletIndexPB* index1 = source_schema_pb.add_index();
+ index1->set_index_id(1001);
+ index1->set_index_name("test_index1");
+ index1->set_index_type(IndexType::BITMAP);
+ index1->add_col_unique_id(2);
+
+ doris::TabletIndexPB* index2 = source_schema_pb.add_index();
+ index2->set_index_id(1002);
+ index2->set_index_name("test_index2");
+ index2->set_index_type(IndexType::INVERTED);
+ index2->add_col_unique_id(2);
+
+ TabletSchemaSPtr local_tablet_schema = std::make_shared<TabletSchema>();
+ local_tablet_schema->init_from_pb(source_schema_pb);
+
+ // reset the index id
+ source_schema_pb.mutable_index(0)->set_index_id(2001);
+ source_schema_pb.mutable_index(1)->set_index_id(2002);
+
+ Status status = _snapshot_mgr->_rename_index_ids(source_schema_pb,
local_tablet_schema);
+ EXPECT_TRUE(status.ok());
+ EXPECT_EQ(source_schema_pb.index_size(), 2);
+ EXPECT_EQ(source_schema_pb.index(0).index_id(), 1001);
+ EXPECT_EQ(source_schema_pb.index(0).index_name(), "test_index1");
+ EXPECT_EQ(source_schema_pb.index(0).index_type(), IndexType::BITMAP);
+ EXPECT_EQ(source_schema_pb.index(1).index_id(), 1002);
+ EXPECT_EQ(source_schema_pb.index(1).index_name(), "test_index2");
+ EXPECT_EQ(source_schema_pb.index(1).index_type(), IndexType::INVERTED);
+}
+
} // namespace doris
diff --git a/be/test/olap/snapshot_manager_test.cpp
b/be/test/olap/snapshot_manager_test.cpp
new file mode 100644
index 00000000000..d7ea196b655
--- /dev/null
+++ b/be/test/olap/snapshot_manager_test.cpp
@@ -0,0 +1,296 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/snapshot_manager.h"
+
+#include <gen_cpp/AgentService_types.h>
+#include <gtest/gtest-message.h>
+#include <gtest/gtest-test-part.h>
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "common/status.h"
+#include "gtest/gtest_pred_impl.h"
+#include "io/fs/local_file_system.h"
+#include "olap/data_dir.h"
+#include "olap/olap_common.h"
+#include "olap/olap_define.h"
+#include "olap/options.h"
+#include "olap/storage_engine.h"
+#include "olap/tablet.h"
+#include "olap/tablet_manager.h"
+#include "olap/tablet_meta.h"
+#include "runtime/exec_env.h"
+#include "util/uid_util.h"
+
+namespace doris {
+class SnapshotManagerTest : public testing::Test {
+public:
+ virtual void SetUp() {
+ _engine_data_path = "./be/test/olap/test_data/converter_test_data/tmp";
+ auto st =
io::global_local_filesystem()->delete_directory(_engine_data_path);
+ ASSERT_TRUE(st.ok()) << st;
+ st =
io::global_local_filesystem()->create_directory(_engine_data_path);
+ ASSERT_TRUE(st.ok()) << st;
+ EXPECT_TRUE(
+
io::global_local_filesystem()->create_directory(_engine_data_path +
"/meta").ok());
+
+ config::tablet_map_shard_size = 1;
+ config::txn_map_shard_size = 1;
+ config::txn_shard_size = 1;
+ EngineOptions options;
+ options.backend_uid = UniqueId::gen_uid();
+ auto engine = std::make_unique<StorageEngine>(options);
+ ExecEnv::GetInstance()->set_storage_engine(std::move(engine));
+ _engine = &ExecEnv::GetInstance()->storage_engine().to_local();
+ _data_dir = new DataDir(*_engine, _engine_data_path, 1000000000);
+ static_cast<void>(_data_dir->init());
+ }
+
+ virtual void TearDown() {
+ SAFE_DELETE(_data_dir);
+
EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_engine_data_path).ok());
+ ExecEnv::GetInstance()->set_storage_engine(nullptr);
+ _engine = nullptr;
+ }
+
+protected:
+ StorageEngine* _engine = nullptr;
+ DataDir* _data_dir = nullptr;
+ std::string _engine_data_path;
+};
+
+TEST_F(SnapshotManagerTest, TestMakeSnapshotInvalidParameters) {
+ TSnapshotRequest request;
+ request.tablet_id = 10001;
+ request.schema_hash = 12345;
+
+ std::string* null_snapshot_path = nullptr;
+ bool allow_incremental_clone = false;
+
+ Status status = _engine->snapshot_mgr()->make_snapshot(request,
null_snapshot_path,
+
&allow_incremental_clone);
+ EXPECT_FALSE(status.ok());
+ EXPECT_EQ(status.code(), ErrorCode::INVALID_ARGUMENT);
+}
+
+TEST_F(SnapshotManagerTest, TestReleaseSnapshotInvalidPath) {
+ std::string invalid_path = "/invalid/snapshot/path";
+
+ Status status = _engine->snapshot_mgr()->release_snapshot(invalid_path);
+ EXPECT_FALSE(status.ok());
+ EXPECT_EQ(status.code(), ErrorCode::CE_CMD_PARAMS_ERROR);
+}
+
+TEST_F(SnapshotManagerTest, TestConvertRowsetIdsInvalidDir) {
+ std::string non_existent_dir = "/non/existent/dir";
+ int64_t tablet_id = 10003;
+ int64_t replica_id = 1;
+ int64_t table_id = 1000;
+ int64_t partition_id = 100;
+ int32_t schema_hash = 54321;
+
+ auto result = _engine->snapshot_mgr()->convert_rowset_ids(
+ non_existent_dir, tablet_id, replica_id, table_id, partition_id,
schema_hash);
+ EXPECT_FALSE(result.has_value());
+ EXPECT_EQ(result.error().code(), ErrorCode::DIR_NOT_EXIST);
+}
+
+TEST_F(SnapshotManagerTest, TestConvertRowsetIdsNormal) {
+ std::string clone_dir = _engine_data_path + "/clone_dir_local";
+
EXPECT_TRUE(io::global_local_filesystem()->create_directory(clone_dir).ok());
+
+ TabletMetaPB tablet_meta_pb;
+ tablet_meta_pb.set_tablet_id(10006);
+ tablet_meta_pb.set_schema_hash(12346);
+ tablet_meta_pb.set_replica_id(1);
+ tablet_meta_pb.set_table_id(1000);
+ tablet_meta_pb.set_partition_id(100);
+
+ TabletSchemaPB* schema_pb = tablet_meta_pb.mutable_schema();
+ schema_pb->set_num_short_key_columns(1);
+ schema_pb->set_num_rows_per_row_block(1024);
+ schema_pb->set_compress_kind(COMPRESS_LZ4);
+
+ // column 0
+ ColumnPB* column = schema_pb->add_column();
+ column->set_unique_id(0);
+ column->set_name("k1");
+ column->set_type("INT");
+ column->set_is_key(true);
+ column->set_is_nullable(false);
+ column->set_length(4);
+ column->set_index_length(4);
+ column->set_precision(0);
+ column->set_frac(0);
+
+ // column 1
+ column = schema_pb->add_column();
+ column->set_unique_id(1);
+ column->set_name("v1");
+ column->set_type("INT");
+ column->set_is_key(false);
+ column->set_is_nullable(true);
+ column->set_length(4);
+ column->set_index_length(4);
+ column->set_precision(0);
+ column->set_frac(0);
+
+ // index 0
+ TabletIndexPB* index_pb = schema_pb->add_index();
+ index_pb->set_index_id(1001);
+ index_pb->set_index_name("test_index");
+ index_pb->set_index_type(IndexType::BITMAP);
+ index_pb->add_col_unique_id(0);
+
+ // rowset meta 0
+ RowsetMetaPB* rowset_meta = tablet_meta_pb.add_rs_metas();
+ rowset_meta->set_rowset_id(10001);
+ rowset_meta->set_rowset_id_v2("02000000000000010001");
+ rowset_meta->set_tablet_id(10006);
+ rowset_meta->set_tablet_schema_hash(12346);
+ rowset_meta->set_rowset_type(BETA_ROWSET);
+ rowset_meta->set_rowset_state(VISIBLE);
+ rowset_meta->set_start_version(0);
+ rowset_meta->set_end_version(1);
+ rowset_meta->set_num_rows(100);
+ rowset_meta->set_total_disk_size(1024);
+ rowset_meta->set_data_disk_size(1000);
+ rowset_meta->set_index_disk_size(24);
+ rowset_meta->set_empty(false);
+
+ TabletSchemaPB* rowset_schema = rowset_meta->mutable_tablet_schema();
+ rowset_schema->CopyFrom(*schema_pb);
+
+ // new ids
+ int64_t tablet_id = 20006;
+ int64_t replica_id = 2;
+ int64_t table_id = 2000;
+ int64_t partition_id = 200;
+ int32_t schema_hash = 65432;
+
+ // save the meta file
+ std::string meta_file = clone_dir + "/" + std::to_string(tablet_id) +
".hdr";
+ EXPECT_TRUE(TabletMeta::save(meta_file, tablet_meta_pb).ok());
+
+ TCreateTabletReq create_tablet_req;
+ create_tablet_req.__set_tablet_id(tablet_id);
+ create_tablet_req.__set_replica_id(replica_id);
+ create_tablet_req.__set_table_id(table_id);
+ create_tablet_req.__set_partition_id(partition_id);
+ create_tablet_req.__set_version(1);
+
+ TTabletSchema tablet_schema;
+ tablet_schema.__set_schema_hash(schema_hash);
+
tablet_schema.__set_short_key_column_count(schema_pb->num_short_key_columns());
+ tablet_schema.__set_keys_type(TKeysType::AGG_KEYS);
+ tablet_schema.__set_storage_type(TStorageType::COLUMN);
+
+ std::vector<TColumn> columns;
+ for (int i = 0; i < schema_pb->column_size(); i++) {
+ const ColumnPB& column_pb = schema_pb->column(i);
+ TColumn tcolumn;
+ tcolumn.__set_column_name(column_pb.name());
+ tcolumn.__set_is_key(column_pb.is_key());
+ TColumnType col_type;
+ col_type.__set_type(TPrimitiveType::INT);
+ tcolumn.__set_column_type(col_type);
+ tcolumn.__set_is_allow_null(column_pb.is_nullable());
+ if (!column_pb.is_key()) {
+ tcolumn.__set_aggregation_type(TAggregationType::SUM);
+ }
+ columns.push_back(tcolumn);
+ }
+ tablet_schema.__set_columns(columns);
+
+ std::vector<TOlapTableIndex> indexes;
+ for (int i = 0; i < schema_pb->index_size(); i++) {
+ TOlapTableIndex tindex;
+ tindex.__set_index_id(2001 + i);
+ tindex.__set_index_name("test_index");
+ tindex.__set_index_type(TIndexType::BITMAP);
+ tindex.__set_columns({"k1"});
+ tindex.__set_column_unique_ids({0});
+ indexes.push_back(tindex);
+ }
+ tablet_schema.__set_indexes(indexes);
+
+ create_tablet_req.__set_tablet_schema(tablet_schema);
+ std::vector<DataDir*> stores;
+ stores.push_back(_data_dir);
+ RuntimeProfile profile("CreateTablet");
+
+ Status status =
_engine->tablet_manager()->create_tablet(create_tablet_req, stores, &profile);
+ EXPECT_TRUE(status.ok()) << "Failed to create tablet: " << status;
+
+ auto result = _engine->snapshot_mgr()->convert_rowset_ids(clone_dir,
tablet_id, replica_id,
+ table_id,
partition_id, schema_hash);
+ EXPECT_TRUE(result.has_value());
+
+ TabletMetaPB converted_meta_pb;
+ EXPECT_TRUE(TabletMeta::load_from_file(meta_file,
&converted_meta_pb).ok());
+
+ EXPECT_EQ(converted_meta_pb.tablet_id(), tablet_id);
+ EXPECT_EQ(converted_meta_pb.schema_hash(), schema_hash);
+ EXPECT_EQ(converted_meta_pb.replica_id(), replica_id);
+ EXPECT_EQ(converted_meta_pb.table_id(), table_id);
+ EXPECT_EQ(converted_meta_pb.partition_id(), partition_id);
+
+ // verify schema
+ EXPECT_EQ(converted_meta_pb.schema().num_short_key_columns(), 1);
+ EXPECT_EQ(converted_meta_pb.schema().column_size(), 2);
+
+ // verify index
+ EXPECT_EQ(converted_meta_pb.schema().index_size(), 1);
+ const TabletIndexPB& converted_index = converted_meta_pb.schema().index(0);
+ EXPECT_EQ(converted_index.index_id(), 2001);
+ EXPECT_EQ(converted_index.index_name(), "test_index");
+ EXPECT_EQ(converted_index.index_type(), IndexType::BITMAP);
+ EXPECT_EQ(converted_index.col_unique_id_size(), 1);
+ EXPECT_EQ(converted_index.col_unique_id(0), 0);
+
+ // verify rowset meta
+ EXPECT_EQ(converted_meta_pb.rs_metas_size(), 1);
+ const RowsetMetaPB& converted_rowset_meta = converted_meta_pb.rs_metas(0);
+ EXPECT_NE(converted_rowset_meta.rowset_id(), 10001);
+ EXPECT_NE(converted_rowset_meta.rowset_id_v2(), "02000000000000010001");
+ EXPECT_EQ(converted_rowset_meta.tablet_id(), tablet_id);
+ EXPECT_EQ(converted_rowset_meta.rowset_type(), BETA_ROWSET);
+ EXPECT_EQ(converted_rowset_meta.rowset_state(), VISIBLE);
+ EXPECT_EQ(converted_rowset_meta.start_version(), 0);
+ EXPECT_EQ(converted_rowset_meta.end_version(), 1);
+ EXPECT_EQ(converted_rowset_meta.num_rows(), 100);
+ EXPECT_EQ(converted_rowset_meta.total_disk_size(), 1024);
+ EXPECT_TRUE(converted_rowset_meta.has_tablet_schema());
+
+ // verify rowset schema
+ const TabletSchemaPB& converted_rowset_schema =
converted_rowset_meta.tablet_schema();
+ EXPECT_EQ(converted_rowset_schema.num_short_key_columns(), 1);
+ EXPECT_EQ(converted_rowset_schema.column_size(), 2);
+ EXPECT_EQ(converted_rowset_schema.index_size(), 1);
+
+ // verify rowset index
+ const TabletIndexPB& converted_rowset_index =
converted_rowset_schema.index(0);
+ EXPECT_EQ(converted_rowset_index.index_id(), 2001);
+ EXPECT_EQ(converted_rowset_index.index_name(), "test_index");
+ EXPECT_EQ(converted_rowset_index.index_type(), IndexType::BITMAP);
+ EXPECT_EQ(converted_rowset_index.col_unique_id_size(), 1);
+ EXPECT_EQ(converted_rowset_index.col_unique_id(0), 0);
+}
+} // namespace doris
diff --git a/be/test/olap/tablet_schema_test.cpp
b/be/test/olap/tablet_schema_test.cpp
index 6b9665d780a..b421e059f27 100644
--- a/be/test/olap/tablet_schema_test.cpp
+++ b/be/test/olap/tablet_schema_test.cpp
@@ -803,4 +803,94 @@ TEST_F(TabletSchemaTest,
test_tablet_schema_need_record_variant_extended_schema)
EXPECT_FALSE(schema_multiple_variants.need_record_variant_extended_schema());
}
-} // namespace doris
\ No newline at end of file
+TEST_F(TabletSchemaTest, test_tablet_schema_get_index) {
+ TabletSchema schema;
+
+ TabletColumn col1;
+ col1.set_unique_id(14001);
+ col1.set_name("test_col1");
+ col1.set_type(FieldType::OLAP_FIELD_TYPE_STRING);
+ schema.append_column(col1);
+
+ TabletColumn col2;
+ col2.set_unique_id(14002);
+ col2.set_name("test_col2");
+ col2.set_type(FieldType::OLAP_FIELD_TYPE_VARCHAR);
+ schema.append_column(col2);
+
+ TabletIndex inverted_index;
+ TabletIndexPB inverted_index_pb;
+ inverted_index_pb.set_index_id(5001);
+ inverted_index_pb.set_index_name("inverted_idx");
+ inverted_index_pb.set_index_type(IndexType::INVERTED);
+ inverted_index_pb.add_col_unique_id(14001);
+ inverted_index.init_from_pb(inverted_index_pb);
+
+ TabletIndex bitmap_index;
+ TabletIndexPB bitmap_index_pb;
+ bitmap_index_pb.set_index_id(5002);
+ bitmap_index_pb.set_index_name("bitmap_idx");
+ bitmap_index_pb.set_index_type(IndexType::BITMAP);
+ bitmap_index_pb.add_col_unique_id(14001);
+ bitmap_index.init_from_pb(bitmap_index_pb);
+
+ TabletIndex ann_index;
+ TabletIndexPB ann_index_pb;
+ ann_index_pb.set_index_id(5003);
+ ann_index_pb.set_index_name("ann_idx");
+ ann_index_pb.set_index_type(IndexType::ANN);
+ ann_index_pb.add_col_unique_id(14002);
+ ann_index.init_from_pb(ann_index_pb);
+
+ TabletIndex ngram_bf_index;
+ TabletIndexPB ngram_bf_index_pb;
+ ngram_bf_index_pb.set_index_id(5004);
+ ngram_bf_index_pb.set_index_name("ngram_bf_idx");
+ ngram_bf_index_pb.set_index_type(IndexType::NGRAM_BF);
+ ngram_bf_index_pb.add_col_unique_id(14002);
+ ngram_bf_index.init_from_pb(ngram_bf_index_pb);
+
+ schema.append_index(std::move(inverted_index));
+ schema.append_index(std::move(bitmap_index));
+ schema.append_index(std::move(ann_index));
+ schema.append_index(std::move(ngram_bf_index));
+
+ const TabletIndex* found_inverted = schema.get_index(14001,
IndexType::INVERTED, "");
+ EXPECT_NE(nullptr, found_inverted);
+ EXPECT_EQ("inverted_idx", found_inverted->index_name());
+ EXPECT_EQ(5001, found_inverted->index_id());
+ const TabletIndex* found_bitmap = schema.get_index(14001,
IndexType::BITMAP, "");
+ EXPECT_NE(nullptr, found_bitmap);
+ EXPECT_EQ("bitmap_idx", found_bitmap->index_name());
+ EXPECT_EQ(5002, found_bitmap->index_id());
+ const TabletIndex* found_ann = schema.get_index(14002, IndexType::ANN, "");
+ EXPECT_NE(nullptr, found_ann);
+ EXPECT_EQ("ann_idx", found_ann->index_name());
+ EXPECT_EQ(5003, found_ann->index_id());
+ const TabletIndex* found_ngram_bf = schema.get_index(14002,
IndexType::NGRAM_BF, "");
+ EXPECT_NE(nullptr, found_ngram_bf);
+ EXPECT_EQ("ngram_bf_idx", found_ngram_bf->index_name());
+ EXPECT_EQ(5004, found_ngram_bf->index_id());
+ const TabletIndex* not_found = schema.get_index(99999,
IndexType::INVERTED, "");
+ EXPECT_EQ(nullptr, not_found);
+ const TabletIndex* empty_suffix = schema.get_index(14001,
IndexType::INVERTED, "");
+ EXPECT_NE(nullptr, empty_suffix);
+ EXPECT_EQ("inverted_idx", empty_suffix->index_name());
+ const TabletIndex* with_suffix = schema.get_index(14001,
IndexType::INVERTED, "test_suffix");
+ EXPECT_EQ(nullptr, with_suffix);
+
+ EXPECT_TRUE(found_inverted->is_inverted_index());
+ EXPECT_EQ(IndexType::INVERTED, found_inverted->index_type());
+ EXPECT_EQ(IndexType::BITMAP, found_bitmap->index_type());
+ EXPECT_EQ(IndexType::ANN, found_ann->index_type());
+ EXPECT_EQ(IndexType::NGRAM_BF, found_ngram_bf->index_type());
+
+ const auto& inverted_col_ids = found_inverted->col_unique_ids();
+ EXPECT_EQ(1, inverted_col_ids.size());
+ EXPECT_EQ(14001, inverted_col_ids[0]);
+ const auto& ann_col_ids = found_ann->col_unique_ids();
+ EXPECT_EQ(1, ann_col_ids.size());
+ EXPECT_EQ(14002, ann_col_ids[0]);
+}
+
+} // namespace doris
diff --git
a/regression-test/suites/backup_restore/test_backup_restore_reset_index_id.groovy
b/regression-test/suites/backup_restore/test_backup_restore_reset_index_id.groovy
new file mode 100644
index 00000000000..6c4fd8c4c01
--- /dev/null
+++
b/regression-test/suites/backup_restore/test_backup_restore_reset_index_id.groovy
@@ -0,0 +1,161 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_backup_restore_reset_index_id", "backup_restore") {
+ String suiteName = "test_backup_restore_reset_index_id"
+ String dbName = "${suiteName}_db"
+ String repoName = "${suiteName}_repo_" +
UUID.randomUUID().toString().replace("-", "")
+ String snapshotName = "${suiteName}_snapshot"
+ String tableName = "${suiteName}_table"
+
+ def syncer = getSyncer()
+ syncer.createS3Repository(repoName)
+ sql "CREATE DATABASE IF NOT EXISTS ${dbName}"
+
+ sql "DROP TABLE IF EXISTS ${dbName}.${tableName}"
+ sql """
+ CREATE TABLE if NOT EXISTS ${dbName}.${tableName}
+ (
+ `test` INT,
+ `id` INT,
+ `username` varchar(32) NULL DEFAULT "",
+ `extra` varchar(32) NULL DEFAULT "",
+ INDEX idx_ngrambf (`username`) USING NGRAM_BF PROPERTIES("bf_size"
= "256", "gram_size" = "3"),
+ INDEX idx_extra_ngram (`extra`) USING NGRAM_BF
PROPERTIES("bf_size" = "256", "gram_size" = "3"),
+ INDEX idx_extra_inverted (`extra`) USING INVERTED
+ )
+ ENGINE=OLAP
+ DUPLICATE KEY(`test`, `id`)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ )
+ """
+ List<String> values = []
+ int numRows = 10;
+ for (int j = 0; j <= numRows; ++j) {
+ values.add("(${j}, ${j}1, \"${j} ${j*10}\", \"index_test\")")
+ }
+ sql "INSERT INTO ${dbName}.${tableName} VALUES ${values.join(",")}"
+
+ def indexes = sql_return_maparray "SHOW INDEX FROM ${dbName}.${tableName}"
+ logger.info("current indexes: ${indexes}")
+
+ def query_table_index_id = { indexName ->
+ def res = sql_return_maparray "SHOW TABLETS FROM
${dbName}.${tableName}"
+ def tabletId = res[0].TabletId
+ res = sql_return_maparray "SHOW TABLET ${tabletId}"
+ def dbId = res[0].DbId
+ def tableId = res[0].TableId
+ res = sql_return_maparray """ SHOW PROC
"/dbs/${dbId}/${tableId}/indexes" """
+ for (def record in res) {
+ if (record.KeyName == indexName) {
+ return record.IndexId
+ }
+ }
+ throw new Exception("index ${indexName} is not exists")
+ }
+
+ def query_tablet_schema_index_id = { indexName ->
+ def metaUrl = sql_return_maparray("SHOW TABLETS FROM
${dbName}.${tableName}").get(0).MetaUrl
+ def (code, out, err) = curl("GET", metaUrl)
+ assertEquals(code, 0)
+ def jsonMeta = parseJson(out.trim())
+
+ for (def index : jsonMeta.schema.index) {
+ if (index.index_name == indexName) {
+ return index.index_id
+ }
+ }
+ throw new Exception("index ${indexName} is not exists")
+ }
+
+ def query_rowset_schema_index_id = { indexName ->
+ def metaUrl = sql_return_maparray("SHOW TABLETS FROM
${dbName}.${tableName}").get(0).MetaUrl
+ def (code, out, err) = curl("GET", metaUrl)
+ assertEquals(code, 0)
+ def jsonMeta = parseJson(out.trim())
+ assertTrue(jsonMeta.rs_metas.size() > 0)
+
+ for (def meta : jsonMeta.rs_metas) {
+ for (def index : meta.tablet_schema.index) {
+ if (index.index_name == indexName) {
+ return index.index_id
+ }
+ }
+ }
+ throw new Exception("index ${indexName} is not exists")
+ }
+
+ try {
+ sql """ ADMIN SET FRONTEND CONFIG ("restore_reset_index_id" = "true")
"""
+ sql """
+ BACKUP SNAPSHOT ${dbName}.${snapshotName}
+ TO `${repoName}`
+ ON (`${tableName}`)
+ """
+
+ syncer.waitSnapshotFinish(dbName)
+
+ def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName)
+ assertTrue(snapshot != null)
+
+ def oldIndexNgramBfId = query_table_index_id("idx_ngrambf")
+ def oldIndexExtraNgram = query_table_index_id("idx_extra_ngram")
+ logger.info("the exists index id of idx_ngrambf is
${oldIndexNgramBfId}")
+ logger.info("the exists index id of idx_ngrambf is
${oldIndexExtraNgram}")
+
+ sql "DROP TABLE ${dbName}.${tableName}"
+
+ sql """
+ RESTORE SNAPSHOT ${dbName}.${snapshotName}
+ FROM `${repoName}`
+ ON (`${tableName}`)
+ PROPERTIES
+ (
+ "backup_timestamp" = "${snapshot}",
+ "reserve_replica" = "true"
+ )
+ """
+
+ syncer.waitAllRestoreFinish(dbName)
+
+ indexes = sql_return_maparray "SHOW INDEX FROM ${dbName}.${tableName}"
+ logger.info("current indexes: ${indexes}")
+
+ def newIndexNgramBfId = query_table_index_id("idx_ngrambf")
+ def newIndexExtraNgram = query_table_index_id("idx_extra_ngram")
+ assertTrue(oldIndexNgramBfId != newIndexNgramBfId, "old index id
${oldIndexNgramBfId}, new index id ${newIndexNgramBfId}")
+ assertTrue(oldIndexExtraNgram != newIndexExtraNgram, "old index id
${oldIndexExtraNgram}, new index id ${newIndexExtraNgram}")
+
+ def indexNgramBfId = query_tablet_schema_index_id("idx_ngrambf")
+ def indexExtraNgram = query_tablet_schema_index_id("idx_extra_ngram")
+ assertEquals(indexNgramBfId, newIndexNgramBfId.toLong(), "tablet index
id ${indexNgramBfId}, new index id ${newIndexNgramBfId}")
+ assertEquals(indexExtraNgram, newIndexExtraNgram.toLong(), "tablet
index id ${indexExtraNgram}, new index id ${newIndexExtraNgram}")
+
+ indexNgramBfId = query_rowset_schema_index_id("idx_ngrambf")
+ indexExtraNgram = query_rowset_schema_index_id("idx_extra_ngram")
+ assertEquals(indexNgramBfId, newIndexNgramBfId.toLong(), "tablet index
id ${indexNgramBfId}, new index id ${newIndexNgramBfId}")
+ assertEquals(indexExtraNgram, newIndexExtraNgram.toLong(), "tablet
index id ${indexExtraNgram}, new index id ${newIndexExtraNgram}")
+ } finally {
+ sql """ ADMIN SET FRONTEND CONFIG ("restore_reset_index_id" = "false")
"""
+ }
+
+ sql "DROP TABLE ${dbName}.${tableName} FORCE"
+ sql "DROP DATABASE ${dbName} FORCE"
+ sql "DROP REPOSITORY `${repoName}`"
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
