This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new dc4d25f4d97 [feature](Cloud) Add one config to control FE's startup
with or without built in vault (#33308)
dc4d25f4d97 is described below
commit dc4d25f4d97b69e8dd6048bc265009bfc46ae70a
Author: AlexYue <[email protected]>
AuthorDate: Tue Apr 9 15:53:27 2024 +0800
[feature](Cloud) Add one config to control FE's startup with or without
built in vault (#33308)
---
cloud/src/common/string_util.h | 2 +-
cloud/src/meta-service/meta_service.cpp | 20 +++------
cloud/src/meta-service/meta_service.h | 2 +
cloud/src/meta-service/meta_service_resource.cpp | 40 +++++++++---------
cloud/test/meta_service_test.cpp | 47 +++++++++++++++++-----
.../org/apache/doris/cloud/catalog/CloudEnv.java | 7 ++++
.../cloud/datasource/CloudInternalCatalog.java | 4 +-
gensrc/proto/cloud.proto | 9 ++---
8 files changed, 77 insertions(+), 54 deletions(-)
diff --git a/cloud/src/common/string_util.h b/cloud/src/common/string_util.h
index ea9929a46e1..af0a5b6bb11 100644
--- a/cloud/src/common/string_util.h
+++ b/cloud/src/common/string_util.h
@@ -22,7 +22,7 @@
namespace doris::cloud {
static inline std::string trim(std::string& str) {
- const std::string drop = "/ \t";
+ constexpr std::string_view drop = "/ \t";
str.erase(str.find_last_not_of(drop) + 1);
return str.erase(0, str.find_first_not_of(drop));
}
diff --git a/cloud/src/meta-service/meta_service.cpp
b/cloud/src/meta-service/meta_service.cpp
index 7e1e5290d1d..e2a19bce339 100644
--- a/cloud/src/meta-service/meta_service.cpp
+++ b/cloud/src/meta-service/meta_service.cpp
@@ -560,6 +560,10 @@ void
MetaServiceImpl::create_tablets(::google::protobuf::RpcController* controll
return;
}
+ // This instance hasn't enable storage vault which means it's using
legacy cloud mode
+ DCHECK(instance.enable_storage_vault())
+ << "Only instances with enable_storage_vault true have vault
name";
+
std::string_view name = request->storage_vault_name();
// Try to use the default vault name if user doesn't specify the vault
name
@@ -573,6 +577,7 @@ void
MetaServiceImpl::create_tablets(::google::protobuf::RpcController* controll
name = instance.default_storage_vault_name();
}
+ // The S3 vault would be stored inside the
instance.storage_vault_names and instance.resource_ids
auto vault_name = std::find_if(
instance.storage_vault_names().begin(),
instance.storage_vault_names().end(),
[&](const auto& candidate_name) { return candidate_name ==
name; });
@@ -583,21 +588,6 @@ void
MetaServiceImpl::create_tablets(::google::protobuf::RpcController* controll
break;
}
- // The S3 vault would be stored inside the instance.obj_info
- auto s3_obj = std::find_if(instance.obj_info().begin(),
instance.obj_info().end(),
- [&](const ObjectStoreInfoPB& obj) {
- if (!obj.has_name()) {
- return false;
- }
- return obj.name() == name;
- });
-
- if (s3_obj != instance.obj_info().end()) {
- response->set_storage_vault_id(s3_obj->id());
- response->set_storage_vault_name(s3_obj->name());
- break;
- }
-
code = MetaServiceCode::INVALID_ARGUMENT;
msg = fmt::format("failed to get vault id, vault name={}", name);
return;
diff --git a/cloud/src/meta-service/meta_service.h
b/cloud/src/meta-service/meta_service.h
index 2e2c2bd4ad8..0efc762f34e 100644
--- a/cloud/src/meta-service/meta_service.h
+++ b/cloud/src/meta-service/meta_service.h
@@ -37,6 +37,8 @@ namespace doris::cloud {
class Transaction;
+constexpr std::string_view BUILT_IN_STORAGE_VAULT_NAME =
"built_in_storage_vault";
+
class MetaServiceImpl : public cloud::MetaService {
public:
MetaServiceImpl(std::shared_ptr<TxnKv> txn_kv,
std::shared_ptr<ResourceManager> resource_mgr,
diff --git a/cloud/src/meta-service/meta_service_resource.cpp
b/cloud/src/meta-service/meta_service_resource.cpp
index 9b2028bab5e..82056a5172c 100644
--- a/cloud/src/meta-service/meta_service_resource.cpp
+++ b/cloud/src/meta-service/meta_service_resource.cpp
@@ -40,9 +40,6 @@ using namespace std::chrono;
namespace doris::cloud {
-const static char* BUILT_IN_STORAGE_VAULT_NAME = "built_in_storage_vault";
-const static char* BUILT_IN_STORAGE_VAULT_ID = "1";
-
static void* run_bthread_work(void* arg) {
auto f = reinterpret_cast<std::function<void()>*>(arg);
(*f)();
@@ -399,11 +396,6 @@ static int add_hdfs_storage_vault(InstanceInfoPB&
instance, Transaction* txn,
std::string vault_id = next_available_vault_id(instance);
storage_vault_key({instance.instance_id(), vault_id}, &key);
hdfs_param.set_id(vault_id);
- if (vault_id == BUILT_IN_STORAGE_VAULT_ID) {
- hdfs_param.set_name(BUILT_IN_STORAGE_VAULT_NAME);
- instance.set_default_storage_vault_name(BUILT_IN_STORAGE_VAULT_NAME);
- instance.set_default_storage_vault_id(BUILT_IN_STORAGE_VAULT_ID);
- }
std::string val = hdfs_param.SerializeAsString();
txn->put(key, val);
LOG_INFO("try to put storage vault_id={}, vault_name={}", vault_id,
hdfs_param.name());
@@ -604,6 +596,11 @@ void
MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont
}
} break;
case AlterObjStoreInfoRequest::ADD_OBJ_INFO: {
+ if (instance.enable_storage_vault()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "Storage vault doesn't support add obj info";
+ return;
+ }
if (!request->obj().has_provider()) {
code = MetaServiceCode::INVALID_ARGUMENT;
msg = "s3 conf lease provider info";
@@ -653,11 +650,6 @@ void
MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont
last_item.set_region(region);
last_item.set_provider(request->obj().provider());
last_item.set_sse_enabled(instance.sse_enabled());
- if (last_item.id() == BUILT_IN_STORAGE_VAULT_ID) {
- last_item.set_name(BUILT_IN_STORAGE_VAULT_NAME);
-
instance.set_default_storage_vault_name(BUILT_IN_STORAGE_VAULT_NAME);
- instance.set_default_storage_vault_id(BUILT_IN_STORAGE_VAULT_ID);
- }
instance.add_obj_info()->CopyFrom(last_item);
} break;
case AlterObjStoreInfoRequest::ADD_HDFS_INFO: {
@@ -912,7 +904,8 @@ void
MetaServiceImpl::update_ak_sk(google::protobuf::RpcController* controller,
}
LOG(INFO) << "instance " << instance_id << " has " <<
instance.obj_info().size()
- << " s3 history info, and instance = " <<
proto_to_json(instance);
+ << " s3 history info, and " << instance.resource_ids_size() << "
vaults "
+ << "instance = " << proto_to_json(instance);
val = instance.SerializeAsString();
if (val.empty()) {
@@ -986,11 +979,6 @@ static int
create_instance_with_object_info(InstanceInfoPB& instance, const Obje
obj_info.set_ctime(time);
obj_info.set_mtime(time);
obj_info.set_sse_enabled(sse_enabled);
- if (obj_info.id() == BUILT_IN_STORAGE_VAULT_ID) {
- obj_info.set_name(BUILT_IN_STORAGE_VAULT_NAME);
- instance.set_default_storage_vault_name(BUILT_IN_STORAGE_VAULT_NAME);
- instance.set_default_storage_vault_id(BUILT_IN_STORAGE_VAULT_ID);
- }
instance.mutable_obj_info()->Add(std::move(obj_info));
return 0;
}
@@ -1019,6 +1007,7 @@ void
MetaServiceImpl::create_instance(google::protobuf::RpcController* controlle
instance.set_name(request->has_name() ? request->name() : "");
instance.set_status(InstanceInfoPB::NORMAL);
instance.set_sse_enabled(request->sse_enabled());
+ instance.set_enable_storage_vault(!request->has_obj_info());
if (request->has_obj_info()) {
if (0 != create_instance_with_object_info(instance,
request->obj_info(),
request->sse_enabled(),
code, msg)) {
@@ -1058,6 +1047,7 @@ void
MetaServiceImpl::create_instance(google::protobuf::RpcController* controlle
if (request->has_hdfs_info()) {
StorageVaultPB hdfs_param;
hdfs_param.mutable_hdfs_info()->MergeFrom(request->hdfs_info());
+ hdfs_param.set_name(BUILT_IN_STORAGE_VAULT_NAME.data());
if (0 != add_hdfs_storage_vault(instance, txn.get(),
std::move(hdfs_param), code, msg)) {
return;
}
@@ -1841,6 +1831,16 @@ void
MetaServiceImpl::get_cluster(google::protobuf::RpcController* controller,
return;
}
+ response->set_enable_storage_vault(instance.enable_storage_vault());
+ if (instance.enable_storage_vault() &&
+ std::find_if(instance.storage_vault_names().begin(),
instance.storage_vault_names().end(),
+ [](const std::string& name) { return name ==
BUILT_IN_STORAGE_VAULT_NAME; }) ==
+ instance.storage_vault_names().end()) {
+ code = MetaServiceCode::STORAGE_VAULT_NOT_FOUND;
+ msg = "instance has no built in storage vault";
+ return;
+ }
+
auto get_cluster_mysql_user = [](const ClusterPB& c,
std::set<std::string>* mysql_users) {
for (int i = 0; i < c.mysql_user_name_size(); i++) {
mysql_users->emplace(c.mysql_user_name(i));
@@ -1868,7 +1868,7 @@ void
MetaServiceImpl::get_cluster(google::protobuf::RpcController* controller,
}
}
- if (response->cluster().size() == 0) {
+ if (response->cluster().empty()) {
ss << "fail to get cluster with " << request->ShortDebugString();
msg = ss.str();
std::replace(msg.begin(), msg.end(), '\n', ' ');
diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp
index cea9304dea1..1186c4bcd2c 100644
--- a/cloud/test/meta_service_test.cpp
+++ b/cloud/test/meta_service_test.cpp
@@ -5238,7 +5238,6 @@ TEST(MetaServiceTest, AddObjInfoTest) {
get_test_instance(instance);
const auto& obj = instance.obj_info().at(0);
ASSERT_EQ(obj.id(), "1");
- ASSERT_EQ(obj.name(), "built_in_storage_vault");
sp->clear_all_call_backs();
sp->clear_trace();
@@ -5676,8 +5675,11 @@ TEST(MetaServiceTest, GetDefaultVaultTest) {
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
InstanceInfoPB i;
get_test_instance(i, instance_id);
- ASSERT_EQ(i.default_storage_vault_id(), "1");
- ASSERT_EQ(i.default_storage_vault_name(), "built_in_storage_vault");
+ // It wouldn't be set
+ ASSERT_EQ(i.default_storage_vault_id(), "");
+ ASSERT_EQ(i.default_storage_vault_name(), "");
+ ASSERT_EQ(i.resource_ids().at(0), "1");
+ ASSERT_EQ(i.storage_vault_names().at(0), "built_in_storage_vault");
sp->clear_all_call_backs();
sp->clear_trace();
sp->disable_processing();
@@ -5728,9 +5730,6 @@ TEST(MetaServiceTest, GetDefaultVaultTest) {
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
InstanceInfoPB i;
get_test_instance(i, instance_id);
- ASSERT_EQ(i.default_storage_vault_id(), "1");
- ASSERT_EQ(i.default_storage_vault_name(), "built_in_storage_vault");
- ASSERT_EQ(i.obj_info().at(0).name(), "built_in_storage_vault");
sp->clear_all_call_backs();
sp->clear_trace();
sp->disable_processing();
@@ -5778,8 +5777,10 @@ TEST(MetaServiceTest, SetDefaultVaultTest) {
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
InstanceInfoPB i;
get_test_instance(i);
- ASSERT_EQ(i.default_storage_vault_id(), "1");
- ASSERT_EQ(i.default_storage_vault_name(), "built_in_storage_vault");
+ ASSERT_EQ(i.default_storage_vault_id(), "");
+ ASSERT_EQ(i.default_storage_vault_name(), "");
+ ASSERT_EQ(i.resource_ids().at(0), "1");
+ ASSERT_EQ(i.storage_vault_names().at(0), "built_in_storage_vault");
for (size_t i = 0; i < 20; i++) {
AlterObjStoreInfoRequest req;
@@ -5970,6 +5971,7 @@ TEST(MetaServiceTest, CreateTabletsVaultsTest) {
instance_key(key_info, &key);
InstanceInfoPB instance;
+ instance.set_enable_storage_vault(true);
val = instance.SerializeAsString();
txn->put(key, val);
ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
@@ -6019,7 +6021,7 @@ TEST(MetaServiceTest, CreateTabletsVaultsTest) {
req.set_cloud_unique_id("test_cloud_unique_id");
req.set_op(AlterObjStoreInfoRequest::ADD_HDFS_INFO);
StorageVaultPB hdfs;
- hdfs.set_name("test_alter_add_hdfs_info");
+ hdfs.set_name("built_in_storage_vault");
HdfsVaultInfo params;
params.mutable_build_conf()->set_fs_name("hdfs://ip:port");
@@ -6034,8 +6036,31 @@ TEST(MetaServiceTest, CreateTabletsVaultsTest) {
InstanceInfoPB i;
get_test_instance(i);
- ASSERT_EQ(i.default_storage_vault_id(), "1");
- ASSERT_EQ(i.default_storage_vault_name(), "built_in_storage_vault");
+ ASSERT_EQ(i.default_storage_vault_id(), "");
+ ASSERT_EQ(i.default_storage_vault_name(), "");
+ ASSERT_EQ(i.resource_ids().at(0), "1");
+ ASSERT_EQ(i.storage_vault_names().at(0), "built_in_storage_vault");
+ }
+
+ // Try to set built_in_storage_vault vault as default
+ {
+ StorageVaultPB hdfs;
+ std::string name = "built_in_storage_vault";
+ hdfs.set_name(std::move(name));
+ HdfsVaultInfo params;
+
+ hdfs.mutable_hdfs_info()->CopyFrom(params);
+ AlterObjStoreInfoRequest set_default_req;
+ set_default_req.set_cloud_unique_id("test_cloud_unique_id");
+ set_default_req.set_op(AlterObjStoreInfoRequest::SET_DEFAULT_VAULT);
+ set_default_req.mutable_hdfs()->CopyFrom(hdfs);
+ AlterObjStoreInfoResponse set_default_res;
+ brpc::Controller cntl;
+ meta_service->alter_obj_store_info(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl),
&set_default_req,
+ &set_default_res, nullptr);
+ ASSERT_EQ(set_default_res.status().code(), MetaServiceCode::OK)
+ << set_default_res.status().msg();
}
// try to use default vault
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudEnv.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudEnv.java
index abcd7f99a9c..f96f5952b75 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudEnv.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudEnv.java
@@ -62,6 +62,8 @@ public class CloudEnv extends Env {
private CloudTabletRebalancer cloudTabletRebalancer;
+ private boolean enableStorageVault;
+
public CloudEnv(boolean isCheckpointCatalog) {
super(isCheckpointCatalog);
this.cloudClusterCheck = new
CloudClusterChecker((CloudSystemInfoService) systemInfo);
@@ -111,6 +113,7 @@ public class CloudEnv extends Env {
Config.cloud_unique_id,
Config.cloud_sql_server_cluster_id, response);
return null;
}
+ this.enableStorageVault = response.getEnableStorageVault();
List<Cloud.NodeInfoPB> allNodes = response.getCluster(0).getNodesList()
.stream().filter(NodeInfoPB::hasNodeType).collect(Collectors.toList());
@@ -431,4 +434,8 @@ public class CloudEnv extends Env {
public void replayUpdateCloudReplica(UpdateCloudReplicaInfo info) throws
MetaNotFoundException {
((CloudInternalCatalog)
getInternalCatalog()).replayUpdateCloudReplica(info);
}
+
+ public boolean getEnableStorageVault() {
+ return this.enableStorageVault;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
index 5c0509831ad..1eaf415b152 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
@@ -39,6 +39,7 @@ import org.apache.doris.catalog.Replica.ReplicaState;
import org.apache.doris.catalog.ReplicaAllocation;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.TabletMeta;
+import org.apache.doris.cloud.catalog.CloudEnv;
import org.apache.doris.cloud.catalog.CloudPartition;
import org.apache.doris.cloud.catalog.CloudReplica;
import org.apache.doris.cloud.persist.UpdateCloudReplicaInfo;
@@ -157,7 +158,7 @@ public class CloudInternalCatalog extends InternalCatalog {
tbl.getEnableUniqueKeyMergeOnWrite(),
tbl.storeRowColumn(), indexMeta.getSchemaVersion());
requestBuilder.addTabletMetas(builder);
}
- if (!storageVaultIdSet) {
+ if (!storageVaultIdSet && ((CloudEnv)
Env.getCurrentEnv()).getEnableStorageVault()) {
requestBuilder.setStorageVaultName(storageVaultName);
}
@@ -165,6 +166,7 @@ public class CloudInternalCatalog extends InternalCatalog {
+ "indexId: {}, vault name {}",
dbId, tbl.getId(), tbl.getName(), partitionId,
partitionName, indexId, storageVaultName);
Cloud.CreateTabletsResponse resp =
sendCreateTabletsRpc(requestBuilder);
+ // If the resp has no vault id set, it means the MS is running
with enable_storage_vault false
if (resp.hasStorageVaultId() && !storageVaultIdSet) {
tbl.setStorageVaultId(resp.getStorageVaultId());
storageVaultIdSet = true;
diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto
index 4e08fd7d599..3054b9245f5 100644
--- a/gensrc/proto/cloud.proto
+++ b/gensrc/proto/cloud.proto
@@ -67,7 +67,7 @@ message InstanceInfoPB {
optional int64 ctime = 5;
optional int64 mtime = 6;
repeated ClusterPB clusters = 7;
- repeated ObjectStoreInfoPB obj_info = 8;
+ repeated ObjectStoreInfoPB obj_info = 8; // Only legacy obj info would be
store in this fields, other objs would be lied in vaults.
repeated StagePB stages = 9;
optional Status status = 10;
optional RamUserPB ram_user = 11;
@@ -78,6 +78,7 @@ message InstanceInfoPB {
repeated string storage_vault_names = 101;
optional string default_storage_vault_id = 102;
optional string default_storage_vault_name = 103;
+ optional bool enable_storage_vault = 104;
}
message StagePB {
@@ -201,11 +202,6 @@ message StorageVaultPB {
reserved 4; // reserved for S3.
}
-message DefaultStorageVaultInfo {
- optional string id = 1;
- optional string name = 2;
-}
-
message HdfsBuildConf {
message HdfsConfKVPair {
required string key = 1;
@@ -997,6 +993,7 @@ message GetClusterStatusResponse {
message GetClusterResponse {
optional MetaServiceResponseStatus status = 1;
repeated ClusterPB cluster = 2;
+ optional bool enable_storage_vault = 3;
}
message GetTabletStatsRequest {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
