This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 2ae498322fb branch-3.1: [feat](storage vault) Check storage vault
connectivity for BE when starting #51175 (#52320)
2ae498322fb is described below
commit 2ae498322fb5e2a45446f67d6ea8f111b56c726b
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Fri Jun 27 10:32:55 2025 +0800
branch-3.1: [feat](storage vault) Check storage vault connectivity for BE
when starting #51175 (#52320)
Cherry-picked from #51175
Co-authored-by: Lei Zhang <[email protected]>
---
be/src/cloud/cloud_storage_engine.cpp | 54 ++++++++++++++++++++++++++---------
be/src/cloud/cloud_storage_engine.h | 3 +-
be/src/cloud/config.cpp | 3 ++
be/src/cloud/config.h | 2 ++
be/src/io/fs/s3_file_system.cpp | 2 ++
5 files changed, 50 insertions(+), 14 deletions(-)
diff --git a/be/src/cloud/cloud_storage_engine.cpp
b/be/src/cloud/cloud_storage_engine.cpp
index 5b46c542965..d8066f8b106 100644
--- a/be/src/cloud/cloud_storage_engine.cpp
+++ b/be/src/cloud/cloud_storage_engine.cpp
@@ -119,12 +119,25 @@ static Status vault_process_error(std::string_view id,
}
struct VaultCreateFSVisitor {
- VaultCreateFSVisitor(const std::string& id, const
cloud::StorageVaultPB_PathFormat& path_format)
- : id(id), path_format(path_format) {}
+ VaultCreateFSVisitor(const std::string& id, const
cloud::StorageVaultPB_PathFormat& path_format,
+ bool check_fs)
+ : id(id), path_format(path_format), check_fs(check_fs) {}
Status operator()(const S3Conf& s3_conf) const {
- LOG(INFO) << "get new s3 info: " << s3_conf.to_string() << "
resource_id=" << id;
+ LOG(INFO) << "get new s3 info: " << s3_conf.to_string() << "
resource_id=" << id
+ << " check_fs: " << check_fs;
auto fs = DORIS_TRY(io::S3FileSystem::create(s3_conf, id));
+ if (check_fs && !s3_conf.client_conf.role_arn.empty()) {
+ bool res = false;
+ // just check connectivity, not care object if exist
+ auto st = fs->exists("not_exist_object", &res);
+ if (!st.ok()) {
+ LOG(FATAL) << "failed to check s3 fs, resource_id: " << id <<
" st: " << st
+ << "s3_conf: " << s3_conf.to_string()
+ << "add enable_check_storage_vault=false to be.conf
to skip the check";
+ }
+ }
+
put_storage_resource(id, {std::move(fs), path_format}, 0);
LOG_INFO("successfully create s3 vault, vault id {}", id);
return Status::OK();
@@ -142,6 +155,7 @@ struct VaultCreateFSVisitor {
const std::string& id;
const cloud::StorageVaultPB_PathFormat& path_format;
+ bool check_fs;
};
struct RefreshFSVaultVisitor {
@@ -176,7 +190,7 @@ struct RefreshFSVaultVisitor {
};
Status CloudStorageEngine::open() {
- sync_storage_vault();
+ sync_storage_vault(config::enable_check_storage_vault);
// TODO(plat1ko): DeleteBitmapTxnManager
@@ -321,13 +335,25 @@ Status CloudStorageEngine::start_bg_threads() {
return Status::OK();
}
-void CloudStorageEngine::sync_storage_vault() {
+void CloudStorageEngine::sync_storage_vault(bool check_storage_vault) {
cloud::StorageVaultInfos vault_infos;
bool enable_storage_vault = false;
- auto st = _meta_mgr->get_storage_vault_info(&vault_infos,
&enable_storage_vault);
- if (!st.ok()) {
- LOG(WARNING) << "failed to get storage vault info. err=" << st;
- return;
+ auto st = Status::OK();
+ while (true) {
+ st = _meta_mgr->get_storage_vault_info(&vault_infos,
&enable_storage_vault);
+ if (st.ok()) {
+ break;
+ }
+
+ if (!check_storage_vault) {
+ LOG(WARNING) << "failed to get storage vault info. err=" << st;
+ return;
+ }
+
+ LOG(WARNING) << "failed to get storage vault info from ms, err=" << st
+ << " sleep 200ms retry or add
enable_check_storage_vault=false to be.conf"
+ << " to skip the check.";
+ std::this_thread::sleep_for(std::chrono::milliseconds(200));
}
if (vault_infos.empty()) {
@@ -337,10 +363,12 @@ void CloudStorageEngine::sync_storage_vault() {
for (auto& [id, vault_info, path_format] : vault_infos) {
auto fs = get_filesystem(id);
- auto status = (fs == nullptr)
- ? std::visit(VaultCreateFSVisitor {id,
path_format}, vault_info)
- : std::visit(RefreshFSVaultVisitor {id,
std::move(fs), path_format},
- vault_info);
+ auto status =
+ (fs == nullptr)
+ ? std::visit(VaultCreateFSVisitor {id, path_format,
check_storage_vault},
+ vault_info)
+ : std::visit(RefreshFSVaultVisitor {id, std::move(fs),
path_format},
+ vault_info);
if (!status.ok()) [[unlikely]] {
LOG(WARNING) << vault_process_error(id, vault_info, std::move(st));
}
diff --git a/be/src/cloud/cloud_storage_engine.h
b/be/src/cloud/cloud_storage_engine.h
index 107e9c98677..9e63be6c36c 100644
--- a/be/src/cloud/cloud_storage_engine.h
+++ b/be/src/cloud/cloud_storage_engine.h
@@ -24,6 +24,7 @@
//#include "cloud/cloud_full_compaction.h"
#include "cloud/cloud_cumulative_compaction_policy.h"
#include "cloud/cloud_tablet.h"
+#include "cloud/config.h"
#include "cloud/schema_cloud_dictionary_cache.h"
#include "cloud_txn_delete_bitmap_cache.h"
#include "io/cache/block_file_cache_factory.h"
@@ -136,7 +137,7 @@ public:
std::shared_ptr<CloudCumulativeCompactionPolicy> cumu_compaction_policy(
std::string_view compaction_policy);
- void sync_storage_vault();
+ void sync_storage_vault(bool check = false);
io::FileCacheBlockDownloader& file_cache_block_downloader() const {
return *_file_cache_block_downloader;
diff --git a/be/src/cloud/config.cpp b/be/src/cloud/config.cpp
index bc5c90e6e94..7198ed128f6 100644
--- a/be/src/cloud/config.cpp
+++ b/be/src/cloud/config.cpp
@@ -87,5 +87,8 @@ DEFINE_mInt32(delete_bitmap_rpc_retry_times, "25");
DEFINE_mInt64(meta_service_rpc_reconnect_interval_ms, "5000");
DEFINE_mInt32(meta_service_conflict_error_retry_times, "10");
+
+DEFINE_Bool(enable_check_storage_vault, "true");
+
#include "common/compile_check_end.h"
} // namespace doris::config
diff --git a/be/src/cloud/config.h b/be/src/cloud/config.h
index 9e724082c9f..e1f9e0de63b 100644
--- a/be/src/cloud/config.h
+++ b/be/src/cloud/config.h
@@ -124,5 +124,7 @@ DECLARE_mInt64(meta_service_rpc_reconnect_interval_ms);
DECLARE_mInt32(meta_service_conflict_error_retry_times);
+DECLARE_Bool(enable_check_storage_vault);
+
#include "common/compile_check_end.h"
} // namespace doris::config
diff --git a/be/src/io/fs/s3_file_system.cpp b/be/src/io/fs/s3_file_system.cpp
index ec1c63c9106..224368bf8d5 100644
--- a/be/src/io/fs/s3_file_system.cpp
+++ b/be/src/io/fs/s3_file_system.cpp
@@ -267,6 +267,8 @@ Status S3FileSystem::exists_impl(const Path& path, bool*
res) const {
CHECK_S3_CLIENT(client);
auto key = DORIS_TRY(get_key(path));
+ VLOG_DEBUG << "key:" << key << " path:" << path;
+
auto resp = client->head_object({.bucket = _bucket, .key = key});
if (resp.resp.status.code == ErrorCode::OK) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
