This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new be1948a7e15 [fix](s3 client)add default ca cert list for s3 client to
avoid problem:'curlCode:77' (#32285) (#32485)
be1948a7e15 is described below
commit be1948a7e158b906b60e530c63d7a8bcce08a11b
Author: Kang <[email protected]>
AuthorDate: Wed Mar 20 10:51:46 2024 +0800
[fix](s3 client)add default ca cert list for s3 client to avoid
problem:'curlCode:77' (#32285) (#32485)
Co-authored-by: ryanzryu <[email protected]>
---
be/src/common/config.cpp | 10 ++++++++++
be/src/common/config.h | 9 +++++++++
be/src/util/s3_util.cpp | 23 +++++++++++++++++++++++
be/src/util/s3_util.h | 2 ++
4 files changed, 44 insertions(+)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 27efa016fab..492a884f5f9 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1136,6 +1136,16 @@ DEFINE_mInt32(query_statistics_reserve_timeout_ms,
"30000");
DEFINE_mBool(check_segment_when_build_rowset_meta, "false");
+DEFINE_mInt32(max_s3_client_retry, "10");
+
+DEFINE_String(trino_connector_plugin_dir, "${DORIS_HOME}/connectors");
+
+// ca_cert_file is in this path by default, Normally no modification is
required
+// ca cert default path is different from different OS
+DEFINE_mString(ca_cert_file_paths,
+
"/etc/pki/tls/certs/ca-bundle.crt;/etc/ssl/certs/ca-certificates.crt;"
+ "/etc/ssl/ca-bundle.pem");
+
// clang-format off
#ifdef BE_TEST
// test s3
diff --git a/be/src/common/config.h b/be/src/common/config.h
index b45c03ed2b9..7af79880fe4 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1189,6 +1189,15 @@ DECLARE_mInt32(query_statistics_reserve_timeout_ms);
DECLARE_mBool(check_segment_when_build_rowset_meta);
+// max s3 client retry times
+DECLARE_mInt32(max_s3_client_retry);
+
+// the directory for storing the trino-connector plugins.
+DECLARE_String(trino_connector_plugin_dir);
+
+// the file paths(one or more) of CA cert, splite using ";" aws s3 lib use it
to init s3client
+DECLARE_mString(ca_cert_file_paths);
+
#ifdef BE_TEST
// test s3
DECLARE_String(test_s3_resource);
diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp
index c2bcb302a4f..25ba69c04e5 100644
--- a/be/src/util/s3_util.cpp
+++ b/be/src/util/s3_util.cpp
@@ -28,6 +28,7 @@
#include <atomic>
#include <cstdlib>
+#include <filesystem>
#include <functional>
#include <ostream>
#include <utility>
@@ -99,6 +100,18 @@ S3ClientFactory::S3ClientFactory() {
return std::make_shared<DorisAWSLogger>(logLevel);
};
Aws::InitAPI(_aws_options);
+ _ca_cert_file_path = get_valid_ca_cert_path();
+}
+
+string S3ClientFactory::get_valid_ca_cert_path() {
+ vector<std::string> vec_ca_file_path =
doris::split(config::ca_cert_file_paths, ";");
+ vector<std::string>::iterator it = vec_ca_file_path.begin();
+ for (; it != vec_ca_file_path.end(); ++it) {
+ if (std::filesystem::exists(*it)) {
+ return *it;
+ }
+ }
+ return "";
}
S3ClientFactory::~S3ClientFactory() {
@@ -142,6 +155,16 @@ std::shared_ptr<Aws::S3::S3Client>
S3ClientFactory::create(const S3Conf& s3_conf
Aws::Client::ClientConfiguration aws_config =
S3ClientFactory::getClientConfiguration();
aws_config.endpointOverride = s3_conf.endpoint;
aws_config.region = s3_conf.region;
+ std::string ca_cert = get_valid_ca_cert_path();
+ if ("" != _ca_cert_file_path) {
+ aws_config.caFile = _ca_cert_file_path;
+ } else {
+ // config::ca_cert_file_paths is valmutable,get newest value if file
path invaild
+ _ca_cert_file_path = get_valid_ca_cert_path();
+ if ("" != _ca_cert_file_path) {
+ aws_config.caFile = _ca_cert_file_path;
+ }
+ }
if (s3_conf.max_connections > 0) {
aws_config.maxConnections = s3_conf.max_connections;
} else {
diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h
index 393b9d7f118..abaf77db6a4 100644
--- a/be/src/util/s3_util.h
+++ b/be/src/util/s3_util.h
@@ -116,10 +116,12 @@ public:
private:
S3ClientFactory();
+ static std::string get_valid_ca_cert_path();
Aws::SDKOptions _aws_options;
std::mutex _lock;
std::unordered_map<uint64_t, std::shared_ptr<Aws::S3::S3Client>> _cache;
+ std::string _ca_cert_file_path;
};
} // end namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
