This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new fb23add819b [fix](s3 client)add default ca cert list for s3 client to
avoid problem:'curlCode:77' (#32285)
fb23add819b is described below
commit fb23add819b7e6caeb5a2bc166a7213312ddde21
Author: ryanzryu <[email protected]>
AuthorDate: Fri Mar 15 22:19:58 2024 +0800
[fix](s3 client)add default ca cert list for s3 client to avoid
problem:'curlCode:77' (#32285)
Co-authored-by: ryanzryu <[email protected]>
---
be/src/common/config.cpp | 6 ++++++
be/src/common/config.h | 3 +++
be/src/util/s3_util.cpp | 23 +++++++++++++++++++++++
be/src/util/s3_util.h | 2 ++
4 files changed, 34 insertions(+)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 2d78a430153..cd31926f197 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1199,6 +1199,12 @@ DEFINE_mInt32(max_s3_client_retry, "10");
DEFINE_String(trino_connector_plugin_dir, "${DORIS_HOME}/connectors");
+// ca_cert_file is in this path by default, Normally no modification is
required
+// ca cert default path is different from different OS
+DEFINE_mString(ca_cert_file_paths,
+
"/etc/pki/tls/certs/ca-bundle.crt;/etc/ssl/certs/ca-certificates.crt;"
+ "/etc/ssl/ca-bundle.pem");
+
// clang-format off
#ifdef BE_TEST
// test s3
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 8958e86c945..c6000fed446 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1277,6 +1277,9 @@ DECLARE_String(tmp_file_dir);
// the directory for storing the trino-connector plugins.
DECLARE_String(trino_connector_plugin_dir);
+// the file paths(one or more) of CA cert, splite using ";" aws s3 lib use it
to init s3client
+DECLARE_mString(ca_cert_file_paths);
+
#ifdef BE_TEST
// test s3
DECLARE_String(test_s3_resource);
diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp
index d09b808868b..063cc16c67b 100644
--- a/be/src/util/s3_util.cpp
+++ b/be/src/util/s3_util.cpp
@@ -30,6 +30,7 @@
#include <atomic>
#include <cstdlib>
+#include <filesystem>
#include <functional>
#include <memory>
#include <ostream>
@@ -114,6 +115,18 @@ S3ClientFactory::S3ClientFactory() {
return std::make_shared<DorisAWSLogger>(logLevel);
};
Aws::InitAPI(_aws_options);
+ _ca_cert_file_path = get_valid_ca_cert_path();
+}
+
+string S3ClientFactory::get_valid_ca_cert_path() {
+ vector<std::string> vec_ca_file_path =
doris::split(config::ca_cert_file_paths, ";");
+ vector<std::string>::iterator it = vec_ca_file_path.begin();
+ for (; it != vec_ca_file_path.end(); ++it) {
+ if (std::filesystem::exists(*it)) {
+ return *it;
+ }
+ }
+ return "";
}
S3ClientFactory::~S3ClientFactory() {
@@ -157,6 +170,16 @@ std::shared_ptr<Aws::S3::S3Client>
S3ClientFactory::create(const S3Conf& s3_conf
Aws::Client::ClientConfiguration aws_config =
S3ClientFactory::getClientConfiguration();
aws_config.endpointOverride = s3_conf.endpoint;
aws_config.region = s3_conf.region;
+ std::string ca_cert = get_valid_ca_cert_path();
+ if ("" != _ca_cert_file_path) {
+ aws_config.caFile = _ca_cert_file_path;
+ } else {
+ // config::ca_cert_file_paths is valmutable,get newest value if file
path invaild
+ _ca_cert_file_path = get_valid_ca_cert_path();
+ if ("" != _ca_cert_file_path) {
+ aws_config.caFile = _ca_cert_file_path;
+ }
+ }
if (s3_conf.max_connections > 0) {
aws_config.maxConnections = s3_conf.max_connections;
} else {
diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h
index 0727128c06b..4479f27a585 100644
--- a/be/src/util/s3_util.h
+++ b/be/src/util/s3_util.h
@@ -138,10 +138,12 @@ public:
private:
S3ClientFactory();
+ static std::string get_valid_ca_cert_path();
Aws::SDKOptions _aws_options;
std::mutex _lock;
std::unordered_map<uint64_t, std::shared_ptr<Aws::S3::S3Client>> _cache;
+ std::string _ca_cert_file_path;
};
} // end namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
