This is an automated email from the ASF dual-hosted git repository.
changchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 0c5f1f5a8e [GLUTEN-8795][CH] Support to use oss with gluten (#8796)
0c5f1f5a8e is described below
commit 0c5f1f5a8e924aeb5502551cdd5514ad2c4061e7
Author: Yan Xin <[email protected]>
AuthorDate: Fri Feb 21 14:03:53 2025 +0800
[GLUTEN-8795][CH] Support to use oss with gluten (#8796)
* CH-8795 Support to use oss with gluten
* use boost to_lower inside of std
---
cpp-ch/local-engine/Common/CHUtil.h | 1 +
.../local-engine/Storages/SubstraitSource/ReadBufferBuilder.cpp | 8 +++++++-
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/cpp-ch/local-engine/Common/CHUtil.h
b/cpp-ch/local-engine/Common/CHUtil.h
index cc03bc0ce6..1521e1e0c2 100644
--- a/cpp-ch/local-engine/Common/CHUtil.h
+++ b/cpp-ch/local-engine/Common/CHUtil.h
@@ -171,6 +171,7 @@ public:
inline static const std::string HADOOP_S3_SECRET_KEY = "fs.s3a.secret.key";
inline static const std::string HADOOP_S3_ENDPOINT = "fs.s3a.endpoint";
inline static const std::string HADOOP_S3_ASSUMED_ROLE =
"fs.s3a.assumed.role.arn";
+ inline static const std::string HADOOP_S3_PATH_STYLE_ACCESS =
"fs.s3a.path.style.access";
inline static const std::string HADOOP_S3_ASSUMED_SESSION_NAME =
"fs.s3a.assumed.role.session.name";
// not hadoop official
inline static const std::string HADOOP_S3_ASSUMED_EXTERNAL_ID =
"fs.s3a.assumed.role.externalId";
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/ReadBufferBuilder.cpp
b/cpp-ch/local-engine/Storages/SubstraitSource/ReadBufferBuilder.cpp
index 09ba291419..043b53d8ac 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/ReadBufferBuilder.cpp
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/ReadBufferBuilder.cpp
@@ -42,6 +42,7 @@
#include <Storages/ObjectStorage/HDFS/ReadBufferFromHDFS.h>
#include <Storages/SubstraitSource/ReadBufferBuilder.h>
#include <Storages/SubstraitSource/SubstraitFileSource.h>
+#include <boost/algorithm/string/case_conv.hpp>
#include <boost/compute/detail/lru_cache.hpp>
#include <sys/stat.h>
#include <Poco/Logger.h>
@@ -567,12 +568,17 @@ private:
std::string ak;
std::string sk;
+ std::string path_style_access;
+ bool addressing_type = false;
tryGetString(settings, BackendInitializerUtil::HADOOP_S3_ACCESS_KEY,
ak);
tryGetString(settings, BackendInitializerUtil::HADOOP_S3_SECRET_KEY,
sk);
+ tryGetString(settings,
BackendInitializerUtil::HADOOP_S3_PATH_STYLE_ACCESS, path_style_access);
const DB::Settings & global_settings =
context->getGlobalContext()->getSettingsRef();
const DB::Settings & local_settings = context->getSettingsRef();
+ boost::algorithm::to_lower(path_style_access);
+ addressing_type = (path_style_access == "false");
DB::S3::ClientSettings client_settings{
- .use_virtual_addressing = false,
+ .use_virtual_addressing = addressing_type,
.disable_checksum =
local_settings[DB::Setting::s3_disable_checksum],
.gcs_issue_compose_request =
context->getConfigRef().getBool("s3.gcs_issue_compose_request", false),
};
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
