This is an automated email from the ASF dual-hosted git repository.
yuanzhou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 6bc71d60ce [GLUTEN-11699][VL] Add support for S3 IMDS configuration
(#11721)
6bc71d60ce is described below
commit 6bc71d60ceb9c37dc9a0cef7b2d5ff3089890361
Author: Shahad Shamsan <[email protected]>
AuthorDate: Tue Mar 10 12:59:14 2026 +0300
[GLUTEN-11699][VL] Add support for S3 IMDS configuration (#11721)
This PR adds support for s3 configuration supported by Velox
Fixes: #11699
---
cpp/velox/utils/ConfigExtractor.cc | 2 ++
.../src/main/scala/org/apache/gluten/config/GlutenConfig.scala | 3 +++
2 files changed, 5 insertions(+)
diff --git a/cpp/velox/utils/ConfigExtractor.cc
b/cpp/velox/utils/ConfigExtractor.cc
index fe35ce9f8e..c6cd5f8b29 100644
--- a/cpp/velox/utils/ConfigExtractor.cc
+++ b/cpp/velox/utils/ConfigExtractor.cc
@@ -70,6 +70,7 @@ void getS3HiveConfig(
{S3Config::Keys::kIamRole, std::make_pair("iam.role", std::nullopt)},
{S3Config::Keys::kIamRoleSessionName,
std::make_pair("iam.role.session.name", "gluten-session")},
{S3Config::Keys::kEndpointRegion, std::make_pair("endpoint.region",
std::nullopt)},
+ {S3Config::Keys::kIMDSEnabled, std::make_pair("aws.imds.enabled",
"true")},
};
// get Velox S3 config key from Spark Suffix.
@@ -124,6 +125,7 @@ void getS3HiveConfig(
setConfigIfPresent(S3Config::Keys::kSocketTimeout);
setConfigIfPresent(S3Config::Keys::kConnectTimeout);
setConfigIfPresent(S3Config::Keys::kEndpointRegion);
+ setConfigIfPresent(S3Config::Keys::kIMDSEnabled);
hiveConfMap[S3Config::kS3LogLevel] =
conf->get<std::string>(kVeloxAwsSdkLogLevel, kVeloxAwsSdkLogLevelDefault);
hiveConfMap[S3Config::baseConfigKey(S3Config::Keys::kUseProxyFromEnv)] =
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
index 8d71e15964..c426ceb821 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
@@ -424,6 +424,8 @@ object GlutenConfig extends ConfigRegistry {
val SPARK_S3_CONNECTION_MAXIMUM: String = HADOOP_PREFIX +
S3_CONNECTION_MAXIMUM
val S3_ENDPOINT_REGION = "fs.s3a.endpoint.region"
val SPARK_S3_ENDPOINT_REGION: String = HADOOP_PREFIX + S3_ENDPOINT_REGION
+ val S3_AWS_IMDS_ENABLED = "fs.s3a.aws.imds.enabled"
+ val SPARK_S3_AWS_IMDS_ENABLED: String = HADOOP_PREFIX + S3_AWS_IMDS_ENABLED
// ABFS config
val ABFS_PREFIX = "fs.azure."
@@ -487,6 +489,7 @@ object GlutenConfig extends ConfigRegistry {
SPARK_S3_RETRY_MAX_ATTEMPTS,
SPARK_S3_CONNECTION_MAXIMUM,
SPARK_S3_ENDPOINT_REGION,
+ SPARK_S3_AWS_IMDS_ENABLED,
"spark.gluten.velox.fs.s3a.connect.timeout",
"spark.gluten.velox.fs.s3a.retry.mode",
"spark.gluten.velox.awsSdkLogLevel",
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
