This is an automated email from the ASF dual-hosted git repository.
mingliang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new f6f52e8718 [GLUTEN-10950][VL][FOLLOWUP] Fix missing time unit in
hadoop-aws time configuration (pre-3.4 versions) (#11329)
f6f52e8718 is described below
commit f6f52e871812ef0ea0434d25b8ff374e46077f85
Author: xumanbu <[email protected]>
AuthorDate: Sat Jan 10 09:19:02 2026 +0800
[GLUTEN-10950][VL][FOLLOWUP] Fix missing time unit in hadoop-aws time
configuration (pre-3.4 versions) (#11329)
---
.../backendsapi/velox/VeloxTransformerApi.scala | 24 +++++++++++++++++-----
1 file changed, 19 insertions(+), 5 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala
index 330086c1e6..a40e9ca6e4 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala
@@ -70,11 +70,25 @@ class VeloxTransformerApi extends TransformerApi with
Logging {
override def postProcessNativeConfig(
nativeConfMap: JMap[String, String],
backendPrefix: String): Unit = {
- // 'spark.hadoop.fs.s3a.connection.timeout' by velox requires time unit,
hadoop-aws versions
- // before 3.4 do not have time unit.
- val s3sConnectionTimeout =
nativeConfMap.get("spark.hadoop.fs.s3a.connection.timeout")
- if (NumberUtils.isCreatable(s3sConnectionTimeout)) {
- nativeConfMap.put("spark.hadoop.fs.s3a.connection.timeout",
s"${s3sConnectionTimeout}ms")
+ // S3A configurations that require time units for Velox.
+ // Hadoop-aws versions before 3.4 do not include time units by default.
+ // scalastyle:off line.size.limit
+ // Reference:
https://hadoop.apache.org/docs/r3.3.6/hadoop-aws/tools/hadoop-aws/#General_S3A_Client_configuration
+ // scalastyle:on line.size.limit
+ val s3aTimeConfigs = Map(
+ "spark.hadoop.fs.s3a.connection.timeout" -> "ms",
+ "spark.hadoop.fs.s3a.connection.establish.timeout" -> "ms",
+ "spark.hadoop.fs.s3a.threads.keepalivetime" -> "s",
+ "spark.hadoop.fs.s3a.multipart.purge.age" -> "s"
+ )
+
+ s3aTimeConfigs.foreach {
+ case (configKey, defaultUnit) =>
+ val configValue = nativeConfMap.get(configKey)
+ if (NumberUtils.isCreatable(configValue)) {
+ // Config is numeric (no unit), append the default unit for backward
compatibility
+ nativeConfMap.put(configKey, s"$configValue$defaultUnit")
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
