This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 43350daa01 [GLUTEN-10094][VL] Fix unsupported OS(<undefined>,
<undefined>) error (#10093)
43350daa01 is described below
commit 43350daa01708633c36e0c6eae8cfc40205a8e1f
Author: Joey <[email protected]>
AuthorDate: Thu Jul 3 17:38:34 2025 +0800
[GLUTEN-10094][VL] Fix unsupported OS(<undefined>, <undefined>) error
(#10093)
* [VL] Fix unsupported OS(<undefined>, <undefined>)
* Add checkUndefined in SparkConfigUtil
* Not call checkUndefined every time
* Address comments
---------
Co-authored-by: Kent Yao <[email protected]>
---
.../org/apache/spark/sql/internal/SparkConfigUtil.scala | 13 ++++++++++---
.../org/apache/gluten/config/SparkConfigUtilSuite.scala | 13 ++++++++++++-
2 files changed, 22 insertions(+), 4 deletions(-)
diff --git
a/gluten-core/src/main/scala/org/apache/spark/sql/internal/SparkConfigUtil.scala
b/gluten-core/src/main/scala/org/apache/spark/sql/internal/SparkConfigUtil.scala
index 47c3a0b7f1..e8e7331f8c 100644
---
a/gluten-core/src/main/scala/org/apache/spark/sql/internal/SparkConfigUtil.scala
+++
b/gluten-core/src/main/scala/org/apache/spark/sql/internal/SparkConfigUtil.scala
@@ -50,15 +50,22 @@ object SparkConfigUtil {
}
def get[T](conf: SparkConf, entry: ConfigEntry[T]): T = {
- entry.valueConverter(conf.get(entry.key, entry.defaultValueString))
+ conf
+ .getOption(entry.key)
+ .map(entry.valueConverter)
+ .getOrElse(entry.defaultValue.getOrElse(None).asInstanceOf[T])
}
def get[T](conf: java.util.Map[String, String], entry: SparkConfigEntry[T]):
T = {
- entry.valueConverter(conf.getOrDefault(entry.key,
entry.defaultValueString))
+ Option(conf.get(entry.key))
+ .map(entry.valueConverter)
+ .getOrElse(entry.defaultValue.getOrElse(None).asInstanceOf[T])
}
def get[T](conf: java.util.Map[String, String], entry: ConfigEntry[T]): T = {
- entry.valueConverter(conf.getOrDefault(entry.key,
entry.defaultValueString))
+ Option(conf.get(entry.key))
+ .map(entry.valueConverter)
+ .getOrElse(entry.defaultValue.getOrElse(None).asInstanceOf[T])
}
def set[T](conf: SparkConf, entry: SparkConfigEntry[T], value: T): SparkConf
= {
diff --git
a/gluten-substrait/src/test/scala/org/apache/gluten/config/SparkConfigUtilSuite.scala
b/gluten-substrait/src/test/scala/org/apache/gluten/config/SparkConfigUtilSuite.scala
index a9a15f46f4..2bcfcb9e67 100644
---
a/gluten-substrait/src/test/scala/org/apache/gluten/config/SparkConfigUtilSuite.scala
+++
b/gluten-substrait/src/test/scala/org/apache/gluten/config/SparkConfigUtilSuite.scala
@@ -17,8 +17,8 @@
package org.apache.gluten.config
import org.apache.spark.SparkConf
+import org.apache.spark.sql.internal.{SparkConfigUtil, SQLConf}
import org.apache.spark.sql.internal.SparkConfigUtil._
-import org.apache.spark.sql.internal.SQLConf
import org.scalatest.funsuite.AnyFunSuiteLike
@@ -31,5 +31,16 @@ class SparkConfigUtilSuite extends AnyFunSuiteLike {
assert(conf.get(GlutenConfig.GLUTEN_UI_ENABLED) === true)
assert(conf.get(GlutenConfig.TEXT_INPUT_ROW_MAX_BLOCK_SIZE) === 8L * 1024)
assert(conf.get(GlutenConfig.SHUFFLE_WRITER_BUFFER_SIZE) === Some(1024 *
1024))
+ assert(conf.get(GlutenConfig.GLUTEN_LOAD_LIB_OS).isEmpty)
+ }
+
+ test("SparkConfigUtil.get for java.util.Map conf") {
+ val conf = new java.util.HashMap[String, String]
+ conf.put(GlutenConfig.SHUFFLE_WRITER_BUFFER_SIZE.key, (1024 *
1024).toString)
+ assert(SparkConfigUtil.get(conf, SQLConf.AUTO_BROADCASTJOIN_THRESHOLD) ===
10L * 1024 * 1024)
+ assert(SparkConfigUtil.get(conf, GlutenConfig.GLUTEN_UI_ENABLED) === true)
+ assert(SparkConfigUtil.get(conf,
GlutenConfig.TEXT_INPUT_ROW_MAX_BLOCK_SIZE) === 8L * 1024)
+ assert(SparkConfigUtil.get(conf, GlutenConfig.SHUFFLE_WRITER_BUFFER_SIZE)
=== Some(1024 * 1024))
+ assert(SparkConfigUtil.get(conf, GlutenConfig.GLUTEN_LOAD_LIB_OS).isEmpty)
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
