This is an automated email from the ASF dual-hosted git repository.
ulyssesyou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 09974d3a4 [CORE] Fix gluten createOptional config contains Some (#5573)
09974d3a4 is described below
commit 09974d3a46278759488a737ad8d929d2a31559dc
Author: Xiduo You <[email protected]>
AuthorDate: Tue Apr 30 14:15:00 2024 +0800
[CORE] Fix gluten createOptional config contains Some (#5573)
---
.../sql/execution/VeloxParquetReadSuite.scala | 8 +++++++
.../spark/sql/internal/GlutenConfigUtil.scala | 26 +++++++++++++++++-----
2 files changed, 29 insertions(+), 5 deletions(-)
diff --git
a/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetReadSuite.scala
b/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetReadSuite.scala
index d5828c738..cb3eeaec6 100644
---
a/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetReadSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetReadSuite.scala
@@ -16,14 +16,22 @@
*/
package org.apache.spark.sql.execution
+import org.apache.gluten.GlutenConfig
import org.apache.gluten.execution.{BasicScanExecTransformer,
VeloxWholeStageTransformerSuite}
+import org.apache.spark.SparkConf
+
import java.io.File
class VeloxParquetReadSuite extends VeloxWholeStageTransformerSuite {
override protected val resourcePath: String = "/parquet-for-read"
override protected val fileFormat: String = "parquet"
+ override protected def sparkConf: SparkConf = {
+ super.sparkConf
+ .set(GlutenConfig.LOAD_QUANTUM.key, "128m")
+ }
+
testWithSpecifiedSparkVersion("read example parquet files", Some("3.5"),
Some("3.5")) {
withTable("test_table") {
val dir = new File(getClass.getResource(resourcePath).getFile)
diff --git
a/shims/common/src/main/scala/org/apache/spark/sql/internal/GlutenConfigUtil.scala
b/shims/common/src/main/scala/org/apache/spark/sql/internal/GlutenConfigUtil.scala
index babb446d0..1a45572ac 100644
---
a/shims/common/src/main/scala/org/apache/spark/sql/internal/GlutenConfigUtil.scala
+++
b/shims/common/src/main/scala/org/apache/spark/sql/internal/GlutenConfigUtil.scala
@@ -21,11 +21,27 @@ import org.apache.spark.internal.config.ConfigReader
import scala.collection.JavaConverters._
object GlutenConfigUtil {
+ private def getConfString(reader: ConfigReader, key: String, value: String):
String = {
+ Option(SQLConf.getConfigEntry(key))
+ .map {
+ _.readFrom(reader) match {
+ case o: Option[_] => o.map(_.toString).getOrElse(value)
+ case null => value
+ case v => v.toString
+ }
+ }
+ .getOrElse(value)
+ }
+
def parseConfig(conf: Map[String, String]): Map[String, String] = {
- val reader = new
ConfigReader(conf.filter(_._1.contains("spark.gluten.")).asJava)
- val glutenConfigEntries =
- SQLConf.getConfigEntries().asScala.filter(e =>
e.key.contains("spark.gluten."))
- val glutenConfig = glutenConfigEntries.map(e => (e.key,
e.readFrom(reader).toString)).toMap
- conf.map(e => (e._1, glutenConfig.getOrElse(e._1, e._2)))
+ val reader = new
ConfigReader(conf.filter(_._1.startsWith("spark.gluten.")).asJava)
+ conf.map {
+ case (k, v) =>
+ if (k.startsWith("spark.gluten.")) {
+ (k, getConfString(reader, k, v))
+ } else {
+ (k, v)
+ }
+ }.toMap
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
