This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 59e02c154 [VL] Follow-ups for #7304 (#7340)
59e02c154 is described below
commit 59e02c1549f0a792bedfbda3994c483783a246d8
Author: Hongze Zhang <[email protected]>
AuthorDate: Wed Sep 25 13:04:51 2024 +0800
[VL] Follow-ups for #7304 (#7340)
---
gluten-arrow/pom.xml | 6 ------
.../scala/org/apache/spark/sql/utils/SparkArrowUtil.scala | 12 +++++++++---
2 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/gluten-arrow/pom.xml b/gluten-arrow/pom.xml
index 2c734bd14..ffba2682e 100644
--- a/gluten-arrow/pom.xml
+++ b/gluten-arrow/pom.xml
@@ -51,12 +51,6 @@
<version>${project.version}</version>
<scope>compile</scope>
</dependency>
- <dependency>
- <groupId>org.apache.gluten</groupId>
- <artifactId>gluten-substrait</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
<dependency>
<groupId>io.glutenproject</groupId>
<artifactId>protobuf-java</artifactId>
diff --git
a/gluten-arrow/src/main/scala/org/apache/spark/sql/utils/SparkArrowUtil.scala
b/gluten-arrow/src/main/scala/org/apache/spark/sql/utils/SparkArrowUtil.scala
index 7c8261c2a..652fbeb48 100644
---
a/gluten-arrow/src/main/scala/org/apache/spark/sql/utils/SparkArrowUtil.scala
+++
b/gluten-arrow/src/main/scala/org/apache/spark/sql/utils/SparkArrowUtil.scala
@@ -16,14 +16,15 @@
*/
package org.apache.spark.sql.utils
-import org.apache.gluten.expression.ConverterUtils
-
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
import org.apache.arrow.vector.complex.MapVector
import org.apache.arrow.vector.types.{DateUnit, FloatingPointPrecision,
IntervalUnit, TimeUnit}
import org.apache.arrow.vector.types.pojo.{ArrowType, Field, FieldType, Schema}
+import java.util.Locale
+
import scala.collection.JavaConverters._
/** Originated from org.apache.spark.sql.util.ArrowUtils. */
@@ -97,7 +98,7 @@ object SparkArrowUtil {
.map(
field =>
toArrowField(
- ConverterUtils.normalizeStructFieldName(field.name),
+ normalizeColName(field.name),
field.dataType,
field.nullable,
timeZoneId))
@@ -162,4 +163,9 @@ object SparkArrowUtil {
StructField(field.getName, dt, field.isNullable)
})
}
+
+ private def normalizeColName(name: String): String = {
+ val caseSensitive = SQLConf.get.caseSensitiveAnalysis
+ if (caseSensitive) name else name.toLowerCase(Locale.ROOT)
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
