This is an automated email from the ASF dual-hosted git repository.
yuanzhou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 4d20123ed5 [GLUTEN-11088][VL] Add config in GlutenCastSuite to cast to
char/varchar (#11181)
4d20123ed5 is described below
commit 4d20123ed55aa34fc93dcf629ca2952ade94c261
Author: Rong Ma <[email protected]>
AuthorDate: Wed Nov 26 16:46:13 2025 +0000
[GLUTEN-11088][VL] Add config in GlutenCastSuite to cast to char/varchar
(#11181)
Need to explicitly set spark.sql.preserveCharVarcharTypeInfo=true for
gluten's test framework. We need to add this into the Gluten test because it
overrides the checkEvaluation that invokes Spark's RowEncoder, which requires
this configuration to be set. But in Vanilla spark, the checkEvaluation method
doesn't invoke RowEncoder.
Related issue: #11088
---
.../org/apache/gluten/utils/velox/VeloxTestSettings.scala | 4 ----
.../spark/sql/catalyst/expressions/GlutenCastSuite.scala | 9 ++++++++-
.../spark/sql/catalyst/expressions/GlutenTryCastSuite.scala | 10 +++++++++-
3 files changed, 17 insertions(+), 6 deletions(-)
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 0bae48e716..536b462521 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -100,8 +100,6 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("cast from timestamp II")
.exclude("SPARK-36286: invalid string cast to timestamp")
.exclude("SPARK-39749: cast Decimal to string")
- // TODO: fix in Spark-4.0
- .exclude("Casting to char/varchar")
enableSuite[GlutenTryCastSuite]
.exclude(
"Process Infinity, -Infinity, NaN in case insensitive manner" // +inf
not supported in folly.
@@ -117,8 +115,6 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("data type casting")
// Revised by setting timezone through config and commented unsupported
cases.
.exclude("cast string to timestamp")
- // TODO: fix in Spark-4.0
- .exclude("Casting to char/varchar")
enableSuite[GlutenCollectionExpressionsSuite]
// Rewrite in Gluten to replace Seq with Array
.exclude("Shuffle")
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
index ef06a98539..c8b8aaf99a 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastSuite.scala
@@ -27,6 +27,14 @@ import java.sql.{Date, Timestamp}
import java.util.{Calendar, TimeZone}
class GlutenCastSuite extends CastWithAnsiOffSuite with GlutenTestsTrait {
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+ // Need to explicitly set spark.sql.preserveCharVarcharTypeInfo=true for
gluten's test
+ // framework. In Gluten, it overrides the checkEvaluation that invokes
Spark's RowEncoder,
+ // which requires this configuration to be set.
+ // In Vanilla spark, the checkEvaluation method doesn't invoke RowEncoder.
+ conf.setConf(SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO, true)
+ }
override def cast(v: Any, targetType: DataType, timeZoneId: Option[String] =
None): Cast = {
v match {
@@ -287,5 +295,4 @@ class GlutenCastSuite extends CastWithAnsiOffSuite with
GlutenTestsTrait {
val d = Decimal(c.getTimeInMillis.toDouble / 1000)
checkEvaluation(cast(d, TimestampType), new Timestamp(c.getTimeInMillis))
}
-
}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala
index 063b646981..fc15ebfeef 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenTryCastSuite.scala
@@ -20,13 +20,21 @@ import org.apache.spark.sql.GlutenTestsTrait
import
org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone,
ALL_TIMEZONES, UTC, UTC_OPT}
import org.apache.spark.sql.catalyst.util.DateTimeUtils.{fromJavaTimestamp,
millisToMicros, TimeZoneUTC}
import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{BinaryType, ByteType, DateType, Decimal,
DecimalType, DoubleType, FloatType, IntegerType, LongType, ShortType,
StringType, TimestampType}
+import org.apache.spark.sql.types._
import org.apache.spark.util.DebuggableThreadUtils
import java.sql.{Date, Timestamp}
import java.util.{Calendar, TimeZone}
class GlutenTryCastSuite extends TryCastSuite with GlutenTestsTrait {
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+ // Need to explicitly set spark.sql.preserveCharVarcharTypeInfo=true for
gluten's test
+ // framework. In Gluten, it overrides the checkEvaluation that invokes
Spark's RowEncoder,
+ // which requires this configuration to be set.
+ // In Vanilla spark, the checkEvaluation method doesn't invoke RowEncoder.
+ conf.setConf(SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO, true)
+ }
testGluten("data type casting") {
val sd = "1970-01-01"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
