This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 49ccdbde3 [GLUTEN-7213][CORE] Make fallback reason for
CheckOverflowInTableInsert clearer (#7248)
49ccdbde3 is described below
commit 49ccdbde349338c87ccc37fa9fc68e8ae4570030
Author: Zhen Wang <[email protected]>
AuthorDate: Sat Sep 14 22:33:03 2024 +0800
[GLUTEN-7213][CORE] Make fallback reason for CheckOverflowInTableInsert
clearer (#7248)
---
.../apache/gluten/expression/ExpressionConverter.scala | 4 ++++
.../expressions/GlutenExpressionMappingSuite.scala | 16 ++++++++++++++++
.../org/apache/gluten/expression/ExpressionNames.scala | 1 +
.../apache/gluten/sql/shims/spark34/Spark34Shims.scala | 3 ++-
.../apache/gluten/sql/shims/spark35/Spark35Shims.scala | 3 ++-
5 files changed, 25 insertions(+), 2 deletions(-)
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
index 98a556c11..0aa676158 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
@@ -481,6 +481,10 @@ object ExpressionConverter extends SQLConfHelper with
Logging {
substraitExprName,
replaceWithExpressionTransformer0(c.child, attributeSeq,
expressionsMap),
c)
+ case c if c.getClass.getSimpleName.equals("CheckOverflowInTableInsert")
=>
+ throw new GlutenNotSupportException(
+ "CheckOverflowInTableInsert is used in ansi mode, but gluten does
not support ANSI mode."
+ )
case b: BinaryArithmetic if DecimalArithmeticUtil.isDecimalArithmetic(b)
=>
DecimalArithmeticUtil.checkAllowDecimalArithmetic()
if (!BackendsApiManager.getSettings.transformCheckOverflow) {
diff --git
a/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenExpressionMappingSuite.scala
b/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenExpressionMappingSuite.scala
index 18f2baf6a..3b52721fe 100644
---
a/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenExpressionMappingSuite.scala
+++
b/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenExpressionMappingSuite.scala
@@ -94,4 +94,20 @@ class GlutenExpressionMappingSuite
}
}
}
+
+ testWithSpecifiedSparkVersion(
+ "GLUTEN-7213: Check fallback reason with CheckOverflowInTableInsert",
+ Some("3.4")) {
+ withTable("t1", "t2") {
+ sql("create table t1 (a float) using parquet")
+ sql("insert into t1 values(1.1)")
+ sql("create table t2 (b decimal(10,4)) using parquet")
+
+ val msg =
+ "CheckOverflowInTableInsert is used in ansi mode, but gluten does not
support ANSI mode."
+ import org.apache.spark.sql.execution.GlutenImplicits._
+ val fallbackSummary = sql("insert overwrite t2 select * from
t1").fallbackSummary
+
assert(fallbackSummary.fallbackNodeToReason.flatMap(_.values).exists(_.contains(msg)))
+ }
+ }
}
diff --git
a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
index c45f0b2d4..0e08e013c 100644
---
a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
+++
b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
@@ -314,6 +314,7 @@ object ExpressionNames {
final val INLINE = "inline"
final val POSEXPLODE = "posexplode"
final val CHECK_OVERFLOW = "check_overflow"
+ final val CHECK_OVERFLOW_IN_TABLE_INSERT = "check_overflow_in_table_insert"
final val MAKE_DECIMAL = "make_decimal"
final val PROMOTE_PRECISION = "promote_precision"
final val SPARK_PARTITION_ID = "spark_partition_id"
diff --git
a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala
b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala
index 5e42f66ba..558d7f60d 100644
---
a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala
+++
b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala
@@ -82,7 +82,8 @@ class Spark34Shims extends SparkShims {
Sig[RoundFloor](ExpressionNames.FLOOR),
Sig[RoundCeil](ExpressionNames.CEIL),
Sig[Mask](ExpressionNames.MASK),
- Sig[ArrayInsert](ExpressionNames.ARRAY_INSERT)
+ Sig[ArrayInsert](ExpressionNames.ARRAY_INSERT),
+
Sig[CheckOverflowInTableInsert](ExpressionNames.CHECK_OVERFLOW_IN_TABLE_INSERT)
)
}
diff --git
a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
index ddb023b5a..4a6590161 100644
---
a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
+++
b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
@@ -82,7 +82,8 @@ class Spark35Shims extends SparkShims {
Sig[TimestampAdd](ExpressionNames.TIMESTAMP_ADD),
Sig[RoundFloor](ExpressionNames.FLOOR),
Sig[RoundCeil](ExpressionNames.CEIL),
- Sig[ArrayInsert](ExpressionNames.ARRAY_INSERT)
+ Sig[ArrayInsert](ExpressionNames.ARRAY_INSERT),
+
Sig[CheckOverflowInTableInsert](ExpressionNames.CHECK_OVERFLOW_IN_TABLE_INSERT)
)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
