This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 40fc52abc1 [GLUTEN-7143][VL] RAS: Fix fallen back plan nodes are not
tagged with meaningful fallback reasons (#7731)
40fc52abc1 is described below
commit 40fc52abc1d1f50bae2655a4fdc24222451b0fbe
Author: Hongze Zhang <[email protected]>
AuthorDate: Thu Oct 31 10:10:19 2024 +0800
[GLUTEN-7143][VL] RAS: Fix fallen back plan nodes are not tagged with
meaningful fallback reasons (#7731)
---
.../org/apache/gluten/expression/ExpressionConverter.scala | 2 +-
.../gluten/extension/columnar/enumerated/RasOffload.scala | 13 ++++++++-----
.../gluten/expressions/GlutenExpressionMappingSuite.scala | 4 ++--
3 files changed, 11 insertions(+), 8 deletions(-)
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
index 72586b0349..f1bf2d00d0 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
@@ -484,7 +484,7 @@ object ExpressionConverter extends SQLConfHelper with
Logging {
c)
case c if c.getClass.getSimpleName.equals("CheckOverflowInTableInsert")
=>
throw new GlutenNotSupportException(
- "CheckOverflowInTableInsert is used in ansi mode, but gluten does
not support ANSI mode."
+ "CheckOverflowInTableInsert is used in ANSI mode, but Gluten does
not support ANSI mode."
)
case b: BinaryArithmetic if DecimalArithmeticUtil.isDecimalArithmetic(b)
=>
DecimalArithmeticUtil.checkAllowDecimalArithmetic()
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffload.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffload.scala
index fc29b36f08..1299fffb99 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffload.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasOffload.scala
@@ -17,7 +17,7 @@
package org.apache.gluten.extension.columnar.enumerated
import org.apache.gluten.extension.GlutenPlan
-import org.apache.gluten.extension.columnar.OffloadSingleNode
+import org.apache.gluten.extension.columnar.{FallbackTags, OffloadSingleNode}
import org.apache.gluten.extension.columnar.rewrite.RewriteSingleNode
import org.apache.gluten.extension.columnar.validator.Validator
import org.apache.gluten.ras.path.Pattern
@@ -93,16 +93,19 @@ object RasOffload {
// 3. Validate current node. If passed, offload it.
validator.validate(from) match {
case Validator.Passed =>
- val offloaded = base.offload(from)
- val offloadedNodes = offloaded.collect[GlutenPlan] { case t:
GlutenPlan => t }
- if (offloadedNodes.exists(!_.doValidate().ok())) {
+ val offloadedPlan = base.offload(from)
+ val offloadedNodes = offloadedPlan.collect[GlutenPlan] { case
t: GlutenPlan => t }
+ val outComes =
offloadedNodes.map(_.doValidate()).filter(!_.ok())
+ if (outComes.nonEmpty) {
// 4. If native validation fails on the offloaded node,
return the
// original one.
+ outComes.foreach(FallbackTags.add(from, _))
from
} else {
- offloaded
+ offloadedPlan
}
case Validator.Failed(reason) =>
+ FallbackTags.add(from, reason)
from
}
}
diff --git
a/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenExpressionMappingSuite.scala
b/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenExpressionMappingSuite.scala
index 3b52721fe6..68b9ab1a36 100644
---
a/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenExpressionMappingSuite.scala
+++
b/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenExpressionMappingSuite.scala
@@ -104,9 +104,9 @@ class GlutenExpressionMappingSuite
sql("create table t2 (b decimal(10,4)) using parquet")
val msg =
- "CheckOverflowInTableInsert is used in ansi mode, but gluten does not
support ANSI mode."
+ "CheckOverflowInTableInsert is used in ANSI mode, but Gluten does not
support ANSI mode."
import org.apache.spark.sql.execution.GlutenImplicits._
- val fallbackSummary = sql("insert overwrite t2 select * from
t1").fallbackSummary
+ val fallbackSummary = sql("insert overwrite t2 select * from
t1").fallbackSummary()
assert(fallbackSummary.fallbackNodeToReason.flatMap(_.values).exists(_.contains(msg)))
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
