tlfua commented on code in PR #7457:
URL: https://github.com/apache/datafusion-comet/pull/7457#discussion_r4596854911


##########
spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala:
##########
@@ -1394,6 +1395,72 @@ object QueryPlanSerde extends Logging with CometExprShim 
with CometTypeShim {
     }
   }
 
+  /**
+   * Whether `expr` cannot raise an error for any row, so native execution may 
evaluate it for
+   * rows that Spark skips. It answers from Spark's semantics, independently 
of the native
+   * `cannot_fail` in `case_when.rs`, which judges the physical expressions it 
is given. It is
+   * conservative: an allowed expression whose children also cannot fail. 
Anything not listed is
+   * treated as able to fail.
+   */
+  def cannotFail(expr: Expression): Boolean = {
+    def numeric: Boolean = expr.dataType.isInstanceOf[NumericType]
+    def nonAnsiNumeric(evalMode: EvalMode.Value): Boolean = evalMode != 
EvalMode.ANSI && numeric
+    // Floating-point arithmetic never overflows into an error, even in ANSI 
mode
+    def floating: Boolean = expr.dataType == FloatType || expr.dataType == 
DoubleType
+    val nodeCannotFail = expr match {
+      case _: Attribute | _: BoundReference | _: Literal => true
+      case _: GetStructField => true
+      case _: BinaryComparison | _: And | _: Or | _: Not | _: IsNull | _: 
IsNotNull => true
+      case _: In | _: InSet | _: Greatest | _: Least => true
+      case _: If | _: CaseWhen | _: Coalesce | _: IsNaN | _: NaNvl => true
+      case _: KnownFloatingPointNormalized | _: NormalizeNaNAndZero => true
+      case _: Substring | _: Upper | _: Lower | _: Length | _: OctetLength | 
_: BitLength => true
+      case _: StringTrim | _: StringTrimLeft | _: StringTrimRight => true
+      case _: StringReplace | _: InitCap => true
+      // These fail only when the result would exceed 2GB, more than a row can 
hold
+      case _: ConcatWs => true
+      case c: Concat => c.children.nonEmpty && c.dataType == StringType
+      case _: Reverse => expr.dataType == StringType
+      case _: Year | _: Quarter | _: Month | _: DayOfMonth | _: DayOfWeek | _: 
WeekDay |
+          _: DayOfYear | _: Hour | _: Minute | _: Second =>
+        true
+      case _: Murmur3Hash | _: XxHash64 | _: Md5 | _: Sha1 | _: Sha2 | _: 
Crc32 => true
+      // Invalid input gives NaN or NULL, and a floating-point floor or ceil 
saturates to a long
+      case _: Floor | _: Ceil | _: Sqrt | _: Exp | _: Log | _: Signum => true
+      // Rounding an integer fails on overflow in ANSI mode; a decimal is 
never treated as safe
+      case r: Round => roundCannotFail(r.dataType, r.ansiEnabled)
+      case r: BRound => roundCannotFail(r.dataType, r.ansiEnabled)
+      case c: Cast => CometCast.cannotFail(c)
+      case _: UnaryPositive => true
+      case u: UnaryMinus => floating || (!u.failOnError && numeric)
+      case a: Abs => floating || (!a.failOnError && numeric)
+      case a: Add => floating || nonAnsiNumeric(a.evalMode)
+      case s: Subtract => floating || nonAnsiNumeric(s.evalMode)
+      case m: Multiply => floating || nonAnsiNumeric(m.evalMode)
+      case d: Divide => nonAnsiNumeric(d.evalMode)
+      case d: IntegralDivide => nonAnsiNumeric(d.evalMode)
+      case r: Remainder => nonAnsiNumeric(r.evalMode)
+      case p: Pmod => nonAnsiNumeric(p.evalMode)
+      case _ => false
+    }
+    nodeCannotFail && expr.children.forall(cannotFail)
+  }
+
+  private def roundCannotFail(dataType: DataType, ansiEnabled: Boolean): 
Boolean =
+    dataType match {
+      case FloatType | DoubleType => true
+      case ByteType | ShortType | IntegerType | LongType => !ansiEnabled
+      case _ => false
+    }
+
+  /**
+   * The first sort key after the first that can fail. Spark's ordering 
evaluates a sort key only
+   * for the rows whose earlier keys tie, but native execution evaluates every 
key over the whole
+   * batch, so such a key may raise on a row that Spark skips (#7246).
+   */
+  def laterSortKeyThatCanFail(sortOrder: Seq[SortOrder]): Option[SortOrder] =
+    sortOrder.drop(1).find(s => !cannotFail(s.child))

Review Comment:
   Thanks, applied as suggested in 
https://github.com/apache/datafusion-comet/pull/7457/commits/201edc6cccfb38f056f57aafade386813bb4dcf2
 : `laterSortKeyThatCanFail` keeps the `cannotFail` allow-list under ANSI and 
otherwise only falls back for `raisesInAnyMode`, with 
`CometCast.failsInAnyMode` pulled out of `cannotFail`. The ANSI-off section of 
`topk_later_key_ansi.sql` now has `date_add(dt, 1)` and `lpad(s, 5)` staying 
native and `element_at(arr, 0)` falling back. I kept the session flag as in 
your sketch and noted the per-expression eval mode, together with the 
expressions the list does not cover yet, on 
https://github.com/apache/datafusion-comet/issues/7585 ( 
https://github.com/apache/datafusion-comet/issues/7585#issuecomment-6942355561 
).
   



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to