andygrove commented on code in PR #7322:
URL: https://github.com/apache/datafusion-comet/pull/7322#discussion_r4629237018


##########
spark/src/main/scala/org/apache/comet/serde/arrays.scala:
##########
@@ -1026,7 +1029,149 @@ object CometFlatten extends 
CometExpressionSerde[Flatten] with ArraysBase {
   }
 }
 
-object CometArrayFilter extends CometExpressionSerde[ArrayFilter] {
+/**
+ * What a serde needs to run the lambda of a higher-order function natively, 
under a DataFusion
+ * higher-order function: the gate on the lambda and its body, and the 
serialization.
+ */
+trait NativeLambdaSerde extends CometTypeShim with CometExprShim {
+
+  // The native lambda has the same semantics as Spark's `filter` for a 
one-parameter lambda whose
+  // body is deterministic, contains no nested lambda, and whose parameter is 
a type the native
+  // side can type-check against the list's element field. `exists` adds a 
condition of its own,
+  // since Spark stops at the first match. It needs bound references: the 
native side re-indexes
+  // the columns the body captures, which it cannot do for unbound ones.
+  protected def nativeLambdaSupported(argument: Expression, body: Expression): 
Boolean =
+    CometConf.COMET_NATIVE_LAMBDA_ENABLED.get() &&
+      body.deterministic &&
+      !body.exists(_.isInstanceOf[HigherOrderFunction]) &&
+      capturesOnlyFixedWidthColumns(body) &&
+      shortCircuitSafe(body) &&
+      !canProduceNullMapKey(argument) &&
+      !canProduceNullMapKey(body) &&
+      !convertsDateTime(body) &&
+      (argument.dataType match {
+        case ArrayType(elementType, _) => isLambdaElementType(elementType)
+        case _ => false
+      })
+
+  /**
+   * Whether `e` has a TRY cast whose map key cast can fail, which makes a 
null key that Arrow
+   * cannot hold (#6172). The native side would have to build that map, so an 
argument or a body
+   * with one stays on the dispatcher, which runs it in Spark's own code, 
where the map never
+   * reaches Arrow.
+   */
+  private def canProduceNullMapKey(e: Expression): Boolean =
+    e.exists {
+      case c: Cast => CometCast.canProduceNullMapKey(c)
+      case _ => false
+    }
+
+  /**
+   * Whether `body` casts between a date or timestamp and a string, date or 
timestamp. The native
+   * cast fails for years beyond chrono's range (about 262,143), which Spark 
supports up to year
+   * 294,247 (#7471), so such a body stays on the dispatcher.
+   */
+  private def convertsDateTime(body: Expression): Boolean = {
+    def isDateTime(dt: DataType): Boolean = dt match {
+      case DateType | TimestampType | TimestampNTZType => true
+      case _ => false
+    }
+    def isTextOrDateTime(dt: DataType): Boolean = dt.isInstanceOf[StringType] 
|| isDateTime(dt)
+    body.exists {
+      case c: Cast if c.child.dataType != c.dataType =>
+        (isDateTime(c.child.dataType) && isTextOrDateTime(c.dataType)) ||
+        (isDateTime(c.dataType) && isTextOrDateTime(c.child.dataType))
+      case _ => false
+    }
+  }
+
+  /**
+   * DataFusion copies each column the body captures once per element of its 
row's array before it
+   * evaluates the body, so a captured string, binary or nested value is 
copied as many times as
+   * the array has elements, and those copies are not reserved with the memory 
pool.
+   */
+  private def capturesOnlyFixedWidthColumns(body: Expression): Boolean =
+    body
+      .collect {
+        case a: AttributeReference => a.dataType
+        case b: BoundReference => b.dataType
+      }
+      .forall(isFixedWidth)
+
+  /**
+   * Spark evaluates `a AND b`, `a OR b` and `x IN (a, b)` lazily, so `x != 0 
AND 100 / x > 1`
+   * never divides by zero. The native side evaluates every operand over the 
whole batch, so the
+   * same lambda raises on the element Spark skips. `CASE WHEN`, `IF` and 
`COALESCE` are not
+   * affected. A body that short-circuits stays on the dispatcher unless none 
of its expressions
+   * can raise.
+   */
+  private def shortCircuitSafe(body: Expression): Boolean =
+    !body.exists {
+      case _: And | _: Or | _: In | _: InSet => true
+      case _ => false
+    } || !body.exists(mayRaise)
+
+  /**
+   * Whether `e` can raise on an input value. This fails closed: an expression 
is assumed to raise
+   * unless it is listed here, since some raise in any mode (`element_at` with 
the index 0), and
+   * arithmetic and casts raise only in the ANSI eval mode they carry. 
`infallible` in
+   * `case_when.rs` is the native counterpart.
+   */
+  protected def mayRaise(e: Expression): Boolean = e match {
+    case _: Literal | _: NamedLambdaVariable | _: AttributeReference | _: 
BoundReference |
+        _: BinaryComparison | _: IsNull | _: IsNotNull | _: Not | _: And | _: 
Or | _: In |
+        _: InSet | _: IsNaN | _: Coalesce | _: If | _: CaseWhen | _: Length | 
_: Upper |
+        _: Lower | _: Substring | _: StartsWith | _: EndsWith | _: Contains =>
+      false
+    case c: Cast => evalMode(c) == CometEvalMode.ANSI

Review Comment:
   Agreed. `mayRaise` now uses the same rules as #7529's `nodeCannotRaise`: 
arithmetic counts as safe only on numbers, a cast only between the types 
`plainCastType` lists, and `GetStructField` is listed. A date cast to a 
timestamp now counts as raising, with or without the #7471 guard. Both PRs are 
mine, so whichever lands second will switch to the one classifier, with 
`NamedLambdaVariable` added. Fixed in 92aa77144.



##########
spark/src/main/scala/org/apache/comet/serde/arrays.scala:
##########
@@ -1026,7 +1029,149 @@ object CometFlatten extends 
CometExpressionSerde[Flatten] with ArraysBase {
   }
 }
 
-object CometArrayFilter extends CometExpressionSerde[ArrayFilter] {
+/**
+ * What a serde needs to run the lambda of a higher-order function natively, 
under a DataFusion
+ * higher-order function: the gate on the lambda and its body, and the 
serialization.
+ */
+trait NativeLambdaSerde extends CometTypeShim with CometExprShim {
+
+  // The native lambda has the same semantics as Spark's `filter` for a 
one-parameter lambda whose
+  // body is deterministic, contains no nested lambda, and whose parameter is 
a type the native
+  // side can type-check against the list's element field. `exists` adds a 
condition of its own,
+  // since Spark stops at the first match. It needs bound references: the 
native side re-indexes
+  // the columns the body captures, which it cannot do for unbound ones.
+  protected def nativeLambdaSupported(argument: Expression, body: Expression): 
Boolean =
+    CometConf.COMET_NATIVE_LAMBDA_ENABLED.get() &&
+      body.deterministic &&
+      !body.exists(_.isInstanceOf[HigherOrderFunction]) &&
+      capturesOnlyFixedWidthColumns(body) &&
+      shortCircuitSafe(body) &&
+      !canProduceNullMapKey(argument) &&
+      !canProduceNullMapKey(body) &&
+      !convertsDateTime(body) &&
+      (argument.dataType match {
+        case ArrayType(elementType, _) => isLambdaElementType(elementType)
+        case _ => false
+      })
+
+  /**
+   * Whether `e` has a TRY cast whose map key cast can fail, which makes a 
null key that Arrow
+   * cannot hold (#6172). The native side would have to build that map, so an 
argument or a body
+   * with one stays on the dispatcher, which runs it in Spark's own code, 
where the map never
+   * reaches Arrow.
+   */
+  private def canProduceNullMapKey(e: Expression): Boolean =
+    e.exists {
+      case c: Cast => CometCast.canProduceNullMapKey(c)
+      case _ => false
+    }
+
+  /**
+   * Whether `body` casts between a date or timestamp and a string, date or 
timestamp. The native
+   * cast fails for years beyond chrono's range (about 262,143), which Spark 
supports up to year
+   * 294,247 (#7471), so such a body stays on the dispatcher.
+   */
+  private def convertsDateTime(body: Expression): Boolean = {

Review Comment:
   Agreed. A body that applies any function to a date or timestamp now stays on 
the dispatcher. The exceptions are comparing it, testing it for NULL, choosing 
it with `IF`, `CASE WHEN` or `COALESCE`, and casting it to a number, which work 
on the stored value. This replaces the cast-only check. 
`array_filter_native_lambda_unsupported_body.sql` pins `year(x) > 2000`, 
`last_day` and `hour` with `expect_dispatch(filter)`, plus two date comparisons 
as `expect_native(filter)`. The tuning page bullet covers #6843. Fixed in 
92aa77144.



##########
spark/src/main/scala/org/apache/comet/serde/arrays.scala:
##########
@@ -1026,7 +1029,149 @@ object CometFlatten extends 
CometExpressionSerde[Flatten] with ArraysBase {
   }
 }
 
-object CometArrayFilter extends CometExpressionSerde[ArrayFilter] {
+/**
+ * What a serde needs to run the lambda of a higher-order function natively, 
under a DataFusion
+ * higher-order function: the gate on the lambda and its body, and the 
serialization.
+ */
+trait NativeLambdaSerde extends CometTypeShim with CometExprShim {
+
+  // The native lambda has the same semantics as Spark's `filter` for a 
one-parameter lambda whose
+  // body is deterministic, contains no nested lambda, and whose parameter is 
a type the native
+  // side can type-check against the list's element field. `exists` adds a 
condition of its own,
+  // since Spark stops at the first match. It needs bound references: the 
native side re-indexes
+  // the columns the body captures, which it cannot do for unbound ones.
+  protected def nativeLambdaSupported(argument: Expression, body: Expression): 
Boolean =
+    CometConf.COMET_NATIVE_LAMBDA_ENABLED.get() &&
+      body.deterministic &&
+      !body.exists(_.isInstanceOf[HigherOrderFunction]) &&
+      capturesOnlyFixedWidthColumns(body) &&
+      shortCircuitSafe(body) &&
+      !canProduceNullMapKey(argument) &&
+      !canProduceNullMapKey(body) &&
+      !convertsDateTime(body) &&
+      (argument.dataType match {
+        case ArrayType(elementType, _) => isLambdaElementType(elementType)
+        case _ => false
+      })
+
+  /**
+   * Whether `e` has a TRY cast whose map key cast can fail, which makes a 
null key that Arrow
+   * cannot hold (#6172). The native side would have to build that map, so an 
argument or a body
+   * with one stays on the dispatcher, which runs it in Spark's own code, 
where the map never
+   * reaches Arrow.
+   */
+  private def canProduceNullMapKey(e: Expression): Boolean =
+    e.exists {
+      case c: Cast => CometCast.canProduceNullMapKey(c)
+      case _ => false
+    }
+
+  /**
+   * Whether `body` casts between a date or timestamp and a string, date or 
timestamp. The native
+   * cast fails for years beyond chrono's range (about 262,143), which Spark 
supports up to year
+   * 294,247 (#7471), so such a body stays on the dispatcher.
+   */
+  private def convertsDateTime(body: Expression): Boolean = {
+    def isDateTime(dt: DataType): Boolean = dt match {
+      case DateType | TimestampType | TimestampNTZType => true
+      case _ => false
+    }
+    def isTextOrDateTime(dt: DataType): Boolean = dt.isInstanceOf[StringType] 
|| isDateTime(dt)
+    body.exists {
+      case c: Cast if c.child.dataType != c.dataType =>
+        (isDateTime(c.child.dataType) && isTextOrDateTime(c.dataType)) ||
+        (isDateTime(c.dataType) && isTextOrDateTime(c.child.dataType))
+      case _ => false
+    }
+  }
+
+  /**
+   * DataFusion copies each column the body captures once per element of its 
row's array before it
+   * evaluates the body, so a captured string, binary or nested value is 
copied as many times as
+   * the array has elements, and those copies are not reserved with the memory 
pool.
+   */
+  private def capturesOnlyFixedWidthColumns(body: Expression): Boolean =
+    body
+      .collect {
+        case a: AttributeReference => a.dataType
+        case b: BoundReference => b.dataType
+      }
+      .forall(isFixedWidth)
+
+  /**
+   * Spark evaluates `a AND b`, `a OR b` and `x IN (a, b)` lazily, so `x != 0 
AND 100 / x > 1`
+   * never divides by zero. The native side evaluates every operand over the 
whole batch, so the
+   * same lambda raises on the element Spark skips. `CASE WHEN`, `IF` and 
`COALESCE` are not
+   * affected. A body that short-circuits stays on the dispatcher unless none 
of its expressions
+   * can raise.
+   */
+  private def shortCircuitSafe(body: Expression): Boolean =
+    !body.exists {
+      case _: And | _: Or | _: In | _: InSet => true

Review Comment:
   You're right. `shortCircuitSafe` now matches only `In` and `InSet`. The 
`AND` and `OR` queries in `array_filter_native_lambda_ansi.sql` are 
`expect_native(filter)` and match Spark, and so does the non-ANSI `v > 0 AND 
element_at(array(threshold, 5), v) > 1`. A new `IN` query with `element_at` as 
a candidate is pinned with `expect_dispatch(filter)`. The tuning page bullet is 
about `IN` now. Fixed in 92aa77144.



##########
native/spark-expr/src/conditional_funcs/case_when.rs:
##########
@@ -451,7 +451,7 @@ pub(super) fn cannot_fail(expr: &Arc<dyn PhysicalExpr>, 
input_schema: &Schema) -
 }
 
 fn infallible(expr: &Arc<dyn PhysicalExpr>, input_schema: &Schema, parsing: 
bool) -> bool {
-    if expr.is::<Column>() || expr.is::<Literal>() {
+    if expr.is::<Column>() || expr.is::<Literal>() || 
expr.is::<LambdaVariable>() {

Review Comment:
   Added the `LambdaVariable` arm. Fixed in 92aa77144.



##########
native/core/src/execution/planner.rs:
##########
@@ -1142,6 +1147,62 @@ impl PhysicalPlanner {
                 ));
                 Ok(expr)
             }
+            ExprStruct::HigherOrderFunc(hof) => {
+                let array =
+                    self.create_expr(hof.array.as_ref().unwrap(), 
Arc::clone(&input_schema))?;
+                // The lambda parameter takes the element field of the array 
as planned, since
+                // DataFusion checks it against the field of the list it 
evaluates.
+                let array_field = array.return_field(input_schema.as_ref())?;
+                let element_field = match array_field.data_type() {
+                    DataType::List(f) | DataType::LargeList(f) => f,
+                    other => {
+                        return Err(GeneralError(format!(
+                            "{} expects an array argument, got {other}",
+                            hof.func
+                        )))
+                    }
+                };
+                // The lambda body sees the enclosing schema with the 
parameter appended, so
+                // `BoundReference`s keep their indexes and a `LambdaVariable` 
is found by name.
+                let mut fields: Vec<FieldRef> = input_schema.fields().to_vec();
+                fields.push(Arc::new(
+                    element_field.as_ref().clone().with_name(&hof.param_name),
+                ));
+                let lambda_schema = Arc::new(Schema::new_with_metadata(
+                    fields,
+                    input_schema.metadata().clone(),
+                ));
+                let body = self.create_expr(hof.body.as_ref().unwrap(), 
lambda_schema)?;
+                let lambda: Arc<dyn PhysicalExpr> =
+                    Arc::new(LambdaExpr::try_new(vec![hof.param_name.clone()], 
body)?);
+                let func = match hof.func.as_str() {

Review Comment:
   Done. The planner resolves the function with 
`self.session_ctx.higher_order_function(&hof.func)?`, and the list of names and 
the two imports are gone. Fixed in 92aa77144.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to