andygrove commented on code in PR #7322:
URL: https://github.com/apache/datafusion-comet/pull/7322#discussion_r4629237018
##########
spark/src/main/scala/org/apache/comet/serde/arrays.scala:
##########
@@ -1026,7 +1029,149 @@ object CometFlatten extends
CometExpressionSerde[Flatten] with ArraysBase {
}
}
-object CometArrayFilter extends CometExpressionSerde[ArrayFilter] {
+/**
+ * What a serde needs to run the lambda of a higher-order function natively,
under a DataFusion
+ * higher-order function: the gate on the lambda and its body, and the
serialization.
+ */
+trait NativeLambdaSerde extends CometTypeShim with CometExprShim {
+
+ // The native lambda has the same semantics as Spark's `filter` for a
one-parameter lambda whose
+ // body is deterministic, contains no nested lambda, and whose parameter is
a type the native
+ // side can type-check against the list's element field. `exists` adds a
condition of its own,
+ // since Spark stops at the first match. It needs bound references: the
native side re-indexes
+ // the columns the body captures, which it cannot do for unbound ones.
+ protected def nativeLambdaSupported(argument: Expression, body: Expression):
Boolean =
+ CometConf.COMET_NATIVE_LAMBDA_ENABLED.get() &&
+ body.deterministic &&
+ !body.exists(_.isInstanceOf[HigherOrderFunction]) &&
+ capturesOnlyFixedWidthColumns(body) &&
+ shortCircuitSafe(body) &&
+ !canProduceNullMapKey(argument) &&
+ !canProduceNullMapKey(body) &&
+ !convertsDateTime(body) &&
+ (argument.dataType match {
+ case ArrayType(elementType, _) => isLambdaElementType(elementType)
+ case _ => false
+ })
+
+ /**
+ * Whether `e` has a TRY cast whose map key cast can fail, which makes a
null key that Arrow
+ * cannot hold (#6172). The native side would have to build that map, so an
argument or a body
+ * with one stays on the dispatcher, which runs it in Spark's own code,
where the map never
+ * reaches Arrow.
+ */
+ private def canProduceNullMapKey(e: Expression): Boolean =
+ e.exists {
+ case c: Cast => CometCast.canProduceNullMapKey(c)
+ case _ => false
+ }
+
+ /**
+ * Whether `body` casts between a date or timestamp and a string, date or
timestamp. The native
+ * cast fails for years beyond chrono's range (about 262,143), which Spark
supports up to year
+ * 294,247 (#7471), so such a body stays on the dispatcher.
+ */
+ private def convertsDateTime(body: Expression): Boolean = {
+ def isDateTime(dt: DataType): Boolean = dt match {
+ case DateType | TimestampType | TimestampNTZType => true
+ case _ => false
+ }
+ def isTextOrDateTime(dt: DataType): Boolean = dt.isInstanceOf[StringType]
|| isDateTime(dt)
+ body.exists {
+ case c: Cast if c.child.dataType != c.dataType =>
+ (isDateTime(c.child.dataType) && isTextOrDateTime(c.dataType)) ||
+ (isDateTime(c.dataType) && isTextOrDateTime(c.child.dataType))
+ case _ => false
+ }
+ }
+
+ /**
+ * DataFusion copies each column the body captures once per element of its
row's array before it
+ * evaluates the body, so a captured string, binary or nested value is
copied as many times as
+ * the array has elements, and those copies are not reserved with the memory
pool.
+ */
+ private def capturesOnlyFixedWidthColumns(body: Expression): Boolean =
+ body
+ .collect {
+ case a: AttributeReference => a.dataType
+ case b: BoundReference => b.dataType
+ }
+ .forall(isFixedWidth)
+
+ /**
+ * Spark evaluates `a AND b`, `a OR b` and `x IN (a, b)` lazily, so `x != 0
AND 100 / x > 1`
+ * never divides by zero. The native side evaluates every operand over the
whole batch, so the
+ * same lambda raises on the element Spark skips. `CASE WHEN`, `IF` and
`COALESCE` are not
+ * affected. A body that short-circuits stays on the dispatcher unless none
of its expressions
+ * can raise.
+ */
+ private def shortCircuitSafe(body: Expression): Boolean =
+ !body.exists {
+ case _: And | _: Or | _: In | _: InSet => true
+ case _ => false
+ } || !body.exists(mayRaise)
+
+ /**
+ * Whether `e` can raise on an input value. This fails closed: an expression
is assumed to raise
+ * unless it is listed here, since some raise in any mode (`element_at` with
the index 0), and
+ * arithmetic and casts raise only in the ANSI eval mode they carry.
`infallible` in
+ * `case_when.rs` is the native counterpart.
+ */
+ protected def mayRaise(e: Expression): Boolean = e match {
+ case _: Literal | _: NamedLambdaVariable | _: AttributeReference | _:
BoundReference |
+ _: BinaryComparison | _: IsNull | _: IsNotNull | _: Not | _: And | _:
Or | _: In |
+ _: InSet | _: IsNaN | _: Coalesce | _: If | _: CaseWhen | _: Length |
_: Upper |
+ _: Lower | _: Substring | _: StartsWith | _: EndsWith | _: Contains =>
+ false
+ case c: Cast => evalMode(c) == CometEvalMode.ANSI
Review Comment:
Agreed. `mayRaise` now uses the same rules as #7529's `nodeCannotRaise`:
arithmetic counts as safe only on numbers, a cast only between the types
`plainCastType` lists, and `GetStructField` is listed. A date cast to a
timestamp now counts as raising, with or without the #7471 guard. Both PRs are
mine, so whichever lands second will switch to the one classifier, with
`NamedLambdaVariable` added. Fixed in 92aa77144.
##########
spark/src/main/scala/org/apache/comet/serde/arrays.scala:
##########
@@ -1026,7 +1029,149 @@ object CometFlatten extends
CometExpressionSerde[Flatten] with ArraysBase {
}
}
-object CometArrayFilter extends CometExpressionSerde[ArrayFilter] {
+/**
+ * What a serde needs to run the lambda of a higher-order function natively,
under a DataFusion
+ * higher-order function: the gate on the lambda and its body, and the
serialization.
+ */
+trait NativeLambdaSerde extends CometTypeShim with CometExprShim {
+
+ // The native lambda has the same semantics as Spark's `filter` for a
one-parameter lambda whose
+ // body is deterministic, contains no nested lambda, and whose parameter is
a type the native
+ // side can type-check against the list's element field. `exists` adds a
condition of its own,
+ // since Spark stops at the first match. It needs bound references: the
native side re-indexes
+ // the columns the body captures, which it cannot do for unbound ones.
+ protected def nativeLambdaSupported(argument: Expression, body: Expression):
Boolean =
+ CometConf.COMET_NATIVE_LAMBDA_ENABLED.get() &&
+ body.deterministic &&
+ !body.exists(_.isInstanceOf[HigherOrderFunction]) &&
+ capturesOnlyFixedWidthColumns(body) &&
+ shortCircuitSafe(body) &&
+ !canProduceNullMapKey(argument) &&
+ !canProduceNullMapKey(body) &&
+ !convertsDateTime(body) &&
+ (argument.dataType match {
+ case ArrayType(elementType, _) => isLambdaElementType(elementType)
+ case _ => false
+ })
+
+ /**
+ * Whether `e` has a TRY cast whose map key cast can fail, which makes a
null key that Arrow
+ * cannot hold (#6172). The native side would have to build that map, so an
argument or a body
+ * with one stays on the dispatcher, which runs it in Spark's own code,
where the map never
+ * reaches Arrow.
+ */
+ private def canProduceNullMapKey(e: Expression): Boolean =
+ e.exists {
+ case c: Cast => CometCast.canProduceNullMapKey(c)
+ case _ => false
+ }
+
+ /**
+ * Whether `body` casts between a date or timestamp and a string, date or
timestamp. The native
+ * cast fails for years beyond chrono's range (about 262,143), which Spark
supports up to year
+ * 294,247 (#7471), so such a body stays on the dispatcher.
+ */
+ private def convertsDateTime(body: Expression): Boolean = {
Review Comment:
Agreed. A body that applies any function to a date or timestamp now stays on
the dispatcher. The exceptions are comparing it, testing it for NULL, choosing
it with `IF`, `CASE WHEN` or `COALESCE`, and casting it to a number, which work
on the stored value. This replaces the cast-only check.
`array_filter_native_lambda_unsupported_body.sql` pins `year(x) > 2000`,
`last_day` and `hour` with `expect_dispatch(filter)`, plus two date comparisons
as `expect_native(filter)`. The tuning page bullet covers #6843. Fixed in
92aa77144.
##########
spark/src/main/scala/org/apache/comet/serde/arrays.scala:
##########
@@ -1026,7 +1029,149 @@ object CometFlatten extends
CometExpressionSerde[Flatten] with ArraysBase {
}
}
-object CometArrayFilter extends CometExpressionSerde[ArrayFilter] {
+/**
+ * What a serde needs to run the lambda of a higher-order function natively,
under a DataFusion
+ * higher-order function: the gate on the lambda and its body, and the
serialization.
+ */
+trait NativeLambdaSerde extends CometTypeShim with CometExprShim {
+
+ // The native lambda has the same semantics as Spark's `filter` for a
one-parameter lambda whose
+ // body is deterministic, contains no nested lambda, and whose parameter is
a type the native
+ // side can type-check against the list's element field. `exists` adds a
condition of its own,
+ // since Spark stops at the first match. It needs bound references: the
native side re-indexes
+ // the columns the body captures, which it cannot do for unbound ones.
+ protected def nativeLambdaSupported(argument: Expression, body: Expression):
Boolean =
+ CometConf.COMET_NATIVE_LAMBDA_ENABLED.get() &&
+ body.deterministic &&
+ !body.exists(_.isInstanceOf[HigherOrderFunction]) &&
+ capturesOnlyFixedWidthColumns(body) &&
+ shortCircuitSafe(body) &&
+ !canProduceNullMapKey(argument) &&
+ !canProduceNullMapKey(body) &&
+ !convertsDateTime(body) &&
+ (argument.dataType match {
+ case ArrayType(elementType, _) => isLambdaElementType(elementType)
+ case _ => false
+ })
+
+ /**
+ * Whether `e` has a TRY cast whose map key cast can fail, which makes a
null key that Arrow
+ * cannot hold (#6172). The native side would have to build that map, so an
argument or a body
+ * with one stays on the dispatcher, which runs it in Spark's own code,
where the map never
+ * reaches Arrow.
+ */
+ private def canProduceNullMapKey(e: Expression): Boolean =
+ e.exists {
+ case c: Cast => CometCast.canProduceNullMapKey(c)
+ case _ => false
+ }
+
+ /**
+ * Whether `body` casts between a date or timestamp and a string, date or
timestamp. The native
+ * cast fails for years beyond chrono's range (about 262,143), which Spark
supports up to year
+ * 294,247 (#7471), so such a body stays on the dispatcher.
+ */
+ private def convertsDateTime(body: Expression): Boolean = {
+ def isDateTime(dt: DataType): Boolean = dt match {
+ case DateType | TimestampType | TimestampNTZType => true
+ case _ => false
+ }
+ def isTextOrDateTime(dt: DataType): Boolean = dt.isInstanceOf[StringType]
|| isDateTime(dt)
+ body.exists {
+ case c: Cast if c.child.dataType != c.dataType =>
+ (isDateTime(c.child.dataType) && isTextOrDateTime(c.dataType)) ||
+ (isDateTime(c.dataType) && isTextOrDateTime(c.child.dataType))
+ case _ => false
+ }
+ }
+
+ /**
+ * DataFusion copies each column the body captures once per element of its
row's array before it
+ * evaluates the body, so a captured string, binary or nested value is
copied as many times as
+ * the array has elements, and those copies are not reserved with the memory
pool.
+ */
+ private def capturesOnlyFixedWidthColumns(body: Expression): Boolean =
+ body
+ .collect {
+ case a: AttributeReference => a.dataType
+ case b: BoundReference => b.dataType
+ }
+ .forall(isFixedWidth)
+
+ /**
+ * Spark evaluates `a AND b`, `a OR b` and `x IN (a, b)` lazily, so `x != 0
AND 100 / x > 1`
+ * never divides by zero. The native side evaluates every operand over the
whole batch, so the
+ * same lambda raises on the element Spark skips. `CASE WHEN`, `IF` and
`COALESCE` are not
+ * affected. A body that short-circuits stays on the dispatcher unless none
of its expressions
+ * can raise.
+ */
+ private def shortCircuitSafe(body: Expression): Boolean =
+ !body.exists {
+ case _: And | _: Or | _: In | _: InSet => true
Review Comment:
You're right. `shortCircuitSafe` now matches only `In` and `InSet`. The
`AND` and `OR` queries in `array_filter_native_lambda_ansi.sql` are
`expect_native(filter)` and match Spark, and so does the non-ANSI `v > 0 AND
element_at(array(threshold, 5), v) > 1`. A new `IN` query with `element_at` as
a candidate is pinned with `expect_dispatch(filter)`. The tuning page bullet is
about `IN` now. Fixed in 92aa77144.
##########
native/spark-expr/src/conditional_funcs/case_when.rs:
##########
@@ -451,7 +451,7 @@ pub(super) fn cannot_fail(expr: &Arc<dyn PhysicalExpr>,
input_schema: &Schema) -
}
fn infallible(expr: &Arc<dyn PhysicalExpr>, input_schema: &Schema, parsing:
bool) -> bool {
- if expr.is::<Column>() || expr.is::<Literal>() {
+ if expr.is::<Column>() || expr.is::<Literal>() ||
expr.is::<LambdaVariable>() {
Review Comment:
Added the `LambdaVariable` arm. Fixed in 92aa77144.
##########
native/core/src/execution/planner.rs:
##########
@@ -1142,6 +1147,62 @@ impl PhysicalPlanner {
));
Ok(expr)
}
+ ExprStruct::HigherOrderFunc(hof) => {
+ let array =
+ self.create_expr(hof.array.as_ref().unwrap(),
Arc::clone(&input_schema))?;
+ // The lambda parameter takes the element field of the array
as planned, since
+ // DataFusion checks it against the field of the list it
evaluates.
+ let array_field = array.return_field(input_schema.as_ref())?;
+ let element_field = match array_field.data_type() {
+ DataType::List(f) | DataType::LargeList(f) => f,
+ other => {
+ return Err(GeneralError(format!(
+ "{} expects an array argument, got {other}",
+ hof.func
+ )))
+ }
+ };
+ // The lambda body sees the enclosing schema with the
parameter appended, so
+ // `BoundReference`s keep their indexes and a `LambdaVariable`
is found by name.
+ let mut fields: Vec<FieldRef> = input_schema.fields().to_vec();
+ fields.push(Arc::new(
+ element_field.as_ref().clone().with_name(&hof.param_name),
+ ));
+ let lambda_schema = Arc::new(Schema::new_with_metadata(
+ fields,
+ input_schema.metadata().clone(),
+ ));
+ let body = self.create_expr(hof.body.as_ref().unwrap(),
lambda_schema)?;
+ let lambda: Arc<dyn PhysicalExpr> =
+ Arc::new(LambdaExpr::try_new(vec![hof.param_name.clone()],
body)?);
+ let func = match hof.func.as_str() {
Review Comment:
Done. The planner resolves the function with
`self.session_ctx.higher_order_function(&hof.func)?`, and the list of names and
the two imports are gone. Fixed in 92aa77144.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]