comphead commented on code in PR #7251:
URL: https://github.com/apache/datafusion-comet/pull/7251#discussion_r4383779954
##########
spark/src/main/scala/org/apache/spark/sql/comet/operators.scala:
##########
@@ -1920,33 +1918,43 @@ object CometExplodeExec extends
CometOperatorSerde[GenerateExec] {
if (!op.generator.deterministic) {
return Unsupported(Some("Only deterministic generators are supported"))
}
- if (op.generator.children.length != 1) {
- return Unsupported(Some("generators with multiple inputs are not
supported"))
- }
- val nodeName = op.generator.nodeName.toLowerCase(Locale.ROOT)
- if (nodeName != "explode" && nodeName != "posexplode") {
- return Unsupported(Some(s"Unsupported generator:
${op.generator.nodeName}"))
- }
- op.generator.children.head.dataType match {
- case _: ArrayType =>
+ op.generator match {
+ // Spark only accepts an array of structs for `inline`, and `stack` is
planned as `inline`;
+ // see `stackAsStructArray`.
+ case _: Inline | _: Stack =>
Compatible()
- case _: MapType =>
- // TODO add support for map types
- // https://github.com/apache/datafusion-comet/issues/2837
- Unsupported(Some("Comet only supports explode/explode_outer for
arrays, not maps"))
+ case explode: ExplodeBase =>
+ explode.child.dataType match {
+ case _: ArrayType =>
+ Compatible()
+ case _: MapType =>
+ // TODO add support for map types
+ // https://github.com/apache/datafusion-comet/issues/2837
+ Unsupported(Some("Comet only supports explode/explode_outer for
arrays, not maps"))
+ case other =>
+ Unsupported(Some(s"Unsupported data type: $other"))
+ }
case other =>
- Unsupported(Some(s"Unsupported data type: $other"))
+ Unsupported(Some(s"Unsupported generator: ${other.nodeName}"))
}
}
override def convert(
op: GenerateExec,
builder: Operator.Builder,
childOp: OperatorOuterClass.Operator*):
Option[OperatorOuterClass.Operator] = {
- val childExpr = op.generator.children.head
+ val (childExpr, isPosExplode, isInline) = op.generator match {
+ case inline: Inline => (inline.child, false, true)
+ case stack: Stack => (stackAsStructArray(stack), false, true)
Review Comment:
Stack will also be supported in DF56
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
