andygrove commented on code in PR #2241:
URL: https://github.com/apache/datafusion-comet/pull/2241#discussion_r2301931560
##########
spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala:
##########
@@ -849,6 +891,41 @@ case class CometExecRule(session: SparkSession) extends
Rule[SparkPlan] {
*/
private def columnarShuffleSupported(s: ShuffleExchangeExec): Boolean = {
+ /**
+ * Determine which data types are supported as data columns in native
shuffle.
+ *
+ * Comet columnar shuffle used native code to convert Spark unsafe rows to
Arrow batches, see
+ * shuffle/row.rs
+ */
+ def supportedSerializableDataType(dt: DataType): Boolean = dt match {
+ case _: BooleanType | _: ByteType | _: ShortType | _: IntegerType | _:
LongType |
+ _: FloatType | _: DoubleType | _: StringType | _: BinaryType | _:
TimestampType |
+ _: TimestampNTZType | _: DecimalType | _: DateType =>
+ true
+ case StructType(fields) =>
+ fields.forall(f => supportedSerializableDataType(f.dataType)) &&
Review Comment:
We should maybe add a check for `fields.nonEmpty`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
