This is an automated email from the ASF dual-hosted git repository.
mbutrovich pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new ad0c2c7ef chore: Move ToPrettyString serde into shim layer (#2549)
ad0c2c7ef is described below
commit ad0c2c7efd40732a7cfdb78c77e143ca8371de50
Author: Andy Grove <[email protected]>
AuthorDate: Tue Oct 14 08:59:27 2025 -0600
chore: Move ToPrettyString serde into shim layer (#2549)
---
.../org/apache/comet/serde/QueryPlanSerde.scala | 40 --------------------
.../org/apache/comet/shims/CometExprShim.scala | 42 ++++++++++++++++++++-
.../org/apache/comet/shims/CometExprShim.scala | 43 ++++++++++++++++++++--
3 files changed, 80 insertions(+), 45 deletions(-)
diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
index 594f144b3..f8395a691 100644
--- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
+++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
@@ -689,46 +689,6 @@ object QueryPlanSerde extends Logging with CometExprShim {
}
versionSpecificExprToProtoInternal(expr, inputs, binding).orElse(expr
match {
- // ToPrettyString is new in Spark 3.5
- case _
- if expr.getClass.getSimpleName == "ToPrettyString" && expr
- .isInstanceOf[UnaryExpression] &&
expr.isInstanceOf[TimeZoneAwareExpression] =>
- val child = expr.asInstanceOf[UnaryExpression].child
- val timezoneId = expr.asInstanceOf[TimeZoneAwareExpression].timeZoneId
-
- val castSupported = CometCast.isSupported(
- child.dataType,
- DataTypes.StringType,
- timezoneId,
- CometEvalMode.TRY)
-
- val isCastSupported = castSupported match {
- case Compatible(_) => true
- case Incompatible(_) => true
- case _ => false
- }
-
- if (isCastSupported) {
- exprToProtoInternal(child, inputs, binding) match {
- case Some(p) =>
- val toPrettyString = ExprOuterClass.ToPrettyString
- .newBuilder()
- .setChild(p)
- .setTimezone(timezoneId.getOrElse("UTC"))
- .setBinaryOutputStyle(binaryOutputStyle)
- .build()
- Some(
- ExprOuterClass.Expr
- .newBuilder()
- .setToPrettyString(toPrettyString)
- .build())
- case _ =>
- withInfo(expr, child)
- None
- }
- } else {
- None
- }
case SortOrder(child, direction, nullOrdering, _) =>
val childExpr = exprToProtoInternal(child, inputs, binding)
diff --git
a/spark/src/main/spark-3.5/org/apache/comet/shims/CometExprShim.scala
b/spark/src/main/spark-3.5/org/apache/comet/shims/CometExprShim.scala
index 600931c34..171216023 100644
--- a/spark/src/main/spark-3.5/org/apache/comet/shims/CometExprShim.scala
+++ b/spark/src/main/spark-3.5/org/apache/comet/shims/CometExprShim.scala
@@ -20,10 +20,13 @@
package org.apache.comet.shims
import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.types.DataTypes
-import org.apache.comet.expressions.CometEvalMode
-import org.apache.comet.serde.CommonStringExprs
+import org.apache.comet.CometSparkSessionExtensions.withInfo
+import org.apache.comet.expressions.{CometCast, CometEvalMode}
+import org.apache.comet.serde.{CommonStringExprs, Compatible, ExprOuterClass,
Incompatible}
import org.apache.comet.serde.ExprOuterClass.{BinaryOutputStyle, Expr}
+import org.apache.comet.serde.QueryPlanSerde.exprToProtoInternal
/**
* `CometExprShim` acts as a shim for parsing expressions from different Spark
versions.
@@ -43,6 +46,41 @@ trait CometExprShim extends CommonStringExprs {
// Right child is the encoding expression.
stringDecode(expr, s.charset, s.bin, inputs, binding)
+ case expr @ ToPrettyString(child, timeZoneId) =>
+ val castSupported = CometCast.isSupported(
+ child.dataType,
+ DataTypes.StringType,
+ timeZoneId,
+ CometEvalMode.TRY)
+
+ val isCastSupported = castSupported match {
+ case Compatible(_) => true
+ case Incompatible(_) => true
+ case _ => false
+ }
+
+ if (isCastSupported) {
+ exprToProtoInternal(child, inputs, binding) match {
+ case Some(p) =>
+ val toPrettyString = ExprOuterClass.ToPrettyString
+ .newBuilder()
+ .setChild(p)
+ .setTimezone(timeZoneId.getOrElse("UTC"))
+ .setBinaryOutputStyle(binaryOutputStyle)
+ .build()
+ Some(
+ ExprOuterClass.Expr
+ .newBuilder()
+ .setToPrettyString(toPrettyString)
+ .build())
+ case _ =>
+ withInfo(expr, child)
+ None
+ }
+ } else {
+ None
+ }
+
case _ => None
}
}
diff --git
a/spark/src/main/spark-4.0/org/apache/comet/shims/CometExprShim.scala
b/spark/src/main/spark-4.0/org/apache/comet/shims/CometExprShim.scala
index 2a7684b86..fc3db183b 100644
--- a/spark/src/main/spark-4.0/org/apache/comet/shims/CometExprShim.scala
+++ b/spark/src/main/spark-4.0/org/apache/comet/shims/CometExprShim.scala
@@ -23,11 +23,13 @@ import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.internal.types.StringTypeWithCollation
-import org.apache.spark.sql.types.{BinaryType, BooleanType, StringType}
+import org.apache.spark.sql.types.{BinaryType, BooleanType, DataTypes,
StringType}
-import org.apache.comet.expressions.CometEvalMode
-import org.apache.comet.serde.CommonStringExprs
+import org.apache.comet.CometSparkSessionExtensions.withInfo
+import org.apache.comet.expressions.{CometCast, CometEvalMode}
+import org.apache.comet.serde.{CommonStringExprs, Compatible, ExprOuterClass,
Incompatible}
import org.apache.comet.serde.ExprOuterClass.{BinaryOutputStyle, Expr}
+import org.apache.comet.serde.QueryPlanSerde.exprToProtoInternal
/**
* `CometExprShim` acts as a shim for parsing expressions from different Spark
versions.
@@ -66,6 +68,41 @@ trait CometExprShim extends CommonStringExprs {
val Seq(bin, charset, _, _) = s.arguments
stringDecode(expr, charset, bin, inputs, binding)
+ case expr @ ToPrettyString(child, timeZoneId) =>
+ val castSupported = CometCast.isSupported(
+ child.dataType,
+ DataTypes.StringType,
+ timeZoneId,
+ CometEvalMode.TRY)
+
+ val isCastSupported = castSupported match {
+ case Compatible(_) => true
+ case Incompatible(_) => true
+ case _ => false
+ }
+
+ if (isCastSupported) {
+ exprToProtoInternal(child, inputs, binding) match {
+ case Some(p) =>
+ val toPrettyString = ExprOuterClass.ToPrettyString
+ .newBuilder()
+ .setChild(p)
+ .setTimezone(timeZoneId.getOrElse("UTC"))
+ .setBinaryOutputStyle(binaryOutputStyle)
+ .build()
+ Some(
+ ExprOuterClass.Expr
+ .newBuilder()
+ .setToPrettyString(toPrettyString)
+ .build())
+ case _ =>
+ withInfo(expr, child)
+ None
+ }
+ } else {
+ None
+ }
+
case _ => None
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
