This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new dd16feff2 chore: Refactor serde for Length (#2547)
dd16feff2 is described below
commit dd16feff2528f4ab088277b51d3755617161a012
Author: Andy Grove <[email protected]>
AuthorDate: Mon Oct 13 07:59:48 2025 -0600
chore: Refactor serde for Length (#2547)
---
.../main/scala/org/apache/comet/serde/QueryPlanSerde.scala | 5 +----
spark/src/main/scala/org/apache/comet/serde/strings.scala | 11 +++++++++--
2 files changed, 10 insertions(+), 6 deletions(-)
diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
index 0a4b61fce..02a5e9d94 100644
--- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
+++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
@@ -166,7 +166,7 @@ object QueryPlanSerde extends Logging with CometExprShim {
classOf[Contains] -> CometScalarFunction("contains"),
classOf[EndsWith] -> CometScalarFunction("ends_with"),
classOf[InitCap] -> CometInitCap,
- classOf[Length] -> CometScalarFunction("length"),
+ classOf[Length] -> CometLength,
classOf[Like] -> CometLike,
classOf[Lower] -> CometLower,
classOf[OctetLength] -> CometScalarFunction("octet_length"),
@@ -891,9 +891,6 @@ object QueryPlanSerde extends Logging with CometExprShim {
withInfo(expr, bloomFilter, value)
None
}
- case l @ Length(child) if child.dataType == BinaryType =>
- withInfo(l, "Length on BinaryType is not supported")
- None
case r @ Reverse(child) if child.dataType.isInstanceOf[ArrayType] =>
convert(r, CometArrayReverse)
case expr =>
diff --git a/spark/src/main/scala/org/apache/comet/serde/strings.scala
b/spark/src/main/scala/org/apache/comet/serde/strings.scala
index 9c85d8d6c..02872eef3 100644
--- a/spark/src/main/scala/org/apache/comet/serde/strings.scala
+++ b/spark/src/main/scala/org/apache/comet/serde/strings.scala
@@ -21,8 +21,8 @@ package org.apache.comet.serde
import java.util.Locale
-import org.apache.spark.sql.catalyst.expressions.{Attribute, Cast, Expression,
InitCap, Like, Literal, Lower, RLike, StringLPad, StringRepeat, StringRPad,
Substring, Upper}
-import org.apache.spark.sql.types.{DataTypes, LongType, StringType}
+import org.apache.spark.sql.catalyst.expressions.{Attribute, Cast, Expression,
InitCap, Length, Like, Literal, Lower, RLike, StringLPad, StringRepeat,
StringRPad, Substring, Upper}
+import org.apache.spark.sql.types.{BinaryType, DataTypes, LongType, StringType}
import org.apache.comet.CometConf
import org.apache.comet.CometSparkSessionExtensions.withInfo
@@ -66,6 +66,13 @@ object CometUpper extends
CometCaseConversionBase[Upper]("upper")
object CometLower extends CometCaseConversionBase[Lower]("lower")
+object CometLength extends CometScalarFunction[Length]("length") {
+ override def getSupportLevel(expr: Length): SupportLevel =
expr.child.dataType match {
+ case _: BinaryType => Unsupported(Some("Length on BinaryType is not
supported"))
+ case _ => Compatible()
+ }
+}
+
object CometInitCap extends CometScalarFunction[InitCap]("initcap") {
override def getSupportLevel(expr: InitCap): SupportLevel = {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
