hvanhovell commented on code in PR #47839:
URL: https://github.com/apache/spark/pull/47839#discussion_r1726147089
##########
connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/connect/columnNodeSupport.scala:
##########
@@ -20,20 +20,30 @@ import scala.jdk.CollectionConverters._
import org.apache.spark.SparkException
import org.apache.spark.connect.proto
+import org.apache.spark.connect.proto.Expression
import
org.apache.spark.connect.proto.Expression.SortOrder.NullOrdering.{SORT_NULLS_FIRST,
SORT_NULLS_LAST}
import
org.apache.spark.connect.proto.Expression.SortOrder.SortDirection.{SORT_DIRECTION_ASCENDING,
SORT_DIRECTION_DESCENDING}
import
org.apache.spark.connect.proto.Expression.Window.WindowFrame.{FrameBoundary,
FrameType}
+import org.apache.spark.sql.{Column, Encoder}
import org.apache.spark.sql.catalyst.trees.{CurrentOrigin, Origin}
import org.apache.spark.sql.connect.common.DataTypeProtoConverter
import
org.apache.spark.sql.connect.common.LiteralValueProtoConverter.toLiteralProtoBuilder
-import org.apache.spark.sql.expressions.ScalaUserDefinedFunction
+import org.apache.spark.sql.expressions.{Aggregator, UserDefinedAggregator,
UserDefinedFunction}
import org.apache.spark.sql.internal._
/**
* Converter for [[ColumnNode]] to [[proto.Expression]] conversions.
*/
object ColumnNodeToProtoConverter extends (ColumnNode => proto.Expression) {
- override def apply(node: ColumnNode): proto.Expression = {
+ def toExpr(column: Column): proto.Expression = apply(column.node, None)
+
+ def toTypedExpr[I](column: Column, encoder: Encoder[I]): proto.Expression = {
+ apply(column.node, Option(encoder))
+ }
+
+ override def apply(node: ColumnNode): Expression = apply(node, None)
+
+ private def apply(node: ColumnNode, e: Option[Encoder[_]]): proto.Expression
= {
Review Comment:
We need the encoder for Aggregator's without an input encoder.
##########
connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/expressions/Aggregator.scala:
##########
Review Comment:
This is shared now.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
