hvanhovell commented on code in PR #40628:
URL: https://github.com/apache/spark/pull/40628#discussion_r1156366512
##########
connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/Dataset.scala:
##########
@@ -2468,6 +2470,153 @@ class Dataset[T] private[sql] (
*/
def transform[U](t: Dataset[T] => Dataset[U]): Dataset[U] = t(this)
+ /**
+ * (Scala-specific) Returns a new Dataset that only contains elements where
`func` returns
+ * `true`.
+ *
+ * @group typedrel
+ * @since 3.5.0
+ */
+ def filter(func: T => Boolean): Dataset[T] = {
+ val udf = ScalarUserDefinedFunction(
+ function = func,
+ inputEncoders = encoder :: Nil,
+ outputEncoder = PrimitiveBooleanEncoder,
+ name = None,
+ nullable = false,
+ deterministic = true)
+ sparkSession.newDataset[T](encoder) { builder =>
+ builder.getFilterBuilder
+ .setInput(plan.getRoot)
+ .setCondition(udf.apply().expr)
+ }
+ }
+
+ /**
+ * (Java-specific) Returns a new Dataset that only contains elements where
`func` returns
+ * `true`.
+ *
+ * @group typedrel
+ * @since 3.5.0
+ */
+ def filter(f: FilterFunction[T]): Dataset[T] = {
+ filter(UdfUtils.filterFuncToScalaFunc(f))
+ }
+
+ /**
+ * (Scala-specific) Returns a new Dataset that contains the result of
applying `func` to each
+ * element.
+ *
+ * @group typedrel
+ * @since 3.5.0
+ */
+ def map[U: Encoder](f: T => U): Dataset[U] = {
+ mapPartitions(UdfUtils.mapFuncToMapPartitionsAdaptor(f))
+ }
+
+ /**
+ * (Java-specific) Returns a new Dataset that contains the result of
applying `func` to each
+ * element.
+ *
+ * @group typedrel
+ * @since 3.5.0
+ */
+ def map[U](f: MapFunction[T, U], encoder: Encoder[U]): Dataset[U] = {
+ map(UdfUtils.mapFunctionToScalaFunc(f))(encoder)
+ }
+
+ /**
+ * (Scala-specific) Returns a new Dataset that contains the result of
applying `func` to each
+ * partition.
+ *
+ * @group typedrel
+ * @since 3.5.0
+ */
+ def mapPartitions[U: Encoder](func: Iterator[T] => Iterator[U]): Dataset[U]
= {
+ val outputEncoder = encoderFor[U]
+ val udf = ScalarUserDefinedFunction(
+ function = func,
+ inputEncoders = encoder :: Nil,
+ outputEncoder = outputEncoder)
+ sparkSession.newDataset(outputEncoder) { builder =>
+ builder.getMapPartitionsBuilder
+ .setInput(plan.getRoot)
+ .setFunc(udf.apply().expr.getCommonInlineUserDefinedFunction)
+ }
+ }
+
+ /**
+ * (Java-specific) Returns a new Dataset that contains the result of
applying `f` to each
+ * partition.
+ *
+ * @group typedrel
+ * @since 3.5.0
+ */
+ def mapPartitions[U](f: MapPartitionsFunction[T, U], encoder: Encoder[U]):
Dataset[U] = {
+ mapPartitions(UdfUtils.mapPartitionsFuncToScalaFunc(f))(encoder)
+ }
+
+ /**
+ * (Scala-specific) Returns a new Dataset by first applying a function to
all elements of this
+ * Dataset, and then flattening the results.
+ *
+ * @group typedrel
+ * @since 3.5.0
+ */
+ def flatMap[U: Encoder](func: T => TraversableOnce[U]): Dataset[U] =
+ mapPartitions(UdfUtils.flatMapFuncToMapPartitionsAdaptor(func))
+
+ /**
+ * (Java-specific) Returns a new Dataset by first applying a function to all
elements of this
+ * Dataset, and then flattening the results.
+ *
+ * @group typedrel
+ * @since 3.5.0
+ */
+ def flatMap[U](f: FlatMapFunction[T, U], encoder: Encoder[U]): Dataset[U] = {
+ flatMap(UdfUtils.flatMapFuncToScalaFunc(f))(encoder)
+ }
+
+ /**
+ * Applies a function `f` to all rows.
+ *
+ * @group action
+ * @since 3.5.0
+ */
+ def foreach(f: T => Unit): Unit = {
+ foreachPartition(UdfUtils.foreachFuncToForeachPartitionsAdaptor(f))
+ }
+
+ /**
+ * (Java-specific) Runs `func` on each element of this Dataset.
+ *
+ * @group action
+ * @since 3.5.0
+ */
+ def foreach(func: ForeachFunction[T]): Unit =
foreach(UdfUtils.foreachFuncToScalaFunc(func))
+
+ /**
+ * Applies a function `f` to each partition of this Dataset.
+ *
+ * @group action
+ * @since 3.5.0
+ */
+ def foreachPartition(f: Iterator[T] => Unit): Unit = {
+ // Delegate to mapPartition followed by a count to drop any result.
+
mapPartitions(UdfUtils.foreachPartitionFuncToMapPartitionsAdaptor(f))(PrimitiveBooleanEncoder)
Review Comment:
There is no count here. However I don't think you need one. If you make the
adaptor return an empty iterator, then no batches are produced on the executor,
subsequently the stream handler will send back an empty batch with a schema. I
think that is as efficient as we are going to get it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
