hvanhovell commented on code in PR #40628:
URL: https://github.com/apache/spark/pull/40628#discussion_r1156367020
##########
connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/Dataset.scala:
##########
@@ -2468,6 +2470,153 @@ class Dataset[T] private[sql] (
*/
def transform[U](t: Dataset[T] => Dataset[U]): Dataset[U] = t(this)
+ /**
+ * (Scala-specific) Returns a new Dataset that only contains elements where
`func` returns
+ * `true`.
+ *
+ * @group typedrel
+ * @since 3.5.0
+ */
+ def filter(func: T => Boolean): Dataset[T] = {
+ val udf = ScalarUserDefinedFunction(
+ function = func,
+ inputEncoders = encoder :: Nil,
+ outputEncoder = PrimitiveBooleanEncoder,
+ name = None,
+ nullable = false,
+ deterministic = true)
+ sparkSession.newDataset[T](encoder) { builder =>
+ builder.getFilterBuilder
+ .setInput(plan.getRoot)
+ .setCondition(udf.apply().expr)
+ }
+ }
+
+ /**
+ * (Java-specific) Returns a new Dataset that only contains elements where
`func` returns
+ * `true`.
+ *
+ * @group typedrel
+ * @since 3.5.0
+ */
+ def filter(f: FilterFunction[T]): Dataset[T] = {
+ filter(UdfUtils.filterFuncToScalaFunc(f))
+ }
+
+ /**
+ * (Scala-specific) Returns a new Dataset that contains the result of
applying `func` to each
+ * element.
+ *
+ * @group typedrel
+ * @since 3.5.0
+ */
+ def map[U: Encoder](f: T => U): Dataset[U] = {
+ mapPartitions(UdfUtils.mapFuncToMapPartitionsAdaptor(f))
+ }
+
+ /**
+ * (Java-specific) Returns a new Dataset that contains the result of
applying `func` to each
+ * element.
+ *
+ * @group typedrel
+ * @since 3.5.0
+ */
+ def map[U](f: MapFunction[T, U], encoder: Encoder[U]): Dataset[U] = {
+ map(UdfUtils.mapFunctionToScalaFunc(f))(encoder)
+ }
+
+ /**
+ * (Scala-specific) Returns a new Dataset that contains the result of
applying `func` to each
+ * partition.
+ *
+ * @group typedrel
+ * @since 3.5.0
+ */
+ def mapPartitions[U: Encoder](func: Iterator[T] => Iterator[U]): Dataset[U]
= {
+ val outputEncoder = encoderFor[U]
+ val udf = ScalarUserDefinedFunction(
+ function = func,
+ inputEncoders = encoder :: Nil,
+ outputEncoder = outputEncoder)
+ sparkSession.newDataset(outputEncoder) { builder =>
+ builder.getMapPartitionsBuilder
+ .setInput(plan.getRoot)
+ .setFunc(udf.apply().expr.getCommonInlineUserDefinedFunction)
+ }
+ }
+
+ /**
+ * (Java-specific) Returns a new Dataset that contains the result of
applying `f` to each
+ * partition.
+ *
+ * @group typedrel
+ * @since 3.5.0
+ */
+ def mapPartitions[U](f: MapPartitionsFunction[T, U], encoder: Encoder[U]):
Dataset[U] = {
+ mapPartitions(UdfUtils.mapPartitionsFuncToScalaFunc(f))(encoder)
+ }
+
+ /**
+ * (Scala-specific) Returns a new Dataset by first applying a function to
all elements of this
+ * Dataset, and then flattening the results.
+ *
+ * @group typedrel
+ * @since 3.5.0
+ */
+ def flatMap[U: Encoder](func: T => TraversableOnce[U]): Dataset[U] =
+ mapPartitions(UdfUtils.flatMapFuncToMapPartitionsAdaptor(func))
+
+ /**
+ * (Java-specific) Returns a new Dataset by first applying a function to all
elements of this
+ * Dataset, and then flattening the results.
+ *
+ * @group typedrel
+ * @since 3.5.0
+ */
+ def flatMap[U](f: FlatMapFunction[T, U], encoder: Encoder[U]): Dataset[U] = {
+ flatMap(UdfUtils.flatMapFuncToScalaFunc(f))(encoder)
+ }
+
+ /**
+ * Applies a function `f` to all rows.
+ *
+ * @group action
+ * @since 3.5.0
+ */
+ def foreach(f: T => Unit): Unit = {
+ foreachPartition(UdfUtils.foreachFuncToForeachPartitionsAdaptor(f))
+ }
+
+ /**
+ * (Java-specific) Runs `func` on each element of this Dataset.
+ *
+ * @group action
+ * @since 3.5.0
+ */
+ def foreach(func: ForeachFunction[T]): Unit =
foreach(UdfUtils.foreachFuncToScalaFunc(func))
+
+ /**
+ * Applies a function `f` to each partition of this Dataset.
+ *
+ * @group action
+ * @since 3.5.0
+ */
+ def foreachPartition(f: Iterator[T] => Unit): Unit = {
+ // Delegate to mapPartition followed by a count to drop any result.
+
mapPartitions(UdfUtils.foreachPartitionFuncToMapPartitionsAdaptor(f))(PrimitiveBooleanEncoder)
Review Comment:
Can you use an empty `RowEncoder` here? That is a bit more expected.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
