venkata91 commented on a change in pull request #33644:
URL: https://github.com/apache/spark/pull/33644#discussion_r692368891
##########
File path: core/src/main/scala/org/apache/spark/api/java/JavaRDDLike.scala
##########
@@ -450,6 +450,19 @@ trait JavaRDDLike[T, This <: JavaRDDLike[T, This]] extends
Serializable {
treeAggregate(zeroValue, seqOp, combOp, 2)
}
+ /**
+ * `org.apache.spark.api.java.JavaRDDLike.treeAggregate` with a parameter to
do the
+ * final aggregation on the executor.
+ */
+ def treeAggregate[U](
+ zeroValue: U,
Review comment:
nit: indent 4 spaces.
##########
File path: core/src/main/scala/org/apache/spark/rdd/RDD.scala
##########
@@ -1201,16 +1201,25 @@ abstract class RDD[T: ClassTag](
jobResult
}
+ def treeAggregate[U: ClassTag](zeroValue: U)(
+ seqOp: (U, T) => U,
+ combOp: (U, U) => U,
Review comment:
nit: same here indent 4 spaces.
##########
File path: core/src/main/scala/org/apache/spark/rdd/RDD.scala
##########
@@ -1201,16 +1201,25 @@ abstract class RDD[T: ClassTag](
jobResult
}
+ def treeAggregate[U: ClassTag](zeroValue: U)(
+ seqOp: (U, T) => U,
+ combOp: (U, U) => U,
+ depth: Int = 2): U = withScope {
+ treeAggregate(zeroValue, seqOp, combOp, depth, finalAggregateOnExecutor =
false)
+ }
+
/**
* Aggregates the elements of this RDD in a multi-level tree pattern.
* This method is semantically identical to
[[org.apache.spark.rdd.RDD#aggregate]].
*
* @param depth suggested depth of the tree (default: 2)
*/
- def treeAggregate[U: ClassTag](zeroValue: U)(
- seqOp: (U, T) => U,
- combOp: (U, U) => U,
- depth: Int = 2): U = withScope {
+ def treeAggregate[U: ClassTag](
+ zeroValue: U,
+ seqOp: (U, T) => U,
Review comment:
nit: indent 4 spaces.
##########
File path: core/src/main/scala/org/apache/spark/rdd/RDD.scala
##########
@@ -1201,16 +1201,25 @@ abstract class RDD[T: ClassTag](
jobResult
}
+ def treeAggregate[U: ClassTag](zeroValue: U)(
+ seqOp: (U, T) => U,
+ combOp: (U, U) => U,
+ depth: Int = 2): U = withScope {
+ treeAggregate(zeroValue, seqOp, combOp, depth, finalAggregateOnExecutor =
false)
+ }
+
/**
* Aggregates the elements of this RDD in a multi-level tree pattern.
* This method is semantically identical to
[[org.apache.spark.rdd.RDD#aggregate]].
*
Review comment:
Isn't this method doc for the above API? Also can we add a new scala doc
for the newly added method accordingly?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
