Github user zsxwing commented on a diff in the pull request:
https://github.com/apache/spark/pull/21913#discussion_r206704241
--- Diff: core/src/main/scala/org/apache/spark/util/ThreadUtils.scala ---
@@ -254,4 +254,49 @@ private[spark] object ThreadUtils {
executor.shutdownNow()
}
}
+
+ /**
+ * Transforms input collection by applying the given function to each
element in parallel fashion.
+ *
+ * @param in - the input collection which should be transformed in
parallel.
+ * @param prefix - the prefix assigned to the underlying thread pool.
+ * @param maxThreads - maximum number of thread can be created during
execution.
+ * @param f - the lambda function will be applied to each element of
`in`.
+ * @tparam I - the type of elements in the input collection.
+ * @tparam O - the type of elements in resulted collection.
+ * @return new collection in which each element was given from the input
collection `in` by
+ * applying the lambda function `f`.
+ */
+ def parmap[I, O](
+ in: TraversableOnce[I],
+ prefix: String,
+ maxThreads: Int)(f: I => O): TraversableOnce[O] = {
+ val pool = newForkJoinPool(prefix, maxThreads)
+ try {
+ implicit val ec = ExecutionContext.fromExecutor(pool)
+ parmap(in)(f)
+ } finally {
+ pool.shutdown()
+ }
+ }
+
+ /**
+ * Transforms input collection by applying the given function to each
element in parallel fashion.
+ *
+ * @param in - the input collection which should be transformed in
parallel.
+ * @param f - the lambda function will be applied to each element of
`in`.
+ * @param ec - an execution context for parallel applying of the given
function `f`.
+ * @tparam I - the type of elements in the input collection.
+ * @tparam O - the type of elements in resulted collection.
+ * @return new collection in which each element was given from the input
collection `in` by
+ * applying the lambda function `f`.
+ */
+ def parmap[I, O](
--- End diff --
You can inline this method to the above one if you revert DStream changes.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
