This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 3696ba8 [SPARK-34310][CORE][SQL][2.4] Replaces map and flatten with
flatMap
3696ba8 is described below
commit 3696ba80a8e772f652eb6d5faba90cd632563416
Author: yangjie01 <[email protected]>
AuthorDate: Tue Feb 2 09:26:36 2021 +0900
[SPARK-34310][CORE][SQL][2.4] Replaces map and flatten with flatMap
### What changes were proposed in this pull request?
Replaces `collection.map(f1).flatten(f2)` with `collection.flatMap` if
possible. it's semantically consistent, but looks simpler.
### Why are the changes needed?
Code Simpilefications.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Pass the Jenkins or GitHub Action
Closes #31417 from LuciferYang/SPARK-34310-24.
Authored-by: yangjie01 <[email protected]>
Signed-off-by: HyukjinKwon <[email protected]>
---
core/src/main/scala/org/apache/spark/rdd/SubtractedRDD.scala | 2 +-
.../spark/sql/execution/datasources/HadoopFileLinesReaderSuite.scala | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/rdd/SubtractedRDD.scala
b/core/src/main/scala/org/apache/spark/rdd/SubtractedRDD.scala
index a733eaa..4cfb3c4 100644
--- a/core/src/main/scala/org/apache/spark/rdd/SubtractedRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/SubtractedRDD.scala
@@ -119,7 +119,7 @@ private[spark] class SubtractedRDD[K: ClassTag, V:
ClassTag, W: ClassTag](
integrate(0, t => getSeq(t._1) += t._2)
// the second dep is rdd2; remove all of its keys
integrate(1, t => map.remove(t._1))
- map.asScala.iterator.map(t => t._2.iterator.map((t._1, _))).flatten
+ map.asScala.iterator.flatMap(t => t._2.iterator.map((t._1, _)))
}
override def clearDependencies() {
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/HadoopFileLinesReaderSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/HadoopFileLinesReaderSuite.scala
index 508614a..7ff7ba0 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/HadoopFileLinesReaderSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/HadoopFileLinesReaderSuite.scala
@@ -36,13 +36,13 @@ class HadoopFileLinesReaderSuite extends SharedSQLContext {
val delimOpt = delimiter.map(_.getBytes(StandardCharsets.UTF_8))
Files.write(path.toPath, text.getBytes(StandardCharsets.UTF_8))
- val lines = ranges.map { case (start, length) =>
+ val lines = ranges.flatMap { case (start, length) =>
val file = PartitionedFile(InternalRow.empty, path.getCanonicalPath,
start, length)
val hadoopConf = conf.getOrElse(spark.sessionState.newHadoopConf())
val reader = new HadoopFileLinesReader(file, delimOpt, hadoopConf)
reader.map(_.toString)
- }.flatten
+ }
lines
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
