This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 78bf448 [SPARK-34275][CORE][SQL][MLLIB][2.4] Replaces filter and size
with count
78bf448 is described below
commit 78bf4480dc4ceb8723b4c425952d8169c8043e5e
Author: yangjie01 <yangji...@baidu.com>
AuthorDate: Thu Jan 28 19:01:45 2021 +0900
[SPARK-34275][CORE][SQL][MLLIB][2.4] Replaces filter and size with count
### What changes were proposed in this pull request?
Use `count` to simplify `find + size(or length)` operation, it's
semantically consistent, but looks simpler.
**Before**
```
seq.filter(p).size
```
**After**
```
seq.count(p)
```
### Why are the changes needed?
Code Simpilefications.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Pass the Jenkins or GitHub Action
Closes #31376 from LuciferYang/SPARK-34275-24.
Authored-by: yangjie01 <yangji...@baidu.com>
Signed-off-by: HyukjinKwon <gurwls...@apache.org>
---
core/src/test/scala/org/apache/spark/SparkContextSuite.scala | 4 ++--
.../scala/org/apache/spark/ml/classification/NaiveBayesSuite.scala | 4 ++--
.../scala/org/apache/spark/sql/TypedImperativeAggregateSuite.scala | 4 ++--
3 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
index e1666a3..663306a 100644
--- a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
@@ -161,7 +161,7 @@ class SparkContextSuite extends SparkFunSuite with
LocalSparkContext with Eventu
}
x
}).count()
- assert(sc.listFiles().filter(_.contains("somesuffix1")).size == 1)
+ assert(sc.listFiles().count(_.contains("somesuffix1")) == 1)
} finally {
sc.stop()
}
@@ -172,7 +172,7 @@ class SparkContextSuite extends SparkFunSuite with
LocalSparkContext with Eventu
try {
sc = new SparkContext(new
SparkConf().setAppName("test").setMaster("local"))
sc.addJar(jarPath.toString)
- assert(sc.listJars().filter(_.contains("TestUDTF.jar")).size == 1)
+ assert(sc.listJars().count(_.contains("TestUDTF.jar")) == 1)
} finally {
sc.stop()
}
diff --git
a/mllib/src/test/scala/org/apache/spark/ml/classification/NaiveBayesSuite.scala
b/mllib/src/test/scala/org/apache/spark/ml/classification/NaiveBayesSuite.scala
index 5f9ab98..603b092 100644
---
a/mllib/src/test/scala/org/apache/spark/ml/classification/NaiveBayesSuite.scala
+++
b/mllib/src/test/scala/org/apache/spark/ml/classification/NaiveBayesSuite.scala
@@ -56,10 +56,10 @@ class NaiveBayesSuite extends MLTest with
DefaultReadWriteTest {
}
def validatePrediction(predictionAndLabels: Seq[Row]): Unit = {
- val numOfErrorPredictions = predictionAndLabels.filter {
+ val numOfErrorPredictions = predictionAndLabels.count {
case Row(prediction: Double, label: Double) =>
prediction != label
- }.length
+ }
// At least 80% of the predictions should be on.
assert(numOfErrorPredictions < predictionAndLabels.length / 5)
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/TypedImperativeAggregateSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/TypedImperativeAggregateSuite.scala
index c5fb173..067169d 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/TypedImperativeAggregateSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/TypedImperativeAggregateSuite.scala
@@ -147,9 +147,9 @@ class TypedImperativeAggregateSuite extends QueryTest with
SharedSQLContext {
val query = df.select(typedMax($"key"), count($"key"), typedMax($"value"),
count($"value"))
val maxKey = nullableData.map(_._1).filter(_ != null).max
- val countKey = nullableData.map(_._1).filter(_ != null).size
+ val countKey = nullableData.map(_._1).count(_ != null)
val maxValue = nullableData.map(_._2).filter(_ != null).max
- val countValue = nullableData.map(_._2).filter(_ != null).size
+ val countValue = nullableData.map(_._2).count(_ != null)
val expected = Seq(Row(maxKey, countKey, maxValue, countValue))
checkAnswer(query, expected)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
