This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new e713d99 [SPARK-37663][CORE][TESTS] Mitigate
ConcurrentModificationException thrown from tests in SparkContextSuite
e713d99 is described below
commit e713d990e50cf59f3cab8d7c54042a278f751ba5
Author: Kousuke Saruta <[email protected]>
AuthorDate: Fri Dec 17 12:21:44 2021 -0800
[SPARK-37663][CORE][TESTS] Mitigate ConcurrentModificationException thrown
from tests in SparkContextSuite
### What changes were proposed in this pull request?
This PR fixes an issue that some tests in `SparkContextSuite` can throw
`ConcurrentModificationException` with Scala 2.13.
https://github.com/apache/spark/runs/4543047740?check_suite_focus=true#step:9:20851
The cause seems to be same as SPARK-37315 (#34583).
> Scala 2.13.7 includes an update to detect ConcurrentModificationException
more precisely.
scala/scala#9786
You can easily reproduce this issue by applying the following diff to
`master`.
```
diff --git a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
index bc809f11cc..e5dde84c6e 100644
--- a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
-1130,9 +1130,11 class SparkContextSuite extends SparkFunSuite with
LocalSparkContext with Eventu
sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0?" +
"invalidParam1=foo&invalidParam2=boo")
assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.7.0.jar")))
-
assert(logAppender.loggingEvents.exists(_.getRenderedMessage.contains(
- "Invalid parameters `invalidParam1,invalidParam2` found in Ivy URI
query " +
- "`invalidParam1=foo&invalidParam2=boo`.")))
+ assert(logAppender.loggingEvents.exists { x =>
+ Thread.sleep(1000)
+ x.getRenderedMessage.contains(
+ "Invalid parameters `invalidParam1,invalidParam2` found in Ivy
URI query " +
+ "`invalidParam1=foo&invalidParam2=boo`.")})
}
}
```
### Why are the changes needed?
Fix the flaky test.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Locally checked that no `ConcurrentModificationException` is thrown even if
a sleep is inserted like the diff shown above.
Closes #34922 from sarutak/fix-concurrent-access-issue.
Authored-by: Kousuke Saruta <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../scala/org/apache/spark/SparkContextSuite.scala | 34 +++++++++++++---------
1 file changed, 20 insertions(+), 14 deletions(-)
diff --git a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
index bc809f1..16e7bc9 100644
--- a/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkContextSuite.scala
@@ -1072,20 +1072,24 @@ class SparkContextSuite extends SparkFunSuite with
LocalSparkContext with Eventu
dependencyJars.foreach(jar =>
assert(sc.listJars().exists(_.contains(jar))))
- assert(logAppender.loggingEvents.count(_.getRenderedMessage.contains(
- "Added dependency jars of Ivy URI " +
- "ivy://org.apache.hive:hive-storage-api:2.7.0?transitive=true")) ==
1)
+ eventually(timeout(10.seconds), interval(1.second)) {
+ assert(logAppender.loggingEvents.count(_.getRenderedMessage.contains(
+ "Added dependency jars of Ivy URI " +
+ "ivy://org.apache.hive:hive-storage-api:2.7.0?transitive=true"))
== 1)
+ }
// test dependency jars exist
sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0?transitive=true")
- assert(logAppender.loggingEvents.count(_.getRenderedMessage.contains(
- "The dependency jars of Ivy URI " +
- "ivy://org.apache.hive:hive-storage-api:2.7.0?transitive=true")) ==
1)
- val existMsg =
logAppender.loggingEvents.filter(_.getRenderedMessage.contains(
- "The dependency jars of Ivy URI " +
- "ivy://org.apache.hive:hive-storage-api:2.7.0?transitive=true"))
- .head.getRenderedMessage
- dependencyJars.foreach(jar => assert(existMsg.contains(jar)))
+ eventually(timeout(10.seconds), interval(1.second)) {
+ assert(logAppender.loggingEvents.count(_.getRenderedMessage.contains(
+ "The dependency jars of Ivy URI " +
+ "ivy://org.apache.hive:hive-storage-api:2.7.0?transitive=true"))
== 1)
+ val existMsg =
logAppender.loggingEvents.filter(_.getRenderedMessage.contains(
+ "The dependency jars of Ivy URI " +
+ "ivy://org.apache.hive:hive-storage-api:2.7.0?transitive=true"))
+ .head.getRenderedMessage
+ dependencyJars.foreach(jar => assert(existMsg.contains(jar)))
+ }
}
}
@@ -1130,9 +1134,11 @@ class SparkContextSuite extends SparkFunSuite with
LocalSparkContext with Eventu
sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0?" +
"invalidParam1=foo&invalidParam2=boo")
assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.7.0.jar")))
- assert(logAppender.loggingEvents.exists(_.getRenderedMessage.contains(
- "Invalid parameters `invalidParam1,invalidParam2` found in Ivy URI
query " +
- "`invalidParam1=foo&invalidParam2=boo`.")))
+ eventually(timeout(10.seconds), interval(1.second)) {
+ assert(logAppender.loggingEvents.exists(_.getRenderedMessage.contains(
+ "Invalid parameters `invalidParam1,invalidParam2` found in Ivy URI
query " +
+ "`invalidParam1=foo&invalidParam2=boo`.")))
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
