This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 7ec3a5730bd [SPARK-38678][TESTS][FOLLOWUP] Enable RocksDB tests in
`AppStatusStoreSuite` and `StreamingQueryStatusListenerSuite` on Apple Silicon
on MacOS
7ec3a5730bd is described below
commit 7ec3a5730bd864089c22e19e823d87b107688378
Author: yangjie01 <[email protected]>
AuthorDate: Wed Apr 13 10:45:22 2022 -0700
[SPARK-38678][TESTS][FOLLOWUP] Enable RocksDB tests in
`AppStatusStoreSuite` and `StreamingQueryStatusListenerSuite` on Apple Silicon
on MacOS
### What changes were proposed in this pull request?
This pr aims to enable RocksDB tests in `AppStatusStoreSuite` and
`StreamingQueryStatusListenerSuite` on Apple Silicon on MacOS, it's a followup
of SPARK-38678.
### Why are the changes needed?
Enable more RocksDB related test on Apple Silicon on MacOS
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
- Pass GA
- Manual test on Apple Silicon environment：
```
build/sbt "core/testOnly *AppStatusStoreSuite*" "sql/testOnly
*StreamingQueryStatusListenerSuite*"
```
All tests passed.
Closes #36139 from LuciferYang/SPARK-38678-FOLLOWUP.
Authored-by: yangjie01 <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../apache/spark/status/AppStatusStoreSuite.scala | 26 +++++++++++++---------
.../ui/StreamingQueryStatusListenerSuite.scala | 1 -
2 files changed, 16 insertions(+), 11 deletions(-)
diff --git
a/core/src/test/scala/org/apache/spark/status/AppStatusStoreSuite.scala
b/core/src/test/scala/org/apache/spark/status/AppStatusStoreSuite.scala
index 70852164b89..b05f2b799d2 100644
--- a/core/src/test/scala/org/apache/spark/status/AppStatusStoreSuite.scala
+++ b/core/src/test/scala/org/apache/spark/status/AppStatusStoreSuite.scala
@@ -90,10 +90,6 @@ class AppStatusStoreSuite extends SparkFunSuite {
if (live) {
return AppStatusStore.createLiveStore(conf)
}
- // LevelDB doesn't support Apple Silicon yet
- if (Utils.isMacOnAppleSilicon && disk) {
- return null
- }
val store: KVStore = if (disk) {
conf.set(HYBRID_STORE_DISK_BACKEND, diskStoreType.toString)
@@ -106,12 +102,22 @@ class AppStatusStoreSuite extends SparkFunSuite {
new AppStatusStore(store)
}
- Seq(
- "disk leveldb" -> createAppStore(disk = true,
HybridStoreDiskBackend.LEVELDB, live = false),
- "disk rocksdb" -> createAppStore(disk = true,
HybridStoreDiskBackend.ROCKSDB, live = false),
- "in memory" -> createAppStore(disk = false, live = false),
- "in memory live" -> createAppStore(disk = false, live = true)
- ).foreach { case (hint, appStore) =>
+ private val cases = {
+ val baseCases = Seq(
+ "disk rocksdb" -> createAppStore(disk = true,
HybridStoreDiskBackend.ROCKSDB, live = false),
+ "in memory" -> createAppStore(disk = false, live = false),
+ "in memory live" -> createAppStore(disk = false, live = true)
+ )
+ if (Utils.isMacOnAppleSilicon) {
+ baseCases
+ } else {
+ Seq(
+ "disk leveldb" -> createAppStore(disk = true,
HybridStoreDiskBackend.LEVELDB, live = false)
+ ) ++ baseCases
+ }
+ }
+
+ cases.foreach { case (hint, appStore) =>
test(s"SPARK-26260: summary should contain only successful tasks' metrics
(store = $hint)") {
assume(appStore != null)
val store = appStore.store
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala
index 1d1b51354f8..1a51b58f4f6 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/streaming/ui/StreamingQueryStatusListenerSuite.scala
@@ -236,7 +236,6 @@ class StreamingQueryStatusListenerSuite extends StreamTest {
}
test("SPARK-38056: test writing StreamingQueryData to a RocksDB store") {
- assume(!Utils.isMacOnAppleSilicon)
val conf = new SparkConf()
.set(HYBRID_STORE_DISK_BACKEND, HybridStoreDiskBackend.ROCKSDB.toString)
val testDir = Utils.createTempDir()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
