This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 34a39a2 [SPARK-38678][TESTS] Enable RocksDB tests on Apple Silicon on
MacOS
34a39a2 is described below
commit 34a39a2c87a31f5bf0d42f9171141eeae198e433
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Mon Mar 28 15:03:28 2022 -0700
[SPARK-38678][TESTS] Enable RocksDB tests on Apple Silicon on MacOS
### What changes were proposed in this pull request?
This PR aims to enable `RocksDB`-related test coverage on Apple Silicon on
MacOS.
- In `kvstore`, remove the following in `RocksDBSuite` and
`RocksDBIteratorSuite` Java test.
```
assumeFalse(SystemUtils.IS_OS_MAC_OSX &&
SystemUtils.OS_ARCH.equals("aarch64"));
```
- Remove `ExtendedRocksDBTest` test tag.
- Remove `ExtendedRocksDBTest` test tag annotation from the following
`core` module tests.
- `HistoryServerDiskManagerSuite`
- `HybridStoreSuite`
- `AppStatusListenerSuite`
- Remove `ExtendedRocksDBTest` test tag annotation from the following
`streaming` module tests.
- `RocksDBStateStoreIntegrationSuite`
- `RocksDBStateStoreSuite`
- `RocksDBSuite`
- Remove `RocksDBStateStoreProvider` exclusion from `providerOptions`.
- `StreamingSessionWindowSuite`
### Why are the changes needed?
RocksDB 7 supports Apple Silicon officially.
### Does this PR introduce _any_ user-facing change?
No. This is a dev-only change.
### How was this patch tested?
Manually test on Apple Silicon environment with Java 17.
```
$ build/sbt "kvstore/test" "core/testOnly *RocksDB*" "sql/testOnly
*RocksDB*" "sql/testOnly *.StreamingSessionWindowSuite"
```
Closes #35993 from dongjoon-hyun/SPARK-38678.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../spark/util/kvstore/RocksDBIteratorSuite.java | 4 ---
.../apache/spark/util/kvstore/RocksDBSuite.java | 3 ---
.../org/apache/spark/tags/ExtendedRocksDBTest.java | 30 ----------------------
.../history/HistoryServerDiskManagerSuite.scala | 3 +--
.../spark/deploy/history/HybridStoreSuite.scala | 3 +--
.../spark/status/AppStatusListenerSuite.scala | 3 +--
.../state/RocksDBStateStoreIntegrationSuite.scala | 2 --
.../streaming/state/RocksDBStateStoreSuite.scala | 2 --
.../execution/streaming/state/RocksDBSuite.scala | 2 --
.../streaming/StreamingSessionWindowSuite.scala | 8 +-----
10 files changed, 4 insertions(+), 56 deletions(-)
diff --git
a/common/kvstore/src/test/java/org/apache/spark/util/kvstore/RocksDBIteratorSuite.java
b/common/kvstore/src/test/java/org/apache/spark/util/kvstore/RocksDBIteratorSuite.java
index d4bfc7e..5450f65 100644
---
a/common/kvstore/src/test/java/org/apache/spark/util/kvstore/RocksDBIteratorSuite.java
+++
b/common/kvstore/src/test/java/org/apache/spark/util/kvstore/RocksDBIteratorSuite.java
@@ -20,11 +20,8 @@ package org.apache.spark.util.kvstore;
import java.io.File;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.SystemUtils;
import org.junit.AfterClass;
-import static org.junit.Assume.assumeFalse;
-
public class RocksDBIteratorSuite extends DBIteratorSuite {
private static File dbpath;
@@ -42,7 +39,6 @@ public class RocksDBIteratorSuite extends DBIteratorSuite {
@Override
protected KVStore createStore() throws Exception {
- assumeFalse(SystemUtils.IS_OS_MAC_OSX &&
SystemUtils.OS_ARCH.equals("aarch64"));
dbpath = File.createTempFile("test.", ".rdb");
dbpath.delete();
db = new RocksDB(dbpath);
diff --git
a/common/kvstore/src/test/java/org/apache/spark/util/kvstore/RocksDBSuite.java
b/common/kvstore/src/test/java/org/apache/spark/util/kvstore/RocksDBSuite.java
index cd18d22..04463ee 100644
---
a/common/kvstore/src/test/java/org/apache/spark/util/kvstore/RocksDBSuite.java
+++
b/common/kvstore/src/test/java/org/apache/spark/util/kvstore/RocksDBSuite.java
@@ -27,14 +27,12 @@ import java.util.stream.StreamSupport;
import com.google.common.collect.ImmutableSet;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.SystemUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.rocksdb.RocksIterator;
import static org.junit.Assert.*;
-import static org.junit.Assume.assumeFalse;
public class RocksDBSuite {
@@ -53,7 +51,6 @@ public class RocksDBSuite {
@Before
public void setup() throws Exception {
- assumeFalse(SystemUtils.IS_OS_MAC_OSX &&
SystemUtils.OS_ARCH.equals("aarch64"));
dbpath = File.createTempFile("test.", ".rdb");
dbpath.delete();
db = new RocksDB(dbpath);
diff --git
a/common/tags/src/test/java/org/apache/spark/tags/ExtendedRocksDBTest.java
b/common/tags/src/test/java/org/apache/spark/tags/ExtendedRocksDBTest.java
deleted file mode 100644
index b4cae4b..0000000
--- a/common/tags/src/test/java/org/apache/spark/tags/ExtendedRocksDBTest.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.tags;
-
-import org.scalatest.TagAnnotation;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-@TagAnnotation
-@Retention(RetentionPolicy.RUNTIME)
-@Target({ElementType.METHOD, ElementType.TYPE})
-public @interface ExtendedRocksDBTest { }
diff --git
a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerDiskManagerSuite.scala
b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerDiskManagerSuite.scala
index c534d66..9d1d741 100644
---
a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerDiskManagerSuite.scala
+++
b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerDiskManagerSuite.scala
@@ -28,7 +28,7 @@ import org.apache.spark.{SparkConf, SparkFunSuite}
import org.apache.spark.internal.config.History._
import org.apache.spark.internal.config.History.HybridStoreDiskBackend
import org.apache.spark.status.KVUtils
-import org.apache.spark.tags.{ExtendedLevelDBTest, ExtendedRocksDBTest}
+import org.apache.spark.tags.ExtendedLevelDBTest
import org.apache.spark.util.{ManualClock, Utils}
import org.apache.spark.util.kvstore.KVStore
@@ -232,7 +232,6 @@ class HistoryServerDiskManagerUseLevelDBSuite extends
HistoryServerDiskManagerSu
override protected def extension: String = ".ldb"
}
-@ExtendedRocksDBTest
class HistoryServerDiskManagerUseRocksDBSuite extends
HistoryServerDiskManagerSuite {
override protected def backend: HybridStoreDiskBackend.Value =
HybridStoreDiskBackend.ROCKSDB
override protected def extension: String = ".rdb"
diff --git
a/core/src/test/scala/org/apache/spark/deploy/history/HybridStoreSuite.scala
b/core/src/test/scala/org/apache/spark/deploy/history/HybridStoreSuite.scala
index c376bba..9c06c99 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/HybridStoreSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/HybridStoreSuite.scala
@@ -28,7 +28,7 @@ import org.scalatest.time.SpanSugar._
import org.apache.spark.SparkFunSuite
import org.apache.spark.status.KVUtils._
-import org.apache.spark.tags.{ExtendedLevelDBTest, ExtendedRocksDBTest}
+import org.apache.spark.tags.ExtendedLevelDBTest
import org.apache.spark.util.kvstore._
abstract class HybridStoreSuite extends SparkFunSuite with BeforeAndAfter with
TimeLimits {
@@ -207,7 +207,6 @@ class LevelDBHybridStoreSuite extends HybridStoreSuite {
}
}
-@ExtendedRocksDBTest
class RocksDBHybridStoreSuite extends HybridStoreSuite {
before {
dbpath = File.createTempFile("test.", ".rdb")
diff --git
a/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
b/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
index 5e2e931..97a59f8 100644
--- a/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala
@@ -37,7 +37,7 @@ import org.apache.spark.scheduler.cluster._
import org.apache.spark.status.ListenerEventsTestHelper._
import org.apache.spark.status.api.v1
import org.apache.spark.storage._
-import org.apache.spark.tags.{ExtendedLevelDBTest, ExtendedRocksDBTest}
+import org.apache.spark.tags.ExtendedLevelDBTest
import org.apache.spark.util.Utils
import org.apache.spark.util.kvstore.{InMemoryStore, KVStore}
@@ -1899,7 +1899,6 @@ class AppStatusListenerWithLevelDBSuite extends
AppStatusListenerSuite {
.set(HYBRID_STORE_DISK_BACKEND, HybridStoreDiskBackend.LEVELDB.toString)
}
-@ExtendedRocksDBTest
class AppStatusListenerWithRocksDBSuite extends AppStatusListenerSuite {
override def conf: SparkConf = super.conf
.set(HYBRID_STORE_DISK_BACKEND, HybridStoreDiskBackend.ROCKSDB.toString)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreIntegrationSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreIntegrationSuite.scala
index 0678cfc..25d0a80 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreIntegrationSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreIntegrationSuite.scala
@@ -27,9 +27,7 @@ import
org.apache.spark.sql.execution.streaming.{MemoryStream, StreamingQueryWra
import org.apache.spark.sql.functions.count
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.streaming._
-import org.apache.spark.tags.ExtendedRocksDBTest
-@ExtendedRocksDBTest
class RocksDBStateStoreIntegrationSuite extends StreamTest {
import testImplicits._
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreSuite.scala
index 5753f88..c93d0f0 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreSuite.scala
@@ -30,11 +30,9 @@ import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.execution.streaming.StatefulOperatorStateInfo
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
-import org.apache.spark.tags.ExtendedRocksDBTest
import org.apache.spark.unsafe.Platform
import org.apache.spark.util.Utils
-@ExtendedRocksDBTest
class RocksDBStateStoreSuite extends
StateStoreSuiteBase[RocksDBStateStoreProvider]
with BeforeAndAfter {
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala
index 91cd91b..35cc8c6 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala
@@ -29,10 +29,8 @@ import org.apache.spark._
import org.apache.spark.sql.catalyst.util.quietly
import org.apache.spark.sql.execution.streaming.CreateAtomicTestManager
import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.tags.ExtendedRocksDBTest
import org.apache.spark.util.{ThreadUtils, Utils}
-@ExtendedRocksDBTest
class RocksDBSuite extends SparkFunSuite {
test("RocksDB: get, put, iterator, commit, load") {
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingSessionWindowSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingSessionWindowSuite.scala
index 3ed23ba..c54a19d 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingSessionWindowSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingSessionWindowSuite.scala
@@ -28,7 +28,6 @@ import org.apache.spark.sql.execution.streaming.MemoryStream
import
org.apache.spark.sql.execution.streaming.state.{HDFSBackedStateStoreProvider,
RocksDBStateStoreProvider}
import org.apache.spark.sql.functions.{count, session_window, sum}
import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.util.Utils
class StreamingSessionWindowSuite extends StreamTest
with BeforeAndAfter with Matchers with Logging {
@@ -44,17 +43,12 @@ class StreamingSessionWindowSuite extends StreamTest
val mergingSessionOptions = Seq(true, false).map { value =>
(SQLConf.STREAMING_SESSION_WINDOW_MERGE_SESSIONS_IN_LOCAL_PARTITION.key,
value)
}
- var providerOptions = Seq(
+ val providerOptions = Seq(
classOf[HDFSBackedStateStoreProvider].getCanonicalName,
classOf[RocksDBStateStoreProvider].getCanonicalName
).map { value =>
(SQLConf.STATE_STORE_PROVIDER_CLASS.key, value.stripSuffix("$"))
}
- // RocksDB doesn't support Apple Silicon yet
- if (Utils.isMacOnAppleSilicon) {
- providerOptions = providerOptions
-
.filterNot(_._2.contains(classOf[RocksDBStateStoreProvider].getSimpleName))
- }
val availableOptions = for (
opt1 <- mergingSessionOptions;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
