This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 34a39a2 [SPARK-38678][TESTS] Enable RocksDB tests on Apple Silicon on MacOS 34a39a2 is described below commit 34a39a2c87a31f5bf0d42f9171141eeae198e433 Author: Dongjoon Hyun <dongj...@apache.org> AuthorDate: Mon Mar 28 15:03:28 2022 -0700 [SPARK-38678][TESTS] Enable RocksDB tests on Apple Silicon on MacOS ### What changes were proposed in this pull request? This PR aims to enable `RocksDB`-related test coverage on Apple Silicon on MacOS. - In `kvstore`, remove the following in `RocksDBSuite` and `RocksDBIteratorSuite` Java test. ``` assumeFalse(SystemUtils.IS_OS_MAC_OSX && SystemUtils.OS_ARCH.equals("aarch64")); ``` - Remove `ExtendedRocksDBTest` test tag. - Remove `ExtendedRocksDBTest` test tag annotation from the following `core` module tests. - `HistoryServerDiskManagerSuite` - `HybridStoreSuite` - `AppStatusListenerSuite` - Remove `ExtendedRocksDBTest` test tag annotation from the following `streaming` module tests. - `RocksDBStateStoreIntegrationSuite` - `RocksDBStateStoreSuite` - `RocksDBSuite` - Remove `RocksDBStateStoreProvider` exclusion from `providerOptions`. - `StreamingSessionWindowSuite` ### Why are the changes needed? RocksDB 7 supports Apple Silicon officially. ### Does this PR introduce _any_ user-facing change? No. This is a dev-only change. ### How was this patch tested? Manually test on Apple Silicon environment with Java 17. ``` $ build/sbt "kvstore/test" "core/testOnly *RocksDB*" "sql/testOnly *RocksDB*" "sql/testOnly *.StreamingSessionWindowSuite" ``` Closes #35993 from dongjoon-hyun/SPARK-38678. Authored-by: Dongjoon Hyun <dongj...@apache.org> Signed-off-by: Dongjoon Hyun <dongj...@apache.org> --- .../spark/util/kvstore/RocksDBIteratorSuite.java | 4 --- .../apache/spark/util/kvstore/RocksDBSuite.java | 3 --- .../org/apache/spark/tags/ExtendedRocksDBTest.java | 30 ---------------------- .../history/HistoryServerDiskManagerSuite.scala | 3 +-- .../spark/deploy/history/HybridStoreSuite.scala | 3 +-- .../spark/status/AppStatusListenerSuite.scala | 3 +-- .../state/RocksDBStateStoreIntegrationSuite.scala | 2 -- .../streaming/state/RocksDBStateStoreSuite.scala | 2 -- .../execution/streaming/state/RocksDBSuite.scala | 2 -- .../streaming/StreamingSessionWindowSuite.scala | 8 +----- 10 files changed, 4 insertions(+), 56 deletions(-) diff --git a/common/kvstore/src/test/java/org/apache/spark/util/kvstore/RocksDBIteratorSuite.java b/common/kvstore/src/test/java/org/apache/spark/util/kvstore/RocksDBIteratorSuite.java index d4bfc7e..5450f65 100644 --- a/common/kvstore/src/test/java/org/apache/spark/util/kvstore/RocksDBIteratorSuite.java +++ b/common/kvstore/src/test/java/org/apache/spark/util/kvstore/RocksDBIteratorSuite.java @@ -20,11 +20,8 @@ package org.apache.spark.util.kvstore; import java.io.File; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.SystemUtils; import org.junit.AfterClass; -import static org.junit.Assume.assumeFalse; - public class RocksDBIteratorSuite extends DBIteratorSuite { private static File dbpath; @@ -42,7 +39,6 @@ public class RocksDBIteratorSuite extends DBIteratorSuite { @Override protected KVStore createStore() throws Exception { - assumeFalse(SystemUtils.IS_OS_MAC_OSX && SystemUtils.OS_ARCH.equals("aarch64")); dbpath = File.createTempFile("test.", ".rdb"); dbpath.delete(); db = new RocksDB(dbpath); diff --git a/common/kvstore/src/test/java/org/apache/spark/util/kvstore/RocksDBSuite.java b/common/kvstore/src/test/java/org/apache/spark/util/kvstore/RocksDBSuite.java index cd18d22..04463ee 100644 --- a/common/kvstore/src/test/java/org/apache/spark/util/kvstore/RocksDBSuite.java +++ b/common/kvstore/src/test/java/org/apache/spark/util/kvstore/RocksDBSuite.java @@ -27,14 +27,12 @@ import java.util.stream.StreamSupport; import com.google.common.collect.ImmutableSet; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.SystemUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.rocksdb.RocksIterator; import static org.junit.Assert.*; -import static org.junit.Assume.assumeFalse; public class RocksDBSuite { @@ -53,7 +51,6 @@ public class RocksDBSuite { @Before public void setup() throws Exception { - assumeFalse(SystemUtils.IS_OS_MAC_OSX && SystemUtils.OS_ARCH.equals("aarch64")); dbpath = File.createTempFile("test.", ".rdb"); dbpath.delete(); db = new RocksDB(dbpath); diff --git a/common/tags/src/test/java/org/apache/spark/tags/ExtendedRocksDBTest.java b/common/tags/src/test/java/org/apache/spark/tags/ExtendedRocksDBTest.java deleted file mode 100644 index b4cae4b..0000000 --- a/common/tags/src/test/java/org/apache/spark/tags/ExtendedRocksDBTest.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.tags; - -import org.scalatest.TagAnnotation; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -@TagAnnotation -@Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.METHOD, ElementType.TYPE}) -public @interface ExtendedRocksDBTest { } diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerDiskManagerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerDiskManagerSuite.scala index c534d66..9d1d741 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerDiskManagerSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerDiskManagerSuite.scala @@ -28,7 +28,7 @@ import org.apache.spark.{SparkConf, SparkFunSuite} import org.apache.spark.internal.config.History._ import org.apache.spark.internal.config.History.HybridStoreDiskBackend import org.apache.spark.status.KVUtils -import org.apache.spark.tags.{ExtendedLevelDBTest, ExtendedRocksDBTest} +import org.apache.spark.tags.ExtendedLevelDBTest import org.apache.spark.util.{ManualClock, Utils} import org.apache.spark.util.kvstore.KVStore @@ -232,7 +232,6 @@ class HistoryServerDiskManagerUseLevelDBSuite extends HistoryServerDiskManagerSu override protected def extension: String = ".ldb" } -@ExtendedRocksDBTest class HistoryServerDiskManagerUseRocksDBSuite extends HistoryServerDiskManagerSuite { override protected def backend: HybridStoreDiskBackend.Value = HybridStoreDiskBackend.ROCKSDB override protected def extension: String = ".rdb" diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HybridStoreSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HybridStoreSuite.scala index c376bba..9c06c99 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/HybridStoreSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/HybridStoreSuite.scala @@ -28,7 +28,7 @@ import org.scalatest.time.SpanSugar._ import org.apache.spark.SparkFunSuite import org.apache.spark.status.KVUtils._ -import org.apache.spark.tags.{ExtendedLevelDBTest, ExtendedRocksDBTest} +import org.apache.spark.tags.ExtendedLevelDBTest import org.apache.spark.util.kvstore._ abstract class HybridStoreSuite extends SparkFunSuite with BeforeAndAfter with TimeLimits { @@ -207,7 +207,6 @@ class LevelDBHybridStoreSuite extends HybridStoreSuite { } } -@ExtendedRocksDBTest class RocksDBHybridStoreSuite extends HybridStoreSuite { before { dbpath = File.createTempFile("test.", ".rdb") diff --git a/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala b/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala index 5e2e931..97a59f8 100644 --- a/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala +++ b/core/src/test/scala/org/apache/spark/status/AppStatusListenerSuite.scala @@ -37,7 +37,7 @@ import org.apache.spark.scheduler.cluster._ import org.apache.spark.status.ListenerEventsTestHelper._ import org.apache.spark.status.api.v1 import org.apache.spark.storage._ -import org.apache.spark.tags.{ExtendedLevelDBTest, ExtendedRocksDBTest} +import org.apache.spark.tags.ExtendedLevelDBTest import org.apache.spark.util.Utils import org.apache.spark.util.kvstore.{InMemoryStore, KVStore} @@ -1899,7 +1899,6 @@ class AppStatusListenerWithLevelDBSuite extends AppStatusListenerSuite { .set(HYBRID_STORE_DISK_BACKEND, HybridStoreDiskBackend.LEVELDB.toString) } -@ExtendedRocksDBTest class AppStatusListenerWithRocksDBSuite extends AppStatusListenerSuite { override def conf: SparkConf = super.conf .set(HYBRID_STORE_DISK_BACKEND, HybridStoreDiskBackend.ROCKSDB.toString) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreIntegrationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreIntegrationSuite.scala index 0678cfc..25d0a80 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreIntegrationSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreIntegrationSuite.scala @@ -27,9 +27,7 @@ import org.apache.spark.sql.execution.streaming.{MemoryStream, StreamingQueryWra import org.apache.spark.sql.functions.count import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.streaming._ -import org.apache.spark.tags.ExtendedRocksDBTest -@ExtendedRocksDBTest class RocksDBStateStoreIntegrationSuite extends StreamTest { import testImplicits._ diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreSuite.scala index 5753f88..c93d0f0 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreSuite.scala @@ -30,11 +30,9 @@ import org.apache.spark.sql.SparkSession import org.apache.spark.sql.execution.streaming.StatefulOperatorStateInfo import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ -import org.apache.spark.tags.ExtendedRocksDBTest import org.apache.spark.unsafe.Platform import org.apache.spark.util.Utils -@ExtendedRocksDBTest class RocksDBStateStoreSuite extends StateStoreSuiteBase[RocksDBStateStoreProvider] with BeforeAndAfter { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala index 91cd91b..35cc8c6 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala @@ -29,10 +29,8 @@ import org.apache.spark._ import org.apache.spark.sql.catalyst.util.quietly import org.apache.spark.sql.execution.streaming.CreateAtomicTestManager import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.tags.ExtendedRocksDBTest import org.apache.spark.util.{ThreadUtils, Utils} -@ExtendedRocksDBTest class RocksDBSuite extends SparkFunSuite { test("RocksDB: get, put, iterator, commit, load") { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingSessionWindowSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingSessionWindowSuite.scala index 3ed23ba..c54a19d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingSessionWindowSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingSessionWindowSuite.scala @@ -28,7 +28,6 @@ import org.apache.spark.sql.execution.streaming.MemoryStream import org.apache.spark.sql.execution.streaming.state.{HDFSBackedStateStoreProvider, RocksDBStateStoreProvider} import org.apache.spark.sql.functions.{count, session_window, sum} import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.util.Utils class StreamingSessionWindowSuite extends StreamTest with BeforeAndAfter with Matchers with Logging { @@ -44,17 +43,12 @@ class StreamingSessionWindowSuite extends StreamTest val mergingSessionOptions = Seq(true, false).map { value => (SQLConf.STREAMING_SESSION_WINDOW_MERGE_SESSIONS_IN_LOCAL_PARTITION.key, value) } - var providerOptions = Seq( + val providerOptions = Seq( classOf[HDFSBackedStateStoreProvider].getCanonicalName, classOf[RocksDBStateStoreProvider].getCanonicalName ).map { value => (SQLConf.STATE_STORE_PROVIDER_CLASS.key, value.stripSuffix("$")) } - // RocksDB doesn't support Apple Silicon yet - if (Utils.isMacOnAppleSilicon) { - providerOptions = providerOptions - .filterNot(_._2.contains(classOf[RocksDBStateStoreProvider].getSimpleName)) - } val availableOptions = for ( opt1 <- mergingSessionOptions; --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org