This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new a826469d0be [SPARK-38851][CORE][TESTS] Refactor `HistoryServerSuite` to add UTs for RocksDB a826469d0be is described below commit a826469d0be60dc344bc38445f621a4ec20861c9 Author: yangjie01 <yangji...@baidu.com> AuthorDate: Mon Apr 18 12:19:20 2022 -0700 [SPARK-38851][CORE][TESTS] Refactor `HistoryServerSuite` to add UTs for RocksDB ### What changes were proposed in this pull request? This pr expand `HistoryServerSuite` to add UTs for RocksDB scenarios. ### Why are the changes needed? Add more UTs for RocksDB on Apple Silicon on MacOS ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? - Pass GA - Manual test on Apple Silicon environment: ``` build/sbt "core/testOnly *RocksDBBackendHistoryServerSuite*" ``` ``` [info] Run completed in 31 seconds, 100 milliseconds. [info] Total number of tests run: 73 [info] Suites: completed 1, aborted 0 [info] Tests: succeeded 73, failed 0, canceled 0, ignored 0, pending 0 [info] All tests passed. ``` Closes #36138 from LuciferYang/SPARK-38851. Authored-by: yangjie01 <yangji...@baidu.com> Signed-off-by: Dongjoon Hyun <dongj...@apache.org> --- .../spark/deploy/history/HistoryServerSuite.scala | 37 ++++++++++++++-------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala index 25f962aaa65..02a32a80ddd 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala @@ -49,6 +49,7 @@ import org.apache.spark.internal.config.Tests.IS_TESTING import org.apache.spark.internal.config.UI._ import org.apache.spark.status.api.v1.ApplicationInfo import org.apache.spark.status.api.v1.JobData +import org.apache.spark.tags.ExtendedLevelDBTest import org.apache.spark.ui.SparkUI import org.apache.spark.util.{ResetSystemProperties, ShutdownHookManager, Utils} @@ -63,8 +64,8 @@ import org.apache.spark.util.{ResetSystemProperties, ShutdownHookManager, Utils} * expectations. However, in general this should be done with extreme caution, as the metrics * are considered part of Spark's public api. */ -class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers with MockitoSugar - with JsonTestUtils with Eventually with WebBrowser with LocalSparkContext +abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers + with MockitoSugar with JsonTestUtils with Eventually with WebBrowser with LocalSparkContext with ResetSystemProperties { private val logDir = getTestResourcePath("spark-events") @@ -75,6 +76,10 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers private var server: HistoryServer = null private var port: Int = -1 + protected def diskBackend: HybridStoreDiskBackend.Value + + def getExpRoot: File = expRoot + def init(extraConf: (String, String)*): Unit = { Utils.deleteRecursively(storeDir) assert(storeDir.mkdir()) @@ -85,11 +90,8 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers .set(LOCAL_STORE_DIR, storeDir.getAbsolutePath()) .set(EVENT_LOG_STAGE_EXECUTOR_METRICS, true) .set(EXECUTOR_PROCESS_TREE_METRICS_ENABLED, true) + .set(HYBRID_STORE_DISK_BACKEND, diskBackend.toString) conf.setAll(extraConf) - // Since LevelDB doesn't support Apple Silicon yet, fallback to in-memory provider - if (Utils.isMacOnAppleSilicon) { - conf.remove(LOCAL_STORE_DIR) - } provider = new FsHistoryProvider(conf) provider.checkForLogs() val securityManager = HistoryServer.createSecurityManager(conf) @@ -393,10 +395,7 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers .set(EVENT_LOG_ENABLED, true) .set(LOCAL_STORE_DIR, storeDir.getAbsolutePath()) .remove(IS_TESTING) - // Since LevelDB doesn't support Apple Silicon yet, fallback to in-memory provider - if (Utils.isMacOnAppleSilicon) { - myConf.remove(LOCAL_STORE_DIR) - } + .set(HYBRID_STORE_DISK_BACKEND, diskBackend.toString) val provider = new FsHistoryProvider(myConf) val securityManager = HistoryServer.createSecurityManager(myConf) @@ -715,9 +714,10 @@ object HistoryServerSuite { // generate the "expected" results for the characterization tests. Just blindly assume the // current behavior is correct, and write out the returned json to the test/resource files - val suite = new HistoryServerSuite - FileUtils.deleteDirectory(suite.expRoot) - suite.expRoot.mkdirs() + // SPARK-38851: Use LevelDB backend because it is the default. + val suite = new LevelDBBackendHistoryServerSuite + FileUtils.deleteDirectory(suite.getExpRoot) + suite.getExpRoot.mkdirs() try { suite.init() suite.cases.foreach { case (name, path) => @@ -792,3 +792,14 @@ class FakeAuthFilter extends Filter { object FakeAuthFilter { val FAKE_HTTP_USER = "HTTP_USER" } + +@ExtendedLevelDBTest +class LevelDBBackendHistoryServerSuite extends HistoryServerSuite { + override protected def diskBackend: History.HybridStoreDiskBackend.Value = + HybridStoreDiskBackend.LEVELDB +} + +class RocksDBBackendHistoryServerSuite extends HistoryServerSuite { + override protected def diskBackend: History.HybridStoreDiskBackend.Value = + HybridStoreDiskBackend.ROCKSDB +} --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org