This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new a826469d0be [SPARK-38851][CORE][TESTS] Refactor `HistoryServerSuite`
to add UTs for RocksDB
a826469d0be is described below
commit a826469d0be60dc344bc38445f621a4ec20861c9
Author: yangjie01 <[email protected]>
AuthorDate: Mon Apr 18 12:19:20 2022 -0700
[SPARK-38851][CORE][TESTS] Refactor `HistoryServerSuite` to add UTs for
RocksDB
### What changes were proposed in this pull request?
This pr expand `HistoryServerSuite` to add UTs for RocksDB scenarios.
### Why are the changes needed?
Add more UTs for RocksDB on Apple Silicon on MacOS
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
- Pass GA
- Manual test on Apple Silicon environment：
```
build/sbt "core/testOnly *RocksDBBackendHistoryServerSuite*"
```
```
[info] Run completed in 31 seconds, 100 milliseconds.
[info] Total number of tests run: 73
[info] Suites: completed 1, aborted 0
[info] Tests: succeeded 73, failed 0, canceled 0, ignored 0, pending 0
[info] All tests passed.
```
Closes #36138 from LuciferYang/SPARK-38851.
Authored-by: yangjie01 <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../spark/deploy/history/HistoryServerSuite.scala | 37 ++++++++++++++--------
1 file changed, 24 insertions(+), 13 deletions(-)
diff --git
a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
index 25f962aaa65..02a32a80ddd 100644
---
a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
+++
b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
@@ -49,6 +49,7 @@ import org.apache.spark.internal.config.Tests.IS_TESTING
import org.apache.spark.internal.config.UI._
import org.apache.spark.status.api.v1.ApplicationInfo
import org.apache.spark.status.api.v1.JobData
+import org.apache.spark.tags.ExtendedLevelDBTest
import org.apache.spark.ui.SparkUI
import org.apache.spark.util.{ResetSystemProperties, ShutdownHookManager,
Utils}
@@ -63,8 +64,8 @@ import org.apache.spark.util.{ResetSystemProperties,
ShutdownHookManager, Utils}
* expectations. However, in general this should be done with extreme
caution, as the metrics
* are considered part of Spark's public api.
*/
-class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with
Matchers with MockitoSugar
- with JsonTestUtils with Eventually with WebBrowser with LocalSparkContext
+abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter
with Matchers
+ with MockitoSugar with JsonTestUtils with Eventually with WebBrowser with
LocalSparkContext
with ResetSystemProperties {
private val logDir = getTestResourcePath("spark-events")
@@ -75,6 +76,10 @@ class HistoryServerSuite extends SparkFunSuite with
BeforeAndAfter with Matchers
private var server: HistoryServer = null
private var port: Int = -1
+ protected def diskBackend: HybridStoreDiskBackend.Value
+
+ def getExpRoot: File = expRoot
+
def init(extraConf: (String, String)*): Unit = {
Utils.deleteRecursively(storeDir)
assert(storeDir.mkdir())
@@ -85,11 +90,8 @@ class HistoryServerSuite extends SparkFunSuite with
BeforeAndAfter with Matchers
.set(LOCAL_STORE_DIR, storeDir.getAbsolutePath())
.set(EVENT_LOG_STAGE_EXECUTOR_METRICS, true)
.set(EXECUTOR_PROCESS_TREE_METRICS_ENABLED, true)
+ .set(HYBRID_STORE_DISK_BACKEND, diskBackend.toString)
conf.setAll(extraConf)
- // Since LevelDB doesn't support Apple Silicon yet, fallback to in-memory
provider
- if (Utils.isMacOnAppleSilicon) {
- conf.remove(LOCAL_STORE_DIR)
- }
provider = new FsHistoryProvider(conf)
provider.checkForLogs()
val securityManager = HistoryServer.createSecurityManager(conf)
@@ -393,10 +395,7 @@ class HistoryServerSuite extends SparkFunSuite with
BeforeAndAfter with Matchers
.set(EVENT_LOG_ENABLED, true)
.set(LOCAL_STORE_DIR, storeDir.getAbsolutePath())
.remove(IS_TESTING)
- // Since LevelDB doesn't support Apple Silicon yet, fallback to in-memory
provider
- if (Utils.isMacOnAppleSilicon) {
- myConf.remove(LOCAL_STORE_DIR)
- }
+ .set(HYBRID_STORE_DISK_BACKEND, diskBackend.toString)
val provider = new FsHistoryProvider(myConf)
val securityManager = HistoryServer.createSecurityManager(myConf)
@@ -715,9 +714,10 @@ object HistoryServerSuite {
// generate the "expected" results for the characterization tests. Just
blindly assume the
// current behavior is correct, and write out the returned json to the
test/resource files
- val suite = new HistoryServerSuite
- FileUtils.deleteDirectory(suite.expRoot)
- suite.expRoot.mkdirs()
+ // SPARK-38851: Use LevelDB backend because it is the default.
+ val suite = new LevelDBBackendHistoryServerSuite
+ FileUtils.deleteDirectory(suite.getExpRoot)
+ suite.getExpRoot.mkdirs()
try {
suite.init()
suite.cases.foreach { case (name, path) =>
@@ -792,3 +792,14 @@ class FakeAuthFilter extends Filter {
object FakeAuthFilter {
val FAKE_HTTP_USER = "HTTP_USER"
}
+
+@ExtendedLevelDBTest
+class LevelDBBackendHistoryServerSuite extends HistoryServerSuite {
+ override protected def diskBackend: History.HybridStoreDiskBackend.Value =
+ HybridStoreDiskBackend.LEVELDB
+}
+
+class RocksDBBackendHistoryServerSuite extends HistoryServerSuite {
+ override protected def diskBackend: History.HybridStoreDiskBackend.Value =
+ HybridStoreDiskBackend.ROCKSDB
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
