LuciferYang commented on code in PR #36119:
URL: https://github.com/apache/spark/pull/36119#discussion_r846706060
##########
core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala:
##########
@@ -1261,74 +1328,6 @@ class FsHistoryProviderSuite extends SparkFunSuite with
Matchers with Logging {
}
}
- test("backwards compatibility with LogInfo from Spark 2.4") {
Review Comment:
The change may as follows:
```
Index:
core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git
a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
---
a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
(revision b57c93b2ab5d6b86cefc8eb0ce8a01c596e0630d)
+++
b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
(revision 5fc7a99519182b336ba5575d7e236182410211b5)
@@ -52,11 +52,24 @@
import org.apache.spark.status.AppStatusStore
import org.apache.spark.status.KVUtils.KVStoreScalaSerializer
import org.apache.spark.status.api.v1.{ApplicationAttemptInfo,
ApplicationInfo}
+import org.apache.spark.tags.ExtendedLevelDBTest
import org.apache.spark.util.{Clock, JsonProtocol, ManualClock, Utils}
import org.apache.spark.util.kvstore.InMemoryStore
import org.apache.spark.util.logging.DriverLogger
-class FsHistoryProviderSuite extends SparkFunSuite with Matchers with
Logging {
+
+@ExtendedLevelDBTest
+class LevelDBBackendFsHistoryProviderSuite extends FsHistoryProviderSuite {
+ override protected def diskBackend: HybridStoreDiskBackend.Value =
+ HybridStoreDiskBackend.LEVELDB
+}
+
+class RocksDBBackendFsHistoryProviderSuite extends FsHistoryProviderSuite {
+ override protected def diskBackend: HybridStoreDiskBackend.Value =
+ HybridStoreDiskBackend.ROCKSDB
+}
+
+abstract class FsHistoryProviderSuite extends SparkFunSuite with Matchers
with Logging {
private var testDir: File = null
override def beforeEach(): Unit = {
@@ -72,6 +85,8 @@
}
}
+ protected def diskBackend: HybridStoreDiskBackend.Value
+
/** Create a fake log file using the new log format used in Spark 1.3+ */
private def newLogFile(
appId: String,
@@ -1651,11 +1666,10 @@
.set(FAST_IN_PROGRESS_PARSING, true)
if (!inMemory) {
- // LevelDB doesn't support Apple Silicon yet
- assume(!Utils.isMacOnAppleSilicon)
conf.set(LOCAL_STORE_DIR, Utils.createTempDir().getAbsolutePath())
}
conf.set(HYBRID_STORE_ENABLED, useHybridStore)
+ conf.set(HYBRID_STORE_DISK_BACKEND.key, diskBackend.toString)
conf
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
