attilapiros commented on a change in pull request #23393:
[SPARK-26288][CORE]add initRegisteredExecutorsDB
URL: https://github.com/apache/spark/pull/23393#discussion_r265189239
##########
File path: core/src/test/scala/org/apache/spark/deploy/worker/WorkerSuite.scala
##########
@@ -245,4 +249,59 @@ class WorkerSuite extends SparkFunSuite with Matchers
with BeforeAndAfter {
ExecutorStateChanged("app1", 0, ExecutorState.EXITED, None, None))
assert(cleanupCalled.get() == value)
}
+
+ test("WorkDirCleanup cleans app dirs and shuffle metadata when " +
+ "spark.shuffle.service.db.enabled=true") {
+ testWorkDirCleanupAndRemoveMetadataWithConfig(true)
+ }
+
+ test("WorkdDirCleanup cleans only app dirs when" +
+ "spark.shuffle.service.db.enabled=false") {
+ testWorkDirCleanupAndRemoveMetadataWithConfig(false)
+ }
+
+ private def testWorkDirCleanupAndRemoveMetadataWithConfig(dbCleanupEnabled:
Boolean) = {
+ val conf = new SparkConf().set("spark.shuffle.service.db.enabled",
dbCleanupEnabled.toString)
+ conf.set("spark.worker.cleanup.appDataTtl", 60.toString)
+ conf.set("spark.shuffle.service.enabled", "true")
+
+ val appId = "app1"
+ val execId = "exec1"
+ val cleanupCalled = new AtomicBoolean(false)
+ when(shuffleService.applicationRemoved(any[String])).thenAnswer(new
Answer[Unit] {
+ override def answer(invocations: InvocationOnMock): Unit = {
+ cleanupCalled.set(true)
+ }
+ })
+ val externalShuffleServiceSupplier = new Supplier[ExternalShuffleService] {
+ override def get: ExternalShuffleService = shuffleService
+ }
+ val worker = makeWorker(conf, externalShuffleServiceSupplier)
+
+ val workDir = Option("/tmp/work").map(new File(_)).get
Review comment:
why this complexity? Is not is just something like:
```
val workDir = new File("/tmp/work")
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
