Github user vanzin commented on a diff in the pull request:
https://github.com/apache/spark/pull/22504#discussion_r230465790
--- Diff:
core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
---
@@ -413,6 +417,66 @@ class FsHistoryProviderSuite extends SparkFunSuite
with BeforeAndAfter with Matc
}
}
+ test("driver log cleaner") {
+ val firstFileModifiedTime = TimeUnit.SECONDS.toMillis(10)
+ val secondFileModifiedTime = TimeUnit.SECONDS.toMillis(20)
+ val maxAge = TimeUnit.SECONDS.toSeconds(40)
+ val clock = new ManualClock(0)
+ val testConf = new SparkConf()
+ testConf.set("spark.history.fs.logDirectory",
+ Utils.createTempDir(namePrefix = "eventLog").getAbsolutePath())
+ testConf.set(DRIVER_LOG_DFS_DIR, testDir.getAbsolutePath())
+ testConf.set(DRIVER_LOG_CLEANER_ENABLED, true)
+ testConf.set(DRIVER_LOG_CLEANER_INTERVAL, maxAge / 4)
+ testConf.set(MAX_DRIVER_LOG_AGE_S, maxAge)
+ val provider = new FsHistoryProvider(testConf, clock)
+
+ val log1 = FileUtils.getFile(testDir, "1" +
DriverLogger.DRIVER_LOG_FILE_SUFFIX)
+ createEmptyFile(log1)
+ clock.setTime(firstFileModifiedTime)
+ log1.setLastModified(clock.getTimeMillis())
+ provider.cleanDriverLogs()
+
+ val log2 = FileUtils.getFile(testDir, "2" +
DriverLogger.DRIVER_LOG_FILE_SUFFIX)
+ createEmptyFile(log2)
+ val log3 = FileUtils.getFile(testDir, "3" +
DriverLogger.DRIVER_LOG_FILE_SUFFIX)
+ createEmptyFile(log3)
+ clock.setTime(secondFileModifiedTime)
+ log2.setLastModified(clock.getTimeMillis())
+ log3.setLastModified(clock.getTimeMillis())
+ provider.cleanDriverLogs()
+
+ // This should not trigger any cleanup
--- End diff --
"This" is the call above this comment, isn't it? Seems like the comment is
in the wrong spot.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
