Github user vanzin commented on a diff in the pull request:
https://github.com/apache/spark/pull/22504#discussion_r228673330
--- Diff:
core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
---
@@ -413,6 +417,68 @@ class FsHistoryProviderSuite extends SparkFunSuite
with BeforeAndAfter with Matc
}
}
+ test("driver log cleaner") {
+ val firstFileModifiedTime = TimeUnit.SECONDS.toMillis(10)
+ val secondFileModifiedTime = TimeUnit.SECONDS.toMillis(20)
+ val maxAge = TimeUnit.SECONDS.toSeconds(40)
+ val clock = new ManualClock(0)
+ val testConf = new SparkConf()
+ testConf.set("spark.history.fs.logDirectory",
+ Utils.createTempDir(namePrefix = "eventLog").getAbsolutePath())
+ testConf.set(DRIVER_LOG_DFS_DIR, testDir.getAbsolutePath())
+ testConf.set(DRIVER_LOG_CLEANER_ENABLED, true)
+ testConf.set(DRIVER_LOG_CLEANER_INTERVAL, maxAge / 4)
+ testConf.set(MAX_DRIVER_LOG_AGE_S, maxAge)
+ val provider = new FsHistoryProvider(testConf, clock)
+
+ val log1 = FileUtils.getFile(testDir, "1" +
DriverLogger.DRIVER_LOG_FILE_SUFFIX)
+ createEmptyFile(log1)
+ val modTime1 = System.currentTimeMillis()
+
+ clock.setTime(modTime1 + firstFileModifiedTime)
+ provider.cleanDriverLogs()
+
+ val log2 = FileUtils.getFile(testDir, "2" +
DriverLogger.DRIVER_LOG_FILE_SUFFIX)
+ createEmptyFile(log2)
+ val log3 = FileUtils.getFile(testDir, "3" +
DriverLogger.DRIVER_LOG_FILE_SUFFIX)
+ createEmptyFile(log3)
+ val modTime2 = System.currentTimeMillis()
+
+ clock.setTime(modTime1 + secondFileModifiedTime)
+ provider.cleanDriverLogs()
+
+ // This should not trigger any cleanup
+ provider.listing.view(classOf[LogInfo]).iterator().asScala.toSeq.size
should be(3)
+
+ // Should trigger cleanup for first file but not second one
+ clock.setTime(modTime1 + firstFileModifiedTime +
TimeUnit.SECONDS.toMillis(maxAge) + 1)
+ provider.cleanDriverLogs()
+ provider.listing.view(classOf[LogInfo]).iterator().asScala.toSeq.size
should be(2)
+ assert(!log1.exists())
+ assert(log2.exists())
+ assert(log3.exists())
+
+ // Should cleanup the second file but not the third file, as
filelength changed.
+ val writer = new OutputStreamWriter(new BufferedOutputStream(new
FileOutputStream(log3)))
--- End diff --
`Files.write` is shorter and nicer.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
