dongjoon-hyun commented on code in PR #36832:
URL: https://github.com/apache/spark/pull/36832#discussion_r895388483
##########
core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala:
##########
@@ -221,6 +222,48 @@ abstract class FsHistoryProviderSuite extends
SparkFunSuite with Matchers with L
}
}
+ test("SPARK-39439: Check final file if in-progress event log file does not
exist") {
+ withTempDir { dir =>
+ val conf = createTestConf()
+ conf.set(HISTORY_LOG_DIR, dir.getAbsolutePath)
+ conf.set(EVENT_LOG_ROLLING_MAX_FILES_TO_RETAIN, 1)
+ conf.set(EVENT_LOG_COMPACTION_SCORE_THRESHOLD, 0.0d)
+ val hadoopConf = SparkHadoopUtil.newConfiguration(conf)
+ val fs = new Path(dir.getAbsolutePath).getFileSystem(hadoopConf)
+ val provider = new FsHistoryProvider(conf)
+
+ val inProgressFile = newLogFile("app1", None, inProgress = true)
+ val logAppender1 = new LogAppender("in-progress and final event log
files does not exist")
+ withLogAppender(logAppender1, level = Some(Level.WARN)) {
+ provider.mergeApplicationListing(
+ EventLogFileReader(fs, new Path(inProgressFile.toURI), None),
+ System.currentTimeMillis,
+ true
+ )
+ }
+ val logs1 =
logAppender1.loggingEvents.map(_.getMessage.getFormattedMessage)
+ .filter(_.contains(s"In-progress event log file does not exist: "))
Review Comment:
nit. `s"` -> `"`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
