This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 3b581e0a758c [SPARK-58468][CORE] Support on-demand loading for single
event logs in SHS
3b581e0a758c is described below
commit 3b581e0a758ca7b3c08af661e2ee75f30d3fdffc
Author: Cheng Pan <[email protected]>
AuthorDate: Fri Jul 31 12:53:53 2026 -0700
[SPARK-58468][CORE] Support on-demand loading for single event logs in SHS
### What changes were proposed in this pull request?
This PR extends History Server on-demand event-log loading to single event
logs.
It adds `spark.history.fs.eventLog.onDemandLoadEnabled`, which is
independent from the existing rolling event-log configuration. When enabled,
the History Server probes completed, compressed, and in-progress single
event-log names after a requested application is absent from the listing.
### Why are the changes needed?
Single event logs remain supported, but on-demand loading currently only
supports rolling event logs. A requested application with a single event log
can therefore return Not Found until periodic scanning discovers its log.
### Does this PR introduce _any_ user-facing change?
Yes. History Server can load single event logs on demand. The new
`spark.history.fs.eventLog.onDemandLoadEnabled` configuration defaults to
`true`.
### How was this patch tested?
Added tests for:
- enabled and disabled single event-log on-demand loading
- uncompressed and LZF-compressed logs
- in-progress logs and cleaner safety
- attempt IDs
- multiple log directories
- scan-disabled directories
Ran the focused `FsHistoryProviderSuite` and
`SingleEventLogFileWriterSuite` tests. Also ran `SparkConfigBindingPolicySuite`.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: gpt-5.6-terra
Closes #57676 from pan3793/shs-on-demand-v1-log-loading.
Authored-by: Cheng Pan <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../spark/deploy/history/EventLogFileWriters.scala | 9 +
.../spark/deploy/history/FsHistoryProvider.scala | 30 +++-
.../org/apache/spark/internal/config/History.scala | 8 +
.../deploy/history/EventLogFileWritersSuite.scala | 14 ++
.../deploy/history/FsHistoryProviderSuite.scala | 186 ++++++++++++++++++++-
docs/monitoring.md | 8 +
6 files changed, 246 insertions(+), 9 deletions(-)
diff --git
a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala
b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala
index 601515e57dc8..a725da2f3a44 100644
---
a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala
+++
b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala
@@ -257,6 +257,15 @@ class SingleEventLogFileWriter(
}
object SingleEventLogFileWriter {
+ /** Returns names for completed and in-progress single event logs. */
+ private[history] def getLogFileNames(appId: String, appAttemptId:
Option[String]): Seq[String] = {
+ val logBaseName = EventLogFileWriter.nameForAppAndAttempt(appId,
appAttemptId)
+ val names = logBaseName +:
CompressionCodec.shortCompressionCodecNames.keys.toSeq.map {
+ codec => s"$logBaseName.$codec"
+ }
+ names ++ names.map(_ + EventLogFileWriter.IN_PROGRESS)
+ }
+
/**
* Return a file-system-safe path to the log file for the given application.
*
diff --git
a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
index a87db3557553..748aec37570d 100644
---
a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
+++
b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
@@ -412,16 +412,11 @@ private[history] class FsHistoryProvider(conf: SparkConf,
clock: Clock)
override def getLastUpdatedTime(): Long = lastScanTime.get()
override def getAppUI(appId: String, attemptId: Option[String]):
Option[LoadedAppUI] = {
- val logPath = RollingEventLogFilesWriter.EVENT_LOG_DIR_NAME_PREFIX +
- EventLogFileWriter.nameForAppAndAttempt(appId, attemptId)
val app = try {
load(appId)
} catch {
- case _: NoSuchElementException if
this.conf.get(EVENT_LOG_ROLLING_ON_DEMAND_LOAD_ENABLED) =>
- loadFromFallbackLocation(appId, attemptId, logPath) match {
- case Some(wrapper) => wrapper
- case None => return None
- }
+ case _: NoSuchElementException if isOnDemandLogLoadEnabled =>
+ loadFromFallbackLocations(appId, attemptId).getOrElse(return None)
case _: NoSuchElementException =>
return None
}
@@ -462,6 +457,27 @@ private[history] class FsHistoryProvider(conf: SparkConf,
clock: Clock)
Some(loadedUI)
}
+ private def loadFromFallbackLocations(
+ appId: String,
+ attemptId: Option[String]): Option[ApplicationInfoWrapper] = {
+ val logPaths = mutable.ArrayBuffer.empty[String]
+ if (conf.get(EVENT_LOG_ROLLING_ON_DEMAND_LOAD_ENABLED)) {
+ logPaths += RollingEventLogFilesWriter.EVENT_LOG_DIR_NAME_PREFIX +
+ EventLogFileWriter.nameForAppAndAttempt(appId, attemptId)
+ }
+ if (conf.get(EVENT_LOG_SINGLE_ON_DEMAND_LOAD_ENABLED)) {
+ logPaths ++= SingleEventLogFileWriter.getLogFileNames(appId, attemptId)
+ }
+ logPaths.iterator.map(loadFromFallbackLocation(appId, attemptId,
_)).collectFirst {
+ case Some(app) => app
+ }
+ }
+
+ private def isOnDemandLogLoadEnabled: Boolean = {
+ conf.get(EVENT_LOG_ROLLING_ON_DEMAND_LOAD_ENABLED) ||
+ conf.get(EVENT_LOG_SINGLE_ON_DEMAND_LOAD_ENABLED)
+ }
+
private def loadFromFallbackLocation(appId: String, attemptId:
Option[String], logPath: String)
: Option[ApplicationInfoWrapper] = {
// Call mergeApplicationListing to populate accurate metadata immediately.
diff --git a/core/src/main/scala/org/apache/spark/internal/config/History.scala
b/core/src/main/scala/org/apache/spark/internal/config/History.scala
index 1936ecf68103..570113d145a1 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/History.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/History.scala
@@ -193,6 +193,14 @@ private[spark] object History {
.booleanConf
.createWithDefault(true)
+ val EVENT_LOG_SINGLE_ON_DEMAND_LOAD_ENABLED =
+ ConfigBuilder("spark.history.fs.eventLog.onDemandLoadEnabled")
+ .doc("Whether to look up single event log locations on demand manner
before listing files.")
+ .version("4.3.0")
+ .withBindingPolicy(ConfigBindingPolicy.NOT_APPLICABLE)
+ .booleanConf
+ .createWithDefault(true)
+
val DRIVER_LOG_CLEANER_ENABLED =
ConfigBuilder("spark.history.fs.driverlog.cleaner.enabled")
.version("3.0.0")
.doc("Specifies whether the History Server should periodically clean up
driver logs from " +
diff --git
a/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileWritersSuite.scala
b/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileWritersSuite.scala
index 00a92c503be4..36c5b1915edd 100644
---
a/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileWritersSuite.scala
+++
b/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileWritersSuite.scala
@@ -352,6 +352,20 @@ class SingleEventLogFileWriterSuite extends
EventLogFileWritersSuite {
"a fine:mind$dollar{bills}.1", None, Some(CompressionCodec.LZ4)))
}
+ test("Event log file names") {
+ Seq(None, Some("attempt1")).foreach { attemptId =>
+ val baseName = attemptId.map(id => s"app1_$id").getOrElse("app1")
+ val names = SingleEventLogFileWriter.getLogFileNames("app1", attemptId)
+ assert(names.head === baseName)
+ assert(names.contains(s"$baseName.inprogress"))
+ assert(names.size === (1 +
CompressionCodec.shortCompressionCodecNames.size) * 2)
+ CompressionCodec.shortCompressionCodecNames.keys.foreach { codec =>
+ assert(names.contains(s"$baseName.$codec"))
+ assert(names.contains(s"$baseName.$codec.inprogress"))
+ }
+ }
+ }
+
override protected def createWriter(
appId: String,
appAttemptId: Option[String],
diff --git
a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
index 873431d7942e..d932265791dc 100644
---
a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
+++
b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
@@ -83,9 +83,10 @@ abstract class FsHistoryProviderSuite extends SparkFunSuite
with Matchers with P
appId: String,
appAttemptId: Option[String],
inProgress: Boolean,
- codec: Option[String] = None): File = {
+ codec: Option[String] = None,
+ logDir: File = testDir): File = {
val ip = if (inProgress) EventLogFileWriter.IN_PROGRESS else ""
- val logUri = SingleEventLogFileWriter.getLogPath(testDir.toURI, appId,
appAttemptId, codec)
+ val logUri = SingleEventLogFileWriter.getLogPath(logDir.toURI, appId,
appAttemptId, codec)
val logPath = new Path(logUri).toUri.getPath + ip
new File(logPath)
}
@@ -1731,6 +1732,7 @@ abstract class FsHistoryProviderSuite extends
SparkFunSuite with Matchers with P
val conf = createTestConf(true)
conf.set(HISTORY_LOG_DIR, dir.getAbsolutePath)
conf.set(EVENT_LOG_ROLLING_ON_DEMAND_LOAD_ENABLED, onDemandEnabled)
+ conf.set(EVENT_LOG_SINGLE_ON_DEMAND_LOAD_ENABLED, false)
val hadoopConf = SparkHadoopUtil.newConfiguration(conf)
val provider = new FsHistoryProvider(conf)
@@ -1759,6 +1761,186 @@ abstract class FsHistoryProviderSuite extends
SparkFunSuite with Matchers with P
}
}
+ test("Support spark.history.fs.eventLog.onDemandLoadEnabled") {
+ Seq(true, false).foreach { onDemandEnabled =>
+ Seq(None, Some(CompressionCodec.LZF)).foreach { codecName =>
+ withTempDir { dir =>
+ val conf = createTestConf(true)
+ conf.set(HISTORY_LOG_DIR, dir.getAbsolutePath)
+ conf.set(EVENT_LOG_ROLLING_ON_DEMAND_LOAD_ENABLED, false)
+ conf.set(EVENT_LOG_SINGLE_ON_DEMAND_LOAD_ENABLED, onDemandEnabled)
+ val provider = new FsHistoryProvider(conf)
+ val appId = s"app1-${codecName.getOrElse("none")}"
+ val logFile = newLogFile(appId, None, inProgress = false, codecName,
dir)
+ val codec = codecName.map(CompressionCodec.createCodec(conf, _))
+ writeFile(logFile, codec,
+ SparkListenerApplicationStart(appId, Some(appId), 1000,
"testuser", None),
+ SparkListenerApplicationEnd(5000))
+
+ assert(provider.getListing().isEmpty)
+ assert(provider.getAppUI(appId, None).isDefined == onDemandEnabled)
+ assert(provider.getListing().length === (if (onDemandEnabled) 1 else
0))
+
+ if (onDemandEnabled) {
+ val appInfo = provider.getListing().next()
+ assert(appInfo.name === appId)
+ assert(appInfo.attempts.head.sparkUser === "testuser")
+ assert(appInfo.attempts.head.completed)
+ assert(appInfo.attempts.head.duration === 4000)
+ }
+
+ assert(provider.getAppUI("nonexist", None).isEmpty)
+ assert(provider.getListing().length === (if (onDemandEnabled) 1 else
0))
+
+ provider.stop()
+ }
+ }
+ }
+ }
+
+ test("Support on-demand loading for in-progress single event logs") {
+ Seq(None, Some(CompressionCodec.LZF)).foreach { codecName =>
+ withTempDir { dir =>
+ val conf = createTestConf(true)
+ conf.set(HISTORY_LOG_DIR, dir.getAbsolutePath)
+ conf.set(EVENT_LOG_ROLLING_ON_DEMAND_LOAD_ENABLED, false)
+ conf.set(EVENT_LOG_SINGLE_ON_DEMAND_LOAD_ENABLED, true)
+ val provider = new FsHistoryProvider(conf)
+ val appId = s"app1-${codecName.getOrElse("none")}"
+ val logFile = newLogFile(appId, None, inProgress = true, codecName,
dir)
+ val codec = codecName.map(CompressionCodec.createCodec(conf, _))
+ writeFile(logFile, codec,
+ SparkListenerApplicationStart(appId, Some(appId), 1000, "testuser",
None))
+
+ assert(provider.getAppUI(appId, None).isDefined)
+ assert(!provider.getListing().next().attempts.head.completed)
+
+ provider.stop()
+ }
+ }
+ }
+
+ test("On-demand loading preserves in-progress single event logs during
cleanup") {
+ withTempDir { dir =>
+ val clock = new ManualClock(0)
+ val conf = createTestConf(true)
+ .set(HISTORY_LOG_DIR, dir.getAbsolutePath)
+ .set(CLEANER_ENABLED, true)
+ .set(MAX_LOG_AGE_S, 0L)
+ .set(EVENT_LOG_ROLLING_ON_DEMAND_LOAD_ENABLED, false)
+ .set(EVENT_LOG_SINGLE_ON_DEMAND_LOAD_ENABLED, true)
+ val provider = new FsHistoryProvider(conf, clock)
+ val logFile = newLogFile("app1", None, inProgress = true, logDir = dir)
+ writeFile(logFile, None,
+ SparkListenerApplicationStart("app1", Some("app1"), 0, "testuser",
None))
+
+ assert(provider.getAppUI("app1", None).isDefined)
+ assert(provider.getListing().next().attempts.head.appSparkVersion ===
SPARK_VERSION)
+ assert(logFile.exists())
+
+ provider.cleanLogs()
+ assert(logFile.exists())
+
+ provider.stop()
+ }
+ }
+
+ test("On-demand loading respects single event log attempts") {
+ withTempDir { dir =>
+ val conf = createTestConf(true)
+ conf.set(HISTORY_LOG_DIR, dir.getAbsolutePath)
+ conf.set(EVENT_LOG_ROLLING_ON_DEMAND_LOAD_ENABLED, false)
+ conf.set(EVENT_LOG_SINGLE_ON_DEMAND_LOAD_ENABLED, true)
+ val provider = new FsHistoryProvider(conf)
+ val logFile = newLogFile("app1", Some("attempt1"), inProgress = false,
logDir = dir)
+ writeFile(logFile, None,
+ SparkListenerApplicationStart("app1", Some("app1"), 1000, "testuser",
Some("attempt1")),
+ SparkListenerApplicationEnd(5000))
+
+ assert(provider.getAppUI("app1", None).isEmpty)
+ assert(provider.getListing().isEmpty)
+ assert(provider.getAppUI("app1", Some("attempt1")).isDefined)
+
+ provider.stop()
+ }
+ }
+
+ test("On-demand loading finds single event logs in all directories") {
+ withTempDir { dir =>
+ val conf = createTestConf(true)
+ conf.set(HISTORY_LOG_DIR,
s"${testDir.getAbsolutePath},${dir.getAbsolutePath}")
+ conf.set(EVENT_LOG_ROLLING_ON_DEMAND_LOAD_ENABLED, false)
+ conf.set(EVENT_LOG_SINGLE_ON_DEMAND_LOAD_ENABLED, true)
+ val provider = new FsHistoryProvider(conf)
+ val logFile = newLogFile("app1", None, inProgress = false, logDir = dir)
+ writeFile(logFile, None,
+ SparkListenerApplicationStart("app1", Some("app1"), 1000, "testuser",
None),
+ SparkListenerApplicationEnd(5000))
+
+ assert(provider.getAppUI("app1", None).isDefined)
+ assert(provider.getListing().next().attempts.head.logSourceFullPath ===
+ Some(dir.getAbsolutePath))
+
+ provider.stop()
+ }
+ }
+
+ test("On-demand loading supports single event logs when scanning is
disabled") {
+ withTempDir { dir =>
+ val conf = createTestConf(true)
+ conf.set(HISTORY_LOG_DIR, dir.getAbsolutePath)
+ conf.set(SCAN_DISABLED_PATH_PATTERNS, Seq("file:.*"))
+ conf.set(EVENT_LOG_ROLLING_ON_DEMAND_LOAD_ENABLED, false)
+ conf.set(EVENT_LOG_SINGLE_ON_DEMAND_LOAD_ENABLED, true)
+ val provider = new FsHistoryProvider(conf)
+ val logFile = newLogFile("app1", None, inProgress = false, logDir = dir)
+ writeFile(logFile, None,
+ SparkListenerApplicationStart("app1", Some("app1"), 1000, "testuser",
None),
+ SparkListenerApplicationEnd(5000))
+
+ provider.checkForLogs()
+ assert(provider.getListing().isEmpty)
+ assert(provider.getAppUI("app1", None).isDefined)
+
+ provider.checkForLogs()
+ assert(provider.getListing().length === 1)
+
+ provider.stop()
+ }
+ }
+
+ test("On-demand loading supports rolling and single event logs together") {
+ withTempDir { dir =>
+ val conf = createTestConf(true)
+ conf.set(HISTORY_LOG_DIR, dir.getAbsolutePath)
+ conf.set(EVENT_LOG_ROLLING_ON_DEMAND_LOAD_ENABLED, true)
+ conf.set(EVENT_LOG_SINGLE_ON_DEMAND_LOAD_ENABLED, true)
+ val hadoopConf = SparkHadoopUtil.newConfiguration(conf)
+ val provider = new FsHistoryProvider(conf)
+
+ val rollingWriter = new RollingEventLogFilesWriter(
+ "app-rolling", None, dir.toURI, conf, hadoopConf)
+ rollingWriter.start()
+ writeEventsToRollingWriter(rollingWriter, Seq(
+ SparkListenerApplicationStart("app-rolling", Some("app-rolling"), 0,
"user", None),
+ SparkListenerJobStart(1, 0, Seq.empty)), rollFile = false)
+ rollingWriter.stop()
+
+ val singleLog = newLogFile("app-single", None, inProgress = false,
logDir = dir)
+ writeFile(singleLog, None,
+ SparkListenerApplicationStart("app-single", Some("app-single"), 0,
"user", None),
+ SparkListenerApplicationEnd(1))
+
+ assert(provider.getAppUI("app-rolling", None).isDefined)
+ assert(provider.getAppUI("app-single", None).isDefined)
+ assert(provider.getListing().length === 2)
+ assert(provider.getAppUI("nonexist", None).isEmpty)
+ assert(provider.getListing().length === 2)
+
+ provider.stop()
+ }
+ }
+
test("SPARK-56278: On-demand loading populates accurate metadata via
mergeApplicationListing") {
withTempDir { dir =>
val conf = createTestConf(true)
diff --git a/docs/monitoring.md b/docs/monitoring.md
index 64552ba28512..a780701f1b60 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -458,6 +458,14 @@ Security options for the Spark History Server are covered
more detail in the
</td>
<td>4.1.0</td>
</tr>
+ <tr>
+ <td>spark.history.fs.eventLog.onDemandLoadEnabled</td>
+ <td>true</td>
+ <td>
+ Whether to look up single event log locations on demand manner before
listing files.
+ </td>
+ <td>4.3.0</td>
+ </tr>
<tr>
<td>spark.history.store.hybridStore.enabled</td>
<td>false</td>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]