This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new e3f783773076 [SPARK-58468][CORE] Support on-demand loading for single 
event logs in SHS
e3f783773076 is described below

commit e3f7837730764d41ffd81be4e9d4f3499b314dc0
Author: Cheng Pan <[email protected]>
AuthorDate: Fri Jul 31 12:53:53 2026 -0700

    [SPARK-58468][CORE] Support on-demand loading for single event logs in SHS
    
    ### What changes were proposed in this pull request?
    
    This PR extends History Server on-demand event-log loading to single event 
logs.
    
    It adds `spark.history.fs.eventLog.onDemandLoadEnabled`, which is 
independent from the existing rolling event-log configuration. When enabled, 
the History Server probes completed, compressed, and in-progress single 
event-log names after a requested application is absent from the listing.
    
    ### Why are the changes needed?
    
    Single event logs remain supported, but on-demand loading currently only 
supports rolling event logs. A requested application with a single event log 
can therefore return Not Found until periodic scanning discovers its log.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. History Server can load single event logs on demand. The new 
`spark.history.fs.eventLog.onDemandLoadEnabled` configuration defaults to 
`true`.
    
    ### How was this patch tested?
    
    Added tests for:
    - enabled and disabled single event-log on-demand loading
    - uncompressed and LZF-compressed logs
    - in-progress logs and cleaner safety
    - attempt IDs
    - multiple log directories
    - scan-disabled directories
    
    Ran the focused `FsHistoryProviderSuite` and 
`SingleEventLogFileWriterSuite` tests. Also ran `SparkConfigBindingPolicySuite`.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: gpt-5.6-terra
    
    Closes #57676 from pan3793/shs-on-demand-v1-log-loading.
    
    Authored-by: Cheng Pan <[email protected]>
    Signed-off-by: Dongjoon Hyun <[email protected]>
    (cherry picked from commit 3b581e0a758ca7b3c08af661e2ee75f30d3fdffc)
    Signed-off-by: Dongjoon Hyun <[email protected]>
---
 .../spark/deploy/history/EventLogFileWriters.scala |   9 +
 .../spark/deploy/history/FsHistoryProvider.scala   |  30 +++-
 .../org/apache/spark/internal/config/History.scala |   8 +
 .../deploy/history/EventLogFileWritersSuite.scala  |  14 ++
 .../deploy/history/FsHistoryProviderSuite.scala    | 186 ++++++++++++++++++++-
 docs/monitoring.md                                 |   8 +
 6 files changed, 246 insertions(+), 9 deletions(-)

diff --git 
a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala 
b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala
index 601515e57dc8..a725da2f3a44 100644
--- 
a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala
+++ 
b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala
@@ -257,6 +257,15 @@ class SingleEventLogFileWriter(
 }
 
 object SingleEventLogFileWriter {
+  /** Returns names for completed and in-progress single event logs. */
+  private[history] def getLogFileNames(appId: String, appAttemptId: 
Option[String]): Seq[String] = {
+    val logBaseName = EventLogFileWriter.nameForAppAndAttempt(appId, 
appAttemptId)
+    val names = logBaseName +: 
CompressionCodec.shortCompressionCodecNames.keys.toSeq.map {
+      codec => s"$logBaseName.$codec"
+    }
+    names ++ names.map(_ + EventLogFileWriter.IN_PROGRESS)
+  }
+
   /**
    * Return a file-system-safe path to the log file for the given application.
    *
diff --git 
a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala 
b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
index d166e61bfb82..81830c24754e 100644
--- 
a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
+++ 
b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
@@ -412,16 +412,11 @@ private[history] class FsHistoryProvider(conf: SparkConf, 
clock: Clock)
   override def getLastUpdatedTime(): Long = lastScanTime.get()
 
   override def getAppUI(appId: String, attemptId: Option[String]): 
Option[LoadedAppUI] = {
-    val logPath = RollingEventLogFilesWriter.EVENT_LOG_DIR_NAME_PREFIX +
-        EventLogFileWriter.nameForAppAndAttempt(appId, attemptId)
     val app = try {
       load(appId)
      } catch {
-      case _: NoSuchElementException if 
this.conf.get(EVENT_LOG_ROLLING_ON_DEMAND_LOAD_ENABLED) =>
-        loadFromFallbackLocation(appId, attemptId, logPath) match {
-          case Some(wrapper) => wrapper
-          case None => return None
-        }
+      case _: NoSuchElementException if isOnDemandLogLoadEnabled =>
+        loadFromFallbackLocations(appId, attemptId).getOrElse(return None)
       case _: NoSuchElementException =>
         return None
     }
@@ -462,6 +457,27 @@ private[history] class FsHistoryProvider(conf: SparkConf, 
clock: Clock)
     Some(loadedUI)
   }
 
+  private def loadFromFallbackLocations(
+      appId: String,
+      attemptId: Option[String]): Option[ApplicationInfoWrapper] = {
+    val logPaths = mutable.ArrayBuffer.empty[String]
+    if (conf.get(EVENT_LOG_ROLLING_ON_DEMAND_LOAD_ENABLED)) {
+      logPaths += RollingEventLogFilesWriter.EVENT_LOG_DIR_NAME_PREFIX +
+        EventLogFileWriter.nameForAppAndAttempt(appId, attemptId)
+    }
+    if (conf.get(EVENT_LOG_SINGLE_ON_DEMAND_LOAD_ENABLED)) {
+      logPaths ++= SingleEventLogFileWriter.getLogFileNames(appId, attemptId)
+    }
+    logPaths.iterator.map(loadFromFallbackLocation(appId, attemptId, 
_)).collectFirst {
+      case Some(app) => app
+    }
+  }
+
+  private def isOnDemandLogLoadEnabled: Boolean = {
+    conf.get(EVENT_LOG_ROLLING_ON_DEMAND_LOAD_ENABLED) ||
+      conf.get(EVENT_LOG_SINGLE_ON_DEMAND_LOAD_ENABLED)
+  }
+
   private def loadFromFallbackLocation(appId: String, attemptId: 
Option[String], logPath: String)
     : Option[ApplicationInfoWrapper] = {
     // Call mergeApplicationListing to populate accurate metadata immediately.
diff --git a/core/src/main/scala/org/apache/spark/internal/config/History.scala 
b/core/src/main/scala/org/apache/spark/internal/config/History.scala
index 1936ecf68103..570113d145a1 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/History.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/History.scala
@@ -193,6 +193,14 @@ private[spark] object History {
       .booleanConf
       .createWithDefault(true)
 
+  val EVENT_LOG_SINGLE_ON_DEMAND_LOAD_ENABLED =
+    ConfigBuilder("spark.history.fs.eventLog.onDemandLoadEnabled")
+      .doc("Whether to look up single event log locations on demand manner 
before listing files.")
+      .version("4.3.0")
+      .withBindingPolicy(ConfigBindingPolicy.NOT_APPLICABLE)
+      .booleanConf
+      .createWithDefault(true)
+
   val DRIVER_LOG_CLEANER_ENABLED = 
ConfigBuilder("spark.history.fs.driverlog.cleaner.enabled")
     .version("3.0.0")
     .doc("Specifies whether the History Server should periodically clean up 
driver logs from " +
diff --git 
a/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileWritersSuite.scala
 
b/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileWritersSuite.scala
index 00a92c503be4..36c5b1915edd 100644
--- 
a/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileWritersSuite.scala
+++ 
b/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileWritersSuite.scala
@@ -352,6 +352,20 @@ class SingleEventLogFileWriterSuite extends 
EventLogFileWritersSuite {
         "a fine:mind$dollar{bills}.1", None, Some(CompressionCodec.LZ4)))
   }
 
+  test("Event log file names") {
+    Seq(None, Some("attempt1")).foreach { attemptId =>
+      val baseName = attemptId.map(id => s"app1_$id").getOrElse("app1")
+      val names = SingleEventLogFileWriter.getLogFileNames("app1", attemptId)
+      assert(names.head === baseName)
+      assert(names.contains(s"$baseName.inprogress"))
+      assert(names.size === (1 + 
CompressionCodec.shortCompressionCodecNames.size) * 2)
+      CompressionCodec.shortCompressionCodecNames.keys.foreach { codec =>
+        assert(names.contains(s"$baseName.$codec"))
+        assert(names.contains(s"$baseName.$codec.inprogress"))
+      }
+    }
+  }
+
   override protected def createWriter(
       appId: String,
       appAttemptId: Option[String],
diff --git 
a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
 
b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
index 829010179bda..8a4fd0ded959 100644
--- 
a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
+++ 
b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
@@ -83,9 +83,10 @@ abstract class FsHistoryProviderSuite extends SparkFunSuite 
with Matchers with P
       appId: String,
       appAttemptId: Option[String],
       inProgress: Boolean,
-      codec: Option[String] = None): File = {
+      codec: Option[String] = None,
+      logDir: File = testDir): File = {
     val ip = if (inProgress) EventLogFileWriter.IN_PROGRESS else ""
-    val logUri = SingleEventLogFileWriter.getLogPath(testDir.toURI, appId, 
appAttemptId, codec)
+    val logUri = SingleEventLogFileWriter.getLogPath(logDir.toURI, appId, 
appAttemptId, codec)
     val logPath = new Path(logUri).toUri.getPath + ip
     new File(logPath)
   }
@@ -1653,6 +1654,7 @@ abstract class FsHistoryProviderSuite extends 
SparkFunSuite with Matchers with P
         val conf = createTestConf(true)
         conf.set(HISTORY_LOG_DIR, dir.getAbsolutePath)
         conf.set(EVENT_LOG_ROLLING_ON_DEMAND_LOAD_ENABLED, onDemandEnabled)
+        conf.set(EVENT_LOG_SINGLE_ON_DEMAND_LOAD_ENABLED, false)
         val hadoopConf = SparkHadoopUtil.newConfiguration(conf)
         val provider = new FsHistoryProvider(conf)
 
@@ -1681,6 +1683,186 @@ abstract class FsHistoryProviderSuite extends 
SparkFunSuite with Matchers with P
     }
   }
 
+  test("Support spark.history.fs.eventLog.onDemandLoadEnabled") {
+    Seq(true, false).foreach { onDemandEnabled =>
+      Seq(None, Some(CompressionCodec.LZF)).foreach { codecName =>
+        withTempDir { dir =>
+          val conf = createTestConf(true)
+          conf.set(HISTORY_LOG_DIR, dir.getAbsolutePath)
+          conf.set(EVENT_LOG_ROLLING_ON_DEMAND_LOAD_ENABLED, false)
+          conf.set(EVENT_LOG_SINGLE_ON_DEMAND_LOAD_ENABLED, onDemandEnabled)
+          val provider = new FsHistoryProvider(conf)
+          val appId = s"app1-${codecName.getOrElse("none")}"
+          val logFile = newLogFile(appId, None, inProgress = false, codecName, 
dir)
+          val codec = codecName.map(CompressionCodec.createCodec(conf, _))
+          writeFile(logFile, codec,
+            SparkListenerApplicationStart(appId, Some(appId), 1000, 
"testuser", None),
+            SparkListenerApplicationEnd(5000))
+
+          assert(provider.getListing().isEmpty)
+          assert(provider.getAppUI(appId, None).isDefined == onDemandEnabled)
+          assert(provider.getListing().length === (if (onDemandEnabled) 1 else 
0))
+
+          if (onDemandEnabled) {
+            val appInfo = provider.getListing().next()
+            assert(appInfo.name === appId)
+            assert(appInfo.attempts.head.sparkUser === "testuser")
+            assert(appInfo.attempts.head.completed)
+            assert(appInfo.attempts.head.duration === 4000)
+          }
+
+          assert(provider.getAppUI("nonexist", None).isEmpty)
+          assert(provider.getListing().length === (if (onDemandEnabled) 1 else 
0))
+
+          provider.stop()
+        }
+      }
+    }
+  }
+
+  test("Support on-demand loading for in-progress single event logs") {
+    Seq(None, Some(CompressionCodec.LZF)).foreach { codecName =>
+      withTempDir { dir =>
+        val conf = createTestConf(true)
+        conf.set(HISTORY_LOG_DIR, dir.getAbsolutePath)
+        conf.set(EVENT_LOG_ROLLING_ON_DEMAND_LOAD_ENABLED, false)
+        conf.set(EVENT_LOG_SINGLE_ON_DEMAND_LOAD_ENABLED, true)
+        val provider = new FsHistoryProvider(conf)
+        val appId = s"app1-${codecName.getOrElse("none")}"
+        val logFile = newLogFile(appId, None, inProgress = true, codecName, 
dir)
+        val codec = codecName.map(CompressionCodec.createCodec(conf, _))
+        writeFile(logFile, codec,
+          SparkListenerApplicationStart(appId, Some(appId), 1000, "testuser", 
None))
+
+        assert(provider.getAppUI(appId, None).isDefined)
+        assert(!provider.getListing().next().attempts.head.completed)
+
+        provider.stop()
+      }
+    }
+  }
+
+  test("On-demand loading preserves in-progress single event logs during 
cleanup") {
+    withTempDir { dir =>
+      val clock = new ManualClock(0)
+      val conf = createTestConf(true)
+        .set(HISTORY_LOG_DIR, dir.getAbsolutePath)
+        .set(CLEANER_ENABLED, true)
+        .set(MAX_LOG_AGE_S, 0L)
+        .set(EVENT_LOG_ROLLING_ON_DEMAND_LOAD_ENABLED, false)
+        .set(EVENT_LOG_SINGLE_ON_DEMAND_LOAD_ENABLED, true)
+      val provider = new FsHistoryProvider(conf, clock)
+      val logFile = newLogFile("app1", None, inProgress = true, logDir = dir)
+      writeFile(logFile, None,
+        SparkListenerApplicationStart("app1", Some("app1"), 0, "testuser", 
None))
+
+      assert(provider.getAppUI("app1", None).isDefined)
+      assert(provider.getListing().next().attempts.head.appSparkVersion === 
SPARK_VERSION)
+      assert(logFile.exists())
+
+      provider.cleanLogs()
+      assert(logFile.exists())
+
+      provider.stop()
+    }
+  }
+
+  test("On-demand loading respects single event log attempts") {
+    withTempDir { dir =>
+      val conf = createTestConf(true)
+      conf.set(HISTORY_LOG_DIR, dir.getAbsolutePath)
+      conf.set(EVENT_LOG_ROLLING_ON_DEMAND_LOAD_ENABLED, false)
+      conf.set(EVENT_LOG_SINGLE_ON_DEMAND_LOAD_ENABLED, true)
+      val provider = new FsHistoryProvider(conf)
+      val logFile = newLogFile("app1", Some("attempt1"), inProgress = false, 
logDir = dir)
+      writeFile(logFile, None,
+        SparkListenerApplicationStart("app1", Some("app1"), 1000, "testuser", 
Some("attempt1")),
+        SparkListenerApplicationEnd(5000))
+
+      assert(provider.getAppUI("app1", None).isEmpty)
+      assert(provider.getListing().isEmpty)
+      assert(provider.getAppUI("app1", Some("attempt1")).isDefined)
+
+      provider.stop()
+    }
+  }
+
+  test("On-demand loading finds single event logs in all directories") {
+    withTempDir { dir =>
+      val conf = createTestConf(true)
+      conf.set(HISTORY_LOG_DIR, 
s"${testDir.getAbsolutePath},${dir.getAbsolutePath}")
+      conf.set(EVENT_LOG_ROLLING_ON_DEMAND_LOAD_ENABLED, false)
+      conf.set(EVENT_LOG_SINGLE_ON_DEMAND_LOAD_ENABLED, true)
+      val provider = new FsHistoryProvider(conf)
+      val logFile = newLogFile("app1", None, inProgress = false, logDir = dir)
+      writeFile(logFile, None,
+        SparkListenerApplicationStart("app1", Some("app1"), 1000, "testuser", 
None),
+        SparkListenerApplicationEnd(5000))
+
+      assert(provider.getAppUI("app1", None).isDefined)
+      assert(provider.getListing().next().attempts.head.logSourceFullPath ===
+        Some(dir.getAbsolutePath))
+
+      provider.stop()
+    }
+  }
+
+  test("On-demand loading supports single event logs when scanning is 
disabled") {
+    withTempDir { dir =>
+      val conf = createTestConf(true)
+      conf.set(HISTORY_LOG_DIR, dir.getAbsolutePath)
+      conf.set(SCAN_DISABLED_PATH_PATTERNS, Seq("file:.*"))
+      conf.set(EVENT_LOG_ROLLING_ON_DEMAND_LOAD_ENABLED, false)
+      conf.set(EVENT_LOG_SINGLE_ON_DEMAND_LOAD_ENABLED, true)
+      val provider = new FsHistoryProvider(conf)
+      val logFile = newLogFile("app1", None, inProgress = false, logDir = dir)
+      writeFile(logFile, None,
+        SparkListenerApplicationStart("app1", Some("app1"), 1000, "testuser", 
None),
+        SparkListenerApplicationEnd(5000))
+
+      provider.checkForLogs()
+      assert(provider.getListing().isEmpty)
+      assert(provider.getAppUI("app1", None).isDefined)
+
+      provider.checkForLogs()
+      assert(provider.getListing().length === 1)
+
+      provider.stop()
+    }
+  }
+
+  test("On-demand loading supports rolling and single event logs together") {
+    withTempDir { dir =>
+      val conf = createTestConf(true)
+      conf.set(HISTORY_LOG_DIR, dir.getAbsolutePath)
+      conf.set(EVENT_LOG_ROLLING_ON_DEMAND_LOAD_ENABLED, true)
+      conf.set(EVENT_LOG_SINGLE_ON_DEMAND_LOAD_ENABLED, true)
+      val hadoopConf = SparkHadoopUtil.newConfiguration(conf)
+      val provider = new FsHistoryProvider(conf)
+
+      val rollingWriter = new RollingEventLogFilesWriter(
+        "app-rolling", None, dir.toURI, conf, hadoopConf)
+      rollingWriter.start()
+      writeEventsToRollingWriter(rollingWriter, Seq(
+        SparkListenerApplicationStart("app-rolling", Some("app-rolling"), 0, 
"user", None),
+        SparkListenerJobStart(1, 0, Seq.empty)), rollFile = false)
+      rollingWriter.stop()
+
+      val singleLog = newLogFile("app-single", None, inProgress = false, 
logDir = dir)
+      writeFile(singleLog, None,
+        SparkListenerApplicationStart("app-single", Some("app-single"), 0, 
"user", None),
+        SparkListenerApplicationEnd(1))
+
+      assert(provider.getAppUI("app-rolling", None).isDefined)
+      assert(provider.getAppUI("app-single", None).isDefined)
+      assert(provider.getListing().length === 2)
+      assert(provider.getAppUI("nonexist", None).isEmpty)
+      assert(provider.getListing().length === 2)
+
+      provider.stop()
+    }
+  }
+
   test("SPARK-56278: On-demand loading populates accurate metadata via 
mergeApplicationListing") {
     withTempDir { dir =>
       val conf = createTestConf(true)
diff --git a/docs/monitoring.md b/docs/monitoring.md
index 64552ba28512..a780701f1b60 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -458,6 +458,14 @@ Security options for the Spark History Server are covered 
more detail in the
     </td>
     <td>4.1.0</td>
   </tr>
+  <tr>
+    <td>spark.history.fs.eventLog.onDemandLoadEnabled</td>
+    <td>true</td>
+    <td>
+      Whether to look up single event log locations on demand manner before 
listing files.
+    </td>
+    <td>4.3.0</td>
+  </tr>
   <tr>
     <td>spark.history.store.hybridStore.enabled</td>
     <td>false</td>


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to