This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 48cd8604f953 [SPARK-46868][CORE] Support Spark Worker Log UI
48cd8604f953 is described below
commit 48cd8604f953dc82cadb6c076914d4d5c69b8126
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Thu Jan 25 19:31:26 2024 -0800
[SPARK-46868][CORE] Support Spark Worker Log UI
### What changes were proposed in this pull request?
This PR aims to support `Spark Worker Log UI` when `SPARK_LOG_DIR` is under
work directory.
### Why are the changes needed?
This is a new feature to allow the users to access the worker log like the
following.
**BEFORE**
**AFTER**
- Worker Page (Worker ID provides a new hyperlink for Log UI)
- Log UI
### Does this PR introduce _any_ user-facing change?
To provide a better UX.
### How was this patch tested?
Manually.
```
$ sbin/start-master.sh
$ SPARK_LOG_DIR=$PWD/work/logs sbin/start-worker.sh spark://$(hostname):7077
```
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #44888 from dongjoon-hyun/SPARK-46868.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../apache/spark/deploy/worker/ui/LogPage.scala | 29 ++++++++++++++++------
.../apache/spark/deploy/worker/ui/WorkerPage.scala | 6 ++++-
2 files changed, 26 insertions(+), 9 deletions(-)
diff --git
a/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala
b/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala
index dd714cdc4437..991c791cc79e 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/ui/LogPage.scala
@@ -30,23 +30,26 @@ import org.apache.spark.util.logging.RollingFileAppender
private[ui] class LogPage(parent: WorkerWebUI) extends WebUIPage("logPage")
with Logging {
private val worker = parent.worker
private val workDir = new File(parent.workDir.toURI.normalize().getPath)
- private val supportedLogTypes = Set("stderr", "stdout")
+ private val supportedLogTypes = Set("stderr", "stdout", "out")
private val defaultBytes = 100 * 1024
def renderLog(request: HttpServletRequest): String = {
val appId = Option(request.getParameter("appId"))
val executorId = Option(request.getParameter("executorId"))
val driverId = Option(request.getParameter("driverId"))
+ val self = Option(request.getParameter("self"))
val logType = request.getParameter("logType")
val offset = Option(request.getParameter("offset")).map(_.toLong)
val byteLength = Option(request.getParameter("byteLength")).map(_.toInt)
.getOrElse(defaultBytes)
- val logDir = (appId, executorId, driverId) match {
- case (Some(a), Some(e), None) =>
+ val logDir = (appId, executorId, driverId, self) match {
+ case (Some(a), Some(e), None, None) =>
s"${workDir.getPath}/$a/$e/"
- case (None, None, Some(d)) =>
+ case (None, None, Some(d), None) =>
s"${workDir.getPath}/$d/"
+ case (None, None, None, Some(_)) =>
+ s"${sys.env.getOrElse("SPARK_LOG_DIR", workDir.getPath)}/"
case _ =>
throw new Exception("Request must specify either application or driver
identifiers")
}
@@ -60,16 +63,19 @@ private[ui] class LogPage(parent: WorkerWebUI) extends
WebUIPage("logPage") with
val appId = Option(request.getParameter("appId"))
val executorId = Option(request.getParameter("executorId"))
val driverId = Option(request.getParameter("driverId"))
+ val self = Option(request.getParameter("self"))
val logType = request.getParameter("logType")
val offset = Option(request.getParameter("offset")).map(_.toLong)
val byteLength = Option(request.getParameter("byteLength")).map(_.toInt)
.getOrElse(defaultBytes)
- val (logDir, params, pageName) = (appId, executorId, driverId) match {
- case (Some(a), Some(e), None) =>
+ val (logDir, params, pageName) = (appId, executorId, driverId, self) match
{
+ case (Some(a), Some(e), None, None) =>
(s"${workDir.getPath}/$a/$e/", s"appId=$a&executorId=$e", s"$a/$e")
- case (None, None, Some(d)) =>
+ case (None, None, Some(d), None) =>
(s"${workDir.getPath}/$d/", s"driverId=$d", d)
+ case (None, None, None, Some(_)) =>
+ (s"${sys.env.getOrElse("SPARK_LOG_DIR", workDir.getPath)}/", "self",
"worker")
case _ =>
throw new Exception("Request must specify either application or driver
identifiers")
}
@@ -138,7 +144,14 @@ private[ui] class LogPage(parent: WorkerWebUI) extends
WebUIPage("logPage") with
}
try {
- val files = RollingFileAppender.getSortedRolledOverFiles(logDirectory,
logType)
+ // Find a log file name
+ val fileName = if (logType.equals("out")) {
+ normalizedLogDir.listFiles.map(_.getName).filter(_.endsWith(".out"))
+ .headOption.getOrElse(logType)
+ } else {
+ logType
+ }
+ val files = RollingFileAppender.getSortedRolledOverFiles(logDirectory,
fileName)
logDebug(s"Sorted log files of type $logType in
$logDirectory:\n${files.mkString("\n")}")
val fileLengths: Seq[Long] = files.map(Utils.getFileLength(_,
worker.conf))
diff --git
a/core/src/main/scala/org/apache/spark/deploy/worker/ui/WorkerPage.scala
b/core/src/main/scala/org/apache/spark/deploy/worker/ui/WorkerPage.scala
index e740b328dd7b..3dd5ebea471c 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/ui/WorkerPage.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/ui/WorkerPage.scala
@@ -78,11 +78,15 @@ private[ui] class WorkerPage(parent: WorkerWebUI) extends
WebUIPage("") {
// For now we only show driver information if the user has submitted
drivers to the cluster.
// This is until we integrate the notion of drivers and applications in
the UI.
+ val workerUrlRef = UIUtils.makeHref(parent.worker.reverseProxy,
workerState.workerId,
+ parent.webUrl)
val content =
<div class="row"> <!-- Worker Details -->
<div class="col-12">
<ul class="list-unstyled">
- <li><strong>ID:</strong> {workerState.workerId}</li>
+ <li><strong>ID:</strong>
+ <a
href={s"$workerUrlRef/logPage/?self&logType=out"}>{workerState.workerId}</a>
+ </li>
<li><strong>
Master URL:</strong> {workerState.masterUrl}
</li>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
