This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new d6fc06bd4515 [SPARK-46870][CORE] Support Spark Master Log UI
d6fc06bd4515 is described below
commit d6fc06bd451586edc5e55068aabecb3dc7ec5849
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Thu Jan 25 21:15:30 2024 -0800
[SPARK-46870][CORE] Support Spark Master Log UI
### What changes were proposed in this pull request?
This PR aims to support `Spark Master` Log UI.
### Why are the changes needed?
This is a new feature to allow the users to access the master log like the
following. The value of `Status`, e.g., `ALIVE`, has a new link for log UI.
**BEFORE**
**AFTER**
### Does this PR introduce _any_ user-facing change?
No. This is a new link and UI.
### How was this patch tested?
Manually.
```
$ sbin/start-master.sh
```
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #44890 from dongjoon-hyun/SPARK-46870.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../apache/spark/deploy/master/ui/LogPage.scala | 125 +++++++++++++++++++++
.../apache/spark/deploy/master/ui/MasterPage.scala | 4 +-
.../spark/deploy/master/ui/MasterWebUI.scala | 1 +
3 files changed, 129 insertions(+), 1 deletion(-)
diff --git
a/core/src/main/scala/org/apache/spark/deploy/master/ui/LogPage.scala
b/core/src/main/scala/org/apache/spark/deploy/master/ui/LogPage.scala
new file mode 100644
index 000000000000..9da05025e1a3
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/LogPage.scala
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.deploy.master.ui
+
+import java.io.File
+import javax.servlet.http.HttpServletRequest
+
+import scala.xml.{Node, Unparsed}
+
+import org.apache.spark.internal.Logging
+import org.apache.spark.ui.{UIUtils, WebUIPage}
+import org.apache.spark.util.Utils
+import org.apache.spark.util.logging.RollingFileAppender
+
+private[ui] class LogPage(parent: MasterWebUI) extends WebUIPage("logPage")
with Logging {
+ private val defaultBytes = 100 * 1024
+
+ def render(request: HttpServletRequest): Seq[Node] = {
+ val logDir = sys.env.getOrElse("SPARK_LOG_DIR", "logs/")
+ val logType = request.getParameter("logType")
+ val offset = Option(request.getParameter("offset")).map(_.toLong)
+ val byteLength = Option(request.getParameter("byteLength")).map(_.toInt)
+ .getOrElse(defaultBytes)
+ val (logText, startByte, endByte, logLength) = getLog(logDir, logType,
offset, byteLength)
+ val curLogLength = endByte - startByte
+ val range =
+ <span id="log-data">
+ Showing {curLogLength} Bytes: {startByte.toString} -
{endByte.toString} of {logLength}
+ </span>
+
+ val moreButton =
+ <button type="button" onclick={"loadMore()"} class="log-more-btn btn
btn-secondary">
+ Load More
+ </button>
+
+ val newButton =
+ <button type="button" onclick={"loadNew()"} class="log-new-btn btn
btn-secondary">
+ Load New
+ </button>
+
+ val alert =
+ <div class="no-new-alert alert alert-info" style="display: none;">
+ End of Log
+ </div>
+
+ val logParams = "?self&logType=%s".format(logType)
+ val jsOnload = "window.onload = " +
+ s"initLogPage('$logParams', $curLogLength, $startByte, $endByte,
$logLength, $byteLength);"
+
+ val content =
+ <script type="module" src={UIUtils.prependBaseUri(request,
"/static/utils.js")}></script> ++
+ <div>
+ <p><a href="/">Back to Master</a></p>
+ {range}
+ <div class="log-content" style="height:80vh; overflow:auto;
padding:5px;">
+ <div>{moreButton}</div>
+ <pre>{logText}</pre>
+ {alert}
+ <div>{newButton}</div>
+ </div>
+ <script>{Unparsed(jsOnload)}</script>
+ </div>
+
+ UIUtils.basicSparkPage(request, content, logType + " log page for master")
+ }
+
+ /** Get the part of the log files given the offset and desired length of
bytes */
+ private def getLog(
+ logDirectory: String,
+ logType: String,
+ offsetOption: Option[Long],
+ byteLength: Int
+ ): (String, Long, Long, Long) = {
+ try {
+ // Find a log file name
+ val fileName = if (logType.equals("out")) {
+ val normalizedUri = new File(logDirectory).toURI.normalize()
+ val normalizedLogDir = new File(normalizedUri.getPath)
+ normalizedLogDir.listFiles.map(_.getName).filter(_.endsWith(".out"))
+ .headOption.getOrElse(logType)
+ } else {
+ logType
+ }
+ val files = RollingFileAppender.getSortedRolledOverFiles(logDirectory,
fileName)
+ logDebug(s"Sorted log files of type $logType in
$logDirectory:\n${files.mkString("\n")}")
+
+ val fileLengths: Seq[Long] = files.map(Utils.getFileLength(_,
parent.master.conf))
+ val totalLength = fileLengths.sum
+ val offset = offsetOption.getOrElse(totalLength - byteLength)
+ val startIndex = {
+ if (offset < 0) {
+ 0L
+ } else if (offset > totalLength) {
+ totalLength
+ } else {
+ offset
+ }
+ }
+ val endIndex = math.min(startIndex + byteLength, totalLength)
+ logDebug(s"Getting log from $startIndex to $endIndex")
+ val logText = Utils.offsetBytes(files, fileLengths, startIndex, endIndex)
+ logDebug(s"Got log of length ${logText.length} bytes")
+ (logText, startIndex, endIndex, totalLength)
+ } catch {
+ case e: Exception =>
+ logError(s"Error getting $logType logs from directory $logDirectory",
e)
+ ("Error getting logs due to exception: " + e.getMessage, 0, 0, 0)
+ }
+ }
+}
diff --git
a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala
b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala
index f1bd1899e496..36a79e060f01 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala
@@ -168,7 +168,9 @@ private[ui] class MasterPage(parent: MasterWebUI) extends
WebUIPage("") {
{state.completedDrivers.count(_.state == DriverState.ERROR)}
Error,
{state.completedDrivers.count(_.state ==
DriverState.RELAUNCHING)} Relaunching)
</li>
- <li><strong>Status:</strong> {state.status}</li>
+ <li><strong>Status:</strong>
+ <a href={"/logPage/?self&logType=out"}>{state.status}</a>
+ </li>
</ul>
</div>
</div>
diff --git
a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala
b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala
index 8f65ca204b3c..d71ef8b9e36e 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/ui/MasterWebUI.scala
@@ -49,6 +49,7 @@ class MasterWebUI(
def initialize(): Unit = {
val masterPage = new MasterPage(this)
attachPage(new ApplicationPage(this))
+ attachPage(new LogPage(this))
attachPage(masterPage)
addStaticHandler(MasterWebUI.STATIC_RESOURCE_DIR)
attachHandler(createRedirectHandler(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
