This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 210d74414fb2 [SPARK-55837][UI] Render Environment page tables
client-side via REST API
210d74414fb2 is described below
commit 210d74414fb2f6b06ec1e3b39391791414859a0d
Author: Kent Yao <[email protected]>
AuthorDate: Thu Mar 5 18:12:25 2026 +0800
[SPARK-55837][UI] Render Environment page tables client-side via REST API
### What changes were proposed in this pull request?
Switches the Environment page from server-side Scala XML table rendering to
client-side DataTables populated via the existing REST API endpoint
(`/api/v1/applications/{appId}/environment`).
**EnvironmentPage.scala:**
- Remove all `UIUtils.listingTable` calls and server-side data preparation
- Remove badge counts from tabs (JS adds them dynamically after data loads)
- Add loading spinner placeholders in each tab pane
- Include `environmentpage.js` and `utils.js` as ES modules
- Enable DataTables via `useDataTables = true` in `headerSparkPage`
**environmentpage.js:**
- Rewrite as ES module importing `getStandAloneAppId` and
`setDataTableDefaults` from `utils.js`
- Fetch environment data via REST API on page load
- Create DataTables with sorting, searching, and pagination for each section
- Use `deferRender: true` for performance with 1000+ row tables (e.g.,
Hadoop Properties)
- Format Resource Profiles with structured executor/task resource text
- Dynamically update badge counts on tabs after data loads
- Persist active tab state in localStorage
- Adjust DataTable columns on tab switch for correct rendering in hidden
panes
### Why are the changes needed?
This continues the modernization of the Spark UI Environment page started
in SPARK-55834. Moving to client-side rendering:
- Reduces server-side HTML payload significantly (especially for 1000+
Hadoop properties)
- Adds built-in search, sort, and pagination via DataTables
- Follows the same pattern used by the Executors page
- Improves user experience for environments with many properties
### Does this PR introduce _any_ user-facing change?
Yes. The Environment page tables now render client-side with DataTables,
providing:
- Built-in search/filter across all columns
- Column sorting
- Pagination (important for large property sets)
- Loading spinners while data fetches
### How was this patch tested?
- `./dev/lint-js` — passed
- `./dev/scalastyle` — passed
- `./build/sbt core/compile` — passed
### Was this patch authored or co-authored using generative AI tooling?
Yes, GitHub Copilot.
Closes #54632 from yaooqinn/SPARK-55837.
Authored-by: Kent Yao <[email protected]>
Signed-off-by: Kent Yao <[email protected]>
---
.../org/apache/spark/ui/static/environmentpage.js | 194 ++++++++++++++++-----
.../org/apache/spark/ui/env/EnvironmentPage.scala | 109 ++----------
2 files changed, 172 insertions(+), 131 deletions(-)
diff --git
a/core/src/main/resources/org/apache/spark/ui/static/environmentpage.js
b/core/src/main/resources/org/apache/spark/ui/static/environmentpage.js
index 28df9824c9b2..bbfa28b27529 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/environmentpage.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/environmentpage.js
@@ -15,9 +15,86 @@
* limitations under the License.
*/
-/* global $ */
+/* global $, uiRoot */
+
+import { getStandAloneAppId, setDataTableDefaults } from "./utils.js";
+
+function createRESTEndPointForEnvironmentPage(appId) {
+ var words = document.baseURI.split("/");
+ var ind = words.indexOf("proxy");
+ var newBaseURI;
+ if (ind > 0) {
+ appId = words[ind + 1];
+ newBaseURI = words.slice(0, ind + 2).join("/");
+ return newBaseURI + "/api/v1/applications/" + appId + "/environment";
+ }
+ ind = words.indexOf("history");
+ if (ind > 0) {
+ appId = words[ind + 1];
+ var attemptId = words[ind + 2];
+ newBaseURI = words.slice(0, ind).join("/");
+ if (isNaN(attemptId)) {
+ return newBaseURI + "/api/v1/applications/" + appId + "/environment";
+ } else {
+ return newBaseURI + "/api/v1/applications/" + appId + "/" + attemptId +
"/environment";
+ }
+ }
+ return uiRoot + "/api/v1/applications/" + appId + "/environment";
+}
+
+function updateBadge(tabId, count) {
+ var tab = document.getElementById(tabId);
+ if (tab) {
+ var existing = tab.querySelector(".badge");
+ if (existing) {
+ existing.textContent = count;
+ } else {
+ var badge = document.createElement("span");
+ badge.className = "badge bg-secondary ms-1";
+ badge.textContent = count;
+ tab.appendChild(badge);
+ }
+ }
+}
+
+function formatResourceProfile(rp) {
+ var lines = ["Executor Reqs:"];
+ var execRes = rp.executorResources || {};
+ Object.keys(execRes).sort().forEach(function (key) {
+ var r = execRes[key];
+ var s = "\t" + r.resourceName + ": [amount: " + r.amount;
+ if (r.discoveryScript) { s += ", discovery: " + r.discoveryScript; }
+ if (r.vendor) { s += ", vendor: " + r.vendor; }
+ s += "]";
+ lines.push(s);
+ });
+ lines.push("Task Reqs:");
+ var taskRes = rp.taskResources || {};
+ Object.keys(taskRes).sort().forEach(function (key) {
+ var r = taskRes[key];
+ lines.push("\t" + r.resourceName + ": [amount: " + r.amount + "]");
+ });
+ return lines.join("\n");
+}
+
+function initDataTable(paneId, tableId, data, columns, dtOpts) {
+ var pane = document.getElementById(paneId);
+ pane.innerHTML = '<table id="' + tableId +
+ '" class="table table-striped compact cell-border"
style="width:100%"></table>';
+ var opts = $.extend({
+ data: data,
+ columns: columns,
+ order: [[0, "asc"]],
+ pageLength: 50,
+ deferRender: true,
+ language: { search: "Search: " }
+ }, dtOpts || {});
+ $("#" + tableId).DataTable(opts);
+}
+
+$(document).ready(function () {
+ setDataTableDefaults();
-$(document).ready(function(){
// Tab state persistence
var storedTab = localStorage.getItem("env-active-tab");
if (storedTab) {
@@ -28,52 +105,87 @@ $(document).ready(function(){
}
document.querySelectorAll('#envTabs button[data-bs-toggle="pill"]')
- .forEach(function(tabEl) {
- tabEl.addEventListener("shown.bs.tab", function(event) {
+ .forEach(function (tabEl) {
+ tabEl.addEventListener("shown.bs.tab", function (event) {
localStorage.setItem("env-active-tab", event.target.id);
+ // Adjust DataTable columns for newly visible tab
+ $(event.target.getAttribute("data-bs-target"))
+ .find(".dataTable").each(function () {
+ $(this).DataTable().columns.adjust();
+ });
});
});
- // Column filter functionality
- $('th').on('click', function(e) {
- let inputBox = $(this).find('.env-table-filter-input');
- if (inputBox.length === 0) {
- $('<input class="env-table-filter-input form-control" type="text">')
- .appendTo(this)
- .focus();
- } else {
- inputBox.toggleClass('d-none');
- inputBox.focus();
- }
- e.stopPropagation();
- });
+ getStandAloneAppId(function (appId) {
+ var endPoint = createRESTEndPointForEnvironmentPage(appId);
+ $.getJSON(endPoint, function (response) {
+ // Runtime Information
+ var runtime = response.runtime || {};
+ var runtimeData = [
+ ["Java Version", runtime.javaVersion || ""],
+ ["Java Home", runtime.javaHome || ""],
+ ["Scala Version", runtime.scalaVersion || ""]
+ ];
+ initDataTable("runtime", "runtime-table", runtimeData, [
+ { title: "Name", width: "35%" },
+ { title: "Value", width: "65%" }
+ ], { paging: false, searching: false, info: false });
+ updateBadge("runtime-tab", runtimeData.length);
- $(document).on('click', function() {
- $('.env-table-filter-input').toggleClass('d-none', true);
- });
+ // Spark Properties
+ var sparkProps = response.sparkProperties || [];
+ initDataTable("spark-props", "spark-props-table", sparkProps, [
+ { title: "Name", width: "35%" },
+ { title: "Value", width: "65%" }
+ ]);
+ updateBadge("spark-props-tab", sparkProps.length);
- $(document).on('input', '.env-table-filter-input', function() {
- const table = $(this).closest('table');
- const filters = table.find('.env-table-filter-input').map(function() {
- const columnIdx = $(this).closest('th').index();
- const searchString = $(this).val().toLowerCase();
- return { columnIdx, searchString };
- }).get();
-
- table.find('tbody tr').each(function() {
- let showRow = true;
- for (const filter of filters) {
- const cellText =
$(this).find('td').eq(filter.columnIdx).text().toLowerCase();
- if (filter.searchString && cellText.indexOf(filter.searchString) ===
-1) {
- showRow = false;
- break;
+ // Resource Profiles
+ var profiles = response.resourceProfiles || [];
+ var rpData = profiles.map(function (rp) {
+ return [String(rp.id), formatResourceProfile(rp)];
+ });
+ initDataTable("resource-profiles", "resource-profiles-table", rpData, [
+ { title: "Resource Profile Id", width: "20%" },
+ {
+ title: "Resource Profile Contents",
+ width: "80%",
+ render: function (data) { return "<pre>" + data + "</pre>"; }
}
- }
- if (showRow) {
- $(this).removeClass('d-none');
- } else {
- $(this).addClass('d-none');
- }
+ ], { paging: false, searching: false, info: false });
+ updateBadge("resource-profiles-tab", rpData.length);
+
+ // Hadoop Properties
+ var hadoopProps = response.hadoopProperties || [];
+ initDataTable("hadoop-props", "hadoop-props-table", hadoopProps, [
+ { title: "Name", width: "35%" },
+ { title: "Value", width: "65%" }
+ ]);
+ updateBadge("hadoop-props-tab", hadoopProps.length);
+
+ // System Properties
+ var systemProps = response.systemProperties || [];
+ initDataTable("system-props", "system-props-table", systemProps, [
+ { title: "Name", width: "35%" },
+ { title: "Value", width: "65%" }
+ ]);
+ updateBadge("system-props-tab", systemProps.length);
+
+ // Metrics Properties
+ var metricsProps = response.metricsProperties || [];
+ initDataTable("metrics-props", "metrics-props-table", metricsProps, [
+ { title: "Name", width: "35%" },
+ { title: "Value", width: "65%" }
+ ]);
+ updateBadge("metrics-props-tab", metricsProps.length);
+
+ // Classpath Entries
+ var classpathEntries = response.classpathEntries || [];
+ initDataTable("classpath", "classpath-table", classpathEntries, [
+ { title: "Resource", width: "35%" },
+ { title: "Source", width: "65%" }
+ ]);
+ updateBadge("classpath-tab", classpathEntries.length);
});
});
});
diff --git a/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala
b/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala
index 423475859d59..4717579aca52 100644
--- a/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala
@@ -17,82 +17,27 @@
package org.apache.spark.ui.env
-import scala.collection.mutable.StringBuilder
import scala.xml.Node
import jakarta.servlet.http.HttpServletRequest
import org.apache.spark.SparkConf
-import org.apache.spark.resource.{ExecutorResourceRequest, TaskResourceRequest}
import org.apache.spark.status.AppStatusStore
import org.apache.spark.ui._
-import org.apache.spark.util.Utils
private[ui] class EnvironmentPage(
parent: EnvironmentTab,
conf: SparkConf,
store: AppStatusStore) extends WebUIPage("") {
- def render(request: HttpServletRequest): Seq[Node] = {
- val appEnv = store.environmentInfo()
- val jvmInformation = Map(
- "Java Version" -> appEnv.runtime.javaVersion,
- "Java Home" -> appEnv.runtime.javaHome,
- "Scala Version" -> appEnv.runtime.scalaVersion)
-
- def constructExecutorRequestString(execReqs: Map[String,
ExecutorResourceRequest]): String = {
- execReqs.map {
- case (_, execReq) =>
- val execStr = new StringBuilder(s"\t${execReq.resourceName}:
[amount: ${execReq.amount}")
- if (execReq.discoveryScript.nonEmpty) {
- execStr ++= s", discovery: ${execReq.discoveryScript}"
- }
- if (execReq.vendor.nonEmpty) {
- execStr ++= s", vendor: ${execReq.vendor}"
- }
- execStr ++= "]"
- execStr.toString()
- }.mkString("\n")
- }
-
- def constructTaskRequestString(taskReqs: Map[String,
TaskResourceRequest]): String = {
- taskReqs.map {
- case (_, taskReq) => s"\t${taskReq.resourceName}: [amount:
${taskReq.amount}]"
- }.mkString("\n")
- }
-
- val resourceProfileInfo = store.resourceProfileInfo().map { rinfo =>
- val einfo = constructExecutorRequestString(rinfo.executorResources)
- val tinfo = constructTaskRequestString(rinfo.taskResources)
- val res = s"Executor Reqs:\n$einfo\nTask Reqs:\n$tinfo"
- (rinfo.id.toString, res)
- }.toMap
+ private val spinner =
+ <div class="text-center p-3">
+ <div class="spinner-border text-primary" role="status">
+ <span class="visually-hidden">Loading...</span>
+ </div>
+ </div>
- val resourceProfileInformationTable =
UIUtils.listingTable(resourceProfileHeader,
- jvmRowDataPre, resourceProfileInfo.toSeq.sortWith(_._1.toInt <
_._1.toInt),
- fixedWidth = true, headerClasses = headerClassesNoSortValues)
- val runtimeInformationTable = UIUtils.listingTable(
- propertyHeader, jvmRow, jvmInformation.toSeq.sorted, fixedWidth = true,
- headerClasses = headerClasses)
- val sparkProperties = Utils.redact(conf, appEnv.sparkProperties.sorted)
- val sparkPropertiesTable = UIUtils.listingTable(propertyHeader,
propertyRow,
- sparkProperties, fixedWidth = true, headerClasses = headerClasses)
- val emptyProperties = collection.Seq.empty[(String, String)]
- val hadoopProperties =
- Utils.redact(conf,
Option(appEnv.hadoopProperties).getOrElse(emptyProperties).sorted)
- val hadoopPropertiesTable = UIUtils.listingTable(propertyHeader,
propertyRow,
- hadoopProperties, fixedWidth = true, headerClasses = headerClasses)
- val systemProperties = Utils.redact(conf, appEnv.systemProperties.sorted)
- val systemPropertiesTable = UIUtils.listingTable(propertyHeader,
propertyRow,
- systemProperties, fixedWidth = true, headerClasses = headerClasses)
- val metricsProperties =
- Utils.redact(conf,
Option(appEnv.metricsProperties).getOrElse(emptyProperties).sorted)
- val metricsPropertiesTable = UIUtils.listingTable(propertyHeader,
propertyRow,
- metricsProperties, fixedWidth = true, headerClasses = headerClasses)
- val classpathEntries = appEnv.classpathEntries.sorted
- val classpathEntriesTable = UIUtils.listingTable(
- classPathHeader, classPathRow, classpathEntries, fixedWidth = true,
- headerClasses = headerClasses)
+ def render(request: HttpServletRequest): Seq[Node] = {
val content =
<span>
<div class="d-flex align-items-start">
@@ -102,93 +47,77 @@ private[ui] class EnvironmentPage(
data-bs-target="#runtime" type="button" role="tab"
aria-controls="runtime" aria-selected="true">
Runtime Information
- <span class="badge bg-secondary
ms-1">{jvmInformation.size}</span>
</button>
<button class="nav-link text-start" id="spark-props-tab"
data-bs-toggle="pill"
data-bs-target="#spark-props" type="button" role="tab"
aria-controls="spark-props" aria-selected="false">
Spark Properties
- <span class="badge bg-secondary
ms-1">{sparkProperties.size}</span>
</button>
<button class="nav-link text-start" id="resource-profiles-tab"
data-bs-toggle="pill"
data-bs-target="#resource-profiles" type="button"
role="tab"
aria-controls="resource-profiles" aria-selected="false">
Resource Profiles
- <span class="badge bg-secondary
ms-1">{resourceProfileInfo.size}</span>
</button>
<button class="nav-link text-start" id="hadoop-props-tab"
data-bs-toggle="pill"
data-bs-target="#hadoop-props" type="button" role="tab"
aria-controls="hadoop-props" aria-selected="false">
Hadoop Properties
- <span class="badge bg-secondary
ms-1">{hadoopProperties.size}</span>
</button>
<button class="nav-link text-start" id="system-props-tab"
data-bs-toggle="pill"
data-bs-target="#system-props" type="button" role="tab"
aria-controls="system-props" aria-selected="false">
System Properties
- <span class="badge bg-secondary
ms-1">{systemProperties.size}</span>
</button>
<button class="nav-link text-start" id="metrics-props-tab"
data-bs-toggle="pill"
data-bs-target="#metrics-props" type="button" role="tab"
aria-controls="metrics-props" aria-selected="false">
Metrics Properties
- <span class="badge bg-secondary
ms-1">{metricsProperties.size}</span>
</button>
<button class="nav-link text-start" id="classpath-tab"
data-bs-toggle="pill"
data-bs-target="#classpath" type="button" role="tab"
aria-controls="classpath" aria-selected="false">
Classpath Entries
- <span class="badge bg-secondary
ms-1">{classpathEntries.size}</span>
</button>
</div>
<div class="tab-content flex-fill" id="envTabContent">
<div class="tab-pane fade show active" id="runtime" role="tabpanel"
aria-labelledby="runtime-tab">
- {runtimeInformationTable}
+ {spinner}
</div>
<div class="tab-pane fade" id="spark-props" role="tabpanel"
aria-labelledby="spark-props-tab">
- {sparkPropertiesTable}
+ {spinner}
</div>
<div class="tab-pane fade" id="resource-profiles" role="tabpanel"
aria-labelledby="resource-profiles-tab">
- {resourceProfileInformationTable}
+ {spinner}
</div>
<div class="tab-pane fade" id="hadoop-props" role="tabpanel"
aria-labelledby="hadoop-props-tab">
- {hadoopPropertiesTable}
+ {spinner}
</div>
<div class="tab-pane fade" id="system-props" role="tabpanel"
aria-labelledby="system-props-tab">
- {systemPropertiesTable}
+ {spinner}
</div>
<div class="tab-pane fade" id="metrics-props" role="tabpanel"
aria-labelledby="metrics-props-tab">
- {metricsPropertiesTable}
+ {spinner}
</div>
<div class="tab-pane fade" id="classpath" role="tabpanel"
aria-labelledby="classpath-tab">
- {classpathEntriesTable}
+ {spinner}
</div>
</div>
</div>
- <script src={UIUtils.prependBaseUri(request,
"/static/environmentpage.js")}></script>
+ <script type="module"
+ src={UIUtils.prependBaseUri(request,
"/static/utils.js")}></script>
+ <script type="module"
+ src={UIUtils.prependBaseUri(request,
"/static/environmentpage.js")}></script>
</span>
- UIUtils.headerSparkPage(request, "Environment", content, parent)
+ UIUtils.headerSparkPage(request, "Environment", content, parent,
useDataTables = true)
}
-
- private def resourceProfileHeader = Seq("Resource Profile Id", "Resource
Profile Contents")
- private def propertyHeader = Seq("Name", "Value")
- private def classPathHeader = Seq("Resource", "Source")
- private def headerClasses = Seq("sorttable_alpha", "sorttable_alpha")
- private def headerClassesNoSortValues = Seq("sorttable_numeric",
"sorttable_nosort")
-
- private def jvmRowDataPre(kv: (String, String)) =
- <tr><td>{kv._1}</td><td><pre>{kv._2}</pre></td></tr>
- private def jvmRow(kv: (String, String)) =
<tr><td>{kv._1}</td><td>{kv._2}</td></tr>
- private def propertyRow(kv: (String, String)) =
<tr><td>{kv._1}</td><td>{kv._2}</td></tr>
- private def classPathRow(data: (String, String)) =
<tr><td>{data._1}</td><td>{data._2}</td></tr>
}
private[ui] class EnvironmentTab(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
