This is an automated email from the ASF dual-hosted git repository.
srowen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 1ebf3e50024 [SPARK-39110][WEBUI] Add metrics properties to environment
tab
1ebf3e50024 is described below
commit 1ebf3e500243003b584d21ff024fba2c11ac7dac
Author: Angerszhuuuu <[email protected]>
AuthorDate: Sun May 8 08:13:03 2022 -0500
[SPARK-39110][WEBUI] Add metrics properties to environment tab
### What changes were proposed in this pull request?
There are three ways to define metrics properties:
1. Default metrics properties
2. metrics.properties file under $SPARK_CONF_DIR
3. spark.metrics.conf to include a metrics properties file
4. spark.metrics.conf.xx.xx
Many new users always confused when test with metrics system, i think we
can add final metrics properties in the environment tab, to let user can
directly know which metrics are working.
<img width="1757" alt="截屏2022-05-06 上午11 23 04"
src="https://user-images.githubusercontent.com/46485123/167062876-c0c98a69-13c7-4a25-bb31-74f1ada88153.png";>
<img width="1786" alt="截屏2022-05-06 上午11 33 00"
src="https://user-images.githubusercontent.com/46485123/167062893-f297eeda-b08f-4c9d-a2a2-a74add97493f.png";>
### Why are the changes needed?
Make user clear about which metrics properties are working
### Does this PR introduce _any_ user-facing change?
user can see working metrics properties in UI environment tag
### How was this patch tested?
MT
Closes #36462 from AngersZhuuuu/SPARK-39110.
Authored-by: Angerszhuuuu <[email protected]>
Signed-off-by: Sean Owen <[email protected]>
---
.../src/main/resources/org/apache/spark/ui/static/webui.js | 1 +
core/src/main/scala/org/apache/spark/SparkContext.scala | 3 ++-
core/src/main/scala/org/apache/spark/SparkEnv.scala | 9 +++++----
.../scala/org/apache/spark/metrics/MetricsSystem.scala | 2 ++
.../scala/org/apache/spark/status/AppStatusListener.scala | 1 +
.../spark/status/api/v1/OneApplicationResource.scala | 1 +
.../main/scala/org/apache/spark/status/api/v1/api.scala | 1 +
.../scala/org/apache/spark/ui/env/EnvironmentPage.scala | 14 ++++++++++++++
.../main/scala/org/apache/spark/util/JsonProtocol.scala | 5 +++++
.../app_environment_expectation.json | 6 ++++++
.../multiple_resource_profiles_expectation.json | 1 +
.../test/resources/spark-events/app-20161116163331-0000 | 2 +-
.../spark/deploy/history/FsHistoryProviderSuite.scala | 3 +++
.../apache/spark/scheduler/EventLoggingListenerSuite.scala | 2 +-
.../scala/org/apache/spark/util/JsonProtocolSuite.scala | 5 +++++
project/MimaExcludes.scala | 5 ++++-
16 files changed, 53 insertions(+), 8 deletions(-)
diff --git a/core/src/main/resources/org/apache/spark/ui/static/webui.js
b/core/src/main/resources/org/apache/spark/ui/static/webui.js
index c149f2d8433..b365082c1e1 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/webui.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/webui.js
@@ -73,6 +73,7 @@ $(function() {
collapseTablePageLoad('collapse-aggregated-sparkProperties','aggregated-sparkProperties');
collapseTablePageLoad('collapse-aggregated-hadoopProperties','aggregated-hadoopProperties');
collapseTablePageLoad('collapse-aggregated-systemProperties','aggregated-systemProperties');
+
collapseTablePageLoad('collapse-aggregated-metricsProperties','aggregated-metricsProperties');
collapseTablePageLoad('collapse-aggregated-classpathEntries','aggregated-classpathEntries');
collapseTablePageLoad('collapse-aggregated-activeJobs','aggregated-activeJobs');
collapseTablePageLoad('collapse-aggregated-completedJobs','aggregated-completedJobs');
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala
b/core/src/main/scala/org/apache/spark/SparkContext.scala
index c6cb5cb5e19..a106d5bacc5 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -2590,7 +2590,8 @@ class SparkContext(config: SparkConf) extends Logging {
val addedFilePaths = addedFiles.keys.toSeq
val addedArchivePaths = addedArchives.keys.toSeq
val environmentDetails = SparkEnv.environmentDetails(conf,
hadoopConfiguration,
- schedulingMode, addedJarPaths, addedFilePaths, addedArchivePaths)
+ schedulingMode, addedJarPaths, addedFilePaths, addedArchivePaths,
+ env.metricsSystem.metricsProperties.asScala.toMap)
val environmentUpdate =
SparkListenerEnvironmentUpdate(environmentDetails)
listenerBus.post(environmentUpdate)
}
diff --git a/core/src/main/scala/org/apache/spark/SparkEnv.scala
b/core/src/main/scala/org/apache/spark/SparkEnv.scala
index 19467e7eca1..66ee959dbd8 100644
--- a/core/src/main/scala/org/apache/spark/SparkEnv.scala
+++ b/core/src/main/scala/org/apache/spark/SparkEnv.scala
@@ -429,14 +429,14 @@ object SparkEnv extends Logging {
* class paths. Map keys define the category, and map values represent the
corresponding
* attributes as a sequence of KV pairs. This is used mainly for
SparkListenerEnvironmentUpdate.
*/
- private[spark]
- def environmentDetails(
+ private[spark] def environmentDetails(
conf: SparkConf,
hadoopConf: Configuration,
schedulingMode: String,
addedJars: Seq[String],
addedFiles: Seq[String],
- addedArchives: Seq[String]): Map[String, Seq[(String, String)]] = {
+ addedArchives: Seq[String],
+ metricsProperties: Map[String, String]): Map[String, Seq[(String,
String)]] = {
import Properties._
val jvmInformation = Seq(
@@ -478,6 +478,7 @@ object SparkEnv extends Logging {
"Spark Properties" -> sparkProperties,
"Hadoop Properties" -> hadoopProperties,
"System Properties" -> otherProperties,
- "Classpath Entries" -> classPaths)
+ "Classpath Entries" -> classPaths,
+ "Metrics Properties" -> metricsProperties.toSeq.sorted)
}
}
diff --git a/core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala
b/core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala
index a5903deeb78..caf3c344d64 100644
--- a/core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala
+++ b/core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala
@@ -233,6 +233,8 @@ private[spark] class MetricsSystem private (
}
}
}
+
+ def metricsProperties(): Properties = metricsConfig.properties
}
private[spark] object MetricsSystem {
diff --git
a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
index add9862c306..ea028dfd11d 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusListener.scala
@@ -179,6 +179,7 @@ private[spark] class AppStatusListener(
details.getOrElse("Spark Properties", Nil),
details.getOrElse("Hadoop Properties", Nil),
details.getOrElse("System Properties", Nil),
+ details.getOrElse("Metrics Properties", Nil),
details.getOrElse("Classpath Entries", Nil),
Nil)
diff --git
a/core/src/main/scala/org/apache/spark/status/api/v1/OneApplicationResource.scala
b/core/src/main/scala/org/apache/spark/status/api/v1/OneApplicationResource.scala
index ef17168ebce..a52539c542e 100644
---
a/core/src/main/scala/org/apache/spark/status/api/v1/OneApplicationResource.scala
+++
b/core/src/main/scala/org/apache/spark/status/api/v1/OneApplicationResource.scala
@@ -111,6 +111,7 @@ private[v1] class AbstractApplicationResource extends
BaseAppResource {
Utils.redact(ui.conf, envInfo.sparkProperties).sortBy(_._1),
Utils.redact(ui.conf, envInfo.hadoopProperties).sortBy(_._1),
Utils.redact(ui.conf, envInfo.systemProperties).sortBy(_._1),
+ Utils.redact(ui.conf, envInfo.metricsProperties).sortBy(_._1),
envInfo.classpathEntries.sortBy(_._1),
resourceProfileInfo)
}
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
index d1a4d602fc5..ba21d5d1f62 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
@@ -459,6 +459,7 @@ class ApplicationEnvironmentInfo private[spark] (
val sparkProperties: Seq[(String, String)],
val hadoopProperties: Seq[(String, String)],
val systemProperties: Seq[(String, String)],
+ val metricsProperties: Seq[(String, String)],
val classpathEntries: Seq[(String, String)],
val resourceProfiles: Seq[ResourceProfileInfo])
diff --git a/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala
b/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala
index 2f5b7311892..c6e224732cb 100644
--- a/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/env/EnvironmentPage.scala
@@ -83,6 +83,9 @@ private[ui] class EnvironmentPage(
val systemPropertiesTable = UIUtils.listingTable(propertyHeader,
propertyRow,
Utils.redact(conf, appEnv.systemProperties.sorted), fixedWidth = true,
headerClasses = headerClasses)
+ val metricsPropertiesTable = UIUtils.listingTable(propertyHeader,
propertyRow,
+ Utils.redact(conf, appEnv.metricsProperties.sorted), fixedWidth = true,
+ headerClasses = headerClasses)
val classpathEntriesTable = UIUtils.listingTable(
classPathHeader, classPathRow, appEnv.classpathEntries.sorted,
fixedWidth = true,
headerClasses = headerClasses)
@@ -143,6 +146,17 @@ private[ui] class EnvironmentPage(
<div class="aggregated-systemProperties collapsible-table collapsed">
{systemPropertiesTable}
</div>
+ <span class="collapse-aggregated-metricsProperties collapse-table"
+ onClick="collapseTable('collapse-aggregated-metricsProperties',
+ 'aggregated-metricsProperties')">
+ <h4>
+ <span class="collapse-table-arrow arrow-closed"></span>
+ <a>Metrics Properties</a>
+ </h4>
+ </span>
+ <div class="aggregated-metricsProperties collapsible-table collapsed">
+ {metricsPropertiesTable}
+ </div>
<span class="collapse-aggregated-classpathEntries collapse-table"
onClick="collapseTable('collapse-aggregated-classpathEntries',
'aggregated-classpathEntries')">
diff --git a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
index 09f1be2076c..ef5a812e4b6 100644
--- a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
@@ -180,12 +180,14 @@ private[spark] object JsonProtocol {
val sparkProperties = mapToJson(environmentDetails("Spark
Properties").toMap)
val hadoopProperties = mapToJson(environmentDetails("Hadoop
Properties").toMap)
val systemProperties = mapToJson(environmentDetails("System
Properties").toMap)
+ val metricsProperties = mapToJson(environmentDetails("Metrics
Properties").toMap)
val classpathEntries = mapToJson(environmentDetails("Classpath
Entries").toMap)
("Event" -> SPARK_LISTENER_EVENT_FORMATTED_CLASS_NAMES.environmentUpdate) ~
("JVM Information" -> jvmInformation) ~
("Spark Properties" -> sparkProperties) ~
("Hadoop Properties" -> hadoopProperties) ~
("System Properties" -> systemProperties) ~
+ ("Metrics Properties"-> metricsProperties) ~
("Classpath Entries" -> classpathEntries)
}
@@ -780,11 +782,14 @@ private[spark] object JsonProtocol {
// For compatible with previous event logs
val hadoopProperties = jsonOption(json \ "Hadoop
Properties").map(mapFromJson(_).toSeq)
.getOrElse(Seq.empty)
+ val metricsProperties = jsonOption(json \ "Metrics
Properties").map(mapFromJson(_).toSeq)
+ .getOrElse(Seq.empty)
val environmentDetails = Map[String, Seq[(String, String)]](
"JVM Information" -> mapFromJson(json \ "JVM Information").toSeq,
"Spark Properties" -> mapFromJson(json \ "Spark Properties").toSeq,
"Hadoop Properties" -> hadoopProperties,
"System Properties" -> mapFromJson(json \ "System Properties").toSeq,
+ "Metrics Properties" -> metricsProperties,
"Classpath Entries" -> mapFromJson(json \ "Classpath Entries").toSeq)
SparkListenerEnvironmentUpdate(environmentDetails)
}
diff --git
a/core/src/test/resources/HistoryServerExpectations/app_environment_expectation.json
b/core/src/test/resources/HistoryServerExpectations/app_environment_expectation.json
index c2616129de9..b03416eec94 100644
---
a/core/src/test/resources/HistoryServerExpectations/app_environment_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/app_environment_expectation.json
@@ -97,6 +97,12 @@
[ "user.name", "jose" ],
[ "user.timezone", "America/Chicago" ]
],
+ "metricsProperties" : [
+ [ "*.sink.servlet.class", "org.apache.spark.metrics.sink.MetricsServlet" ],
+ [ "*.sink.servlet.path", "/metrics/json" ],
+ [ "applications.sink.servlet.path", "/metrics/applications/json" ],
+ [ "master.sink.servlet.path", "/metrics/master/json" ]
+ ],
"classpathEntries" : [
[
"/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/RoaringBitmap-0.5.11.jar",
"System Classpath" ],
[
"/Users/Jose/IdeaProjects/spark/assembly/target/scala-2.11/jars/antlr4-runtime-4.5.3.jar",
"System Classpath" ],
diff --git
a/core/src/test/resources/HistoryServerExpectations/multiple_resource_profiles_expectation.json
b/core/src/test/resources/HistoryServerExpectations/multiple_resource_profiles_expectation.json
index 5c1e4cc2337..6b2c18bb0fa 100644
---
a/core/src/test/resources/HistoryServerExpectations/multiple_resource_profiles_expectation.json
+++
b/core/src/test/resources/HistoryServerExpectations/multiple_resource_profiles_expectation.json
@@ -7,6 +7,7 @@
"sparkProperties" : [ ],
"hadoopProperties" : [ ],
"systemProperties" : [ ],
+ "metricsProperties": [ ],
"classpathEntries" : [ ],
"resourceProfiles" : [ {
"id" : 0,
diff --git a/core/src/test/resources/spark-events/app-20161116163331-0000
b/core/src/test/resources/spark-events/app-20161116163331-0000
index 8f77fdd34f4..3e7565beff8 100755
--- a/core/src/test/resources/spark-events/app-20161116163331-0000
+++ b/core/src/test/resources/spark-events/app-20161116163331-0000
@@ -1,6 +1,6 @@
{"Event":"SparkListenerLogStart","Spark Version":"2.1.0-SNAPSHOT"}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor
ID":"driver","Host":"172.22.0.167","Port":51475},"Maximum
Memory":908381388,"Timestamp":1479335611477,"Maximum Onheap
Memory":384093388,"Maximum Offheap Memory":524288000}
-{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java
Home":"/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre","Java
Version":"1.8.0_92 (Oracle Corporation)","Scala Version":"version
2.11.8"},"Spark
Properties":{"spark.blacklist.task.maxTaskAttemptsPerExecutor":"3","spark.blacklist.enabled":"TRUE","spark.driver.host":"172.22.0.167","spark.blacklist.task.maxTaskAttemptsPerNode":"3","spark.eventLog.enabled":"TRUE","spark.driver.port":"51459","spark.repl.clas
[...]
+{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java
Home":"/Library/Java/JavaVirtualMachines/jdk1.8.0_92.jdk/Contents/Home/jre","Java
Version":"1.8.0_92 (Oracle Corporation)","Scala Version":"version
2.11.8"},"Spark
Properties":{"spark.blacklist.task.maxTaskAttemptsPerExecutor":"3","spark.blacklist.enabled":"TRUE","spark.driver.host":"172.22.0.167","spark.blacklist.task.maxTaskAttemptsPerNode":"3","spark.eventLog.enabled":"TRUE","spark.driver.port":"51459","spark.repl.clas
[...]
{"Event":"SparkListenerApplicationStart","App Name":"Spark shell","App
ID":"app-20161116163331-0000","Timestamp":1479335609916,"User":"jose"}
{"Event":"SparkListenerExecutorAdded","Timestamp":1479335615320,"Executor
ID":"3","Executor Info":{"Host":"172.22.0.167","Total Cores":4,"Log
Urls":{"stdout":"http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stdout","stderr":"http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stderr"}}}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor
ID":"3","Host":"172.22.0.167","Port":51485},"Maximum
Memory":908381388,"Timestamp":1479335615387,"Maximum Onheap
Memory":384093388,"Maximum Offheap Memory":524288000}
diff --git
a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
index 2d802b81763..78abbe93b79 100644
---
a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
+++
b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
@@ -829,6 +829,7 @@ abstract class FsHistoryProviderSuite extends SparkFunSuite
with Matchers with L
"Hadoop Properties" -> Seq.empty,
"JVM Information" -> Seq.empty,
"System Properties" -> Seq.empty,
+ "Metrics Properties" -> Seq.empty,
"Classpath Entries" -> Seq.empty
)),
SparkListenerApplicationEnd(System.currentTimeMillis()))
@@ -1088,6 +1089,7 @@ abstract class FsHistoryProviderSuite extends
SparkFunSuite with Matchers with L
"Hadoop Properties" -> Seq.empty,
"JVM Information" -> Seq.empty,
"System Properties" -> Seq.empty,
+ "Metrics Properties" -> Seq.empty,
"Classpath Entries" -> Seq.empty
)),
SparkListenerApplicationEnd(5L)
@@ -1573,6 +1575,7 @@ abstract class FsHistoryProviderSuite extends
SparkFunSuite with Matchers with L
"Hadoop Properties" -> Seq.empty,
"JVM Information" -> Seq.empty,
"System Properties" -> Seq.empty,
+ "Metrics Properties" -> Seq.empty,
"Classpath Entries" -> Seq.empty
)),
SparkListenerApplicationEnd(System.currentTimeMillis()))
diff --git
a/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala
b/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala
index d7cd63a7c21..d790c35a330 100644
---
a/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala
+++
b/core/src/test/scala/org/apache/spark/scheduler/EventLoggingListenerSuite.scala
@@ -91,7 +91,7 @@ class EventLoggingListenerSuite extends SparkFunSuite with
LocalSparkContext wit
.set(key, secretPassword)
val hadoopconf = SparkHadoopUtil.get.newConfiguration(new SparkConf())
val envDetails = SparkEnv.environmentDetails(
- conf, hadoopconf, "FIFO", Seq.empty, Seq.empty, Seq.empty)
+ conf, hadoopconf, "FIFO", Seq.empty, Seq.empty, Seq.empty, Map.empty)
val event = SparkListenerEnvironmentUpdate(envDetails)
val redactedProps = EventLoggingListener
.redactEvent(conf, event).environmentDetails("Spark Properties").toMap
diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
index 5800dbda297..572ac6fc8bb 100644
--- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
@@ -78,6 +78,8 @@ class JsonProtocolSuite extends SparkFunSuite {
"Spark Properties" -> Seq(("Job throughput", "80000 jobs/s, regardless
of job type")),
"Hadoop Properties" -> Seq(("hadoop.tmp.dir", "/usr/local/hadoop/tmp")),
"System Properties" -> Seq(("Username", "guest"), ("Password", "guest")),
+ "Metrics Properties" ->
+ Seq(("*.sink.servlet.class",
"org.apache.spark.metrics.sink.MetricsServlet")),
"Classpath Entries" -> Seq(("Super library", "/tmp/super_library"))
))
val blockManagerAdded = SparkListenerBlockManagerAdded(1L,
@@ -2048,6 +2050,9 @@ private[spark] object JsonProtocolSuite extends
Assertions {
| "Username": "guest",
| "Password": "guest"
| },
+ | "Metrics Properties": {
+ | "*.sink.servlet.class":
"org.apache.spark.metrics.sink.MetricsServlet"
+ | },
| "Classpath Entries": {
| "Super library": "/tmp/super_library"
| }
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index f0010dfcd83..927384d4f1e 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -37,7 +37,10 @@ object MimaExcludes {
// Exclude rules for 3.4.x
lazy val v34excludes = v33excludes ++ Seq(
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.recommendation.ALS.checkedCast"),
-
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.recommendation.ALSModel.checkedCast")
+
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.recommendation.ALSModel.checkedCast"),
+
+ // [SPARK-39110] Show metrics properties in HistoryServer environment tab
+
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.status.api.v1.ApplicationEnvironmentInfo.this")
)
// Exclude rules for 3.3.x from 3.2.0
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
![https://user-images.githubusercontent.com/46485123/167062876-c0c98a69-13c7-4a25-bb31-74f1ada88153.png"](https://user-images.githubusercontent.com/46485123/167062876-c0c98a69-13c7-4a25-bb31-74f1ada88153.png"){kind=link}
![https://user-images.githubusercontent.com/46485123/167062893-f297eeda-b08f-4c9d-a2a2-a74add97493f.png"](https://user-images.githubusercontent.com/46485123/167062893-f297eeda-b08f-4c9d-a2a2-a74add97493f.png"){kind=link}