This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 1787a5261e87 [SPARK-46887][DOCS] Document a few missed `spark.ui.*`
configs to `Configuration` page
1787a5261e87 is described below
commit 1787a5261e87e0214a3f803f6534c5e52a0138e6
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Sat Jan 27 16:48:26 2024 -0800
[SPARK-46887][DOCS] Document a few missed `spark.ui.*` configs to
`Configuration` page
### What changes were proposed in this pull request?
This PR aims to document a few missed `spark.ui.*` configurations for
Apache Spark 4. This PR focuses only public configurations and excludes
`internal` configuration like `spark.ui.jettyStopTimeout`.
### Why are the changes needed?
To improve documentations.
After this PR, I verified the following configurations are documented at
least once in `Configuration` or `Security` page.
```
$ git grep 'ConfigBuilder("spark.ui.'
core/src/main/scala/org/apache/spark/internal/config/Status.scala: val
LIVE_ENTITY_UPDATE_PERIOD = ConfigBuilder("spark.ui.liveUpdate.period")
core/src/main/scala/org/apache/spark/internal/config/Status.scala: val
LIVE_ENTITY_UPDATE_MIN_FLUSH_PERIOD =
ConfigBuilder("spark.ui.liveUpdate.minFlushPeriod")
core/src/main/scala/org/apache/spark/internal/config/Status.scala: val
MAX_RETAINED_JOBS = ConfigBuilder("spark.ui.retainedJobs")
core/src/main/scala/org/apache/spark/internal/config/Status.scala: val
MAX_RETAINED_STAGES = ConfigBuilder("spark.ui.retainedStages")
core/src/main/scala/org/apache/spark/internal/config/Status.scala: val
MAX_RETAINED_TASKS_PER_STAGE = ConfigBuilder("spark.ui.retainedTasks")
core/src/main/scala/org/apache/spark/internal/config/Status.scala: val
MAX_RETAINED_DEAD_EXECUTORS = ConfigBuilder("spark.ui.retainedDeadExecutors")
core/src/main/scala/org/apache/spark/internal/config/Status.scala: val
MAX_RETAINED_ROOT_NODES = ConfigBuilder("spark.ui.dagGraph.retainedRootRDDs")
core/src/main/scala/org/apache/spark/internal/config/Status.scala: val
LIVE_UI_LOCAL_STORE_DIR = ConfigBuilder("spark.ui.store.path")
core/src/main/scala/org/apache/spark/internal/config/UI.scala: val
UI_SHOW_CONSOLE_PROGRESS = ConfigBuilder("spark.ui.showConsoleProgress")
core/src/main/scala/org/apache/spark/internal/config/UI.scala:
ConfigBuilder("spark.ui.consoleProgress.update.interval")
core/src/main/scala/org/apache/spark/internal/config/UI.scala: val
UI_ENABLED = ConfigBuilder("spark.ui.enabled")
core/src/main/scala/org/apache/spark/internal/config/UI.scala: val UI_PORT
= ConfigBuilder("spark.ui.port")
core/src/main/scala/org/apache/spark/internal/config/UI.scala: val
UI_FILTERS = ConfigBuilder("spark.ui.filters")
core/src/main/scala/org/apache/spark/internal/config/UI.scala: val
UI_ALLOW_FRAMING_FROM = ConfigBuilder("spark.ui.allowFramingFrom")
core/src/main/scala/org/apache/spark/internal/config/UI.scala: val
UI_REVERSE_PROXY = ConfigBuilder("spark.ui.reverseProxy")
core/src/main/scala/org/apache/spark/internal/config/UI.scala: val
UI_REVERSE_PROXY_URL = ConfigBuilder("spark.ui.reverseProxyUrl")
core/src/main/scala/org/apache/spark/internal/config/UI.scala: val
UI_KILL_ENABLED = ConfigBuilder("spark.ui.killEnabled")
core/src/main/scala/org/apache/spark/internal/config/UI.scala: val
UI_THREAD_DUMPS_ENABLED = ConfigBuilder("spark.ui.threadDumpsEnabled")
core/src/main/scala/org/apache/spark/internal/config/UI.scala: val
UI_FLAMEGRAPH_ENABLED = ConfigBuilder("spark.ui.threadDump.flamegraphEnabled")
core/src/main/scala/org/apache/spark/internal/config/UI.scala: val
UI_HEAP_HISTOGRAM_ENABLED = ConfigBuilder("spark.ui.heapHistogramEnabled")
core/src/main/scala/org/apache/spark/internal/config/UI.scala: val
UI_PROMETHEUS_ENABLED = ConfigBuilder("spark.ui.prometheus.enabled")
core/src/main/scala/org/apache/spark/internal/config/UI.scala: val
UI_X_XSS_PROTECTION = ConfigBuilder("spark.ui.xXssProtection")
core/src/main/scala/org/apache/spark/internal/config/UI.scala: val
UI_X_CONTENT_TYPE_OPTIONS =
ConfigBuilder("spark.ui.xContentTypeOptions.enabled")
core/src/main/scala/org/apache/spark/internal/config/UI.scala: val
UI_STRICT_TRANSPORT_SECURITY = ConfigBuilder("spark.ui.strictTransportSecurity")
core/src/main/scala/org/apache/spark/internal/config/UI.scala: val
UI_REQUEST_HEADER_SIZE = ConfigBuilder("spark.ui.requestHeaderSize")
core/src/main/scala/org/apache/spark/internal/config/UI.scala: val
UI_TIMELINE_ENABLED = ConfigBuilder("spark.ui.timelineEnabled")
core/src/main/scala/org/apache/spark/internal/config/UI.scala: val
UI_TIMELINE_TASKS_MAXIMUM = ConfigBuilder("spark.ui.timeline.tasks.maximum")
core/src/main/scala/org/apache/spark/internal/config/UI.scala: val
UI_TIMELINE_JOBS_MAXIMUM = ConfigBuilder("spark.ui.timeline.jobs.maximum")
core/src/main/scala/org/apache/spark/internal/config/UI.scala: val
UI_TIMELINE_STAGES_MAXIMUM = ConfigBuilder("spark.ui.timeline.stages.maximum")
core/src/main/scala/org/apache/spark/internal/config/UI.scala: val
UI_TIMELINE_EXECUTORS_MAXIMUM =
ConfigBuilder("spark.ui.timeline.executors.maximum")
core/src/main/scala/org/apache/spark/internal/config/UI.scala: val
UI_VIEW_ACLS = ConfigBuilder("spark.ui.view.acls")
core/src/main/scala/org/apache/spark/internal/config/UI.scala: val
UI_VIEW_ACLS_GROUPS = ConfigBuilder("spark.ui.view.acls.groups")
core/src/main/scala/org/apache/spark/internal/config/UI.scala: val
PROXY_REDIRECT_URI = ConfigBuilder("spark.ui.proxyRedirectUri")
core/src/main/scala/org/apache/spark/internal/config/UI.scala: val
CUSTOM_EXECUTOR_LOG_URL = ConfigBuilder("spark.ui.custom.executor.log.url")
core/src/main/scala/org/apache/spark/internal/config/UI.scala: val
UI_SQL_GROUP_SUB_EXECUTION_ENABLED =
ConfigBuilder("spark.ui.groupSQLSubExecutionEnabled")
core/src/main/scala/org/apache/spark/internal/config/UI.scala: val
UI_JETTY_STOP_TIMEOUT = ConfigBuilder("spark.ui.jettyStopTimeout")
```
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Manual review.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #44913 from dongjoon-hyun/SPARK-46887.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../org/apache/spark/internal/config/UI.scala | 2 +
docs/configuration.md | 48 ++++++++++++++++++++++
2 files changed, 50 insertions(+)
diff --git a/core/src/main/scala/org/apache/spark/internal/config/UI.scala
b/core/src/main/scala/org/apache/spark/internal/config/UI.scala
index 086c83552732..0af38a2eb90b 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/UI.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/UI.scala
@@ -93,6 +93,7 @@ private[spark] object UI {
.createWithDefault(true)
val UI_THREAD_DUMPS_ENABLED = ConfigBuilder("spark.ui.threadDumpsEnabled")
+ .doc("Whether to show a link for executor thread dumps in Stages and
Executor pages.")
.version("1.2.0")
.booleanConf
.createWithDefault(true)
@@ -104,6 +105,7 @@ private[spark] object UI {
.createWithDefault(true)
val UI_HEAP_HISTOGRAM_ENABLED =
ConfigBuilder("spark.ui.heapHistogramEnabled")
+ .doc("Whether to show a link for executor heap histogram in Executor
page.")
.version("3.5.0")
.booleanConf
.createWithDefault(true)
diff --git a/docs/configuration.md b/docs/configuration.md
index beb52c62d6c2..e771c323d369 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -1425,6 +1425,14 @@ Apart from these, the following properties are also
available, and may be useful
</td>
<td>2.1.0</td>
</tr>
+<tr>
+ <td><code>spark.ui.groupSQLSubExecutionEnabled</code></td>
+ <td>true</td>
+ <td>
+ Whether to group sub executions together in SQL UI when they belong to the
same root execution
+ </td>
+ <td>3.4.0</td>
+</tr>
<tr>
<td><code>spark.ui.enabled</code></td>
<td>true</td>
@@ -1450,6 +1458,30 @@ Apart from these, the following properties are also
available, and may be useful
</td>
<td>1.0.0</td>
</tr>
+<tr>
+ <td><code>spark.ui.threadDumpsEnabled</code></td>
+ <td>true</td>
+ <td>
+ Whether to show a link for executor thread dumps in Stages and Executor
pages.
+ </td>
+ <td>1.2.0</td>
+</tr>
+<tr>
+ <td><code>spark.ui.threadDump.flamegraphEnabled</code></td>
+ <td>true</td>
+ <td>
+ Whether to render the Flamegraph for executor thread dumps.
+ </td>
+ <td>4.0.0</td>
+</tr>
+<tr>
+ <td><code>spark.ui.heapHistogramEnabled</code></td>
+ <td>true</td>
+ <td>
+ Whether to show a link for executor heap histogram in Executor page.
+ </td>
+ <td>3.5.0</td>
+</tr>
<tr>
<td><code>spark.ui.liveUpdate.period</code></td>
<td>100ms</td>
@@ -1568,6 +1600,14 @@ Apart from these, the following properties are also
available, and may be useful
</td>
<td>1.2.1</td>
</tr>
+<tr>
+ <td><code>spark.ui.consoleProgress.update.interval</code></td>
+ <td>200</td>
+ <td>
+ An interval in milliseconds to update the progress bar in the console.
+ </td>
+ <td>2.1.0</td>
+</tr>
<tr>
<td><code>spark.ui.custom.executor.log.url</code></td>
<td>(none)</td>
@@ -1584,6 +1624,14 @@ Apart from these, the following properties are also
available, and may be useful
</td>
<td>3.0.0</td>
</tr>
+<tr>
+ <td><code>spark.ui.prometheus.enabled</code></td>
+ <td>true</td>
+ <td>
+ Expose executor metrics at /metrics/executors/prometheus at driver web
page.
+ </td>
+ <td>3.0.0</td>
+</tr>
<tr>
<td><code>spark.worker.ui.retainedExecutors</code></td>
<td>1000</td>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
