This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new e60d8fc [SPARK-27045][SQL] SQL tab in UI shows actual SQL instead of
callsite in case of SparkSQLDriver
e60d8fc is described below
commit e60d8fce0b0cf2a6d766ea2fc5f994546550570a
Author: Ajith <[email protected]>
AuthorDate: Tue Mar 12 16:14:29 2019 -0700
[SPARK-27045][SQL] SQL tab in UI shows actual SQL instead of callsite in
case of SparkSQLDriver
## What changes were proposed in this pull request?
When we run sql in spark via SparkSQLDriver (thrift server, spark-sql), SQL
string is siet via ``setJobDescription``. the SparkUI SQL tab must show SQL
instead of stacktrace in case ``setJobDescription`` is set which is more useful
to end user. Instead it currently shows in description column the callsite
shortform which is less useful
## How was this patch tested?
Manually:
Closes #23958 from ajithme/sqlui.
Authored-by: Ajith <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../org/apache/spark/sql/internal/StaticSQLConf.scala | 6 ++++++
.../org/apache/spark/sql/execution/SQLExecution.scala | 15 ++++++++++++++-
.../apache/spark/sql/execution/ui/AllExecutionsPage.scala | 5 +++--
3 files changed, 23 insertions(+), 3 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
index fc07efb..e12f05b 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
@@ -132,4 +132,10 @@ object StaticSQLConf {
.intConf
.createWithDefault(1000)
+ val SQL_EVENT_TRUNCATE_LENGTH =
buildStaticConf("spark.sql.event.truncate.length")
+ .doc("Threshold of SQL length beyond which it will be truncated before
adding to " +
+ "event. Defaults to no truncation. If set to 0, callsite will be logged
instead.")
+ .intConf
+ .checkValue(_ >= 0, "Must be set greater or equal to zero")
+ .createWithDefault(Int.MaxValue)
}
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala
index 5b38fe5..ca66337 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecution.scala
@@ -20,9 +20,12 @@ package org.apache.spark.sql.execution
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.atomic.AtomicLong
+import org.apache.spark.SparkContext
import org.apache.spark.internal.config.Tests.IS_TESTING
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.execution.ui.{SparkListenerSQLExecutionEnd,
SparkListenerSQLExecutionStart}
+import org.apache.spark.sql.internal.StaticSQLConf.SQL_EVENT_TRUNCATE_LENGTH
+import org.apache.spark.util.Utils
object SQLExecution {
@@ -71,13 +74,23 @@ object SQLExecution {
// streaming queries would give us call site like "run at <unknown>:0"
val callSite = sc.getCallSite()
+ val truncateLength = sc.conf.get(SQL_EVENT_TRUNCATE_LENGTH)
+
+ val desc =
Option(sc.getLocalProperty(SparkContext.SPARK_JOB_DESCRIPTION))
+ .filter(_ => truncateLength > 0)
+ .map { sqlStr =>
+ val redactedStr = Utils
+ .redact(sparkSession.sessionState.conf.stringRedactionPattern,
sqlStr)
+ redactedStr.substring(0, Math.min(truncateLength,
redactedStr.length))
+ }.getOrElse(callSite.shortForm)
+
withSQLConfPropagated(sparkSession) {
var ex: Option[Exception] = None
val startTime = System.nanoTime()
try {
sc.listenerBus.post(SparkListenerSQLExecutionStart(
executionId = executionId,
- description = callSite.shortForm,
+ description = desc,
details = callSite.longForm,
physicalPlanDescription = queryExecution.toString,
// `queryExecution.executedPlan` triggers query planning. If it
fails, the exception
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/AllExecutionsPage.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/AllExecutionsPage.scala
index 43ff1e1..824c094 100644
---
a/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/AllExecutionsPage.scala
+++
b/sql/core/src/main/scala/org/apache/spark/sql/execution/ui/AllExecutionsPage.scala
@@ -388,14 +388,15 @@ private[ui] class ExecutionPagedTable(
+details
</span> ++
<div class="stage-details collapsed">
- <pre>{execution.details}</pre>
+ <pre>{execution.description}<br></br>{execution.details}</pre>
</div>
} else {
Nil
}
val desc = if (execution.description != null &&
execution.description.nonEmpty) {
- <a href={executionURL(execution.executionId)}>{execution.description}</a>
+ <a href={executionURL(execution.executionId)} class="description-input">
+ {execution.description}</a>
} else {
<a href={executionURL(execution.executionId)}>{execution.executionId}</a>
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
