This is an automated email from the ASF dual-hosted git repository.
ulyssesyou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new e7c4201 [KYUUBI #2119][FOLLOWUP] Support output progress bar in Spark
engine
e7c4201 is described below
commit e7c420122635f4b195d27253622303c8d4732609
Author: sychen <[email protected]>
AuthorDate: Mon Mar 21 11:01:42 2022 +0800
[KYUUBI #2119][FOLLOWUP] Support output progress bar in Spark engine
### _Why are the changes needed?_
Based on this https://github.com/apache/incubator-kyuubi/pull/2162
Add several configuration items to control the output of the progress bar.
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run
test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests)
locally before make a pull request
Closes #2168 from cxzl25/KYUUBI-2119-FOLLOWUP.
Closes #2119
36db8c5f [sychen] Add configuration progress bar
Authored-by: sychen <[email protected]>
Signed-off-by: ulysses-you <[email protected]>
---
docs/deployment/settings.md | 3 +++
.../apache/spark/kyuubi/SQLOperationListener.scala | 18 ++++++++++++++----
.../spark/kyuubi/SparkConsoleProgressBar.scala | 8 ++++----
.../org/apache/kyuubi/config/KyuubiConf.scala | 22 ++++++++++++++++++++++
.../kyuubi/config/AllKyuubiConfiguration.scala | 4 ++--
5 files changed, 45 insertions(+), 10 deletions(-)
diff --git a/docs/deployment/settings.md b/docs/deployment/settings.md
index 3e1ec78..ce3b93c 100644
--- a/docs/deployment/settings.md
+++ b/docs/deployment/settings.md
@@ -333,6 +333,9 @@ Key | Default | Meaning | Type | Since
<code>kyuubi.session.engine.request.timeout</code>|<div style='width:
65pt;word-wrap: break-word;white-space: normal'>PT1M</div>|<div style='width:
170pt;word-wrap: break-word;white-space: normal'>The timeout of awaiting
response after sending request to remote sql query engine</div>|<div
style='width: 30pt'>duration</div>|<div style='width: 20pt'>1.4.0</div>
<code>kyuubi.session.engine.share.level</code>|<div style='width:
65pt;word-wrap: break-word;white-space: normal'>USER</div>|<div style='width:
170pt;word-wrap: break-word;white-space: normal'>(deprecated) - Using
kyuubi.engine.share.level instead</div>|<div style='width:
30pt'>string</div>|<div style='width: 20pt'>1.0.0</div>
<code>kyuubi.session.engine.spark.main.resource</code>|<div style='width:
65pt;word-wrap: break-word;white-space: normal'><undefined></div>|<div
style='width: 170pt;word-wrap: break-word;white-space: normal'>The package used
to create Spark SQL engine remote application. If it is undefined, Kyuubi will
use the default</div>|<div style='width: 30pt'>string</div>|<div style='width:
20pt'>1.0.0</div>
+<code>kyuubi.session.engine.spark.progress.timeFormat</code>|<div
style='width: 65pt;word-wrap: break-word;white-space: normal'>yyyy-MM-dd
HH:mm:ss.SSS</div>|<div style='width: 170pt;word-wrap: break-word;white-space:
normal'>The time format of the progress bar</div>|<div style='width:
30pt'>string</div>|<div style='width: 20pt'>1.6.0</div>
+<code>kyuubi.session.engine.spark.progress.update.interval</code>|<div
style='width: 65pt;word-wrap: break-word;white-space: normal'>PT1S</div>|<div
style='width: 170pt;word-wrap: break-word;white-space: normal'>Update period of
progress bar.</div>|<div style='width: 30pt'>duration</div>|<div style='width:
20pt'>1.6.0</div>
+<code>kyuubi.session.engine.spark.showProgress</code>|<div style='width:
65pt;word-wrap: break-word;white-space: normal'>false</div>|<div style='width:
170pt;word-wrap: break-word;white-space: normal'>When true, show the progress
bar in the spark engine log.</div>|<div style='width: 30pt'>boolean</div>|<div
style='width: 20pt'>1.6.0</div>
<code>kyuubi.session.engine.startup.error.max.size</code>|<div style='width:
65pt;word-wrap: break-word;white-space: normal'>8192</div>|<div style='width:
170pt;word-wrap: break-word;white-space: normal'>During engine bootstrapping,
if error occurs, using this config to limit the length error
message(characters).</div>|<div style='width: 30pt'>int</div>|<div
style='width: 20pt'>1.1.0</div>
<code>kyuubi.session.engine.startup.maxLogLines</code>|<div style='width:
65pt;word-wrap: break-word;white-space: normal'>10</div>|<div style='width:
170pt;word-wrap: break-word;white-space: normal'>The maximum number of engine
log lines when errors occur during engine startup phase. Note that this max
lines is for client-side to help track engine startup issue.</div>|<div
style='width: 30pt'>int</div>|<div style='width: 20pt'>1.4.0</div>
<code>kyuubi.session.engine.startup.waitCompletion</code>|<div style='width:
65pt;word-wrap: break-word;white-space: normal'>true</div>|<div style='width:
170pt;word-wrap: break-word;white-space: normal'>Whether to wait for completion
after engine starts. If false, the startup process will be destroyed after the
engine is started. Note that only use it when the driver is not running
locally, such as yarn-cluster mode; Otherwise, the engine will be
killed.</div>|<div style='width: 30pt'>b [...]
diff --git
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SQLOperationListener.scala
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SQLOperationListener.scala
index d59d8f5..42b49ea 100644
---
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SQLOperationListener.scala
+++
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SQLOperationListener.scala
@@ -25,6 +25,8 @@ import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd
import org.apache.kyuubi.Logging
+import org.apache.kyuubi.config.KyuubiConf
+import org.apache.kyuubi.config.KyuubiConf.{ENGINE_SPARK_SHOW_PROGRESS,
ENGINE_SPARK_SHOW_PROGRESS_TIME_FORMAT,
ENGINE_SPARK_SHOW_PROGRESS_UPDATE_INTERVAL}
import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_STATEMENT_ID_KEY
import
org.apache.kyuubi.engine.spark.KyuubiSparkUtil.SPARK_SQL_EXECUTION_ID_KEY
import org.apache.kyuubi.operation.Operation
@@ -46,9 +48,17 @@ class SQLOperationListener(
private var executionId: Option[Long] = None
private val liveStages = new ConcurrentHashMap[StageAttempt, StageInfo]()
- private val consoleProgressBar = new SparkConsoleProgressBar(
- operation,
- liveStages)
+ private val conf: KyuubiConf = operation.getSession.sessionManager.getConf
+ private val consoleProgressBar =
+ if (conf.get(ENGINE_SPARK_SHOW_PROGRESS)) {
+ Some(new SparkConsoleProgressBar(
+ operation,
+ liveStages,
+ conf.get(ENGINE_SPARK_SHOW_PROGRESS_UPDATE_INTERVAL),
+ conf.get(ENGINE_SPARK_SHOW_PROGRESS_TIME_FORMAT)))
+ } else {
+ None
+ }
def getExecutionId: Option[Long] = executionId
@@ -148,7 +158,7 @@ class SQLOperationListener(
case sqlExecutionEnd: SparkListenerSQLExecutionEnd
if executionId.contains(sqlExecutionEnd.executionId) =>
spark.sparkContext.removeSparkListener(this)
- consoleProgressBar.finish()
+ consoleProgressBar.foreach(_.finish())
case _ =>
}
}
diff --git
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkConsoleProgressBar.scala
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkConsoleProgressBar.scala
index eefdc3b..fc2ebd5 100644
---
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkConsoleProgressBar.scala
+++
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkConsoleProgressBar.scala
@@ -29,12 +29,12 @@ import org.apache.kyuubi.operation.Operation
class SparkConsoleProgressBar(
operation: Operation,
- liveStages: ConcurrentHashMap[StageAttempt, StageInfo])
+ liveStages: ConcurrentHashMap[StageAttempt, StageInfo],
+ updatePeriodMSec: Long,
+ timeFormat: String)
extends Logging {
// Carriage return
private val CR = '\r'
- // Update period of progress bar, in milliseconds
- private val updatePeriodMSec = 200L
// Delay to show up a progress bar, in milliseconds
private val firstDelayMSec = 500L
@@ -44,7 +44,7 @@ class SparkConsoleProgressBar(
private var lastUpdateTime = 0L
private var lastProgressBar = ""
- val dtFormatter: DateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd
HH:mm:ss.SSS")
+ val dtFormatter: DateTimeFormatter = DateTimeFormatter.ofPattern(timeFormat)
.withLocale(Locale.getDefault).withZone(ZoneId.systemDefault)
// Schedule a refresh thread to run periodically
diff --git
a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
index 7c0a084..513c840 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
@@ -1267,4 +1267,26 @@ object KyuubiConf {
.stringConf
.toSequence()
.createWithDefault(Seq("java", "sun", "os", "jdk"))
+
+ val ENGINE_SPARK_SHOW_PROGRESS: ConfigEntry[Boolean] =
+ buildConf("session.engine.spark.showProgress")
+ .doc("When true, show the progress bar in the spark engine log.")
+ .version("1.6.0")
+ .booleanConf
+ .createWithDefault(false)
+
+ val ENGINE_SPARK_SHOW_PROGRESS_UPDATE_INTERVAL: ConfigEntry[Long] =
+ buildConf("session.engine.spark.progress.update.interval")
+ .doc("Update period of progress bar.")
+ .version("1.6.0")
+ .timeConf
+ .checkValue(_ >= 200, "Minimum 200 milliseconds")
+ .createWithDefault(1000)
+
+ val ENGINE_SPARK_SHOW_PROGRESS_TIME_FORMAT: ConfigEntry[String] =
+ buildConf("session.engine.spark.progress.timeFormat")
+ .doc("The time format of the progress bar")
+ .version("1.6.0")
+ .stringConf
+ .createWithDefault("yyyy-MM-dd HH:mm:ss.SSS")
}
diff --git
a/kyuubi-server/src/test/scala/org/apache/kyuubi/config/AllKyuubiConfiguration.scala
b/kyuubi-server/src/test/scala/org/apache/kyuubi/config/AllKyuubiConfiguration.scala
index 0468ea0..946cf00 100644
---
a/kyuubi-server/src/test/scala/org/apache/kyuubi/config/AllKyuubiConfiguration.scala
+++
b/kyuubi-server/src/test/scala/org/apache/kyuubi/config/AllKyuubiConfiguration.scala
@@ -35,12 +35,12 @@ import org.apache.kyuubi.zookeeper.ZookeeperConf
*
* To run the entire test suite:
* {{{
- * build/mvn clean install -Pflink-provided,spark-provided
-DwildcardSuites=org.apache.kyuubi.config.AllKyuubiConfiguration
+ * build/mvn clean install -Pflink-provided,spark-provided,hive-provided
-DwildcardSuites=org.apache.kyuubi.config.AllKyuubiConfiguration
* }}}
*
* To re-generate golden files for entire suite, run:
* {{{
- * KYUUBI_UPDATE=1 build/mvn clean install -Pflink-provided,spark-provided
-DwildcardSuites=org.apache.kyuubi.config.AllKyuubiConfiguration
+ * KYUUBI_UPDATE=1 build/mvn clean install
-Pflink-provided,spark-provided,hive-provided
-DwildcardSuites=org.apache.kyuubi.config.AllKyuubiConfiguration
* }}}
*/
// scalastyle:on line.size.limit
