This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 7631e7d8b [KYUUBI #5009] Pass Spark Engine Log Path to Spark Conf
7631e7d8b is described below
commit 7631e7d8b8ca66b635f9d761c41d40e2bb73ea10
Author: zwangsheng <[email protected]>
AuthorDate: Fri Jul 21 11:21:07 2023 +0800
[KYUUBI #5009] Pass Spark Engine Log Path to Spark Conf
### _Why are the changes needed?_
Close #5009
When Kyuubi Server Log is Huge, it's difficult to find `Spark Engine Log
Path` in logs.
Here pass the path to spark conf, user can find engine log path in spark ui
or spark history server.
Submit Command Like:
```shell
XXXX/bin/spark-submit \
--class org.apache.kyuubi.engine.spark.SparkSQLEngine \
--conf
spark.kyuubi.engine.engineLog.path=XXXX/kyuubi-spark-sql-engine.log.0 \
--proxy-user kyuubi
XXXX/target/kyuubi-spark-sql-engine_2.12-1.8.0-SNAPSHOT.jar
```
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [ ] [Run
test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests)
locally before make a pull request
Closes #5011 from zwangsheng/KYUUBI_5009.
Closes #5009
36c772209 [zwangsheng] fix compile
1c20f9264 [zwangsheng] retest
70568c758 [zwangsheng] Fix Unit Test
2bc465740 [zwangsheng] try to fix unit test
2197b3503 [zwangsheng] Narrow the scope of access
a44eefc5c [zwangsheng] [KYUUBI #5009]Pass Spark Engine Log Path to Spark
COnf
Authored-by: zwangsheng <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
---
.../src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala | 2 ++
.../apache/kyuubi/engine/spark/SparkBatchProcessBuilder.scala | 4 ++--
.../org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala | 11 ++++++++---
.../apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala | 11 ++++++++++-
4 files changed, 22 insertions(+), 6 deletions(-)
diff --git
a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala
b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala
index d0e6bae6b..f1e49d687 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala
@@ -359,4 +359,6 @@ object ProcBuilder extends Logging {
private val PROC_BUILD_LOGGER = new
NamedThreadFactory("process-logger-capture", daemon = true)
private val UNCAUGHT_ERROR = new RuntimeException("Uncaught error")
+
+ private[engine] val KYUUBI_ENGINE_LOG_PATH_KEY =
"kyuubi.engine.engineLog.path"
}
diff --git
a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkBatchProcessBuilder.scala
b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkBatchProcessBuilder.scala
index 0d20068de..7f1be93b5 100644
---
a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkBatchProcessBuilder.scala
+++
b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkBatchProcessBuilder.scala
@@ -36,7 +36,7 @@ class SparkBatchProcessBuilder(
extends SparkProcessBuilder(proxyUser, conf, batchId, extraEngineLog) {
import SparkProcessBuilder._
- override protected val commands: Array[String] = {
+ override protected lazy val commands: Array[String] = {
val buffer = new ArrayBuffer[String]()
buffer += executable
Option(mainClass).foreach { cla =>
@@ -51,7 +51,7 @@ class SparkBatchProcessBuilder(
// tag batch application
KyuubiApplicationManager.tagApplication(batchId, "spark",
clusterManager(), batchKyuubiConf)
- (batchKyuubiConf.getAll ++ sparkAppNameConf()).foreach { case (k, v) =>
+ (batchKyuubiConf.getAll ++ sparkAppNameConf() ++
engineLogPathConf()).foreach { case (k, v) =>
buffer += CONF
buffer += s"${convertConfigKey(k)}=$v"
}
diff --git
a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala
b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala
index 5bfe506da..5998d5d4e 100644
---
a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala
+++
b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala
@@ -30,6 +30,7 @@ import org.apache.kyuubi._
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.engine.{ApplicationManagerInfo,
KyuubiApplicationManager, ProcBuilder}
import
org.apache.kyuubi.engine.KubernetesApplicationOperation.{KUBERNETES_SERVICE_HOST,
KUBERNETES_SERVICE_PORT}
+import org.apache.kyuubi.engine.ProcBuilder.KYUUBI_ENGINE_LOG_PATH_KEY
import org.apache.kyuubi.ha.HighAvailabilityConf
import org.apache.kyuubi.ha.client.AuthTypes
import org.apache.kyuubi.operation.log.OperationLog
@@ -99,7 +100,7 @@ class SparkProcessBuilder(
}
}
- override protected val commands: Array[String] = {
+ override protected lazy val commands: Array[String] = {
// complete `spark.master` if absent on kubernetes
completeMasterUrl(conf)
@@ -116,8 +117,8 @@ class SparkProcessBuilder(
== AuthTypes.KERBEROS) {
allConf = allConf ++ zkAuthKeytabFileConf(allConf)
}
-
- allConf.foreach { case (k, v) =>
+ // pass spark engine log path to spark conf
+ (allConf ++ engineLogPathConf).foreach { case (k, v) =>
buffer += CONF
buffer += s"${convertConfigKey(k)}=$v"
}
@@ -244,6 +245,10 @@ class SparkProcessBuilder(
}
}
}
+
+ private[spark] def engineLogPathConf(): Map[String, String] = {
+ Map(KYUUBI_ENGINE_LOG_PATH_KEY -> engineLog.getAbsolutePath)
+ }
}
object SparkProcessBuilder {
diff --git
a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala
b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala
index ad8324c85..cd549c0f3 100644
---
a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala
+++
b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala
@@ -29,6 +29,8 @@ import org.scalatestplus.mockito.MockitoSugar
import org.apache.kyuubi.{KerberizedTestHelper, KyuubiSQLException, Utils}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.config.KyuubiConf.{ENGINE_LOG_TIMEOUT,
ENGINE_SPARK_MAIN_RESOURCE}
+import org.apache.kyuubi.engine.ProcBuilder.KYUUBI_ENGINE_LOG_PATH_KEY
+import org.apache.kyuubi.engine.spark.SparkProcessBuilder.CONF
import org.apache.kyuubi.ha.HighAvailabilityConf
import org.apache.kyuubi.ha.client.AuthTypes
import org.apache.kyuubi.service.ServiceUtils
@@ -296,9 +298,16 @@ class SparkProcessBuilderSuite extends
KerberizedTestHelper with MockitoSugar {
assert(!c3.contains(s"spark.kubernetes.driverEnv.SPARK_USER_NAME=$proxyName"))
assert(!c3.contains(s"spark.executorEnv.SPARK_USER_NAME=$proxyName"))
}
+
+ test("[KYUUBI #5009] Test pass spark engine log path to spark conf") {
+ val b1 = new SparkProcessBuilder("kyuubi", conf)
+ assert(
+ b1.toString.contains(
+ s"$CONF
spark.$KYUUBI_ENGINE_LOG_PATH_KEY=${b1.engineLog.getAbsolutePath}"))
+ }
}
class FakeSparkProcessBuilder(config: KyuubiConf)
extends SparkProcessBuilder("fake", config) {
- override protected val commands: Array[String] = Array("ls")
+ override protected lazy val commands: Array[String] = Array("ls")
}
