This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 14818cfb4 [KYUUBI #5065] Call destroy first on killing Spark startup
process to allows it release temp files
14818cfb4 is described below
commit 14818cfb43f28c906744d1341e8cf1551c7bb2e8
Author: liupeiyue <[email protected]>
AuthorDate: Thu Jul 20 17:58:40 2023 +0800
[KYUUBI #5065] Call destroy first on killing Spark startup process to
allows it release temp files
### _Why are the changes needed?_
fix bug https://github.com/apache/kyuubi/issues/5065
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [ ] [Run
test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests)
locally before make a pull request
Closes #5066 from ASiegeLion/master.
Closes #5065
08d1ac077 [Cheng Pan] Update
kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala
bf908f5af [Cheng Pan] Update
kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala
9144582f9 [Cheng Pan] Update
kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala
f1c95e409 [liupeiyue] [KYUUBI-#5065] Call destroy first on killing Spark
startup process to allows it release temp files
907123a93 [liupeiyue] [KYUUBI-#5065] Call destroy first on killing Spark
startup process to allows it release temp files
f30a9fc39 [liupeiyue] [KYUUBI-#5065] Call destroy first on killing Spark
startup process to allows it release temp files
449be44d7 [文艺攻城狮] Update
kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala
987ffc7fe [文艺攻城狮] Update
kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
995386f98 [文艺攻城狮] Update
kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
ad3d11191 [liupeiyue] [KYUUBI-#5065]destroy the spark engine release the
submitted temp files
Lead-authored-by: liupeiyue <[email protected]>
Co-authored-by: 文艺攻城狮 <[email protected]>
Co-authored-by: Cheng Pan <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
---
docs/deployment/settings.md | 1 +
.../src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala | 9 +++++++++
.../main/scala/org/apache/kyuubi/engine/ProcBuilder.scala | 12 +++++++++++-
3 files changed, 21 insertions(+), 1 deletion(-)
diff --git a/docs/deployment/settings.md b/docs/deployment/settings.md
index 5d795864d..b1b8af143 100644
--- a/docs/deployment/settings.md
+++ b/docs/deployment/settings.md
@@ -419,6 +419,7 @@ You can configure the Kyuubi properties in
`$KYUUBI_HOME/conf/kyuubi-defaults.co
| kyuubi.session.engine.spark.progress.timeFormat | yyyy-MM-dd
HH:mm:ss.SSS | The time format of the progress bar
[...]
| kyuubi.session.engine.spark.progress.update.interval | PT1S
| Update period of progress bar.
[...]
| kyuubi.session.engine.spark.showProgress | false
| When true, show the progress bar in the Spark's engine log.
[...]
+| kyuubi.session.engine.startup.destroy.timeout | PT5S
| Engine startup process destroy wait time, if the process does not stop
after this time, force destroy instead. This configuration only takes effect
when `kyuubi.session.engine.startup.waitCompletion=false`.
[...]
| kyuubi.session.engine.startup.error.max.size | 8192
| During engine bootstrapping, if anderror occurs, using this config to limit
the length of error message(characters).
[...]
| kyuubi.session.engine.startup.maxLogLines | 10
| The maximum number of engine log lines when errors occur during the engine
startup phase. Note that this config effects on client-side to help track
engine startup issues.
[...]
| kyuubi.session.engine.startup.waitCompletion | true
| Whether to wait for completion after the engine starts. If false, the
startup process will be destroyed after the engine is started. Note that only
use it when the driver is not running locally, such as in yarn-cluster mode;
Otherwise, the engine will be killed.
[...]
diff --git
a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
index 116455da7..65f03c654 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
@@ -1513,6 +1513,15 @@ object KyuubiConf {
.booleanConf
.createWithDefault(true)
+ val SESSION_ENGINE_STARTUP_DESTROY_TIMEOUT: ConfigEntry[Long] =
+ buildConf("kyuubi.session.engine.startup.destroy.timeout")
+ .doc("Engine startup process destroy wait time, if the process does not
" +
+ "stop after this time, force destroy instead. This configuration only
" +
+ s"takes effect when
`${SESSION_ENGINE_STARTUP_WAIT_COMPLETION.key}=false`.")
+ .version("1.8.0")
+ .timeConf
+ .createWithDefault(Duration.ofSeconds(5).toMillis)
+
val SESSION_ENGINE_LAUNCH_ASYNC: ConfigEntry[Boolean] =
buildConf("kyuubi.session.engine.launch.async")
.doc("When opening kyuubi session, whether to launch the backend engine
asynchronously." +
diff --git
a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala
b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala
index a44fe06bc..d0e6bae6b 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala
@@ -21,6 +21,7 @@ import java.io.{File, FilenameFilter, IOException}
import java.net.URI
import java.nio.charset.StandardCharsets
import java.nio.file.{Files, Path, Paths}
+import java.util.concurrent.TimeUnit
import scala.collection.JavaConverters._
@@ -156,6 +157,9 @@ trait ProcBuilder {
private val engineLogMaxLines =
conf.get(KyuubiConf.SESSION_ENGINE_STARTUP_MAX_LOG_LINES)
+ private val engineStartupDestroyTimeout =
+ conf.get(KyuubiConf.SESSION_ENGINE_STARTUP_DESTROY_TIMEOUT)
+
protected val lastRowsOfLog: EvictingQueue[String] =
EvictingQueue.create(engineLogMaxLines)
// Visible for test
@volatile private[kyuubi] var logCaptureThreadReleased: Boolean = true
@@ -257,7 +261,13 @@ trait ProcBuilder {
logCaptureThread = null
}
if (destroyProcess && process != null) {
- process.destroyForcibly()
+ process.destroy()
+ if (!process.waitFor(engineStartupDestroyTimeout,
TimeUnit.MILLISECONDS)) {
+ warn("Engine startup process does not exit after " +
+ s"$engineStartupDestroyTimeout ms, try to forcibly kill. " +
+ "Staging files generated by the process may be retained!")
+ process.destroyForcibly()
+ }
process = null
}
}
