turboFei commented on code in PR #5986:
URL: https://github.com/apache/kyuubi/pull/5986#discussion_r1466805488
##########
externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSQLSessionManager.scala:
##########
@@ -181,22 +183,45 @@ class SparkSQLSessionManager private (name: String,
spark: SparkSession)
} catch {
case e: KyuubiSQLException =>
warn(s"Error closing session ${sessionHandle}", e)
+ } finally {
+ if (getSessionConf(KyuubiConf.OPERATION_RESULT_SAVE_TO_FILE, spark)) {
+ val sessionSavePath = getSessionResultSavePath(sessionHandle)
+ try {
+ val path = new Path(sessionSavePath)
+ val fs = path.getFileSystem(spark.sparkContext.hadoopConfiguration)
+ if (fs.exists(path)) {
+ fs.delete(path, true)
+ info(s"Deleted session result path: $sessionSavePath")
+ }
+ } catch {
+ case e: Throwable => error(s"Error cleaning session result path:
$sessionSavePath", e)
+ }
+ }
}
if (shareLevel == ShareLevel.CONNECTION) {
info("Session stopped due to shared level is Connection.")
stopSession()
}
- if (conf.get(OPERATION_RESULT_SAVE_TO_FILE)) {
- val path = new Path(s"${conf.get(OPERATION_RESULT_SAVE_TO_FILE_DIR)}/" +
- s"$engineId/${sessionHandle.identifier}")
- path.getFileSystem(spark.sparkContext.hadoopConfiguration).delete(path,
true)
- info(s"Delete session result file $path")
- }
}
private def stopSession(): Unit = {
SparkSQLEngine.currentEngine.foreach(_.stop())
}
override protected def isServer: Boolean = false
+
+ private[spark] def getEngineResultSavePath(): String = {
+ Seq(conf.get(OPERATION_RESULT_SAVE_TO_FILE_DIR),
engineId).mkString(Path.SEPARATOR)
Review Comment:
addressed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
