pan3793 commented on code in PR #6876:
URL: https://github.com/apache/kyuubi/pull/6876#discussion_r1899887111
##########
kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala:
##########
@@ -266,6 +274,40 @@ class SparkProcessBuilder(
map.result().toMap
}
+ def prepareK8sFileUploadPath(): Map[String, String] = {
+ kubernetesFileUploadPath() match {
+ case Some(uploadPathPattern) if isK8sClusterMode =>
+ val today = LocalDate.now()
+ val uploadPath = uploadPathPattern
+ .replace("{{YEAR}}", today.format(YEAR_FMT))
+ .replace("{{MONTH}}", today.format(MONTH_FMT))
+ .replace("{{DAY}}", today.format(DAY_FMT))
+
+ if (conf.get(KUBERNETES_SPARK_AUTO_CREATE_FILE_UPLOAD_PATH)) {
+ val hadoopConf = KyuubiHadoopUtils.newHadoopConf(conf, loadDefaults
= false)
+ val path = new Path(uploadPath)
+ var fs: FileSystem = null
+ try {
+ fs = path.getFileSystem(hadoopConf)
+ if (!fs.exists(path)) {
+ info(s"Try creating $KUBERNETES_FILE_UPLOAD_PATH: $uploadPath")
+ fs.mkdirs(path, KUBERNETES_UPLOAD_PATH_PERMISSION)
+ }
+ } catch {
+ case ioe: IOException =>
+ warn(s"Failed to create $KUBERNETES_FILE_UPLOAD_PATH:
$uploadPath", ioe)
+ } finally {
+ if (fs != null) {
+ Utils.tryLogNonFatalError(fs.close())
+ }
+ }
+ }
+ Map(KUBERNETES_FILE_UPLOAD_PATH -> uploadPath)
Review Comment:
let me also leave a comment on the code to mention that
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscr...@kyuubi.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscr...@kyuubi.apache.org
For additional commands, e-mail: notifications-h...@kyuubi.apache.org
