turboFei commented on code in PR #6876:
URL: https://github.com/apache/kyuubi/pull/6876#discussion_r1899868699
##########
kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala:
##########
@@ -266,6 +274,40 @@ class SparkProcessBuilder(
map.result().toMap
}
+ def prepareK8sFileUploadPath(): Map[String, String] = {
+ kubernetesFileUploadPath() match {
+ case Some(uploadPathPattern) if isK8sClusterMode =>
+ val today = LocalDate.now()
+ val uploadPath = uploadPathPattern
+ .replace("{{YEAR}}", today.format(YEAR_FMT))
+ .replace("{{MONTH}}", today.format(MONTH_FMT))
+ .replace("{{DAY}}", today.format(DAY_FMT))
+
+ if (conf.get(KUBERNETES_SPARK_AUTO_CREATE_FILE_UPLOAD_PATH)) {
+ val hadoopConf = KyuubiHadoopUtils.newHadoopConf(conf, loadDefaults
= false)
+ val path = new Path(uploadPath)
+ var fs: FileSystem = null
+ try {
+ fs = path.getFileSystem(hadoopConf)
+ if (!fs.exists(path)) {
+ info(s"Try creating $KUBERNETES_FILE_UPLOAD_PATH: $uploadPath")
+ fs.mkdirs(path, KUBERNETES_UPLOAD_PATH_PERMISSION)
+ }
+ } catch {
+ case ioe: IOException =>
+ warn(s"Failed to create $KUBERNETES_FILE_UPLOAD_PATH:
$uploadPath", ioe)
+ } finally {
+ if (fs != null) {
+ Utils.tryLogNonFatalError(fs.close())
+ }
+ }
+ }
+ Map(KUBERNETES_FILE_UPLOAD_PATH -> uploadPath)
Review Comment:
so since this PR, it will replace the `uploadPathPattern` pattern if it is
set and is k8s cluster mode anyway.
The only usage for `KUBERNETES_SPARK_AUTO_CREATE_FILE_UPLOAD_PATH` is to
create the path.
Seems not necessary? I saw Spark will create the dir for
`KUBERNETES_FILE_UPLOAD_PATH`.
https://github.com/apache/spark/blob/c4145db09ffeb0d0f9714a517ff9bacb018483e3/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala#L314-L321
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscr...@kyuubi.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscr...@kyuubi.apache.org
For additional commands, e-mail: notifications-h...@kyuubi.apache.org
