This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new a42eef9e029a [SPARK-48077][K8S] Promote `KubernetesClientUtils` to `DeveloperApi` a42eef9e029a is described below commit a42eef9e029a388559e461f856af435457406a6d Author: Dongjoon Hyun <dh...@apple.com> AuthorDate: Wed May 1 18:10:53 2024 -0700 [SPARK-48077][K8S] Promote `KubernetesClientUtils` to `DeveloperApi` ### What changes were proposed in this pull request? This PR aims to promote `KubernetesClientUtils` to `DeveloperApi`. ### Why are the changes needed? Since `Apache Spark Kubernetes Operator` requires this, we had better maintain it as a developer API officially from `Apache Spark 4.0.0`. - https://github.com/apache/spark-kubernetes-operator/pull/10 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Pass the CIs. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #46327 from dongjoon-hyun/SPARK-48077. Authored-by: Dongjoon Hyun <dh...@apple.com> Signed-off-by: Dongjoon Hyun <dh...@apple.com> --- .../spark/deploy/k8s/submit/KubernetesClientUtils.scala | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientUtils.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientUtils.scala index 930588fb0077..d6b1da39bcbb 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientUtils.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientUtils.scala @@ -28,6 +28,7 @@ import scala.jdk.CollectionConverters._ import io.fabric8.kubernetes.api.model.{ConfigMap, ConfigMapBuilder, KeyToPath} import org.apache.spark.SparkConf +import org.apache.spark.annotation.{DeveloperApi, Since, Unstable} import org.apache.spark.deploy.k8s.{Config, Constants, KubernetesUtils} import org.apache.spark.deploy.k8s.Config.{KUBERNETES_DNS_SUBDOMAIN_NAME_MAX_LENGTH, KUBERNETES_NAMESPACE} import org.apache.spark.deploy.k8s.Constants.ENV_SPARK_CONF_DIR @@ -35,16 +36,26 @@ import org.apache.spark.internal.{Logging, MDC} import org.apache.spark.internal.LogKeys.{CONFIG, PATH, PATHS} import org.apache.spark.util.ArrayImplicits._ -private[spark] object KubernetesClientUtils extends Logging { +/** + * :: DeveloperApi :: + * + * A utility class used for K8s operations internally and Spark K8s operator. + */ +@Unstable +@DeveloperApi +object KubernetesClientUtils extends Logging { // Config map name can be KUBERNETES_DNS_SUBDOMAIN_NAME_MAX_LENGTH chars at max. + @Since("3.3.0") def configMapName(prefix: String): String = { val suffix = "-conf-map" s"${prefix.take(KUBERNETES_DNS_SUBDOMAIN_NAME_MAX_LENGTH - suffix.length)}$suffix" } + @Since("3.1.0") val configMapNameExecutor: String = configMapName(s"spark-exec-${KubernetesUtils.uniqueID()}") + @Since("3.1.0") val configMapNameDriver: String = configMapName(s"spark-drv-${KubernetesUtils.uniqueID()}") private def buildStringFromPropertiesMap(configMapName: String, @@ -62,6 +73,7 @@ private[spark] object KubernetesClientUtils extends Logging { /** * Build, file -> 'file's content' map of all the selected files in SPARK_CONF_DIR. */ + @Since("3.1.1") def buildSparkConfDirFilesMap( configMapName: String, sparkConf: SparkConf, @@ -77,6 +89,7 @@ private[spark] object KubernetesClientUtils extends Logging { } } + @Since("3.1.0") def buildKeyToPathObjects(confFilesMap: Map[String, String]): Seq[KeyToPath] = { confFilesMap.map { case (fileName: String, _: String) => @@ -89,6 +102,7 @@ private[spark] object KubernetesClientUtils extends Logging { * Build a Config Map that will hold the content for environment variable SPARK_CONF_DIR * on remote pods. */ + @Since("3.1.0") def buildConfigMap(configMapName: String, confFileMap: Map[String, String], withLabels: Map[String, String] = Map()): ConfigMap = { val configMapNameSpace = --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org