This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new a42eef9e029a [SPARK-48077][K8S] Promote `KubernetesClientUtils` to
`DeveloperApi`
a42eef9e029a is described below
commit a42eef9e029a388559e461f856af435457406a6d
Author: Dongjoon Hyun <dh...@apple.com>
AuthorDate: Wed May 1 18:10:53 2024 -0700
[SPARK-48077][K8S] Promote `KubernetesClientUtils` to `DeveloperApi`
### What changes were proposed in this pull request?
This PR aims to promote `KubernetesClientUtils` to `DeveloperApi`.
### Why are the changes needed?
Since `Apache Spark Kubernetes Operator` requires this, we had better
maintain it as a developer API officially from `Apache Spark 4.0.0`.
- https://github.com/apache/spark-kubernetes-operator/pull/10
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass the CIs.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #46327 from dongjoon-hyun/SPARK-48077.
Authored-by: Dongjoon Hyun <dh...@apple.com>
Signed-off-by: Dongjoon Hyun <dh...@apple.com>
---
.../spark/deploy/k8s/submit/KubernetesClientUtils.scala | 16 +++++++++++++++-
1 file changed, 15 insertions(+), 1 deletion(-)
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientUtils.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientUtils.scala
index 930588fb0077..d6b1da39bcbb 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientUtils.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientUtils.scala
@@ -28,6 +28,7 @@ import scala.jdk.CollectionConverters._
import io.fabric8.kubernetes.api.model.{ConfigMap, ConfigMapBuilder, KeyToPath}
import org.apache.spark.SparkConf
+import org.apache.spark.annotation.{DeveloperApi, Since, Unstable}
import org.apache.spark.deploy.k8s.{Config, Constants, KubernetesUtils}
import
org.apache.spark.deploy.k8s.Config.{KUBERNETES_DNS_SUBDOMAIN_NAME_MAX_LENGTH,
KUBERNETES_NAMESPACE}
import org.apache.spark.deploy.k8s.Constants.ENV_SPARK_CONF_DIR
@@ -35,16 +36,26 @@ import org.apache.spark.internal.{Logging, MDC}
import org.apache.spark.internal.LogKeys.{CONFIG, PATH, PATHS}
import org.apache.spark.util.ArrayImplicits._
-private[spark] object KubernetesClientUtils extends Logging {
+/**
+ * :: DeveloperApi ::
+ *
+ * A utility class used for K8s operations internally and Spark K8s operator.
+ */
+@Unstable
+@DeveloperApi
+object KubernetesClientUtils extends Logging {
// Config map name can be KUBERNETES_DNS_SUBDOMAIN_NAME_MAX_LENGTH chars at
max.
+ @Since("3.3.0")
def configMapName(prefix: String): String = {
val suffix = "-conf-map"
s"${prefix.take(KUBERNETES_DNS_SUBDOMAIN_NAME_MAX_LENGTH -
suffix.length)}$suffix"
}
+ @Since("3.1.0")
val configMapNameExecutor: String =
configMapName(s"spark-exec-${KubernetesUtils.uniqueID()}")
+ @Since("3.1.0")
val configMapNameDriver: String =
configMapName(s"spark-drv-${KubernetesUtils.uniqueID()}")
private def buildStringFromPropertiesMap(configMapName: String,
@@ -62,6 +73,7 @@ private[spark] object KubernetesClientUtils extends Logging {
/**
* Build, file -> 'file's content' map of all the selected files in
SPARK_CONF_DIR.
*/
+ @Since("3.1.1")
def buildSparkConfDirFilesMap(
configMapName: String,
sparkConf: SparkConf,
@@ -77,6 +89,7 @@ private[spark] object KubernetesClientUtils extends Logging {
}
}
+ @Since("3.1.0")
def buildKeyToPathObjects(confFilesMap: Map[String, String]): Seq[KeyToPath]
= {
confFilesMap.map {
case (fileName: String, _: String) =>
@@ -89,6 +102,7 @@ private[spark] object KubernetesClientUtils extends Logging {
* Build a Config Map that will hold the content for environment variable
SPARK_CONF_DIR
* on remote pods.
*/
+ @Since("3.1.0")
def buildConfigMap(configMapName: String, confFileMap: Map[String, String],
withLabels: Map[String, String] = Map()): ConfigMap = {
val configMapNameSpace =
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
