This is an automated email from the ASF dual-hosted git repository.

sarutak pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new bcd57be745f0 [SPARK-60566][CONNECT][K8S] Allow to launch 
SparkConnectServer in KUBERNETES cluster mode
bcd57be745f0 is described below

commit bcd57be745f0a75e75e8e72be59b80c1e30caae8
Author: Bo Yang <[email protected]>
AuthorDate: Fri Oct 30 22:24:38 2026 +0900

    [SPARK-60566][CONNECT][K8S] Allow to launch SparkConnectServer in 
KUBERNETES cluster mode
    
    ### What changes were proposed in this pull request?
    
    This PR proposes to allow to launch SparkConnectServer in KUBERNETES 
cluster mode, following YARN example https://github.com/apache/spark/pull/54004.
    
    Jira ticket: https://issues.apache.org/jira/browse/SPARK-60566: Allow to 
launch SparkConnectServer in KUBERNETES mode.
    
    This PR is a small change so users who already launch jobs with 
spark-submit in K8s cluster mode can start SparkConnectServer the same way, as 
YARN already allows. It is not to impact start-connect-server.sh, which people 
will still use for a long running SparkConnect server.
    
    ### Why are the changes needed?
    People can utilize their Kubernetes cluster for running SparkConnectServer 
to support SparkConnect use scenarios: remote connecting to SparkConnect 
server, use different programming languages in SparkConnect client side, etc.
    
    ### Does this PR introduce _any_ user-facing change?
    Yes, user will be able to submit a spark application with 
SparkConnectServer as main class in KUBERNETES cluster mode.
    
    ### How was this patch tested?
    I confirmed that SparkConnectServer launched on my Kubernetes cluster with 
`spark-submit --deploy-mode cluster` and stopped with `kubectl delete 
<driver-pod>`. Also added unit test in SparkSubmitSuite.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    Yes, co-authored with Claude Opus 5.5
    
    Closes #59878 from hiboyang/bo-dev-02.
    
    Authored-by: Bo Yang <[email protected]>
    Signed-off-by: Kousuke Saruta <[email protected]>
    (cherry picked from commit df7d31752b3d1ee8bfc31993938aca60b0998f13)
    Signed-off-by: Kousuke Saruta <[email protected]>
---
 .../org/apache/spark/deploy/SparkSubmit.scala      |  9 +++++-
 .../org/apache/spark/deploy/SparkSubmitSuite.scala | 32 ++++++++++++++++++++++
 2 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala 
b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
index d1e63fab00ff..fe8d090affee 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -313,8 +313,15 @@ private[spark] class SparkSubmit extends Logging {
       case (YARN, CLUSTER) if isConnectServer(args.mainClass) =>
         logInfo("SparkConnectServer is starting in cluster deploy mode. " +
           "Use `yarn application -kill` command or YARN client API to stop the 
server.")
+      case (KUBERNETES, CLUSTER) if isConnectServer(args.mainClass) =>
+        logInfo("SparkConnectServer is starting in cluster deploy mode. " +
+          "Use `spark-submit --master <k8s-master-url> --kill 
<namespace>:<driver-pod-name>` " +
+          "or delete the driver pod to stop the server. Consider setting " +
+          "spark.kubernetes.submission.waitAppCompletion=false so that this 
launcher " +
+          "does not keep running and logging the driver pod status.")
       case (_, CLUSTER) if isConnectServer(args.mainClass) =>
-        error("Launching Spark Connect server in cluster deploy mode is 
supported only for YARN")
+        error("Launching Spark Connect server in cluster deploy mode is 
supported only for " +
+          "YARN and Kubernetes")
       case _ =>
     }
 
diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala 
b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
index 2627633126cd..b5ba2c0af3bd 100644
--- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
@@ -464,6 +464,38 @@ class SparkSubmitSuite
     conf.get("spark.kubernetes.driver.container.image") should be ("bar")
   }
 
+  test("handles Spark Connect server in k8s cluster mode") {
+    val connectServerClass = 
"org.apache.spark.sql.connect.service.SparkConnectServer"
+    val clArgs = Seq(
+      "--deploy-mode", "cluster",
+      "--master", "k8s://host:port",
+      "--class", connectServerClass,
+      "--conf", "spark.kubernetes.namespace=spark",
+      "--conf", "spark.kubernetes.driver.container.image=bar",
+      "spark-internal")
+    val appArgs = new SparkSubmitArguments(clArgs)
+    val (childArgs, _, _, mainClass) = submit.prepareSubmitEnvironment(appArgs)
+
+    val childArgsMap = childArgs.grouped(2).map(a => a(0) -> a(1)).toMap
+    // spark-internal is not a real resource, so no primary resource is passed 
to the driver.
+    childArgsMap.get("--primary-java-resource") should be (None)
+    childArgsMap.get("--main-class") should be (Some(connectServerClass))
+    mainClass should be (KUBERNETES_CLUSTER_SUBMIT_CLASS)
+
+    // Other cluster managers still reject the Spark Connect server in cluster 
mode.
+    val standaloneArgs = Seq(
+      "--deploy-mode", "cluster",
+      "--master", "spark://host:port",
+      "--class", connectServerClass,
+      "spark-internal")
+    val e = intercept[SparkException] {
+      submit.prepareSubmitEnvironment(new SparkSubmitArguments(standaloneArgs))
+    }
+    assert(e.getMessage.contains(
+      "Launching Spark Connect server in cluster deploy mode is supported only 
for " +
+        "YARN and Kubernetes"))
+  }
+
   test("SPARK-35084: include jars of the --packages" +
     " in k8s client mode & driver runs inside a POD") {
     val unusedJar = TestUtils.createJarWithClasses(Seq.empty)


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to