This is an automated email from the ASF dual-hosted git repository.
sarutak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new df7d31752b3d [SPARK-60566][CONNECT][K8S] Allow to launch
SparkConnectServer in KUBERNETES cluster mode
df7d31752b3d is described below
commit df7d31752b3d1ee8bfc31993938aca60b0998f13
Author: Bo Yang <[email protected]>
AuthorDate: Fri Oct 30 22:24:38 2026 +0900
[SPARK-60566][CONNECT][K8S] Allow to launch SparkConnectServer in
KUBERNETES cluster mode
### What changes were proposed in this pull request?
This PR proposes to allow to launch SparkConnectServer in KUBERNETES
cluster mode, following YARN example https://github.com/apache/spark/pull/54004.
Jira ticket: https://issues.apache.org/jira/browse/SPARK-60566: Allow to
launch SparkConnectServer in KUBERNETES mode.
This PR is a small change so users who already launch jobs with
spark-submit in K8s cluster mode can start SparkConnectServer the same way, as
YARN already allows. It is not to impact start-connect-server.sh, which people
will still use for a long running SparkConnect server.
### Why are the changes needed?
People can utilize their Kubernetes cluster for running SparkConnectServer
to support SparkConnect use scenarios: remote connecting to SparkConnect
server, use different programming languages in SparkConnect client side, etc.
### Does this PR introduce _any_ user-facing change?
Yes, user will be able to submit a spark application with
SparkConnectServer as main class in KUBERNETES cluster mode.
### How was this patch tested?
I confirmed that SparkConnectServer launched on my Kubernetes cluster with
`spark-submit --deploy-mode cluster` and stopped with `kubectl delete
<driver-pod>`. Also added unit test in SparkSubmitSuite.
### Was this patch authored or co-authored using generative AI tooling?
Yes, co-authored with Claude Opus 5.5
Closes #59878 from hiboyang/bo-dev-02.
Authored-by: Bo Yang <[email protected]>
Signed-off-by: Kousuke Saruta <[email protected]>
---
.../org/apache/spark/deploy/SparkSubmit.scala | 9 +++++-
.../org/apache/spark/deploy/SparkSubmitSuite.scala | 32 ++++++++++++++++++++++
2 files changed, 40 insertions(+), 1 deletion(-)
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
index d1e63fab00ff..fe8d090affee 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -313,8 +313,15 @@ private[spark] class SparkSubmit extends Logging {
case (YARN, CLUSTER) if isConnectServer(args.mainClass) =>
logInfo("SparkConnectServer is starting in cluster deploy mode. " +
"Use `yarn application -kill` command or YARN client API to stop the
server.")
+ case (KUBERNETES, CLUSTER) if isConnectServer(args.mainClass) =>
+ logInfo("SparkConnectServer is starting in cluster deploy mode. " +
+ "Use `spark-submit --master <k8s-master-url> --kill
<namespace>:<driver-pod-name>` " +
+ "or delete the driver pod to stop the server. Consider setting " +
+ "spark.kubernetes.submission.waitAppCompletion=false so that this
launcher " +
+ "does not keep running and logging the driver pod status.")
case (_, CLUSTER) if isConnectServer(args.mainClass) =>
- error("Launching Spark Connect server in cluster deploy mode is
supported only for YARN")
+ error("Launching Spark Connect server in cluster deploy mode is
supported only for " +
+ "YARN and Kubernetes")
case _ =>
}
diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
index 2627633126cd..b5ba2c0af3bd 100644
--- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
@@ -464,6 +464,38 @@ class SparkSubmitSuite
conf.get("spark.kubernetes.driver.container.image") should be ("bar")
}
+ test("handles Spark Connect server in k8s cluster mode") {
+ val connectServerClass =
"org.apache.spark.sql.connect.service.SparkConnectServer"
+ val clArgs = Seq(
+ "--deploy-mode", "cluster",
+ "--master", "k8s://host:port",
+ "--class", connectServerClass,
+ "--conf", "spark.kubernetes.namespace=spark",
+ "--conf", "spark.kubernetes.driver.container.image=bar",
+ "spark-internal")
+ val appArgs = new SparkSubmitArguments(clArgs)
+ val (childArgs, _, _, mainClass) = submit.prepareSubmitEnvironment(appArgs)
+
+ val childArgsMap = childArgs.grouped(2).map(a => a(0) -> a(1)).toMap
+ // spark-internal is not a real resource, so no primary resource is passed
to the driver.
+ childArgsMap.get("--primary-java-resource") should be (None)
+ childArgsMap.get("--main-class") should be (Some(connectServerClass))
+ mainClass should be (KUBERNETES_CLUSTER_SUBMIT_CLASS)
+
+ // Other cluster managers still reject the Spark Connect server in cluster
mode.
+ val standaloneArgs = Seq(
+ "--deploy-mode", "cluster",
+ "--master", "spark://host:port",
+ "--class", connectServerClass,
+ "spark-internal")
+ val e = intercept[SparkException] {
+ submit.prepareSubmitEnvironment(new SparkSubmitArguments(standaloneArgs))
+ }
+ assert(e.getMessage.contains(
+ "Launching Spark Connect server in cluster deploy mode is supported only
for " +
+ "YARN and Kubernetes"))
+ }
+
test("SPARK-35084: include jars of the --packages" +
" in k8s client mode & driver runs inside a POD") {
val unusedJar = TestUtils.createJarWithClasses(Seq.empty)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]