This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new a2897d2 [SPARK-31601][K8S] Fix
spark.kubernetes.executor.podNamePrefix to work
a2897d2 is described below
commit a2897d2d31f2a572affd1ed35a6e2808c99b5bd2
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Thu Apr 30 09:15:12 2020 +0530
[SPARK-31601][K8S] Fix spark.kubernetes.executor.podNamePrefix to work
This PR aims to fix `spark.kubernetes.executor.podNamePrefix` to work.
Currently, the configuration is broken like the following.
```
bin/spark-submit \
--master k8s://$K8S_MASTER \
--deploy-mode cluster \
--name spark-pi \
--class org.apache.spark.examples.SparkPi \
-c spark.kubernetes.container.image=spark:pr \
-c spark.kubernetes.driver.pod.name=mypod \
-c spark.kubernetes.executor.podNamePrefix=mypod \
local:///opt/spark/examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar
```
**BEFORE SPARK-31601**
```
pod/mypod 1/1 Running 0 9s
pod/spark-pi-7469dd71c499fafb-exec-1 1/1 Running 0 4s
pod/spark-pi-7469dd71c499fafb-exec-2 1/1 Running 0 4s
```
**AFTER SPARK-31601**
```
pod/mypod 1/1 Running 0 8s
pod/mypod-exec-1 1/1 Running 0 3s
pod/mypod-exec-2 1/1 Running 0 3s
```
Yes. This is a bug fix. The conf will work as described in the
documentation.
Pass the Jenkins and run the above comment manually.
Closes #28401 from dongjoon-hyun/SPARK-31601.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Prashant Sharma <[email protected]>
(cherry picked from commit 85dad37f69ebb617c8ac015dbbbda11054170298)
Signed-off-by: Prashant Sharma <[email protected]>
(cherry picked from commit 82b8f7fc9d21f4ac506d8cd613158f0511f5cb1d)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala | 1 -
.../apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala | 2 --
2 files changed, 3 deletions(-)
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala
index 575bc54..1948e8f 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala
@@ -133,7 +133,6 @@ private[spark] class BasicDriverFeatureStep(
val additionalProps = mutable.Map(
KUBERNETES_DRIVER_POD_NAME.key -> driverPodName,
"spark.app.id" -> conf.appId,
- KUBERNETES_EXECUTOR_POD_NAME_PREFIX.key -> conf.appResourceNamePrefix,
KUBERNETES_DRIVER_SUBMIT_CHECK.key -> "true")
val resolvedSparkJars = KubernetesUtils.resolveFileUrisAndPath(
diff --git
a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
index 0968cce..a5d25bb 100644
---
a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
+++
b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
@@ -128,7 +128,6 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite {
val expectedSparkConf = Map(
KUBERNETES_DRIVER_POD_NAME.key -> "spark-driver-pod",
"spark.app.id" -> APP_ID,
- KUBERNETES_EXECUTOR_POD_NAME_PREFIX.key -> RESOURCE_NAME_PREFIX,
"spark.kubernetes.submitInDriver" -> "true")
assert(featureStep.getAdditionalPodSystemProperties() ===
expectedSparkConf)
}
@@ -205,7 +204,6 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite {
val expectedSparkConf = Map(
KUBERNETES_DRIVER_POD_NAME.key -> "spark-driver-pod",
"spark.app.id" -> APP_ID,
- KUBERNETES_EXECUTOR_POD_NAME_PREFIX.key -> RESOURCE_NAME_PREFIX,
"spark.kubernetes.submitInDriver" -> "true",
"spark.jars" -> "/opt/spark/jar1.jar,hdfs:///opt/spark/jar2.jar",
"spark.files" -> "https://localhost:9000/file1.txt,/opt/spark/file2.txt";)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
