This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new f97c3b3 [SPARK-38379][K8S] Fix Kubernetes Client mode when mounting
persistent volume with storage class
f97c3b3 is described below
commit f97c3b37b85fecd78627e7f85da1f2edbcc75910
Author: Thomas Graves <[email protected]>
AuthorDate: Wed Mar 9 21:06:25 2022 -0800
[SPARK-38379][K8S] Fix Kubernetes Client mode when mounting persistent
volume with storage class
### What changes were proposed in this pull request?
Running spark-shell in client mode on Kubernetes cluster when mounting
persistent volumes with a storage class results in a big warning being thrown
on startup.
https://issues.apache.org/jira/browse/SPARK-38379
The issue here is there is a race condition between when spark.app.id is
set in SparkContext and when its used, so change to use the KubernetesConf
appId, which is what is used to set spark.app.id.
### Why are the changes needed?
Throws big warning to user and I believe the label is wrong as well.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Unit test added. The test fails without the fix.
Also manually tested on real k8s cluster.
Closes #35792 from tgravescs/fixVolk8s.
Authored-by: Thomas Graves <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit f286416ee16e878de3c70a31cef20549b33aaa0a)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../k8s/features/MountVolumesFeatureStep.scala | 2 +-
.../features/MountVolumesFeatureStepSuite.scala | 25 ++++++++++++++++++++++
2 files changed, 26 insertions(+), 1 deletion(-)
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala
index 4e16473..78dd6ec 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala
@@ -85,7 +85,7 @@ private[spark] class MountVolumesFeatureStep(conf:
KubernetesConf)
.withApiVersion("v1")
.withNewMetadata()
.withName(claimName)
- .addToLabels(SPARK_APP_ID_LABEL, conf.sparkConf.getAppId)
+ .addToLabels(SPARK_APP_ID_LABEL, conf.appId)
.endMetadata()
.withNewSpec()
.withStorageClassName(storageClass.get)
diff --git
a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStepSuite.scala
b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStepSuite.scala
index 38f8fac..468d1dd 100644
---
a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStepSuite.scala
+++
b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStepSuite.scala
@@ -89,6 +89,31 @@ class MountVolumesFeatureStepSuite extends SparkFunSuite {
assert(executorPVC.getClaimName ===
s"pvc-spark-${KubernetesTestConf.EXECUTOR_ID}")
}
+ test("SPARK-32713 Mounts parameterized persistentVolumeClaims in executors
with storage class") {
+ val volumeConf = KubernetesVolumeSpec(
+ "testVolume",
+ "/tmp",
+ "",
+ true,
+ KubernetesPVCVolumeConf("pvc-spark-SPARK_EXECUTOR_ID", Some("fast"),
Some("512mb"))
+ )
+ val driverConf = KubernetesTestConf.createDriverConf(volumes =
Seq(volumeConf))
+ val driverStep = new MountVolumesFeatureStep(driverConf)
+ val driverPod = driverStep.configurePod(SparkPod.initialPod())
+
+ assert(driverPod.pod.getSpec.getVolumes.size() === 1)
+ val driverPVC =
driverPod.pod.getSpec.getVolumes.get(0).getPersistentVolumeClaim
+ assert(driverPVC.getClaimName === "pvc-spark-SPARK_EXECUTOR_ID")
+
+ val executorConf = KubernetesTestConf.createExecutorConf(volumes =
Seq(volumeConf))
+ val executorStep = new MountVolumesFeatureStep(executorConf)
+ val executorPod = executorStep.configurePod(SparkPod.initialPod())
+
+ assert(executorPod.pod.getSpec.getVolumes.size() === 1)
+ val executorPVC =
executorPod.pod.getSpec.getVolumes.get(0).getPersistentVolumeClaim
+ assert(executorPVC.getClaimName ===
s"pvc-spark-${KubernetesTestConf.EXECUTOR_ID}")
+ }
+
test("Create and mounts persistentVolumeClaims in driver") {
val volumeConf = KubernetesVolumeSpec(
"testVolume",
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
