This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new b7356b6682b0 [SPARK-46783][K8S][TESTS] Use `built-in` storage classes
in PVTestsSuite
b7356b6682b0 is described below
commit b7356b6682b0b34883a2df0e8fef8467bff2a199
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Sat Jan 20 02:10:59 2024 -0800
[SPARK-46783][K8S][TESTS] Use `built-in` storage classes in PVTestsSuite
### What changes were proposed in this pull request?
This PR aims to use `built-in` storage classes in PVTestsSuite.
### Why are the changes needed?
`Minikube` already has `built-in` storage classe, `standard`, for
`hostpath`. We don't need to create a new one.
```
$ minikube kubectl get storageclass
NAME PROVISIONER RECLAIMPOLICY
VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
standard (default) k8s.io/minikube-hostpath Delete
Immediate false 109m
```
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass the CIs.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #44809 from dongjoon-hyun/SPARK-46783.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../deploy/k8s/integrationtest/PVTestsSuite.scala | 41 +++-------------------
1 file changed, 4 insertions(+), 37 deletions(-)
diff --git
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala
index 127731bd6eb4..3ffe0f9dc5c6 100644
---
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala
+++
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala
@@ -19,41 +19,17 @@ package org.apache.spark.deploy.k8s.integrationtest
import scala.jdk.CollectionConverters._
import io.fabric8.kubernetes.api.model._
-import io.fabric8.kubernetes.api.model.storage.StorageClassBuilder
import org.scalatest.concurrent.{Eventually, PatienceConfiguration}
import org.scalatest.time.{Milliseconds, Span}
import org.apache.spark.deploy.k8s.integrationtest.KubernetesSuite._
+import
org.apache.spark.deploy.k8s.integrationtest.backend.minikube.MinikubeTestBackend
private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite =>
import PVTestsSuite._
- private def setupLocalStorageClass(): Unit = {
- val scBuilder = new StorageClassBuilder()
- .withKind("StorageClass")
- .withApiVersion("storage.k8s.io/v1")
- .withNewMetadata()
- .withName(STORAGE_NAME)
- .endMetadata()
- .withProvisioner("kubernetes.io/no-provisioner")
- .withVolumeBindingMode("WaitForFirstConsumer")
- try {
- kubernetesTestComponents
- .kubernetesClient
- .storage()
- .v1()
- .storageClasses()
- .create(scBuilder.build())
- } catch {
- case e: io.fabric8.kubernetes.client.KubernetesClientException =>
- // Ignore storage class error sometimes we have a dangling class
- }
- }
-
private def setupLocalStorage(): Unit = {
-
- setupLocalStorageClass()
-
+ val storageClassName = if (testBackend == MinikubeTestBackend) "standard"
else "hostpath"
val pvBuilder = new PersistentVolumeBuilder()
.withKind("PersistentVolume")
.withApiVersion("v1")
@@ -64,7 +40,7 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite
=>
.withCapacity(Map("storage" -> new Quantity("1Gi")).asJava)
.withAccessModes("ReadWriteOnce")
.withPersistentVolumeReclaimPolicy("Retain")
- .withStorageClassName("test-local-storage")
+ .withStorageClassName(storageClassName)
.withLocal(new LocalVolumeSourceBuilder().withPath(VM_PATH).build())
.withNewNodeAffinity()
.withNewRequired()
@@ -86,7 +62,7 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite
=>
.endMetadata()
.withNewSpec()
.withAccessModes("ReadWriteOnce")
- .withStorageClassName("test-local-storage")
+ .withStorageClassName(storageClassName)
.withResources(new VolumeResourceRequirementsBuilder()
.withRequests(Map("storage" -> new Quantity("1Gi")).asJava).build())
.endSpec()
@@ -116,14 +92,6 @@ private[spark] trait PVTestsSuite { k8sSuite:
KubernetesSuite =>
.persistentVolumes()
.withName(PV_NAME)
.delete()
-
- kubernetesTestComponents
- .kubernetesClient
- .storage()
- .v1()
- .storageClasses()
- .withName(STORAGE_NAME)
- .delete()
}
private def checkPVs(pod: Pod, file: String) = {
@@ -237,7 +205,6 @@ private[spark] trait PVTestsSuite { k8sSuite:
KubernetesSuite =>
}
private[spark] object PVTestsSuite {
- val STORAGE_NAME = "test-local-storage"
val PV_NAME = "test-local-pv"
val PVC_NAME = "test-local-pvc"
val CONTAINER_MOUNT_PATH = "/opt/spark/pv-tests"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
