This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new 137528aeab18 [SPARK-44495][INFRA][K8S][3.4] Use the latest minikube in
K8s IT
137528aeab18 is described below
commit 137528aeab189422675a9655da42d440d1605381
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Sun Jan 21 00:19:23 2024 -0800
[SPARK-44495][INFRA][K8S][3.4] Use the latest minikube in K8s IT
### What changes were proposed in this pull request?
This is a backport of #44813 .
This PR aims to recover GitHub Action K8s IT to use the latest Minikube and
to make it sure that Apache Spark K8s module are tested with all Minikubes
without any issues.
**BEFORE**
- Minikube: v1.30.1
- K8s: v1.26.3
**AFTER**
- Minikube: v1.32.0
- K8s: v1.28.3
### Why are the changes needed?
- Previously, it was pinned due to the failure.
- After this PR, we will track the latest Minikube and K8s version always.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass the CIs.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #44820 from dongjoon-hyun/SPARK-44495-3.4.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.github/workflows/build_and_test.yml | 6 ++----
.../spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala | 2 ++
.../org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala | 4 +++-
.../apache/spark/deploy/k8s/integrationtest/VolcanoTestsSuite.scala | 4 ++--
4 files changed, 9 insertions(+), 7 deletions(-)
diff --git a/.github/workflows/build_and_test.yml
b/.github/workflows/build_and_test.yml
index 21e7ffb6b9ce..13527119e51a 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -957,9 +957,7 @@ jobs:
- name: start minikube
run: |
# See more in "Installation" https://minikube.sigs.k8s.io/docs/start/
- # curl -LO
https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64
- # TODO(SPARK-44495): Resume to use the latest minikube for
k8s-integration-tests.
- curl -LO
https://storage.googleapis.com/minikube/releases/v1.30.1/minikube-linux-amd64
+ curl -LO
https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64
sudo install minikube-linux-amd64 /usr/local/bin/minikube
# Github Action limit cpu:2, memory: 6947MB, limit to 2U6G for
better resource statistic
minikube start --cpus 2 --memory 6144
@@ -977,7 +975,7 @@ jobs:
kubectl create clusterrolebinding serviceaccounts-cluster-admin
--clusterrole=cluster-admin --group=system:serviceaccounts || true
kubectl apply -f
https://raw.githubusercontent.com/volcano-sh/volcano/v1.7.0/installer/volcano-development.yaml
|| true
eval $(minikube docker-env)
- build/sbt -Psparkr -Pkubernetes -Pvolcano
-Pkubernetes-integration-tests -Dspark.kubernetes.test.driverRequestCores=0.5
-Dspark.kubernetes.test.executorRequestCores=0.2
-Dspark.kubernetes.test.volcanoMaxConcurrencyJobNum=1 -Dtest.exclude.tags=local
"kubernetes-integration-tests/test"
+ build/sbt -Psparkr -Pkubernetes -Pvolcano
-Pkubernetes-integration-tests
-Dspark.kubernetes.test.volcanoMaxConcurrencyJobNum=1 -Dtest.exclude.tags=local
"kubernetes-integration-tests/test"
- name: Upload Spark on K8S integration tests log files
if: failure()
uses: actions/upload-artifact@v3
diff --git
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala
index 4fdb89eab6eb..3686ff212bc9 100644
---
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala
+++
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala
@@ -76,6 +76,8 @@ private[spark] class KubernetesTestComponents(defaultClient:
DefaultKubernetesCl
.set(UI_ENABLED.key, "true")
.set("spark.kubernetes.submission.waitAppCompletion", "false")
.set("spark.kubernetes.authenticate.driver.serviceAccountName",
serviceAccountName)
+ .set("spark.kubernetes.driver.request.cores", "0.2")
+ .set("spark.kubernetes.executor.request.cores", "0.2")
}
}
diff --git
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala
index 4f6eb1b94849..61a25d9c406e 100644
---
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala
+++
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala
@@ -24,6 +24,7 @@ import org.scalatest.concurrent.{Eventually,
PatienceConfiguration}
import org.scalatest.time.{Milliseconds, Span}
import org.apache.spark.deploy.k8s.integrationtest.KubernetesSuite._
+import
org.apache.spark.deploy.k8s.integrationtest.backend.minikube.MinikubeTestBackend
private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite =>
import PVTestsSuite._
@@ -53,6 +54,7 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite
=>
setupLocalStorageClass()
+ val hostname = if (testBackend == MinikubeTestBackend) "minikube" else
"docker-desktop"
val pvBuilder = new PersistentVolumeBuilder()
.withKind("PersistentVolume")
.withApiVersion("v1")
@@ -71,7 +73,7 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite
=>
.withMatchExpressions(new NodeSelectorRequirementBuilder()
.withKey("kubernetes.io/hostname")
.withOperator("In")
- .withValues("minikube", "m01", "docker-for-desktop",
"docker-desktop")
+ .withValues(hostname)
.build()).build())
.endRequired()
.endNodeAffinity()
diff --git
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolcanoTestsSuite.scala
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolcanoTestsSuite.scala
index f37a7644a948..a8338e2faa3c 100644
---
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolcanoTestsSuite.scala
+++
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolcanoTestsSuite.scala
@@ -467,8 +467,8 @@ private[spark] object VolcanoTestsSuite extends
SparkFunSuite {
val DRIVER_PG_TEMPLATE_MEMORY_3G = new File(
getClass.getResource("/volcano/driver-podgroup-template-memory-3g.yml").getFile
).getAbsolutePath
- val DRIVER_REQUEST_CORES =
sys.props.get(CONFIG_DRIVER_REQUEST_CORES).getOrElse("1")
- val EXECUTOR_REQUEST_CORES =
sys.props.get(CONFIG_EXECUTOR_REQUEST_CORES).getOrElse("1")
+ val DRIVER_REQUEST_CORES =
sys.props.get(CONFIG_DRIVER_REQUEST_CORES).getOrElse("0.2")
+ val EXECUTOR_REQUEST_CORES =
sys.props.get(CONFIG_EXECUTOR_REQUEST_CORES).getOrElse("0.2")
val VOLCANO_MAX_JOB_NUM =
sys.props.get(CONFIG_KEY_VOLCANO_MAX_JOB_NUM).getOrElse("2")
val TEMP_DIR = "/tmp/"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
