This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push:
new 687c2979959f [SPARK-44495][INFRA][K8S][3.5] Use the latest minikube in
K8s IT
687c2979959f is described below
commit 687c2979959f13ac2c32e596f745f79284144735
Author: Dongjoon Hyun <dh...@apple.com>
AuthorDate: Sat Jan 20 18:53:21 2024 -0800
[SPARK-44495][INFRA][K8S][3.5] Use the latest minikube in K8s IT
### What changes were proposed in this pull request?
This is a backport of #44813 .
This PR aims to recover GitHub Action K8s IT to use the latest Minikube and
to make it sure that Apache Spark K8s module are tested with all Minikubes
without any issues.
**BEFORE**
- Minikube: v1.30.1
- K8s: v1.26.3
**AFTER**
- Minikube: v1.32.0
- K8s: v1.28.3
### Why are the changes needed?
- Previously, it was pinned due to the failure.
- After this PR, we will track the latest Minikube and K8s version always.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass the CIs.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #44819 from dongjoon-hyun/SPARK-44495-3.5.
Authored-by: Dongjoon Hyun <dh...@apple.com>
Signed-off-by: Dongjoon Hyun <dh...@apple.com>
---
.github/workflows/build_and_test.yml | 6 ++----
.../spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala | 2 ++
.../org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala | 4 +++-
.../apache/spark/deploy/k8s/integrationtest/VolcanoTestsSuite.scala | 4 ++--
4 files changed, 9 insertions(+), 7 deletions(-)
diff --git a/.github/workflows/build_and_test.yml
b/.github/workflows/build_and_test.yml
index b0b72a0d1a68..ad8685754b31 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -1016,9 +1016,7 @@ jobs:
- name: start minikube
run: |
# See more in "Installation" https://minikube.sigs.k8s.io/docs/start/
- # curl -LO
https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64
- # TODO(SPARK-44495): Resume to use the latest minikube for
k8s-integration-tests.
- curl -LO
https://storage.googleapis.com/minikube/releases/v1.30.1/minikube-linux-amd64
+ curl -LO
https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64
sudo install minikube-linux-amd64 /usr/local/bin/minikube
# Github Action limit cpu:2, memory: 6947MB, limit to 2U6G for
better resource statistic
minikube start --cpus 2 --memory 6144
@@ -1036,7 +1034,7 @@ jobs:
kubectl create clusterrolebinding serviceaccounts-cluster-admin
--clusterrole=cluster-admin --group=system:serviceaccounts || true
kubectl apply -f
https://raw.githubusercontent.com/volcano-sh/volcano/v1.7.0/installer/volcano-development.yaml
|| true
eval $(minikube docker-env)
- build/sbt -Psparkr -Pkubernetes -Pvolcano
-Pkubernetes-integration-tests -Dspark.kubernetes.test.driverRequestCores=0.5
-Dspark.kubernetes.test.executorRequestCores=0.2
-Dspark.kubernetes.test.volcanoMaxConcurrencyJobNum=1 -Dtest.exclude.tags=local
"kubernetes-integration-tests/test"
+ build/sbt -Psparkr -Pkubernetes -Pvolcano
-Pkubernetes-integration-tests
-Dspark.kubernetes.test.volcanoMaxConcurrencyJobNum=1 -Dtest.exclude.tags=local
"kubernetes-integration-tests/test"
- name: Upload Spark on K8S integration tests log files
if: failure()
uses: actions/upload-artifact@v3
diff --git
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala
index 4aba11bdb9d8..4ebf44ce9a4b 100644
---
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala
+++
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala
@@ -75,6 +75,8 @@ private[spark] class KubernetesTestComponents(val
kubernetesClient: KubernetesCl
.set(UI_ENABLED.key, "true")
.set("spark.kubernetes.submission.waitAppCompletion", "false")
.set("spark.kubernetes.authenticate.driver.serviceAccountName",
serviceAccountName)
+ .set("spark.kubernetes.driver.request.cores", "0.2")
+ .set("spark.kubernetes.executor.request.cores", "0.2")
}
}
diff --git
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala
index f8e76012638c..a699ef674cdc 100644
---
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala
+++
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala
@@ -24,6 +24,7 @@ import org.scalatest.concurrent.{Eventually,
PatienceConfiguration}
import org.scalatest.time.{Milliseconds, Span}
import org.apache.spark.deploy.k8s.integrationtest.KubernetesSuite._
+import
org.apache.spark.deploy.k8s.integrationtest.backend.minikube.MinikubeTestBackend
private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite =>
import PVTestsSuite._
@@ -54,6 +55,7 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite
=>
setupLocalStorageClass()
+ val hostname = if (testBackend == MinikubeTestBackend) "minikube" else
"docker-desktop"
val pvBuilder = new PersistentVolumeBuilder()
.withKind("PersistentVolume")
.withApiVersion("v1")
@@ -72,7 +74,7 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite
=>
.withMatchExpressions(new NodeSelectorRequirementBuilder()
.withKey("kubernetes.io/hostname")
.withOperator("In")
- .withValues("minikube", "m01", "docker-for-desktop",
"docker-desktop")
+ .withValues(hostname)
.build()).build())
.endRequired()
.endNodeAffinity()
diff --git
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolcanoTestsSuite.scala
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolcanoTestsSuite.scala
index 06d6f7dc100f..e7143e32db61 100644
---
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolcanoTestsSuite.scala
+++
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolcanoTestsSuite.scala
@@ -496,8 +496,8 @@ private[spark] object VolcanoTestsSuite extends
SparkFunSuite {
val DRIVER_PG_TEMPLATE_MEMORY_3G = new File(
getClass.getResource("/volcano/driver-podgroup-template-memory-3g.yml").getFile
).getAbsolutePath
- val DRIVER_REQUEST_CORES =
sys.props.get(CONFIG_DRIVER_REQUEST_CORES).getOrElse("1")
- val EXECUTOR_REQUEST_CORES =
sys.props.get(CONFIG_EXECUTOR_REQUEST_CORES).getOrElse("1")
+ val DRIVER_REQUEST_CORES =
sys.props.get(CONFIG_DRIVER_REQUEST_CORES).getOrElse("0.2")
+ val EXECUTOR_REQUEST_CORES =
sys.props.get(CONFIG_EXECUTOR_REQUEST_CORES).getOrElse("0.2")
val VOLCANO_MAX_JOB_NUM =
sys.props.get(CONFIG_KEY_VOLCANO_MAX_JOB_NUM).getOrElse("2")
val TEMP_DIR = "/tmp/"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
