This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 0a0a131e987c [SPARK-46755][K8S][INFRA][DOCS] Upgrade `Volcano` to 1.8.2 0a0a131e987c is described below commit 0a0a131e987ca3c44c11d78f8916f077be58a14c Author: Dongjoon Hyun <dh...@apple.com> AuthorDate: Sat Jan 20 01:50:51 2024 -0800 [SPARK-46755][K8S][INFRA][DOCS] Upgrade `Volcano` to 1.8.2 ### What changes were proposed in this pull request? This PR aims to upgrade `Volcano` to 1.8.1 in K8s integration test document and GitHub Action job. ### Why are the changes needed? To bring the latest bug fixes in addition to the test coverage for Volcano scheduler 1.8.2. - https://github.com/volcano-sh/volcano/releases/tag/v1.8.2 Note that the following `Volcano` bug fix reveals our test suite issue. So, it's disabled in Volcano testing. - https://github.com/volcano-sh/volcano/issues/3271 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Pass the CIs. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #44779 from dongjoon-hyun/SPARK-46755. Authored-by: Dongjoon Hyun <dh...@apple.com> Signed-off-by: Dongjoon Hyun <dh...@apple.com> --- .github/workflows/build_and_test.yml | 2 +- resource-managers/kubernetes/integration-tests/README.md | 6 +++--- .../org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala | 3 +++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 51bbdb9fcb35..99bb2b12e083 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -1082,7 +1082,7 @@ jobs: export PVC_TESTS_VM_PATH=$PVC_TMP_DIR minikube mount ${PVC_TESTS_HOST_PATH}:${PVC_TESTS_VM_PATH} --gid=0 --uid=185 & kubectl create clusterrolebinding serviceaccounts-cluster-admin --clusterrole=cluster-admin --group=system:serviceaccounts || true - kubectl apply -f https://raw.githubusercontent.com/volcano-sh/volcano/v1.8.1/installer/volcano-development.yaml || true + kubectl apply -f https://raw.githubusercontent.com/volcano-sh/volcano/v1.8.2/installer/volcano-development.yaml || true eval $(minikube docker-env) build/sbt -Phadoop-3 -Psparkr -Pkubernetes -Pvolcano -Pkubernetes-integration-tests -Dspark.kubernetes.test.driverRequestCores=0.5 -Dspark.kubernetes.test.executorRequestCores=0.2 -Dspark.kubernetes.test.volcanoMaxConcurrencyJobNum=1 -Dtest.exclude.tags=local "kubernetes-integration-tests/test" - name: Upload Spark on K8S integration tests log files diff --git a/resource-managers/kubernetes/integration-tests/README.md b/resource-managers/kubernetes/integration-tests/README.md index 7298cde0ad3f..c0d92d988b1a 100644 --- a/resource-managers/kubernetes/integration-tests/README.md +++ b/resource-managers/kubernetes/integration-tests/README.md @@ -330,11 +330,11 @@ You can also specify your specific dockerfile to build JVM/Python/R based image ## Requirements - A minimum of 6 CPUs and 9G of memory is required to complete all Volcano test cases. -- Volcano v1.8.1. +- Volcano v1.8.2. ## Installation - kubectl apply -f https://raw.githubusercontent.com/volcano-sh/volcano/v1.8.1/installer/volcano-development.yaml + kubectl apply -f https://raw.githubusercontent.com/volcano-sh/volcano/v1.8.2/installer/volcano-development.yaml ## Run tests @@ -355,5 +355,5 @@ You can also specify `volcano` tag to only run Volcano test: ## Cleanup Volcano - kubectl delete -f https://raw.githubusercontent.com/volcano-sh/volcano/v1.8.1/installer/volcano-development.yaml + kubectl delete -f https://raw.githubusercontent.com/volcano-sh/volcano/v1.8.2/installer/volcano-development.yaml diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala index fe1a24e3e1e6..a6a9b4856202 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala @@ -136,6 +136,7 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite => } test("PVs with local hostpath storage on statefulsets", k8sTestTag, MinikubeTag) { + assume(this.getClass.getSimpleName == "KubernetesSuite") sparkAppConf .set(s"spark.kubernetes.driver.volumes.persistentVolumeClaim.data.mount.path", CONTAINER_MOUNT_PATH) @@ -167,6 +168,7 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite => } test("PVs with local hostpath and storageClass on statefulsets", k8sTestTag, MinikubeTag) { + assume(this.getClass.getSimpleName == "KubernetesSuite") sparkAppConf .set(s"spark.kubernetes.driver.volumes.persistentVolumeClaim.data.mount.path", CONTAINER_MOUNT_PATH) @@ -201,6 +203,7 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite => } test("PVs with local storage", k8sTestTag, MinikubeTag) { + assume(this.getClass.getSimpleName == "KubernetesSuite") sparkAppConf .set(s"spark.kubernetes.driver.volumes.persistentVolumeClaim.data.mount.path", CONTAINER_MOUNT_PATH) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org