This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 7c19df6fb2f [SPARK-40302][K8S][TESTS] Add `YuniKornSuite`
7c19df6fb2f is described below
commit 7c19df6fb2f684a80ea3366fd365a6bbc13421b3
Author: Dongjoon Hyun <dongj...@apache.org>
AuthorDate: Thu Sep 1 09:26:45 2022 -0700
[SPARK-40302][K8S][TESTS] Add `YuniKornSuite`
This PR aims the followings.
1. Add `YuniKornSuite` integration test suite which extends
`KubernetesSuite` on Apache YuniKorn scheduler.
2. Support `--default-exclude-tags` command to override
`test.default.exclude.tags`.
To improve test coverage.
No. This is a test suite addition.
Since this requires `Apache YuniKorn` installation, the test suite is
disabled by default.
So, CI K8s integration test should pass without running this suite.
In order to run the tests, we need to override `test.default.exclude.tags`
like the following.
**SBT**
```
$ build/sbt -Psparkr -Pkubernetes -Pkubernetes-integration-tests \
-Dspark.kubernetes.test.deployMode=docker-desktop
"kubernetes-integration-tests/test" \
-Dtest.exclude.tags=minikube,local \
-Dtest.default.exclude.tags=
```
**MAVEN**
```
$ dev/dev-run-integration-tests.sh --deploy-mode docker-desktop \
--exclude-tag minikube,local \
--default-exclude-tags ''
```
Closes #37753 from dongjoon-hyun/SPARK-40302.
Authored-by: Dongjoon Hyun <dongj...@apache.org>
Signed-off-by: Dongjoon Hyun <dongj...@apache.org>
(cherry picked from commit b2e38e16bfc547a62957e0a67085985b3c65d525)
Signed-off-by: Dongjoon Hyun <dongj...@apache.org>
---
project/SparkBuild.scala | 3 ++-
.../dev/dev-run-integration-tests.sh | 10 ++++++++
.../kubernetes/integration-tests/pom.xml | 5 ++--
.../deploy/k8s/integrationtest/YuniKornTag.java | 27 ++++++++++++++++++++
.../deploy/k8s/integrationtest/YuniKornSuite.scala | 29 ++++++++++++++++++++++
5 files changed, 71 insertions(+), 3 deletions(-)
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 934fa4a1fdd..f830e64edfc 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -1135,7 +1135,8 @@ object CopyDependencies {
object TestSettings {
import BuildCommons._
- private val defaultExcludedTags = Seq("org.apache.spark.tags.ChromeUITest")
+ private val defaultExcludedTags = Seq("org.apache.spark.tags.ChromeUITest",
+ "org.apache.spark.deploy.k8s.integrationtest.YuniKornTag")
lazy val settings = Seq (
// Fork new JVMs for tests and set Java options for those
diff --git
a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
index 5f94203c0e2..f5f93adeddf 100755
---
a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
+++
b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh
@@ -37,6 +37,7 @@ SERVICE_ACCOUNT=
CONTEXT=
INCLUDE_TAGS="k8s"
EXCLUDE_TAGS=
+DEFAULT_EXCLUDE_TAGS="N/A"
JAVA_VERSION="8"
BUILD_DEPENDENCIES_MVN_FLAG="-am"
HADOOP_PROFILE="hadoop-3"
@@ -101,6 +102,10 @@ while (( "$#" )); do
EXCLUDE_TAGS="$2"
shift
;;
+ --default-exclude-tags)
+ DEFAULT_EXCLUDE_TAGS="$2"
+ shift
+ ;;
--base-image-name)
BASE_IMAGE_NAME="$2"
shift
@@ -180,6 +185,11 @@ then
properties=( ${properties[@]} -Dtest.exclude.tags=$EXCLUDE_TAGS )
fi
+if [ "$DEFAULT_EXCLUDE_TAGS" != "N/A" ];
+then
+ properties=( ${properties[@]}
-Dtest.default.exclude.tags=$DEFAULT_EXCLUDE_TAGS )
+fi
+
BASE_IMAGE_NAME=${BASE_IMAGE_NAME:-spark}
JVM_IMAGE_NAME=${JVM_IMAGE_NAME:-${BASE_IMAGE_NAME}}
PYTHON_IMAGE_NAME=${PYTHON_IMAGE_NAME:-${BASE_IMAGE_NAME}-py}
diff --git a/resource-managers/kubernetes/integration-tests/pom.xml
b/resource-managers/kubernetes/integration-tests/pom.xml
index 40e578f9a7e..516c92b1df6 100644
--- a/resource-managers/kubernetes/integration-tests/pom.xml
+++ b/resource-managers/kubernetes/integration-tests/pom.xml
@@ -46,6 +46,7 @@
<spark.kubernetes.test.dockerFile>Dockerfile.java17</spark.kubernetes.test.dockerFile>
<test.exclude.tags></test.exclude.tags>
+
<test.default.exclude.tags>org.apache.spark.deploy.k8s.integrationtest.YuniKornTag</test.default.exclude.tags>
<test.include.tags></test.include.tags>
<volcano.exclude>**/*Volcano*.scala</volcano.exclude>
</properties>
@@ -137,7 +138,7 @@
<argument>${spark.kubernetes.test.dockerFile}</argument>
<argument>--test-exclude-tags</argument>
- <argument>"${test.exclude.tags}"</argument>
+
<argument>"${test.exclude.tags},${test.default.exclude.tags}"</argument>
</arguments>
</configuration>
</execution>
@@ -179,7 +180,7 @@
<spark.kubernetes.test.pythonImage>${spark.kubernetes.test.pythonImage}</spark.kubernetes.test.pythonImage>
<spark.kubernetes.test.rImage>${spark.kubernetes.test.rImage}</spark.kubernetes.test.rImage>
</systemProperties>
- <tagsToExclude>${test.exclude.tags}</tagsToExclude>
+
<tagsToExclude>${test.exclude.tags},${test.default.exclude.tags}</tagsToExclude>
<tagsToInclude>${test.include.tags}</tagsToInclude>
</configuration>
<executions>
diff --git
a/resource-managers/kubernetes/integration-tests/src/test/java/org/apache/spark/deploy/k8s/integrationtest/YuniKornTag.java
b/resource-managers/kubernetes/integration-tests/src/test/java/org/apache/spark/deploy/k8s/integrationtest/YuniKornTag.java
new file mode 100644
index 00000000000..cc21cad7aad
--- /dev/null
+++
b/resource-managers/kubernetes/integration-tests/src/test/java/org/apache/spark/deploy/k8s/integrationtest/YuniKornTag.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.deploy.k8s.integrationtest;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.ElementType;
+
+@org.scalatest.TagAnnotation
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.METHOD, ElementType.TYPE})
+public @interface YuniKornTag {}
diff --git
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/YuniKornSuite.scala
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/YuniKornSuite.scala
new file mode 100644
index 00000000000..5a3c063efa1
--- /dev/null
+++
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/YuniKornSuite.scala
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.deploy.k8s.integrationtest
+
+@YuniKornTag
+class YuniKornSuite extends KubernetesSuite {
+
+ override protected def setUpTest(): Unit = {
+ super.setUpTest()
+ sparkAppConf
+ .set("spark.kubernetes.scheduler.name", "yunikorn")
+ .set("spark.kubernetes.driver.annotation.yunikorn.apache.org/app-id",
"{{APP_ID}}")
+ .set("spark.kubernetes.executor.annotation.yunikorn.apache.org/app-id",
"{{APP_ID}}")
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
