This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.3 by this push: new 399c397e703 [SPARK-40304][K8S][TESTS] Add `decomTestTag` to K8s Integration Test 399c397e703 is described below commit 399c397e7035665c928b1d439a860f9e7b1ce3b3 Author: Dongjoon Hyun <dongj...@apache.org> AuthorDate: Thu Sep 1 09:34:55 2022 -0700 [SPARK-40304][K8S][TESTS] Add `decomTestTag` to K8s Integration Test ### What changes were proposed in this pull request? This PR aims to add a new test tag, `decomTestTag`, to K8s Integration Test. ### Why are the changes needed? Decommission-related tests took over 6 minutes (`363s`). It would be helpful we can run them selectively. ``` [info] - Test basic decommissioning (44 seconds, 51 milliseconds) [info] - Test basic decommissioning with shuffle cleanup (44 seconds, 450 milliseconds) [info] - Test decommissioning with dynamic allocation & shuffle cleanups (2 minutes, 43 seconds) [info] - Test decommissioning timeouts (44 seconds, 389 milliseconds) [info] - SPARK-37576: Rolling decommissioning (1 minute, 8 seconds) ``` ### Does this PR introduce _any_ user-facing change? No, this is a test-only change. ### How was this patch tested? Pass the CIs and test manually. ``` $ build/sbt -Psparkr -Pkubernetes -Pkubernetes-integration-tests \ -Dspark.kubernetes.test.deployMode=docker-desktop "kubernetes-integration-tests/test" \ -Dtest.exclude.tags=minikube,local,decom ... [info] KubernetesSuite: [info] - Run SparkPi with no resources (12 seconds, 441 milliseconds) [info] - Run SparkPi with no resources & statefulset allocation (11 seconds, 949 milliseconds) [info] - Run SparkPi with a very long application name. (11 seconds, 999 milliseconds) [info] - Use SparkLauncher.NO_RESOURCE (11 seconds, 846 milliseconds) [info] - Run SparkPi with a master URL without a scheme. (11 seconds, 176 milliseconds) [info] - Run SparkPi with an argument. (11 seconds, 868 milliseconds) [info] - Run SparkPi with custom labels, annotations, and environment variables. (11 seconds, 858 milliseconds) [info] - All pods have the same service account by default (11 seconds, 5 milliseconds) [info] - Run extraJVMOptions check on driver (5 seconds, 757 milliseconds) [info] - Verify logging configuration is picked from the provided SPARK_CONF_DIR/log4j2.properties (12 seconds, 467 milliseconds) [info] - Run SparkPi with env and mount secrets. (21 seconds, 119 milliseconds) [info] - Run PySpark on simple pi.py example (13 seconds, 129 milliseconds) [info] - Run PySpark to test a pyfiles example (14 seconds, 937 milliseconds) [info] - Run PySpark with memory customization (12 seconds, 195 milliseconds) [info] - Run in client mode. (11 seconds, 343 milliseconds) [info] - Start pod creation from template (11 seconds, 975 milliseconds) [info] - SPARK-38398: Schedule pod creation from template (11 seconds, 901 milliseconds) [info] - Run SparkR on simple dataframe.R example (14 seconds, 305 milliseconds) ... ``` Closes #37755 from dongjoon-hyun/SPARK-40304. Authored-by: Dongjoon Hyun <dongj...@apache.org> Signed-off-by: Dongjoon Hyun <dongj...@apache.org> (cherry picked from commit fd0498f81df72c196f19a5b26053660f6f3f4d70) Signed-off-by: Dongjoon Hyun <dongj...@apache.org> --- .../deploy/k8s/integrationtest/DecommissionSuite.scala | 13 +++++++------ .../spark/deploy/k8s/integrationtest/KubernetesSuite.scala | 1 + 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DecommissionSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DecommissionSuite.scala index 5d1a57fb46e..81f4660afe9 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DecommissionSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DecommissionSuite.scala @@ -34,7 +34,7 @@ import org.apache.spark.internal.config.PLUGINS private[spark] trait DecommissionSuite { k8sSuite: KubernetesSuite => import DecommissionSuite._ - import KubernetesSuite.k8sTestTag + import KubernetesSuite.{decomTestTag, k8sTestTag} def runDecommissionTest(f: () => Unit): Unit = { val logConfFilePath = s"${sparkHomeDir.toFile}/conf/log4j2.properties" @@ -61,7 +61,7 @@ private[spark] trait DecommissionSuite { k8sSuite: KubernetesSuite => } } - test("Test basic decommissioning", k8sTestTag) { + test("Test basic decommissioning", k8sTestTag, decomTestTag) { runDecommissionTest(() => { sparkAppConf .set(config.DECOMMISSION_ENABLED.key, "true") @@ -91,7 +91,7 @@ private[spark] trait DecommissionSuite { k8sSuite: KubernetesSuite => }) } - test("Test basic decommissioning with shuffle cleanup", k8sTestTag) { + test("Test basic decommissioning with shuffle cleanup", k8sTestTag, decomTestTag) { runDecommissionTest(() => { sparkAppConf .set(config.DECOMMISSION_ENABLED.key, "true") @@ -122,7 +122,8 @@ private[spark] trait DecommissionSuite { k8sSuite: KubernetesSuite => }) } - test("Test decommissioning with dynamic allocation & shuffle cleanups", k8sTestTag) { + test("Test decommissioning with dynamic allocation & shuffle cleanups", + k8sTestTag, decomTestTag) { runDecommissionTest(() => { sparkAppConf .set(config.DECOMMISSION_ENABLED.key, "true") @@ -183,7 +184,7 @@ private[spark] trait DecommissionSuite { k8sSuite: KubernetesSuite => }) } - test("Test decommissioning timeouts", k8sTestTag) { + test("Test decommissioning timeouts", k8sTestTag, decomTestTag) { runDecommissionTest(() => { sparkAppConf .set(config.DECOMMISSION_ENABLED.key, "true") @@ -216,7 +217,7 @@ private[spark] trait DecommissionSuite { k8sSuite: KubernetesSuite => }) } - test("SPARK-37576: Rolling decommissioning", k8sTestTag) { + test("SPARK-37576: Rolling decommissioning", k8sTestTag, decomTestTag) { runDecommissionTest(() => { sparkAppConf .set("spark.kubernetes.container.image", pyImage) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala index 3db51b28600..3d7a9313031 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala @@ -612,6 +612,7 @@ private[spark] object KubernetesSuite { val k8sTestTag = Tag("k8s") val localTestTag = Tag("local") val schedulingTestTag = Tag("schedule") + val decomTestTag = Tag("decom") val rTestTag = Tag("r") val MinikubeTag = Tag("minikube") val SPARK_PI_MAIN_CLASS: String = "org.apache.spark.examples.SparkPi" --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org