This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new f05f2c2 [SPARK-38081][K8S][TESTS] Support `cloud`-backend in K8s IT
with SBT
f05f2c2 is described below
commit f05f2c2e7f3a5939156f12576f0a8097cdec9772
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Tue Feb 1 08:30:39 2022 -0800
[SPARK-38081][K8S][TESTS] Support `cloud`-backend in K8s IT with SBT
This PR aims to
- Support `cloud` backend in K8s IT with SBT (Image
building/pushing/testing)
- Add a new K8s test tag, `local`, and apply it to a test case using local
HTTP server.
To run K8s IT in the cloud environment more easily.
No
Manually test like the following.
```
$ build/sbt -Psparkr -Pkubernetes -Pkubernetes-integration-tests
-Dtest.exclude.tags=minikube,local -Dspark.kubernetes.test.deployMode=cloud
-Dspark.kubernetes.test.master=k8s://....eks.amazonaws.com
-Dspark.kubernetes.test.namespace=spark-cloud-test
-Dspark.kubernetes.test.imageRepo=...
-Dspark.kubernetes.test.imageTag=2022-02-01 "kubernetes-integration-tests/test"
...
[info] KubernetesSuite:
[info] - Run SparkPi with no resources (26 seconds, 678 milliseconds)
[info] - Run SparkPi with no resources & statefulset allocation (18
seconds, 617 milliseconds)
[info] - Run SparkPi with a very long application name. (17 seconds, 205
milliseconds)
[info] - Use SparkLauncher.NO_RESOURCE (17 seconds, 555 milliseconds)
[info] - Run SparkPi with a master URL without a scheme. (17 seconds, 478
milliseconds)
[info] - Run SparkPi with an argument. (17 seconds, 518 milliseconds)
[info] - Run SparkPi with custom labels, annotations, and environment
variables. (17 seconds, 648 milliseconds)
[info] - All pods have the same service account by default (17 seconds, 800
milliseconds)
[info] - Run extraJVMOptions check on driver (11 seconds, 141 milliseconds)
[info] - Verify logging configuration is picked from the provided
SPARK_CONF_DIR/log4j2.properties (25 seconds, 608 milliseconds)
[info] - Run SparkPi with env and mount secrets. (27 seconds, 114
milliseconds)
[info] - Run PySpark on simple pi.py example (42 seconds, 929 milliseconds)
[info] - Run PySpark to test a pyfiles example (19 seconds, 914
milliseconds)
[info] - Run PySpark with memory customization (16 seconds, 985
milliseconds)
[info] - Run in client mode. (10 seconds, 42 milliseconds)
[info] - Start pod creation from template (16 seconds, 207 milliseconds)
[info] - Test basic decommissioning (49 seconds, 519 milliseconds)
[info] - Test basic decommissioning with shuffle cleanup (49 seconds, 472
milliseconds)
[info] - Test decommissioning with dynamic allocation & shuffle cleanups (2
minutes, 49 seconds)
[info] - Test decommissioning timeouts (50 seconds, 423 milliseconds)
[info] - SPARK-37576: Rolling decommissioning (1 minute, 13 seconds)
[info] - Run SparkR on simple dataframe.R example (51 seconds, 712
milliseconds)
[info] Run completed in 15 minutes, 50 seconds.
[info] Total number of tests run: 22
[info] Suites: completed 1, aborted 0
[info] Tests: succeeded 22, failed 0, canceled 0, ignored 0, pending 0
[info] All tests passed.
[success] Total time: 1920 s (32:00), completed Jan 31, 2022 11:56:38 PM
```
Closes #35376 from dongjoon-hyun/SPARK-38081.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit c37c6c393dc0302abd41b0c4c41002710ba52270)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
project/SparkBuild.scala | 12 +++++++++++-
resource-managers/kubernetes/integration-tests/README.md | 2 +-
.../spark/deploy/k8s/integrationtest/BasicTestsSuite.scala | 4 ++--
.../spark/deploy/k8s/integrationtest/KubernetesSuite.scala | 1 +
4 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 6aeaa8c..8c1dbce 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -614,6 +614,7 @@ object KubernetesIntegrationTests {
val dockerBuild = TaskKey[Unit]("docker-imgs", "Build the docker images for
ITs.")
val runITs = TaskKey[Unit]("run-its", "Only run ITs, skip image build.")
+ val imageRepo = sys.props.getOrElse("spark.kubernetes.test.imageRepo",
"docker.io/kubespark")
val imageTag = sys.props.get("spark.kubernetes.test.imageTag")
val namespace = sys.props.get("spark.kubernetes.test.namespace")
val deployMode = sys.props.get("spark.kubernetes.test.deployMode")
@@ -629,15 +630,23 @@ object KubernetesIntegrationTests {
val dockerTool = s"$sparkHome/bin/docker-image-tool.sh"
val bindingsDir =
s"$sparkHome/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings"
val cmd = Seq(dockerTool,
+ "-r", imageRepo,
"-t", imageTag.getOrElse("dev"),
"-p", s"$bindingsDir/python/Dockerfile",
"-R", s"$bindingsDir/R/Dockerfile") ++
- (if (deployMode == Some("docker-for-desktop")) Seq.empty else
Seq("-m")) :+
+ (if (deployMode != Some("minikube")) Seq.empty else Seq("-m")) :+
"build"
val ec = Process(cmd).!
if (ec != 0) {
throw new IllegalStateException(s"Process '${cmd.mkString(" ")}'
exited with $ec.")
}
+ if (deployMode == Some("cloud")) {
+ val cmd = Seq(dockerTool, "-r", imageRepo, "-t",
imageTag.getOrElse("dev"), "push")
+ val ret = Process(cmd).!
+ if (ret != 0) {
+ throw new IllegalStateException(s"Process '${cmd.mkString(" ")}'
exited with $ret.")
+ }
+ }
}
shouldBuildImage = true
},
@@ -650,6 +659,7 @@ object KubernetesIntegrationTests {
(Test / test) := (Test / test).dependsOn(dockerBuild).value,
(Test / javaOptions) ++= Seq(
s"-Dspark.kubernetes.test.deployMode=${deployMode.getOrElse("minikube")}",
+ s"-Dspark.kubernetes.test.imageRepo=${imageRepo}",
s"-Dspark.kubernetes.test.imageTag=${imageTag.getOrElse("dev")}",
s"-Dspark.kubernetes.test.unpackSparkDir=$sparkHome"
),
diff --git a/resource-managers/kubernetes/integration-tests/README.md
b/resource-managers/kubernetes/integration-tests/README.md
index 9d3efa4..3a81033 100644
--- a/resource-managers/kubernetes/integration-tests/README.md
+++ b/resource-managers/kubernetes/integration-tests/README.md
@@ -180,7 +180,7 @@ to the wrapper scripts and using the wrapper scripts will
simply set these appro
<tr>
<td><code>spark.kubernetes.test.master</code></td>
<td>
- When using the <code>cloud-url</code> backend must be specified to
indicate the K8S master URL to communicate
+ When using the <code>cloud</code> backend must be specified to indicate
the K8S master URL to communicate
with.
</td>
<td></td>
diff --git
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
index 1c12123..743fbf0 100644
---
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
+++
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
@@ -24,7 +24,7 @@ import org.apache.spark.launcher.SparkLauncher
private[spark] trait BasicTestsSuite { k8sSuite: KubernetesSuite =>
import BasicTestsSuite._
- import KubernetesSuite.k8sTestTag
+ import KubernetesSuite.{k8sTestTag, localTestTag}
test("Run SparkPi with no resources", k8sTestTag) {
runSparkPiAndVerifyCompletion()
@@ -99,7 +99,7 @@ private[spark] trait BasicTestsSuite { k8sSuite:
KubernetesSuite =>
expectedJVMValue = Seq("(spark.test.foo,spark.test.bar)"))
}
- test("Run SparkRemoteFileTest using a remote data file", k8sTestTag) {
+ test("Run SparkRemoteFileTest using a remote data file", k8sTestTag,
localTestTag) {
assert(sys.props.contains("spark.test.home"), "spark.test.home is not
set!")
TestUtils.withHttpServer(sys.props("spark.test.home")) { baseURL =>
sparkAppConf
diff --git
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
index 7b0da1a..c3d41a7 100644
---
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
+++
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
@@ -567,6 +567,7 @@ class KubernetesSuite extends SparkFunSuite
private[spark] object KubernetesSuite {
val k8sTestTag = Tag("k8s")
val pvTestTag = Tag("persistentVolume")
+ val localTestTag = Tag("local")
val rTestTag = Tag("r")
val MinikubeTag = Tag("minikube")
val SPARK_PI_MAIN_CLASS: String = "org.apache.spark.examples.SparkPi"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
