This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 6062e2bd8025 [SPARK-55196][K8S][TESTS] Add `commandTestTag` to K8s
Integration Test
6062e2bd8025 is described below
commit 6062e2bd80255c4d5b9e1a94f0d02e3e6c3068da
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Mon Jan 26 20:07:14 2026 +0900
[SPARK-55196][K8S][TESTS] Add `commandTestTag` to K8s Integration Test
### What changes were proposed in this pull request?
This PR aims to add a new test tag, `commandTestTag`, to K8s Integration
Test.
### Why are the changes needed?
`o.a.s.deploy.k8s.integrationtest.Utils.executeCommand` requires a live pod
access like `cat`, `env`, `ls`, and `df` commands . We had better distinguish
this kind of test group.
```
$ cd
resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/
$ git grep executeCommand
PVTestsSuite.scala: val contents = Utils.executeCommand("cat",
s"$CONTAINER_MOUNT_PATH/$file")
SecretsTestsSuite.scala: val env = Utils.executeCommand("env")
SecretsTestsSuite.scala: val files = Utils.executeCommand("ls",
s"$SECRET_MOUNT_PATH")
SecretsTestsSuite.scala: .executeCommand("cat",
s"$SECRET_MOUNT_PATH/$ENV_SECRET_KEY_1")
SecretsTestsSuite.scala: .executeCommand("cat",
s"$SECRET_MOUNT_PATH/$ENV_SECRET_KEY_2")
Utils.scala: def executeCommand(cmd: String*)(
VolumeSuite.scala: assert(Utils.executeCommand("df",
path).contains(expected))
```
### Does this PR introduce _any_ user-facing change?
No, this is a test-only change.
### How was this patch tested?
Pass the CIs.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #53975 from dongjoon-hyun/SPARK-55196.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../spark/deploy/k8s/integrationtest/KubernetesSuite.scala | 1 +
.../spark/deploy/k8s/integrationtest/PVTestsSuite.scala | 2 +-
.../deploy/k8s/integrationtest/SecretsTestsSuite.scala | 2 +-
.../spark/deploy/k8s/integrationtest/VolumeSuite.scala | 13 +++++++------
4 files changed, 10 insertions(+), 8 deletions(-)
diff --git
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
index 2450d50366a2..71fa9e653e52 100644
---
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
+++
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
@@ -642,6 +642,7 @@ class KubernetesSuite extends SparkFunSuite
private[spark] object KubernetesSuite {
val k8sTestTag = Tag("k8s")
val localTestTag = Tag("local")
+ val commandTestTag = Tag("command")
val pvTestTag = Tag("pv")
val schedulingTestTag = Tag("schedule")
val decomTestTag = Tag("decom")
diff --git
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala
index 17c5c1055d4d..78ccc5a3ce17 100644
---
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala
+++
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala
@@ -178,7 +178,7 @@ private[spark] trait PVTestsSuite { k8sSuite:
KubernetesSuite =>
}
}
- test("PVs with local storage", k8sTestTag, pvTestTag) {
+ test("PVs with local storage", k8sTestTag, pvTestTag, commandTestTag) {
assume(this.getClass.getSimpleName == "KubernetesSuite")
sparkAppConf
.set(s"spark.kubernetes.driver.volumes.persistentVolumeClaim.data.mount.path",
diff --git
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SecretsTestsSuite.scala
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SecretsTestsSuite.scala
index 234f627202c1..245466677e4c 100644
---
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SecretsTestsSuite.scala
+++
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SecretsTestsSuite.scala
@@ -57,7 +57,7 @@ private[spark] trait SecretsTestsSuite { k8sSuite:
KubernetesSuite =>
.delete()
}
- test("Run SparkPi with env and mount secrets.", k8sTestTag) {
+ test("Run SparkPi with env and mount secrets.", k8sTestTag, commandTestTag) {
createTestSecret()
sparkAppConf
.set(s"spark.kubernetes.driver.secrets.$ENV_SECRET_NAME",
SECRET_MOUNT_PATH)
diff --git
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolumeSuite.scala
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolumeSuite.scala
index 745e294598ef..62ca2a7b142a 100644
---
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolumeSuite.scala
+++
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolumeSuite.scala
@@ -36,7 +36,7 @@ private[spark] trait VolumeSuite { k8sSuite: KubernetesSuite
=>
}
}
- test("A driver-only Spark job with a tmpfs-backed localDir volume",
k8sTestTag) {
+ test("A driver-only Spark job with a tmpfs-backed localDir volume",
k8sTestTag, commandTestTag) {
sparkAppConf
.set("spark.kubernetes.driver.master", "local[10]")
.set("spark.kubernetes.local.dirs.tmpfs", "true")
@@ -57,7 +57,8 @@ private[spark] trait VolumeSuite { k8sSuite: KubernetesSuite
=>
executorPatience = IGNORE)
}
- test("A driver-only Spark job with a tmpfs-backed emptyDir data volume",
k8sTestTag) {
+ test("A driver-only Spark job with a tmpfs-backed emptyDir data volume",
k8sTestTag,
+ commandTestTag) {
sparkAppConf
.set("spark.kubernetes.driver.master", "local[10]")
.set("spark.kubernetes.driver.volumes.emptyDir.data.mount.path", "/data")
@@ -78,7 +79,7 @@ private[spark] trait VolumeSuite { k8sSuite: KubernetesSuite
=>
executorPatience = IGNORE)
}
- test("A driver-only Spark job with a disk-backed emptyDir volume",
k8sTestTag) {
+ test("A driver-only Spark job with a disk-backed emptyDir volume",
k8sTestTag, commandTestTag) {
sparkAppConf
.set("spark.kubernetes.driver.master", "local[10]")
.set("spark.kubernetes.driver.volumes.emptyDir.data.mount.path", "/data")
@@ -98,7 +99,7 @@ private[spark] trait VolumeSuite { k8sSuite: KubernetesSuite
=>
executorPatience = IGNORE)
}
- test("A driver-only Spark job with an OnDemand PVC volume", k8sTestTag) {
+ test("A driver-only Spark job with an OnDemand PVC volume", k8sTestTag,
commandTestTag) {
val storageClassName = testBackend match {
case MinikubeTestBackend => "standard"
case _ => "hostpath"
@@ -126,7 +127,7 @@ private[spark] trait VolumeSuite { k8sSuite:
KubernetesSuite =>
executorPatience = IGNORE)
}
- test("A Spark job with tmpfs-backed localDir volumes", k8sTestTag) {
+ test("A Spark job with tmpfs-backed localDir volumes", k8sTestTag,
commandTestTag) {
sparkAppConf
.set("spark.kubernetes.local.dirs.tmpfs", "true")
runSparkApplicationAndVerifyCompletion(
@@ -150,7 +151,7 @@ private[spark] trait VolumeSuite { k8sSuite:
KubernetesSuite =>
isJVM = true)
}
- test("A Spark job with two executors with OnDemand PVC volumes", k8sTestTag)
{
+ test("A Spark job with two executors with OnDemand PVC volumes", k8sTestTag,
commandTestTag) {
val storageClassName = testBackend match {
case MinikubeTestBackend => "standard"
case _ => "hostpath"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
