This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-kubernetes-operator.git
The following commit(s) were added to refs/heads/main by this push:
new a433342e [SPARK-60056] Split `kueue` E2E test group into `kueue` and
`kueue-pods-ready`
a433342e is described below
commit a433342ef66f8ca7d957a5344e03b40ecd57fa2c
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Wed Oct 7 14:29:57 2026 -0700
[SPARK-60056] Split `kueue` E2E test group into `kueue` and
`kueue-pods-ready`
### What changes were proposed in this pull request?
This PR aims to split the `kueue` E2E test group into two groups, `kueue`
and `kueue-pods-ready`, which run in parallel as separate matrix jobs.
- `kueue-pods-ready` has the two steps which wait for the shortened
`waitForPodsReady` timeout of Kueue:
- `spark-cluster-whose-pods-are-not-ready-is-evicted-by-kueue`
- `spark-application-whose-executors-are-not-ready-is-evicted-by-kueue`
- `kueue` keeps the other nine steps.
- The fixtures used only by the moved steps are moved together, and
`kueue-queues.yaml` is shared via a relative path.
### Why are the changes needed?
The `kueue` job is the longest job of the CI and determines the wall-clock
time of the whole workflow. Its E2E test step alone takes about 630 seconds,
while the second longest E2E job takes about 480 seconds in total. The two
moved steps take about 300 seconds and the remaining steps take about 330
seconds, so the split balances them.
### Does this PR introduce _any_ user-facing change?
No. This is a test-only change.
### How was this patch tested?
Pass the CIs.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
Closes #953 from dongjoon-hyun/SPARK-60056.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.github/workflows/build_and_test.yml | 3 +
tests/e2e/kueue-pods-ready/chainsaw-test.yaml | 331 +++++++++++++++++++++
.../spark-cluster-example-pods-not-ready.yaml | 0
.../spark-cluster-example-pods-ready.yaml | 0
.../spark-example-executors-not-ready.yaml | 0
tests/e2e/kueue/chainsaw-test.yaml | 307 -------------------
6 files changed, 334 insertions(+), 307 deletions(-)
diff --git a/.github/workflows/build_and_test.yml
b/.github/workflows/build_and_test.yml
index 57c5d92d..28d40d2b 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -167,6 +167,9 @@ jobs:
- kubernetes-version: "1.37.0"
mode: kueue
test-group: kueue
+ - kubernetes-version: "1.37.0"
+ mode: kueue
+ test-group: kueue-pods-ready
steps:
- name: Checkout repository
uses: actions/checkout@v7
diff --git a/tests/e2e/kueue-pods-ready/chainsaw-test.yaml
b/tests/e2e/kueue-pods-ready/chainsaw-test.yaml
new file mode 100644
index 00000000..af705d25
--- /dev/null
+++ b/tests/e2e/kueue-pods-ready/chainsaw-test.yaml
@@ -0,0 +1,331 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+ name: spark-operator-kueue-pods-ready
+spec:
+ namespace: default
+ steps:
+ - name: spark-cluster-whose-pods-are-not-ready-is-evicted-by-kueue
+ # Each step uses its own queue objects, so that a step never waits for
Kueue to finish
+ # deleting the objects of another step.
+ bindings:
+ - name: FLAVOR
+ value: spark-e2e-pods-ready-flavor
+ - name: CLUSTER_QUEUE
+ value: spark-e2e-pods-ready-cluster-queue
+ - name: LOCAL_QUEUE
+ value: spark-e2e-pods-ready-queue
+ try:
+ # The eviction below needs the waitForPodsReady timeout which the CI
shortens to 2 minutes,
+ # rather than the default of 30 minutes.
+ - script:
+ timeout: 30s
+ content: |
+ kubectl get configmap kueue-manager-config -n kueue-system -o json \
+ | jq -r '.data["controller_manager_config.yaml"] | if
contains("\nwaitForPodsReady:\n timeout: 2m\n") then "SHORTENED" else
"DEFAULT" end'
+ check:
+ (contains($stdout, 'SHORTENED')): true
+ - apply:
+ file: ../kueue/kueue-queues.yaml
+ # The quota fits both clusters (1 CPU for each master and worker)
+ - script:
+ timeout: 30s
+ env:
+ - name: CLUSTER_QUEUE
+ value: ($CLUSTER_QUEUE)
+ content: |
+ kubectl patch clusterqueue "$CLUSTER_QUEUE" --type json -p \
+ '[{"op": "replace", "path":
"/spec/resourceGroups/0/flavors/0/resources/0/nominalQuota", "value": "4"},
+ {"op": "replace", "path":
"/spec/resourceGroups/0/flavors/0/resources/1/nominalQuota", "value": "8Gi"}]'
+ - apply:
+ file: spark-cluster-example-pods-ready.yaml
+ - assert:
+ timeout: 60s
+ resource:
+ apiVersion: kueue.x-k8s.io/v1beta2
+ kind: Workload
+ metadata:
+ name: sparkcluster-spark-cluster-kueue-pods-ready-test
+ namespace: default
+ status:
+ (conditions[?type == 'Admitted' && status == 'True'] | length(@)):
1
+ # The node labels and tolerations of the assigned ResourceFlavor are
injected into the master
+ # and the worker pods.
+ - assert:
+ timeout: 5m
+ resource:
+ apiVersion: apps/v1
+ kind: StatefulSet
+ metadata:
+ name: spark-cluster-kueue-pods-ready-test-master
+ namespace: default
+ spec:
+ template:
+ spec:
+ nodeSelector:
+ kubernetes.io/os: linux
+ (tolerations[?key == 'spark.apache.org/kueue-e2e'] |
length(@)): 1
+ - assert:
+ timeout: 5m
+ resource:
+ apiVersion: apps/v1
+ kind: StatefulSet
+ metadata:
+ name: spark-cluster-kueue-pods-ready-test-worker
+ namespace: default
+ spec:
+ template:
+ spec:
+ nodeSelector:
+ kubernetes.io/os: linux
+ (tolerations[?key == 'spark.apache.org/kueue-e2e'] |
length(@)): 1
+ # Admitted after the cluster above, so that the waitForPodsReady timeout
of Kueue, which the CI
+ # shortens to 2 minutes, has elapsed for both clusters once this one is
evicted.
+ - apply:
+ file: spark-cluster-example-pods-not-ready.yaml
+ # RunningHealthy does not wait for the master and the worker, so the
operator reports that they
+ # are ready once they are.
+ - assert:
+ timeout: 5m
+ resource:
+ apiVersion: kueue.x-k8s.io/v1beta2
+ kind: Workload
+ metadata:
+ name: sparkcluster-spark-cluster-kueue-pods-ready-test
+ namespace: default
+ status:
+ (conditions[?type == 'PodsReady' && status == 'True' && reason ==
'Started'] | length(@)): 1
+ - assert:
+ timeout: 5m
+ resource:
+ apiVersion: kueue.x-k8s.io/v1beta2
+ kind: Workload
+ metadata:
+ name: sparkcluster-spark-cluster-kueue-pods-not-ready-test
+ namespace: default
+ status:
+ (conditions[?type == 'Evicted' && status == 'True' && reason ==
'PodsReadyTimeout'] | length(@)): 1
+ # Like on a preemption, the operator releases the master and worker first
and then the Workload
+ # once its requeue backoff, which the CI shortens to 10 seconds, elapses,
and the cluster is
+ # queued again.
+ - assert:
+ timeout: 60s
+ resource:
+ apiVersion: v1
+ kind: Event
+ metadata:
+ namespace: default
+ involvedObject:
+ kind: SparkCluster
+ name: spark-cluster-kueue-pods-not-ready-test
+ reason: Suspended
+ (contains(message || '', 'evicted by Kueue')): true
+ (contains(message || '', 'PodsReadyTimeout')): true
+ - assert:
+ timeout: 5m
+ resource:
+ apiVersion: v1
+ kind: Event
+ metadata:
+ namespace: default
+ involvedObject:
+ kind: SparkCluster
+ name: spark-cluster-kueue-pods-not-ready-test
+ reason: Submitted
+ (contains(message || '', 'after its eviction')): true
+ # The cluster whose master and worker are ready is not evicted
+ - assert:
+ timeout: 30s
+ resource:
+ apiVersion: kueue.x-k8s.io/v1beta2
+ kind: Workload
+ metadata:
+ name: sparkcluster-spark-cluster-kueue-pods-ready-test
+ namespace: default
+ status:
+ (conditions[?type == 'Evicted' && status == 'True'] | length(@)): 0
+ # Nor was it evicted before, which the check above misses once the evicted
Workload is replaced
+ # by a new one of the same name
+ - error:
+ timeout: 10s
+ resource:
+ apiVersion: v1
+ kind: Event
+ metadata:
+ namespace: default
+ involvedObject:
+ kind: SparkCluster
+ name: spark-cluster-kueue-pods-ready-test
+ reason: Suspended
+ catch:
+ - describe:
+ apiVersion: spark.apache.org/v1
+ kind: SparkCluster
+ namespace: default
+ - describe:
+ apiVersion: kueue.x-k8s.io/v1beta2
+ kind: Workload
+ namespace: default
+ - events:
+ namespace: default
+ finally:
+ - script:
+ timeout: 120s
+ env:
+ - name: FLAVOR
+ value: ($FLAVOR)
+ - name: CLUSTER_QUEUE
+ value: ($CLUSTER_QUEUE)
+ - name: LOCAL_QUEUE
+ value: ($LOCAL_QUEUE)
+ content: |
+ kubectl delete sparkcluster spark-cluster-kueue-pods-ready-test
spark-cluster-kueue-pods-not-ready-test -n default --ignore-not-found=true
+ # Kueue finalizers may delay the deletion, which later steps do not
depend on.
+ kubectl delete localqueue "$LOCAL_QUEUE" -n default
--ignore-not-found=true --wait=false
+ kubectl delete clusterqueue "$CLUSTER_QUEUE" --ignore-not-found=true
--wait=false
+ kubectl delete resourceflavor "$FLAVOR" --ignore-not-found=true
--wait=false
+ - name: spark-application-whose-executors-are-not-ready-is-evicted-by-kueue
+ # Each step uses its own queue objects, so that a step never waits for
Kueue to finish
+ # deleting the objects of another step.
+ bindings:
+ - name: FLAVOR
+ value: spark-e2e-app-pods-ready-flavor
+ - name: CLUSTER_QUEUE
+ value: spark-e2e-app-pods-ready-cluster-queue
+ - name: LOCAL_QUEUE
+ value: spark-e2e-app-pods-ready-queue
+ try:
+ # The eviction below needs the waitForPodsReady timeout which the CI
shortens to 2 minutes,
+ # rather than the default of 30 minutes.
+ - script:
+ timeout: 30s
+ content: |
+ kubectl get configmap kueue-manager-config -n kueue-system -o json \
+ | jq -r '.data["controller_manager_config.yaml"] | if
contains("\nwaitForPodsReady:\n timeout: 2m\n") then "SHORTENED" else
"DEFAULT" end'
+ check:
+ (contains($stdout, 'SHORTENED')): true
+ - apply:
+ file: ../kueue/kueue-queues.yaml
+ # The quota fits the application (1 CPU for the driver and 1 for the
executor)
+ - script:
+ timeout: 30s
+ env:
+ - name: CLUSTER_QUEUE
+ value: ($CLUSTER_QUEUE)
+ content: |
+ kubectl patch clusterqueue "$CLUSTER_QUEUE" --type json -p \
+ '[{"op": "replace", "path":
"/spec/resourceGroups/0/flavors/0/resources/0/nominalQuota", "value": "2"},
+ {"op": "replace", "path":
"/spec/resourceGroups/0/flavors/0/resources/1/nominalQuota", "value": "8Gi"}]'
+ - apply:
+ file: spark-example-executors-not-ready.yaml
+ - assert:
+ timeout: 5m
+ resource:
+ apiVersion: kueue.x-k8s.io/v1beta2
+ kind: Workload
+ metadata:
+ name: sparkapplication-spark-job-kueue-executors-not-ready-test
+ namespace: default
+ status:
+ (conditions[?type == 'Evicted' && status == 'True' && reason ==
'PodsReadyTimeout'] | length(@)): 1
+ # Like a running SparkCluster, the operator releases the driver and
executor first and then the
+ # Workload once its requeue backoff, which the CI shortens to 10 seconds,
elapses, and the
+ # application is queued again with a new attempt.
+ - assert:
+ timeout: 60s
+ resource:
+ apiVersion: v1
+ kind: Event
+ metadata:
+ namespace: default
+ involvedObject:
+ kind: SparkApplication
+ name: spark-job-kueue-executors-not-ready-test
+ reason: Suspended
+ (contains(message || '', 'evicted by Kueue')): true
+ (contains(message || '', 'PodsReadyTimeout')): true
+ - assert:
+ timeout: 5m
+ resource:
+ apiVersion: v1
+ kind: Event
+ metadata:
+ namespace: default
+ involvedObject:
+ kind: SparkApplication
+ name: spark-job-kueue-executors-not-ready-test
+ reason: Submitted
+ (contains(message || '', 'after its eviction')): true
+ # The new attempt does not count as a restart, even with the default
restartPolicy Never, and
+ # runs again on the quota which Kueue admits its new Workload into. It is
evicted again once the
+ # waitForPodsReady timeout elapses, since its executor is never scheduled.
+ - assert:
+ timeout: 5m
+ resource:
+ apiVersion: spark.apache.org/v1
+ kind: SparkApplication
+ metadata:
+ name: spark-job-kueue-executors-not-ready-test
+ namespace: default
+ status:
+ currentState:
+ currentStateSummary: RunningHealthy
+ currentAttemptSummary:
+ attemptInfo:
+ id: 1
+ restartCounter: 0
+ - assert:
+ timeout: 60s
+ resource:
+ apiVersion: kueue.x-k8s.io/v1beta2
+ kind: Workload
+ metadata:
+ name: sparkapplication-spark-job-kueue-executors-not-ready-test
+ namespace: default
+ status:
+ (conditions[?type == 'Admitted' && status == 'True'] | length(@)):
1
+ (conditions[?type == 'Evicted' && status == 'True'] | length(@)): 0
+ catch:
+ - describe:
+ apiVersion: spark.apache.org/v1
+ kind: SparkApplication
+ namespace: default
+ - describe:
+ apiVersion: kueue.x-k8s.io/v1beta2
+ kind: Workload
+ namespace: default
+ - events:
+ namespace: default
+ finally:
+ - script:
+ timeout: 120s
+ env:
+ - name: FLAVOR
+ value: ($FLAVOR)
+ - name: CLUSTER_QUEUE
+ value: ($CLUSTER_QUEUE)
+ - name: LOCAL_QUEUE
+ value: ($LOCAL_QUEUE)
+ content: |
+ kubectl delete sparkapplication
spark-job-kueue-executors-not-ready-test -n default --ignore-not-found=true
+ # Kueue finalizers may delay the deletion, which later steps do not
depend on.
+ kubectl delete localqueue "$LOCAL_QUEUE" -n default
--ignore-not-found=true --wait=false
+ kubectl delete clusterqueue "$CLUSTER_QUEUE" --ignore-not-found=true
--wait=false
+ kubectl delete resourceflavor "$FLAVOR" --ignore-not-found=true
--wait=false
diff --git a/tests/e2e/kueue/spark-cluster-example-pods-not-ready.yaml
b/tests/e2e/kueue-pods-ready/spark-cluster-example-pods-not-ready.yaml
similarity index 100%
rename from tests/e2e/kueue/spark-cluster-example-pods-not-ready.yaml
rename to tests/e2e/kueue-pods-ready/spark-cluster-example-pods-not-ready.yaml
diff --git a/tests/e2e/kueue/spark-cluster-example-pods-ready.yaml
b/tests/e2e/kueue-pods-ready/spark-cluster-example-pods-ready.yaml
similarity index 100%
rename from tests/e2e/kueue/spark-cluster-example-pods-ready.yaml
rename to tests/e2e/kueue-pods-ready/spark-cluster-example-pods-ready.yaml
diff --git a/tests/e2e/kueue/spark-example-executors-not-ready.yaml
b/tests/e2e/kueue-pods-ready/spark-example-executors-not-ready.yaml
similarity index 100%
rename from tests/e2e/kueue/spark-example-executors-not-ready.yaml
rename to tests/e2e/kueue-pods-ready/spark-example-executors-not-ready.yaml
diff --git a/tests/e2e/kueue/chainsaw-test.yaml
b/tests/e2e/kueue/chainsaw-test.yaml
index ce84f314..ca1dcac9 100644
--- a/tests/e2e/kueue/chainsaw-test.yaml
+++ b/tests/e2e/kueue/chainsaw-test.yaml
@@ -939,313 +939,6 @@ spec:
kubectl delete clusterqueue "$CLUSTER_QUEUE" --ignore-not-found=true
--wait=false
kubectl delete resourceflavor "$FLAVOR" --ignore-not-found=true
--wait=false
kubectl delete workloadpriorityclass "$LOW_PRIORITY_CLASS"
"$PRIORITY_CLASS" --ignore-not-found=true --wait=false
- - name: spark-cluster-whose-pods-are-not-ready-is-evicted-by-kueue
- # Each step uses its own queue objects, so that a step never waits for
Kueue to finish
- # deleting the objects of another step.
- bindings:
- - name: FLAVOR
- value: spark-e2e-pods-ready-flavor
- - name: CLUSTER_QUEUE
- value: spark-e2e-pods-ready-cluster-queue
- - name: LOCAL_QUEUE
- value: spark-e2e-pods-ready-queue
- try:
- # The eviction below needs the waitForPodsReady timeout which the CI
shortens to 2 minutes,
- # rather than the default of 30 minutes.
- - script:
- timeout: 30s
- content: |
- kubectl get configmap kueue-manager-config -n kueue-system -o json \
- | jq -r '.data["controller_manager_config.yaml"] | if
contains("\nwaitForPodsReady:\n timeout: 2m\n") then "SHORTENED" else
"DEFAULT" end'
- check:
- (contains($stdout, 'SHORTENED')): true
- - apply:
- file: kueue-queues.yaml
- # The quota fits both clusters (1 CPU for each master and worker)
- - script:
- timeout: 30s
- env:
- - name: CLUSTER_QUEUE
- value: ($CLUSTER_QUEUE)
- content: |
- kubectl patch clusterqueue "$CLUSTER_QUEUE" --type json -p \
- '[{"op": "replace", "path":
"/spec/resourceGroups/0/flavors/0/resources/0/nominalQuota", "value": "4"},
- {"op": "replace", "path":
"/spec/resourceGroups/0/flavors/0/resources/1/nominalQuota", "value": "8Gi"}]'
- - apply:
- file: spark-cluster-example-pods-ready.yaml
- - assert:
- timeout: 60s
- resource:
- apiVersion: kueue.x-k8s.io/v1beta2
- kind: Workload
- metadata:
- name: sparkcluster-spark-cluster-kueue-pods-ready-test
- namespace: default
- status:
- (conditions[?type == 'Admitted' && status == 'True'] | length(@)):
1
- # The node labels and tolerations of the assigned ResourceFlavor are
injected into the master
- # and the worker pods.
- - assert:
- timeout: 5m
- resource:
- apiVersion: apps/v1
- kind: StatefulSet
- metadata:
- name: spark-cluster-kueue-pods-ready-test-master
- namespace: default
- spec:
- template:
- spec:
- nodeSelector:
- kubernetes.io/os: linux
- (tolerations[?key == 'spark.apache.org/kueue-e2e'] |
length(@)): 1
- - assert:
- timeout: 5m
- resource:
- apiVersion: apps/v1
- kind: StatefulSet
- metadata:
- name: spark-cluster-kueue-pods-ready-test-worker
- namespace: default
- spec:
- template:
- spec:
- nodeSelector:
- kubernetes.io/os: linux
- (tolerations[?key == 'spark.apache.org/kueue-e2e'] |
length(@)): 1
- # Admitted after the cluster above, so that the waitForPodsReady timeout
of Kueue, which the CI
- # shortens to 2 minutes, has elapsed for both clusters once this one is
evicted.
- - apply:
- file: spark-cluster-example-pods-not-ready.yaml
- # RunningHealthy does not wait for the master and the worker, so the
operator reports that they
- # are ready once they are.
- - assert:
- timeout: 5m
- resource:
- apiVersion: kueue.x-k8s.io/v1beta2
- kind: Workload
- metadata:
- name: sparkcluster-spark-cluster-kueue-pods-ready-test
- namespace: default
- status:
- (conditions[?type == 'PodsReady' && status == 'True' && reason ==
'Started'] | length(@)): 1
- - assert:
- timeout: 5m
- resource:
- apiVersion: kueue.x-k8s.io/v1beta2
- kind: Workload
- metadata:
- name: sparkcluster-spark-cluster-kueue-pods-not-ready-test
- namespace: default
- status:
- (conditions[?type == 'Evicted' && status == 'True' && reason ==
'PodsReadyTimeout'] | length(@)): 1
- # Like on a preemption, the operator releases the master and worker first
and then the Workload
- # once its requeue backoff, which the CI shortens to 10 seconds, elapses,
and the cluster is
- # queued again.
- - assert:
- timeout: 60s
- resource:
- apiVersion: v1
- kind: Event
- metadata:
- namespace: default
- involvedObject:
- kind: SparkCluster
- name: spark-cluster-kueue-pods-not-ready-test
- reason: Suspended
- (contains(message || '', 'evicted by Kueue')): true
- (contains(message || '', 'PodsReadyTimeout')): true
- - assert:
- timeout: 5m
- resource:
- apiVersion: v1
- kind: Event
- metadata:
- namespace: default
- involvedObject:
- kind: SparkCluster
- name: spark-cluster-kueue-pods-not-ready-test
- reason: Submitted
- (contains(message || '', 'after its eviction')): true
- # The cluster whose master and worker are ready is not evicted
- - assert:
- timeout: 30s
- resource:
- apiVersion: kueue.x-k8s.io/v1beta2
- kind: Workload
- metadata:
- name: sparkcluster-spark-cluster-kueue-pods-ready-test
- namespace: default
- status:
- (conditions[?type == 'Evicted' && status == 'True'] | length(@)): 0
- # Nor was it evicted before, which the check above misses once the evicted
Workload is replaced
- # by a new one of the same name
- - error:
- timeout: 10s
- resource:
- apiVersion: v1
- kind: Event
- metadata:
- namespace: default
- involvedObject:
- kind: SparkCluster
- name: spark-cluster-kueue-pods-ready-test
- reason: Suspended
- catch:
- - describe:
- apiVersion: spark.apache.org/v1
- kind: SparkCluster
- namespace: default
- - describe:
- apiVersion: kueue.x-k8s.io/v1beta2
- kind: Workload
- namespace: default
- - events:
- namespace: default
- finally:
- - script:
- timeout: 120s
- env:
- - name: FLAVOR
- value: ($FLAVOR)
- - name: CLUSTER_QUEUE
- value: ($CLUSTER_QUEUE)
- - name: LOCAL_QUEUE
- value: ($LOCAL_QUEUE)
- content: |
- kubectl delete sparkcluster spark-cluster-kueue-pods-ready-test
spark-cluster-kueue-pods-not-ready-test -n default --ignore-not-found=true
- # Kueue finalizers may delay the deletion, which later steps do not
depend on.
- kubectl delete localqueue "$LOCAL_QUEUE" -n default
--ignore-not-found=true --wait=false
- kubectl delete clusterqueue "$CLUSTER_QUEUE" --ignore-not-found=true
--wait=false
- kubectl delete resourceflavor "$FLAVOR" --ignore-not-found=true
--wait=false
- - name: spark-application-whose-executors-are-not-ready-is-evicted-by-kueue
- # Each step uses its own queue objects, so that a step never waits for
Kueue to finish
- # deleting the objects of another step.
- bindings:
- - name: FLAVOR
- value: spark-e2e-app-pods-ready-flavor
- - name: CLUSTER_QUEUE
- value: spark-e2e-app-pods-ready-cluster-queue
- - name: LOCAL_QUEUE
- value: spark-e2e-app-pods-ready-queue
- try:
- # The eviction below needs the waitForPodsReady timeout which the CI
shortens to 2 minutes,
- # rather than the default of 30 minutes.
- - script:
- timeout: 30s
- content: |
- kubectl get configmap kueue-manager-config -n kueue-system -o json \
- | jq -r '.data["controller_manager_config.yaml"] | if
contains("\nwaitForPodsReady:\n timeout: 2m\n") then "SHORTENED" else
"DEFAULT" end'
- check:
- (contains($stdout, 'SHORTENED')): true
- - apply:
- file: kueue-queues.yaml
- # The quota fits the application (1 CPU for the driver and 1 for the
executor)
- - script:
- timeout: 30s
- env:
- - name: CLUSTER_QUEUE
- value: ($CLUSTER_QUEUE)
- content: |
- kubectl patch clusterqueue "$CLUSTER_QUEUE" --type json -p \
- '[{"op": "replace", "path":
"/spec/resourceGroups/0/flavors/0/resources/0/nominalQuota", "value": "2"},
- {"op": "replace", "path":
"/spec/resourceGroups/0/flavors/0/resources/1/nominalQuota", "value": "8Gi"}]'
- - apply:
- file: spark-example-executors-not-ready.yaml
- - assert:
- timeout: 5m
- resource:
- apiVersion: kueue.x-k8s.io/v1beta2
- kind: Workload
- metadata:
- name: sparkapplication-spark-job-kueue-executors-not-ready-test
- namespace: default
- status:
- (conditions[?type == 'Evicted' && status == 'True' && reason ==
'PodsReadyTimeout'] | length(@)): 1
- # Like a running SparkCluster, the operator releases the driver and
executor first and then the
- # Workload once its requeue backoff, which the CI shortens to 10 seconds,
elapses, and the
- # application is queued again with a new attempt.
- - assert:
- timeout: 60s
- resource:
- apiVersion: v1
- kind: Event
- metadata:
- namespace: default
- involvedObject:
- kind: SparkApplication
- name: spark-job-kueue-executors-not-ready-test
- reason: Suspended
- (contains(message || '', 'evicted by Kueue')): true
- (contains(message || '', 'PodsReadyTimeout')): true
- - assert:
- timeout: 5m
- resource:
- apiVersion: v1
- kind: Event
- metadata:
- namespace: default
- involvedObject:
- kind: SparkApplication
- name: spark-job-kueue-executors-not-ready-test
- reason: Submitted
- (contains(message || '', 'after its eviction')): true
- # The new attempt does not count as a restart, even with the default
restartPolicy Never, and
- # runs again on the quota which Kueue admits its new Workload into. It is
evicted again once the
- # waitForPodsReady timeout elapses, since its executor is never scheduled.
- - assert:
- timeout: 5m
- resource:
- apiVersion: spark.apache.org/v1
- kind: SparkApplication
- metadata:
- name: spark-job-kueue-executors-not-ready-test
- namespace: default
- status:
- currentState:
- currentStateSummary: RunningHealthy
- currentAttemptSummary:
- attemptInfo:
- id: 1
- restartCounter: 0
- - assert:
- timeout: 60s
- resource:
- apiVersion: kueue.x-k8s.io/v1beta2
- kind: Workload
- metadata:
- name: sparkapplication-spark-job-kueue-executors-not-ready-test
- namespace: default
- status:
- (conditions[?type == 'Admitted' && status == 'True'] | length(@)):
1
- (conditions[?type == 'Evicted' && status == 'True'] | length(@)): 0
- catch:
- - describe:
- apiVersion: spark.apache.org/v1
- kind: SparkApplication
- namespace: default
- - describe:
- apiVersion: kueue.x-k8s.io/v1beta2
- kind: Workload
- namespace: default
- - events:
- namespace: default
- finally:
- - script:
- timeout: 120s
- env:
- - name: FLAVOR
- value: ($FLAVOR)
- - name: CLUSTER_QUEUE
- value: ($CLUSTER_QUEUE)
- - name: LOCAL_QUEUE
- value: ($LOCAL_QUEUE)
- content: |
- kubectl delete sparkapplication
spark-job-kueue-executors-not-ready-test -n default --ignore-not-found=true
- # Kueue finalizers may delay the deletion, which later steps do not
depend on.
- kubectl delete localqueue "$LOCAL_QUEUE" -n default
--ignore-not-found=true --wait=false
- kubectl delete clusterqueue "$CLUSTER_QUEUE" --ignore-not-found=true
--wait=false
- kubectl delete resourceflavor "$FLAVOR" --ignore-not-found=true
--wait=false
- name: pod-set-updates-of-admission-checks-reach-the-pods
# Each step uses its own queue objects, so that a step never waits for
Kueue to finish
# deleting the objects of another step.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
