This is an automated email from the ASF dual-hosted git repository.
o-nikolas pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 2b8a3716385 Give EKS delete operations longer to ride out
ResourceInUseException (#73219)
2b8a3716385 is described below
commit 2b8a3716385d6f686ed8634b447cca3b9ea8b1cd
Author: Vincent <[email protected]>
AuthorDate: Tue Sep 15 15:53:57 2026 -0400
Give EKS delete operations longer to ride out ResourceInUseException
(#73219)
EKS keeps rejecting DeleteFargateProfile and DeleteCluster with
ResourceInUseException, saying the cluster has an update in progress, for
over
ten minutes after a Fargate profile becomes ACTIVE. DescribeCluster reports
the
cluster as ACTIVE throughout, so no sensor or waiter can detect the state
and
wait it out beforehand; the operator's retry window is the only thing that
covers it, and five minutes was not enough.
---
.../src/airflow/providers/amazon/aws/utils/__init__.py | 13 +++++++------
.../amazon/aws/example_eks_with_fargate_in_one_step.py | 6 ++++--
.../system/amazon/aws/example_eks_with_fargate_profile.py | 6 ++++--
3 files changed, 15 insertions(+), 10 deletions(-)
diff --git
a/providers/amazon/src/airflow/providers/amazon/aws/utils/__init__.py
b/providers/amazon/src/airflow/providers/amazon/aws/utils/__init__.py
index b4bf0d4f7c3..63c52998397 100644
--- a/providers/amazon/src/airflow/providers/amazon/aws/utils/__init__.py
+++ b/providers/amazon/src/airflow/providers/amazon/aws/utils/__init__.py
@@ -36,12 +36,13 @@ if TYPE_CHECKING:
log = logging.getLogger(__name__)
-# AWS briefly rejects a delete call with ResourceInUseException while the
target resource is still
-# settling from a prior operation (e.g. EKS finalizing a nodegroup removal
before the cluster can be
-# deleted). Retry with exponential backoff (1s, 2s, 4s, ... capped at
RESOURCE_IN_USE_RETRY_MAX_WAIT
-# per wait) until RESOURCE_IN_USE_RETRY_TIMEOUT elapses, then give up and
re-raise. This rides out the
-# settling window without hanging a genuinely wedged resource for long.
-RESOURCE_IN_USE_RETRY_TIMEOUT = 300
+# AWS rejects a delete call with ResourceInUseException while the target
resource is still settling
+# from a prior operation (e.g. EKS finalizing a nodegroup removal before the
cluster can be deleted).
+# Retry with exponential backoff (1s, 2s, 4s, ... capped at
RESOURCE_IN_USE_RETRY_MAX_WAIT per wait)
+# until RESOURCE_IN_USE_RETRY_TIMEOUT elapses, then give up and re-raise. The
window has to cover an
+# EKS cluster reporting "an update in progress" for over ten minutes after a
Fargate profile became
+# ACTIVE, which is not observable through DescribeCluster (the cluster still
reports ACTIVE).
+RESOURCE_IN_USE_RETRY_TIMEOUT = 900
RESOURCE_IN_USE_RETRY_MAX_WAIT = 60
diff --git
a/providers/amazon/tests/system/amazon/aws/example_eks_with_fargate_in_one_step.py
b/providers/amazon/tests/system/amazon/aws/example_eks_with_fargate_in_one_step.py
index 894539c0ac6..430d04d3f7b 100644
---
a/providers/amazon/tests/system/amazon/aws/example_eks_with_fargate_in_one_step.py
+++
b/providers/amazon/tests/system/amazon/aws/example_eks_with_fargate_in_one_step.py
@@ -123,8 +123,10 @@ with DAG(
target_state=FargateProfileStates.ACTIVE,
)
- # The cluster can still report an update in progress after the fargate
profile turns ACTIVE, and EKS
- # rejects the DeleteFargateProfile call below with ResourceInUseException
while that update runs.
+ # EKS can reject the DeleteFargateProfile call below with
ResourceInUseException for over ten minutes
+ # after the profile turns ACTIVE, saying the cluster has an update in
progress. DescribeCluster does
+ # not expose that update, so this sensor only catches a visibly UPDATING
cluster; riding out the rest
+ # is left to the delete operator's ResourceInUseException retry window.
await_cluster_stable = EksClusterStateSensor(
task_id="await_cluster_stable",
trigger_rule=TriggerRule.ALL_DONE,
diff --git
a/providers/amazon/tests/system/amazon/aws/example_eks_with_fargate_profile.py
b/providers/amazon/tests/system/amazon/aws/example_eks_with_fargate_profile.py
index 96ba6ba8c50..ae12ba0329c 100644
---
a/providers/amazon/tests/system/amazon/aws/example_eks_with_fargate_profile.py
+++
b/providers/amazon/tests/system/amazon/aws/example_eks_with_fargate_profile.py
@@ -139,8 +139,10 @@ with DAG(
target_state=FargateProfileStates.ACTIVE,
)
- # The cluster can still report an update in progress after the fargate
profile turns ACTIVE, and EKS
- # rejects DeleteFargateProfile with ResourceInUseException while that
update runs.
+ # EKS can reject DeleteFargateProfile with ResourceInUseException for over
ten minutes after the
+ # profile turns ACTIVE, saying the cluster has an update in progress.
DescribeCluster does not expose
+ # that update, so this sensor only catches a visibly UPDATING cluster;
riding out the rest is left to
+ # the delete operator's ResourceInUseException retry window.
await_cluster_stable_before_profile_delete = EksClusterStateSensor(
task_id="await_cluster_stable_before_profile_delete",
trigger_rule=TriggerRule.ALL_DONE,
