This is an automated email from the ASF dual-hosted git repository.
eladkal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new ea44ed9f54 Increase `waiter_max_attempts` default value in
`EcsRunTaskOperator` (#33712)
ea44ed9f54 is described below
commit ea44ed9f54f6c0083aa6283b2f3f3712bc710a1f
Author: mjsqu <[email protected]>
AuthorDate: Wed Aug 30 22:48:45 2023 +1200
Increase `waiter_max_attempts` default value in `EcsRunTaskOperator`
(#33712)
* Replace waiter_max_attempts default with sys.maxsize
* Update ecs tests to use sys.maxsize as waiter_max_attempts
* Add note regarding change to waiter_max_attempts for EcsRunTaskOperator
* Add clarity on EcsRunTaskOperator fix
* Set max_attempts to large value that can be handled by C int in timedelta
---------
Co-authored-by: Hussein Awala <[email protected]>
Co-authored-by: Elad Kalif <[email protected]>
---
airflow/providers/amazon/CHANGELOG.rst | 1 +
airflow/providers/amazon/aws/operators/ecs.py | 6 +++---
tests/providers/amazon/aws/operators/test_ecs.py | 4 +---
3 files changed, 5 insertions(+), 6 deletions(-)
diff --git a/airflow/providers/amazon/CHANGELOG.rst
b/airflow/providers/amazon/CHANGELOG.rst
index 3a61e959d8..b937a158ec 100644
--- a/airflow/providers/amazon/CHANGELOG.rst
+++ b/airflow/providers/amazon/CHANGELOG.rst
@@ -25,6 +25,7 @@
Changelog
---------
+* ``A bug intoduced in provider-amazon version 8.0.0 caused all
'EcsRunTaskOperator' tasks to detach from the ECS task after 10 minutes and
fail - even if the ECS task was still running. In this version we are fixing it
by returning the default 'waiter_max_attempts' value to 'sys.maxsize'``
8.6.0
.....
diff --git a/airflow/providers/amazon/aws/operators/ecs.py
b/airflow/providers/amazon/aws/operators/ecs.py
index ed8e5ccec3..ad500db40a 100644
--- a/airflow/providers/amazon/aws/operators/ecs.py
+++ b/airflow/providers/amazon/aws/operators/ecs.py
@@ -18,7 +18,6 @@
from __future__ import annotations
import re
-import sys
import warnings
from datetime import timedelta
from functools import cached_property
@@ -476,7 +475,9 @@ class EcsRunTaskOperator(EcsBaseOperator):
number_logs_exception: int = 10,
wait_for_completion: bool = True,
waiter_delay: int = 6,
- waiter_max_attempts: int = 100,
+ waiter_max_attempts: int = 1000000 * 365 * 24 * 60 * 10,
+ # Set the default waiter duration to 1M years (attempts*delay)
+ # Airflow execution_timeout handles task timeout
deferrable: bool = conf.getboolean("operators", "default_deferrable",
fallback=False),
**kwargs,
):
@@ -665,7 +666,6 @@ class EcsRunTaskOperator(EcsBaseOperator):
return
waiter = self.client.get_waiter("tasks_stopped")
- waiter.config.max_attempts = sys.maxsize # timeout is managed by
airflow
waiter.wait(
cluster=self.cluster,
tasks=[self.arn],
diff --git a/tests/providers/amazon/aws/operators/test_ecs.py
b/tests/providers/amazon/aws/operators/test_ecs.py
index 9fad66ec2a..c3a33ab47f 100644
--- a/tests/providers/amazon/aws/operators/test_ecs.py
+++ b/tests/providers/amazon/aws/operators/test_ecs.py
@@ -17,7 +17,6 @@
# under the License.
from __future__ import annotations
-import sys
from copy import deepcopy
from unittest import mock
from unittest.mock import MagicMock, PropertyMock
@@ -348,9 +347,8 @@ class TestEcsRunTaskOperator(EcsBaseTestCase):
self.ecs._wait_for_task_ended()
client_mock.get_waiter.assert_called_once_with("tasks_stopped")
client_mock.get_waiter.return_value.wait.assert_called_once_with(
- cluster="c", tasks=["arn"], WaiterConfig={"Delay": 6,
"MaxAttempts": 100}
+ cluster="c", tasks=["arn"], WaiterConfig={"Delay": 6,
"MaxAttempts": 1000000 * 365 * 24 * 60 * 10}
)
- assert sys.maxsize ==
client_mock.get_waiter.return_value.config.max_attempts
@mock.patch.object(EcsBaseOperator, "client")
def test_check_success_tasks_raises_failed_to_start(self, client_mock):
