eladkal commented on code in PR #27784:
URL: https://github.com/apache/airflow/pull/27784#discussion_r1026949048
##########
airflow/providers/amazon/aws/hooks/emr.py:
##########
@@ -202,57 +202,6 @@ def conn(self):
"""Get the underlying boto3 EmrServerlessAPIService client (cached)"""
return super().conn
- # This method should be replaced with boto waiters which would implement
timeouts and backoff nicely.
- def waiter(
Review Comment:
Needs to have deprecation warning before we can remove it
Also we need ro make it backward compatible by calling the function in the
new location
##########
airflow/providers/amazon/aws/hooks/emr.py:
##########
@@ -202,57 +202,6 @@ def conn(self):
"""Get the underlying boto3 EmrServerlessAPIService client (cached)"""
return super().conn
- # This method should be replaced with boto waiters which would implement
timeouts and backoff nicely.
- def waiter(
- self,
- get_state_callable: Callable,
- get_state_args: dict,
- parse_response: list,
- desired_state: set,
- failure_states: set,
- object_type: str,
- action: str,
- countdown: int = 25 * 60,
- check_interval_seconds: int = 60,
- ) -> None:
- """
- Will run the sensor until it turns True.
-
- :param get_state_callable: A callable to run until it returns True
- :param get_state_args: Arguments to pass to get_state_callable
- :param parse_response: Dictionary keys to extract state from response
of get_state_callable
- :param desired_state: Wait until the getter returns this value
- :param failure_states: A set of states which indicate failure and
should throw an
- exception if any are reached before the desired_state
- :param object_type: Used for the reporting string. What are you
waiting for? (application, job, etc)
- :param action: Used for the reporting string. What action are you
waiting for? (created, deleted, etc)
- :param countdown: Total amount of time the waiter should wait for the
desired state
- before timing out (in seconds). Defaults to 25 * 60 seconds.
- :param check_interval_seconds: Number of seconds waiter should wait
before attempting
- to retry get_state_callable. Defaults to 60 seconds.
- """
- response = get_state_callable(**get_state_args)
- state: str = self.get_state(response, parse_response)
- while state not in desired_state:
- if state in failure_states:
- raise AirflowException(f"{object_type.title()} reached failure
state {state}.")
- if countdown >= check_interval_seconds:
- countdown -= check_interval_seconds
- self.log.info("Waiting for %s to be %s.", object_type.lower(),
action.lower())
- sleep(check_interval_seconds)
- state = self.get_state(get_state_callable(**get_state_args),
parse_response)
- else:
- message = f"{object_type.title()} still not {action.lower()}
after the allocated time limit."
- self.log.error(message)
- raise RuntimeError(message)
-
- def get_state(self, response, keys) -> str:
Review Comment:
Same
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
