codenamestif commented on a change in pull request #17626:
URL: https://github.com/apache/airflow/pull/17626#discussion_r721462625
##########
File path: airflow/providers/amazon/aws/operators/ecs.py
##########
@@ -80,6 +84,73 @@ def list_tasks(self, cluster: str, launchType: str,
desiredStatus: str, family:
...
+class ECSTaskLogFetcher(Thread):
+ """
+ Fetches Cloudwatch log events with specific interval as a thread
+ and sends the log events to the info channel of the provided logger.
+ """
+
+ def __init__(
+ self,
+ *,
+ aws_conn_id: Optional[str] = 'aws_default',
+ region_name: Optional[str] = None,
+ log_group: str,
+ log_stream_name: str,
+ fetch_interval: timedelta,
+ logger: Logger,
+ ):
+ super().__init__()
+ self._event = Event()
+
+ self.fetch_interval = fetch_interval
+
+ self.logger = logger
+ self.log_group = log_group
+ self.log_stream_name = log_stream_name
+
+ self.hook = AwsLogsHook(aws_conn_id=aws_conn_id,
region_name=region_name)
+
+ def run(self) -> None:
+ logs_to_skip = 0
+ while not self.is_stopped():
+ log_events = self._get_log_events(logs_to_skip)
+ for log_event in log_events:
+ self.logger.info(self._event_to_str(log_event))
+ logs_to_skip += 1
+ time.sleep(self.fetch_interval.total_seconds())
+
+ def _get_log_events(self, skip: int = 0) -> Generator:
+ try:
+ yield from self.hook.get_log_events(self.log_group,
self.log_stream_name, skip=skip)
+ except ClientError as error:
+ if error.response['Error']['Code'] != 'ResourceNotFoundException':
Review comment:
@zachliu i can only guess what `describe_tasks` returns in case of the
edge case (#15000), but i guess if arn is presented and task is not started,
then there should be either empty list of tasks in response and it's handled
and the proper error is thrown or there is a non empty list of tasks and their
status is handled by current logic. The missing part of handling empty list of
tasks can be added and i think it's more clear than relying on cloudwatch logs.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
