codenamestif commented on a change in pull request #17626:
URL: https://github.com/apache/airflow/pull/17626#discussion_r721510086
##########
File path: airflow/providers/amazon/aws/operators/ecs.py
##########
@@ -80,6 +84,73 @@ def list_tasks(self, cluster: str, launchType: str,
desiredStatus: str, family:
...
+class ECSTaskLogFetcher(Thread):
+ """
+ Fetches Cloudwatch log events with specific interval as a thread
+ and sends the log events to the info channel of the provided logger.
+ """
+
+ def __init__(
+ self,
+ *,
+ aws_conn_id: Optional[str] = 'aws_default',
+ region_name: Optional[str] = None,
+ log_group: str,
+ log_stream_name: str,
+ fetch_interval: timedelta,
+ logger: Logger,
+ ):
+ super().__init__()
+ self._event = Event()
+
+ self.fetch_interval = fetch_interval
+
+ self.logger = logger
+ self.log_group = log_group
+ self.log_stream_name = log_stream_name
+
+ self.hook = AwsLogsHook(aws_conn_id=aws_conn_id,
region_name=region_name)
+
+ def run(self) -> None:
+ logs_to_skip = 0
+ while not self.is_stopped():
+ log_events = self._get_log_events(logs_to_skip)
+ for log_event in log_events:
+ self.logger.info(self._event_to_str(log_event))
+ logs_to_skip += 1
+ time.sleep(self.fetch_interval.total_seconds())
+
+ def _get_log_events(self, skip: int = 0) -> Generator:
+ try:
+ yield from self.hook.get_log_events(self.log_group,
self.log_stream_name, skip=skip)
+ except ClientError as error:
+ if error.response['Error']['Code'] != 'ResourceNotFoundException':
Review comment:
Alright, i can prepare a small pr with a check for existence of the log
group of the task. Just to restore previous behaviour with better error
message. Like if the task is stopped, logging is enabled and there is no log
group, then we throw the error. But then we need to exclude the provider from
release @potiuk.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
