codenamestif commented on a change in pull request #17626:
URL: https://github.com/apache/airflow/pull/17626#discussion_r721432621
##########
File path: airflow/providers/amazon/aws/operators/ecs.py
##########
@@ -80,6 +84,73 @@ def list_tasks(self, cluster: str, launchType: str,
desiredStatus: str, family:
...
+class ECSTaskLogFetcher(Thread):
+ """
+ Fetches Cloudwatch log events with specific interval as a thread
+ and sends the log events to the info channel of the provided logger.
+ """
+
+ def __init__(
+ self,
+ *,
+ aws_conn_id: Optional[str] = 'aws_default',
+ region_name: Optional[str] = None,
+ log_group: str,
+ log_stream_name: str,
+ fetch_interval: timedelta,
+ logger: Logger,
+ ):
+ super().__init__()
+ self._event = Event()
+
+ self.fetch_interval = fetch_interval
+
+ self.logger = logger
+ self.log_group = log_group
+ self.log_stream_name = log_stream_name
+
+ self.hook = AwsLogsHook(aws_conn_id=aws_conn_id,
region_name=region_name)
+
+ def run(self) -> None:
+ logs_to_skip = 0
+ while not self.is_stopped():
+ log_events = self._get_log_events(logs_to_skip)
+ for log_event in log_events:
+ self.logger.info(self._event_to_str(log_event))
+ logs_to_skip += 1
+ time.sleep(self.fetch_interval.total_seconds())
+
+ def _get_log_events(self, skip: int = 0) -> Generator:
+ try:
+ yield from self.hook.get_log_events(self.log_group,
self.log_stream_name, skip=skip)
+ except ClientError as error:
+ if error.response['Error']['Code'] != 'ResourceNotFoundException':
Review comment:
@zachliu that's a good question. Now i realise that this case is not
handled at all. And looks like handling of `fail-to-start` was implicitly
relied the availability of the log stream.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
