nikhi-suthar commented on code in PR #26269:
URL: https://github.com/apache/airflow/pull/26269#discussion_r979203551
##########
airflow/providers/amazon/aws/hooks/glue.py:
##########
@@ -151,85 +153,150 @@ def print_job_logs(
job_failed: bool = False,
next_token: Optional[str] = None,
) -> Optional[str]:
- """Prints the batch of logs to the Airflow task log and returns
nextToken."""
- log_client = boto3.client('logs')
- response = {}
-
- filter_pattern = FAILURE_LOG_FILTER if job_failed else
DEFAULT_LOG_FILTER
+ """Prints the batch of Glue cloudwatch logs to the Airflow task log
and returns nextToken."""
+ credentials = self.get_credentials(region_name=self.conn_region_name)
+ log_client = boto3.client(
+ 'logs',
+ region_name=self.conn_region_name,
+ aws_access_key_id=credentials.access_key,
+ aws_secret_access_key=credentials.secret_key,
+ )
log_group_prefix = self.conn.get_job_run(JobName=job_name,
RunId=run_id)['JobRun']['LogGroupName']
log_group_suffix = FAILURE_LOG_SUFFIX if job_failed else
DEFAULT_LOG_SUFFIX
log_group_name = f'{log_group_prefix}/{log_group_suffix}'
-
try:
- if next_token:
- response = log_client.filter_log_events(
- logGroupName=log_group_name,
- logStreamNames=[run_id],
- filterPattern=filter_pattern,
- nextToken=next_token,
- )
- else:
- response = log_client.filter_log_events(
- logGroupName=log_group_name,
- logStreamNames=[run_id],
- filterPattern=filter_pattern,
- )
- if len(response['events']):
+ self.log.info('Glue Job Run Logs')
+ response = log_client.get_log_events(logGroupName=log_group_name,
logStreamName=run_id)
Review Comment:
filter_log_events will not give logs and also in the case of a custom log,
it will not be useful. get_logs_events I have tested and it is working fine.
Agree with you regarding extra filter constants, I will drop and push new
changes.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
