potiuk commented on a change in pull request #10956:
URL: https://github.com/apache/airflow/pull/10956#discussion_r499532238
##########
File path: airflow/jobs/scheduler_job.py
##########
@@ -867,122 +631,41 @@ def process_file(
except Exception: # pylint: disable=broad-except
self.log.exception("Failed at reloading the DAG file %s",
file_path)
Stats.incr('dag_file_refresh_error', 1, 1)
- return [], 0
+ return 0, 0
if len(dagbag.dags) > 0:
self.log.info("DAG(s) %s retrieved from %s", dagbag.dags.keys(),
file_path)
else:
self.log.warning("No viable dags retrieved from %s", file_path)
self.update_import_errors(session, dagbag)
- return [], len(dagbag.import_errors)
+ return 0, len(dagbag.import_errors)
try:
- self.execute_on_failure_callbacks(dagbag,
failure_callback_requests)
+ self.execute_callbacks(dagbag, callback_requests)
Review comment:
The `except Exception` here should moved to inside of the
execute_callbacks, so that failure in one callback will not drop all other
callbacks.
##########
File path: airflow/jobs/scheduler_job.py
##########
@@ -867,122 +631,41 @@ def process_file(
except Exception: # pylint: disable=broad-except
self.log.exception("Failed at reloading the DAG file %s",
file_path)
Stats.incr('dag_file_refresh_error', 1, 1)
- return [], 0
+ return 0, 0
if len(dagbag.dags) > 0:
self.log.info("DAG(s) %s retrieved from %s", dagbag.dags.keys(),
file_path)
else:
self.log.warning("No viable dags retrieved from %s", file_path)
self.update_import_errors(session, dagbag)
- return [], len(dagbag.import_errors)
+ return 0, len(dagbag.import_errors)
try:
- self.execute_on_failure_callbacks(dagbag,
failure_callback_requests)
+ self.execute_callbacks(dagbag, callback_requests)
Review comment:
The `except Exception` here should be moved to inside of the
execute_callbacks, so that failure in one callback will not drop all other
callbacks.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
