AutomationDev85 commented on code in PR #30704:
URL: https://github.com/apache/airflow/pull/30704#discussion_r1171453144
##########
airflow/jobs/scheduler_job_runner.py:
##########
@@ -1083,8 +1083,13 @@ def _do_scheduling(self, session: Session) -> int:
callback_tuples = self._schedule_all_dag_runs(guard, dag_runs,
session)
# Send the callbacks after we commit to ensure the context is up to
date when it gets run
+ # cache saves time during scheduling of many dag_runs for same dag
+ cached_dags: dict = {}
for dag_run, callback_to_run in callback_tuples:
- dag = self.dagbag.get_dag(dag_run.dag_id, session=session)
+ if dag_run.dag_id not in cached_dags.keys():
+ cached_dags[dag_run.dag_id] =
self.dagbag.get_dag(dag_run.dag_id, session=session)
+
+ dag = cached_dags[dag_run.dag_id]
Review Comment:
Thanks for your feedback! This is a good idea, i added a function to the
helpers, not sure is this is the right place but looking forward for your
feedback :)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
