AutomationDev85 commented on code in PR #30704:
URL: https://github.com/apache/airflow/pull/30704#discussion_r1172229514
##########
airflow/jobs/scheduler_job_runner.py:
##########
@@ -1083,8 +1084,13 @@ def _do_scheduling(self, session: Session) -> int:
callback_tuples = self._schedule_all_dag_runs(guard, dag_runs,
session)
# Send the callbacks after we commit to ensure the context is up to
date when it gets run
+ # cache saves time during scheduling of many dag_runs for same dag
+ cached_dags: dict = {}
for dag_run, callback_to_run in callback_tuples:
- dag = self.dagbag.get_dag(dag_run.dag_id, session=session)
+ dag = get_value_with_cache(
+ cached_dags, dag_run.dag_id, self.dagbag.get_dag,
dag_run.dag_id, session=session
+ )
Review Comment:
Thanks for the feedback! Using now lambda.
##########
airflow/jobs/scheduler_job_runner.py:
##########
@@ -1348,8 +1354,14 @@ def _update_state(dag: DAG, dag_run: DagRun):
tags={"dag_id": dag.dag_id},
)
+ # cache saves time during scheduling of many dag_runs for same dag
+ cached_dags: dict = {}
+
for dag_run in dag_runs:
- dag = dag_run.dag = self.dagbag.get_dag(dag_run.dag_id,
session=session)
+ dag = dag_run.dag = get_value_with_cache(
+ cached_dags, dag_run.dag_id, self.dagbag.get_dag,
dag_run.dag_id, session=session
+ )
Review Comment:
Thanks for the feedback! Using now lambda.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
