This is an automated email from the ASF dual-hosted git repository.
dstandish pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 2035dc71be Support deferrable operators in dag test (#34585)
2035dc71be is described below
commit 2035dc71be68b6d8dcf4536370c6e21ee0fd9585
Author: Daniel Standish <[email protected]>
AuthorDate: Mon Sep 25 09:27:03 2023 -0700
Support deferrable operators in dag test (#34585)
You need to run triggerer separately, but now at least dag test will resume
the task after deferral.
---
airflow/models/dag.py | 16 ++++++++++++++--
airflow/models/taskinstance.py | 3 +++
2 files changed, 17 insertions(+), 2 deletions(-)
diff --git a/airflow/models/dag.py b/airflow/models/dag.py
index 4baf2dbf32..da734152c8 100644
--- a/airflow/models/dag.py
+++ b/airflow/models/dag.py
@@ -27,6 +27,7 @@ import os
import pathlib
import pickle
import sys
+import time
import traceback
import warnings
import weakref
@@ -123,7 +124,7 @@ from airflow.utils.sqlalchemy import (
tuple_in_condition,
with_row_locks,
)
-from airflow.utils.state import DagRunState, TaskInstanceState
+from airflow.utils.state import DagRunState, State, TaskInstanceState
from airflow.utils.trigger_rule import TriggerRule
from airflow.utils.types import NOTSET, ArgNotSet, DagRunType, EdgeInfoType
@@ -2811,8 +2812,19 @@ class DAG(LoggingMixin):
# for task readiness and dependency management. This is notably faster
# than creating a BackfillJob and allows us to surface logs to the user
while dr.state == DagRunState.RUNNING:
+ session.expire_all()
schedulable_tis, _ = dr.update_state(session=session)
- for ti in schedulable_tis:
+ for s in schedulable_tis:
+ s.state = TaskInstanceState.SCHEDULED
+ session.commit()
+ # triggerer may mark tasks scheduled so we read from DB
+ all_tis = set(dr.get_task_instances(session=session))
+ scheduled_tis = {x for x in all_tis if x.state ==
TaskInstanceState.SCHEDULED}
+ ids_unrunnable = {x for x in all_tis if x.state not in
State.finished} - scheduled_tis
+ if not scheduled_tis and ids_unrunnable:
+ self.log.warning("No tasks to run. unrunnable tasks: %s",
ids_unrunnable)
+ time.sleep(1)
+ for ti in scheduled_tis:
try:
add_logger_if_needed(ti)
ti.task = tasks[ti.task_id]
diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py
index 98da686184..039fe68f98 100644
--- a/airflow/models/taskinstance.py
+++ b/airflow/models/taskinstance.py
@@ -1361,6 +1361,9 @@ class TaskInstance(Base, LoggingMixin):
# can be changed when calling 'run'
self.test_mode = False
+ def __hash__(self):
+ return hash((self.task_id, self.dag_id, self.run_id, self.map_index))
+
@property
def stats_tags(self) -> dict[str, str]:
"""Returns task instance tags."""