This is an automated email from the ASF dual-hosted git repository.

ephraimanierobi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new fe27031382 Fix MappedTaskGroup tasks not respecting upstream 
dependency (#33732)
fe27031382 is described below

commit fe27031382e2034b59a23db1c6b9bdbfef259137
Author: Ephraim Anierobi <[email protected]>
AuthorDate: Tue Aug 29 17:48:43 2023 +0100

    Fix MappedTaskGroup tasks not respecting upstream dependency (#33732)
    
    * Fix MappedTaskGroup tasks not respecting upstream dependency
    
    When a MappedTaskGroup has upstream dependencies, the tasks in the group 
don't wait for the upstream tasks
    before they start running, this causes the tasks to fail.
    From my investigation, the tasks inside the MappedTaskGroup don't have 
upstream tasks while the
    MappedTaskGroup has the upstream tasks properly set. Due to this, the 
task's dependencies are met even though the Group has
    upstreams that haven't finished.
    The Fix was to set upstreams after creating the task group with the factory
    Closes: https://github.com/apache/airflow/issues/33446
    
    * set the relationship in __exit__
---
 airflow/utils/task_group.py         |  7 ++++--
 tests/decorators/test_task_group.py | 46 +++++++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+), 2 deletions(-)

diff --git a/airflow/utils/task_group.py b/airflow/utils/task_group.py
index e892f0e85d..0b41929f36 100644
--- a/airflow/utils/task_group.py
+++ b/airflow/utils/task_group.py
@@ -566,8 +566,6 @@ class MappedTaskGroup(TaskGroup):
     def __init__(self, *, expand_input: ExpandInput, **kwargs: Any) -> None:
         super().__init__(**kwargs)
         self._expand_input = expand_input
-        for op, _ in expand_input.iter_references():
-            self.set_upstream(op)
 
     def iter_mapped_dependencies(self) -> Iterator[Operator]:
         """Upstream dependencies that provide XComs used by this mapped task 
group."""
@@ -620,6 +618,11 @@ class MappedTaskGroup(TaskGroup):
             (g._expand_input.get_total_map_length(run_id, session=session) for 
g in groups),
         )
 
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        for op, _ in self._expand_input.iter_references():
+            self.set_upstream(op)
+        super().__exit__(exc_type, exc_val, exc_tb)
+
 
 class TaskGroupContext:
     """TaskGroup context is used to keep the current TaskGroup when TaskGroup 
is used as ContextManager."""
diff --git a/tests/decorators/test_task_group.py 
b/tests/decorators/test_task_group.py
index 3462c3a1d8..4c741ef1c1 100644
--- a/tests/decorators/test_task_group.py
+++ b/tests/decorators/test_task_group.py
@@ -191,6 +191,52 @@ def test_expand_kwargs_create_mapped():
     assert saved == {"a": 1, "b": MappedArgument(input=tg._expand_input, 
key="b")}
 
 
+def test_task_group_expand_kwargs_with_upstream(dag_maker, session, caplog):
+    with dag_maker() as dag:
+
+        @dag.task
+        def t1():
+            return [{"a": 1}, {"a": 2}]
+
+        @task_group("tg1")
+        def tg1(a, b):
+            @dag.task()
+            def t2():
+                return [a, b]
+
+            t2()
+
+        tg1.expand_kwargs(t1())
+
+    dr = dag_maker.create_dagrun()
+    dr.task_instance_scheduling_decisions()
+    assert "Cannot expand" not in caplog.text
+    assert "missing upstream values: ['expand_kwargs() argument']" not in 
caplog.text
+
+
+def test_task_group_expand_with_upstream(dag_maker, session, caplog):
+    with dag_maker() as dag:
+
+        @dag.task
+        def t1():
+            return [1, 2, 3]
+
+        @task_group("tg1")
+        def tg1(a, b):
+            @dag.task()
+            def t2():
+                return [a, b]
+
+            t2()
+
+        tg1.partial(a=1).expand(b=t1())
+
+    dr = dag_maker.create_dagrun()
+    dr.task_instance_scheduling_decisions()
+    assert "Cannot expand" not in caplog.text
+    assert "missing upstream values: ['b']" not in caplog.text
+
+
 def test_override_dag_default_args():
     @dag(
         dag_id="test_dag",

Reply via email to