This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 848a3969e6 Classic TaskGroup setup/teardown (#29891)
848a3969e6 is described below
commit 848a3969e65681ea7581a63f9dee3aba52dd316b
Author: Ephraim Anierobi <[email protected]>
AuthorDate: Tue Mar 14 07:56:05 2023 +0100
Classic TaskGroup setup/teardown (#29891)
* Classic TaskGroup setup/teardown
This implements classic TaskGroup setup/teardown. Ensures that nested
TaskGroups are taken into account
* fixup! Classic TaskGroup setup/teardown
---
airflow/decorators/task_group.py | 2 +
airflow/utils/task_group.py | 33 +++-
tests/decorators/test_setup_teardown.py | 314 +++++++++++++++++++++++++++++++-
3 files changed, 343 insertions(+), 6 deletions(-)
diff --git a/airflow/decorators/task_group.py b/airflow/decorators/task_group.py
index 2aa714be3b..1431a10452 100644
--- a/airflow/decorators/task_group.py
+++ b/airflow/decorators/task_group.py
@@ -182,6 +182,8 @@ def task_group(
ui_color: str = "CornflowerBlue",
ui_fgcolor: str = "#000",
add_suffix_on_collision: bool = False,
+ setup: bool = False,
+ teardown: bool = False,
) -> Callable[[Callable[FParams, FReturn]], _TaskGroupFactory[FParams,
FReturn]]:
...
diff --git a/airflow/utils/task_group.py b/airflow/utils/task_group.py
index 8883fa659e..85249d3a98 100644
--- a/airflow/utils/task_group.py
+++ b/airflow/utils/task_group.py
@@ -92,11 +92,18 @@ class TaskGroup(DAGNode):
ui_color: str = "CornflowerBlue",
ui_fgcolor: str = "#000",
add_suffix_on_collision: bool = False,
+ setup: bool = False,
+ teardown: bool = False,
):
from airflow.models.dag import DagContext
+ if setup and teardown:
+ raise AirflowException("Cannot set both setup and teardown to
True")
+
self.prefix_group_id = prefix_group_id
self.default_args = copy.deepcopy(default_args or {})
+ self.setup = setup
+ self.teardown = teardown
dag = dag or DagContext.get_current_dag()
@@ -231,15 +238,37 @@ class TaskGroup(DAGNode):
if task.children:
raise AirflowException("Cannot add a non-empty TaskGroup")
- if SetupTeardownContext.is_setup:
+ is_setup, is_teardown = self._check_is_setup_teardown(task)
+
+ if SetupTeardownContext.is_setup or is_setup:
if isinstance(task, AbstractOperator):
setattr(task, "_is_setup", True)
- elif SetupTeardownContext.is_teardown:
+ elif SetupTeardownContext.is_teardown or is_teardown:
if isinstance(task, AbstractOperator):
setattr(task, "_is_teardown", True)
self.children[key] = task
+ def _check_is_setup_teardown(self, task_):
+ """Check if setup or teardown is set for the task"""
+ from airflow.models.abstractoperator import AbstractOperator
+
+ def _find_setup_teardown(tg):
+ setup, teardown = tg.setup, tg.teardown
+ while tg and tg.parent_group:
+ if setup or teardown:
+ break
+ tg = tg.parent_group
+ setup, teardown = tg.setup, tg.teardown
+ return setup, teardown
+
+ if isinstance(task_, TaskGroup):
+ return _find_setup_teardown(task_)
+ if isinstance(task_, AbstractOperator):
+ tg = task_.task_group
+ return _find_setup_teardown(tg)
+ return False, False
+
def _remove(self, task: DAGNode) -> None:
key = task.node_id
diff --git a/tests/decorators/test_setup_teardown.py
b/tests/decorators/test_setup_teardown.py
index 4dedd2421d..ec2d1773e4 100644
--- a/tests/decorators/test_setup_teardown.py
+++ b/tests/decorators/test_setup_teardown.py
@@ -17,7 +17,12 @@
# under the License.
from __future__ import annotations
+import pytest
+
+from airflow import AirflowException
from airflow.decorators import setup, task, task_group, teardown
+from airflow.operators.bash import BashOperator
+from airflow.utils.task_group import TaskGroup
class TestSetupTearDownTask:
@@ -59,8 +64,6 @@ class TestSetupTearDownTask:
assert setup_task._is_setup
def test_marking_operator_as_setup_task(self, dag_maker):
- from airflow.operators.bash import BashOperator
-
with dag_maker() as dag:
BashOperator.as_setup(task_id="mytask", bash_command='echo "I am a
setup task"')
@@ -91,7 +94,7 @@ class TestSetupTearDownTask:
teardown_task = dag.task_group.children["mytask"]
assert teardown_task._is_teardown
- def test_setup_taskgroup(self, dag_maker):
+ def test_setup_taskgroup_decorator(self, dag_maker):
@setup
@task_group
def mygroup():
@@ -110,7 +113,7 @@ class TestSetupTearDownTask:
setup_task = setup_task_group.children["mygroup.mytask"]
assert setup_task._is_setup
- def test_teardown_taskgroup(self, dag_maker):
+ def test_teardown_taskgroup_decorator(self, dag_maker):
@teardown
@task_group
def mygroup():
@@ -128,3 +131,306 @@ class TestSetupTearDownTask:
assert len(teardown_task_group.children) == 1
teardown_task = teardown_task_group.children["mygroup.mytask"]
assert teardown_task._is_teardown
+
+ def test_setup_taskgroup_classic(self, dag_maker):
+ with dag_maker() as dag:
+ with TaskGroup("mygroup", setup=True):
+ BashOperator(task_id="mytask", bash_command='echo "I am a
setup task"')
+
+ assert len(dag.task_group.children) == 1
+ setup_task_group = dag.task_group.children["mygroup"]
+ assert len(setup_task_group.children) == 1
+ setup_task = setup_task_group.children["mygroup.mytask"]
+ assert setup_task._is_setup
+
+ def test_teardown_taskgroup_classic(self, dag_maker):
+ with dag_maker() as dag:
+ with TaskGroup("mygroup", teardown=True):
+ BashOperator(task_id="mytask", bash_command='echo "I am a
setup task"')
+
+ assert len(dag.task_group.children) == 1
+ teardown_task_group = dag.task_group.children["mygroup"]
+ assert len(teardown_task_group.children) == 1
+ teardown_task = teardown_task_group.children["mygroup.mytask"]
+ assert teardown_task._is_teardown
+
+ def test_setup_taskgroup_decorator_with_subgroup(self, dag_maker):
+ @setup
+ @task_group
+ def mygroup():
+ @task
+ def mytask():
+ print("I am a setup task")
+
+ @task_group
+ def subgroup():
+ @task
+ def mytask2():
+ print("I am a task")
+
+ mytask2()
+
+ mytask()
+ subgroup()
+
+ with dag_maker() as dag:
+ mygroup()
+
+ assert len(dag.task_group.children) == 1
+ setup_task_group = dag.task_group.children["mygroup"]
+ assert len(setup_task_group.children) == 2
+ setup_task = setup_task_group.children["mygroup.mytask"]
+ assert setup_task._is_setup
+ subgroup_task_group = setup_task_group.children["mygroup.subgroup"]
+ assert len(subgroup_task_group.children) == 1
+ subgroup_task =
subgroup_task_group.children["mygroup.subgroup.mytask2"]
+ assert subgroup_task._is_setup
+
+ def test_teardown_taskgroup_decorator_with_subgroup(self, dag_maker):
+ @teardown
+ @task_group
+ def mygroup():
+ @task
+ def mytask():
+ print("I am a teardown task")
+
+ @task_group
+ def subgroup():
+ @task
+ def mytask2():
+ print("I am a task")
+
+ mytask2()
+
+ mytask()
+ subgroup()
+
+ with dag_maker() as dag:
+ mygroup()
+
+ assert len(dag.task_group.children) == 1
+ teardown_task_group = dag.task_group.children["mygroup"]
+ assert len(teardown_task_group.children) == 2
+ teardown_task = teardown_task_group.children["mygroup.mytask"]
+ assert teardown_task._is_teardown
+ subgroup_task_group = teardown_task_group.children["mygroup.subgroup"]
+ assert len(subgroup_task_group.children) == 1
+ subgroup_task =
subgroup_task_group.children["mygroup.subgroup.mytask2"]
+ assert subgroup_task._is_teardown
+
+ def test_teardown_taskgroup_with_subgroup_classic(self, dag_maker):
+ with dag_maker() as dag:
+ with TaskGroup("mygroup", teardown=True):
+
+ @task
+ def mytask():
+ print("I am a teardown task")
+
+ with TaskGroup("subgroup"):
+
+ @task
+ def mytask2():
+ print("I am a teardown task")
+
+ mytask2()
+
+ mytask()
+
+ assert len(dag.task_group.children) == 1
+ teardown_task_group = dag.task_group.children["mygroup"]
+ assert len(teardown_task_group.children) == 2
+ teardown_task = teardown_task_group.children["mygroup.mytask"]
+ assert teardown_task._is_teardown
+ subgroup_task_group = teardown_task_group.children["mygroup.subgroup"]
+ assert len(subgroup_task_group.children) == 1
+ subgroup_task =
subgroup_task_group.children["mygroup.subgroup.mytask2"]
+ assert subgroup_task._is_teardown
+
+ def test_setup_taskgroup_with_subgroup_classic(self, dag_maker):
+ with dag_maker() as dag:
+ with TaskGroup("mygroup", setup=True):
+
+ @task
+ def mytask():
+ print("I am a setup task")
+
+ with TaskGroup("subgroup"):
+
+ @task
+ def mytask2():
+ print("I am a setup task")
+
+ mytask2()
+
+ mytask()
+
+ assert len(dag.task_group.children) == 1
+ setup_task_group = dag.task_group.children["mygroup"]
+ assert len(setup_task_group.children) == 2
+ setup_task = setup_task_group.children["mygroup.mytask"]
+ assert setup_task._is_setup
+ subgroup_task_group = setup_task_group.children["mygroup.subgroup"]
+ assert len(subgroup_task_group.children) == 1
+ subgroup_task =
subgroup_task_group.children["mygroup.subgroup.mytask2"]
+ assert subgroup_task._is_setup
+
+ def test_setup_taskgroup_with_subgroup_being_the_setup_decorated(self,
dag_maker):
+ """Here, the subgroup is the setup taskgroup while the parent is not"""
+
+ with dag_maker() as dag:
+
+ @task_group
+ def mygroup():
+ @task
+ def mytask():
+ print("I am a task")
+
+ @setup
+ @task_group
+ def subgroup():
+ @task
+ def mytask2():
+ print("I am a setup task")
+
+ mytask2()
+
+ subgroup()
+ mytask()
+
+ mygroup()
+ assert len(dag.task_group.children) == 1
+ normal_task_group = dag.task_group.children["mygroup"]
+ assert len(normal_task_group.children) == 2
+ normal_task = normal_task_group.children["mygroup.mytask"]
+ assert not hasattr(normal_task, "_is_setup")
+ setup_task_group = normal_task_group.children["mygroup.subgroup"]
+ assert len(setup_task_group.children) == 1
+ subgroup_task = setup_task_group.children["mygroup.subgroup.mytask2"]
+ assert subgroup_task._is_setup
+
+ def
test_teardown_taskgroup_with_subgroup_being_the_teardown_decorated(self,
dag_maker):
+ """Here, the subgroup is the teardown taskgroup while the parent is
not"""
+
+ @task_group
+ def mygroup():
+ @task
+ def mytask():
+ print("I am not a teardown task")
+
+ @teardown
+ @task_group
+ def subgroup():
+ @task
+ def mytask2():
+ print("I am a teardown task")
+
+ mytask2()
+
+ mytask()
+ subgroup()
+
+ with dag_maker() as dag:
+ mygroup()
+
+ assert len(dag.task_group.children) == 1
+ normal_task_group = dag.task_group.children["mygroup"]
+ assert len(normal_task_group.children) == 2
+ normal_task = normal_task_group.children["mygroup.mytask"]
+ assert not hasattr(normal_task, "_is_teardown")
+ teardown_task_group = normal_task_group.children["mygroup.subgroup"]
+ assert len(teardown_task_group.children) == 1
+ subgroup_task =
teardown_task_group.children["mygroup.subgroup.mytask2"]
+ assert subgroup_task._is_teardown
+
+ def test_setup_taskgroup_with_subgroup_being_the_setup_classic(self,
dag_maker):
+ """Here, the subgroup is the setup taskgroup while the parent is not"""
+ with dag_maker() as dag:
+ with TaskGroup("mygroup"):
+
+ @task
+ def mytask():
+ print("I am not a setup task")
+
+ with TaskGroup("subgroup", setup=True):
+
+ @task
+ def mytask2():
+ print("I am a setup task")
+
+ mytask2()
+
+ mytask()
+
+ assert len(dag.task_group.children) == 1
+ normal_task_group = dag.task_group.children["mygroup"]
+ assert len(normal_task_group.children) == 2
+ normal_task = normal_task_group.children["mygroup.mytask"]
+ assert not hasattr(normal_task, "_is_setup")
+ setup_task_group = normal_task_group.children["mygroup.subgroup"]
+ assert len(setup_task_group.children) == 1
+ subgroup_task = setup_task_group.children["mygroup.subgroup.mytask2"]
+ assert subgroup_task._is_setup
+
+ def test_teardown_taskgroup_with_subgroup_being_the_teardown_classic(self,
dag_maker):
+ """Here, the subgroup is the teardown taskgroup while the parent is
not"""
+ with dag_maker() as dag:
+ with TaskGroup("mygroup"):
+
+ @task
+ def mytask():
+ print("I am not a teardown task")
+
+ with TaskGroup("subgroup", teardown=True):
+
+ @task
+ def mytask2():
+ print("I am a teardown task")
+
+ mytask2()
+
+ mytask()
+
+ assert len(dag.task_group.children) == 1
+ normal_task_group = dag.task_group.children["mygroup"]
+ assert len(normal_task_group.children) == 2
+ normal_task = normal_task_group.children["mygroup.mytask"]
+ assert not hasattr(normal_task, "_is_teardown")
+ teardown_task_group = normal_task_group.children["mygroup.subgroup"]
+ assert len(teardown_task_group.children) == 1
+ subgroup_task =
teardown_task_group.children["mygroup.subgroup.mytask2"]
+ assert subgroup_task._is_teardown
+
+ def test_setup_taskgroup_using_alternative_syntax(self, dag_maker):
+ """Here, the subgroup is the setup taskgroup while the parent is not"""
+ with dag_maker() as dag:
+ tg1 = TaskGroup("mygroup")
+
+ @task(task_group=tg1)
+ def mytask():
+ print("I am not a setup task")
+
+ tg2 = TaskGroup("subgroup", setup=True, parent_group=tg1)
+
+ @task(task_group=tg2)
+ def mytask2():
+ print("I am a setup task")
+
+ mytask2()
+
+ mytask()
+
+ assert len(dag.task_group.children) == 1
+ normal_task_group = dag.task_group.children["mygroup"]
+ assert len(normal_task_group.children) == 2
+ normal_task = normal_task_group.children["mygroup.mytask"]
+ assert not hasattr(normal_task, "_is_setup")
+ setup_task_group = normal_task_group.children["mygroup.subgroup"]
+ assert len(setup_task_group.children) == 1
+ subgroup_task = setup_task_group.children["mygroup.subgroup.mytask2"]
+ assert subgroup_task._is_setup
+
+ def test_setup_teardown_are_mutually_exclusive_on_taskgroup(self,
dag_maker):
+ """Test that setup and teardown are mutually exclusive on TaskGroup"""
+ with dag_maker():
+ with pytest.raises(AirflowException, match="Cannot set both setup
and teardown to True"):
+ TaskGroup("mygroup", setup=True, teardown=True)