This is an automated email from the ASF dual-hosted git repository.
uranusjr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 58e264c83f Add support for tabs (and other UX components) to docs
(#36041)
58e264c83f is described below
commit 58e264c83fed1ca42486302600288230b944ab06
Author: Josh Fell <[email protected]>
AuthorDate: Wed Dec 6 03:44:18 2023 -0500
Add support for tabs (and other UX components) to docs (#36041)
---
...hon_operator.py => example_python_decorator.py} | 45 +--
airflow/example_dags/example_python_operator.py | 76 ++---
.../example_dags/example_short_circuit_operator.py | 6 +-
docs/apache-airflow/howto/operator/python.rst | 341 ++++++++++++++-------
docs/conf.py | 6 +
docs/sphinx_design/static/custom.css | 33 ++
setup.py | 1 +
.../endpoints/test_task_instance_endpoint.py | 30 +-
8 files changed, 329 insertions(+), 209 deletions(-)
diff --git a/airflow/example_dags/example_python_operator.py
b/airflow/example_dags/example_python_decorator.py
similarity index 77%
copy from airflow/example_dags/example_python_operator.py
copy to airflow/example_dags/example_python_decorator.py
index 7518206b41..8a57175dd2 100644
--- a/airflow/example_dags/example_python_operator.py
+++ b/airflow/example_dags/example_python_decorator.py
@@ -28,26 +28,21 @@ from pprint import pprint
import pendulum
-from airflow.decorators import task
-from airflow.models.dag import DAG
-from airflow.operators.python import ExternalPythonOperator,
PythonVirtualenvOperator, is_venv_installed
+from airflow.decorators import dag, task
+from airflow.operators.python import is_venv_installed
log = logging.getLogger(__name__)
PATH_TO_PYTHON_BINARY = sys.executable
-def x():
- pass
-
-
-with DAG(
- dag_id="example_python_operator",
+@dag(
schedule=None,
start_date=pendulum.datetime(2021, 1, 1, tz="UTC"),
catchup=False,
tags=["example"],
-) as dag:
+)
+def example_python_decorator():
# [START howto_operator_python]
@task(task_id="print_the_context")
def print_context(ds=None, **kwargs):
@@ -69,14 +64,13 @@ with DAG(
# [START howto_operator_python_kwargs]
# Generate 5 sleeping tasks, sleeping from 0.0 to 0.4 seconds respectively
- for i in range(5):
-
- @task(task_id=f"sleep_for_{i}")
- def my_sleeping_function(random_base):
- """This is a function that will run within the DAG execution"""
- time.sleep(random_base)
+ @task
+ def my_sleeping_function(random_base):
+ """This is a function that will run within the DAG execution"""
+ time.sleep(random_base)
- sleeping_task = my_sleeping_function(random_base=i / 10)
+ for i in range(5):
+ sleeping_task =
my_sleeping_function.override(task_id=f"sleep_for_{i}")(random_base=i / 10)
run_this >> log_the_sql >> sleeping_task
# [END howto_operator_python_kwargs]
@@ -135,20 +129,7 @@ with DAG(
external_python_task = callable_external_python()
# [END howto_operator_external_python]
- # [START howto_operator_external_python_classic]
- external_classic = ExternalPythonOperator(
- task_id="external_python_classic",
- python=PATH_TO_PYTHON_BINARY,
- python_callable=x,
- )
- # [END howto_operator_external_python_classic]
+ run_this >> external_python_task >> virtualenv_task
- # [START howto_operator_python_venv_classic]
- virtual_classic = PythonVirtualenvOperator(
- task_id="virtualenv_classic",
- requirements="colorama==0.4.0",
- python_callable=x,
- )
- # [END howto_operator_python_venv_classic]
- run_this >> external_classic >> external_python_task >> virtual_classic
+example_python_decorator()
diff --git a/airflow/example_dags/example_python_operator.py
b/airflow/example_dags/example_python_operator.py
index 7518206b41..47cad6bd4b 100644
--- a/airflow/example_dags/example_python_operator.py
+++ b/airflow/example_dags/example_python_operator.py
@@ -16,8 +16,8 @@
# specific language governing permissions and limitations
# under the License.
"""
-Example DAG demonstrating the usage of the TaskFlow API to execute Python
functions natively and within a
-virtual environment.
+Example DAG demonstrating the usage of the classic Python operators to execute
Python functions natively and
+within a virtual environment.
"""
from __future__ import annotations
@@ -28,55 +28,58 @@ from pprint import pprint
import pendulum
-from airflow.decorators import task
from airflow.models.dag import DAG
-from airflow.operators.python import ExternalPythonOperator,
PythonVirtualenvOperator, is_venv_installed
+from airflow.operators.python import (
+ ExternalPythonOperator,
+ PythonOperator,
+ PythonVirtualenvOperator,
+ is_venv_installed,
+)
log = logging.getLogger(__name__)
PATH_TO_PYTHON_BINARY = sys.executable
-def x():
- pass
-
-
with DAG(
dag_id="example_python_operator",
schedule=None,
start_date=pendulum.datetime(2021, 1, 1, tz="UTC"),
catchup=False,
tags=["example"],
-) as dag:
+):
# [START howto_operator_python]
- @task(task_id="print_the_context")
def print_context(ds=None, **kwargs):
"""Print the Airflow context and ds variable from the context."""
pprint(kwargs)
print(ds)
return "Whatever you return gets printed in the logs"
- run_this = print_context()
+ run_this = PythonOperator(task_id="print_the_context",
python_callable=print_context)
# [END howto_operator_python]
# [START howto_operator_python_render_sql]
- @task(task_id="log_sql_query", templates_dict={"query": "sql/sample.sql"},
templates_exts=[".sql"])
def log_sql(**kwargs):
logging.info("Python task decorator query: %s",
str(kwargs["templates_dict"]["query"]))
- log_the_sql = log_sql()
+ log_the_sql = PythonOperator(
+ task_id="log_sql_query",
+ python_callable=log_sql,
+ templates_dict={"query": "sql/sample.sql"},
+ templates_exts=[".sql"],
+ )
# [END howto_operator_python_render_sql]
# [START howto_operator_python_kwargs]
# Generate 5 sleeping tasks, sleeping from 0.0 to 0.4 seconds respectively
- for i in range(5):
-
- @task(task_id=f"sleep_for_{i}")
- def my_sleeping_function(random_base):
- """This is a function that will run within the DAG execution"""
- time.sleep(random_base)
+ def my_sleeping_function(random_base):
+ """This is a function that will run within the DAG execution"""
+ time.sleep(random_base)
- sleeping_task = my_sleeping_function(random_base=i / 10)
+ for i in range(5):
+ sleeping_task = PythonOperator(
+ task_id=f"sleep_for_{i}", python_callable=my_sleeping_function,
op_kwargs={"random_base": i / 10}
+ )
run_this >> log_the_sql >> sleeping_task
# [END howto_operator_python_kwargs]
@@ -85,9 +88,6 @@ with DAG(
log.warning("The virtalenv_python example task requires virtualenv,
please install it.")
else:
# [START howto_operator_python_venv]
- @task.virtualenv(
- task_id="virtualenv_python", requirements=["colorama==0.4.0"],
system_site_packages=False
- )
def callable_virtualenv():
"""
Example function that will be performed in a virtual environment.
@@ -108,13 +108,17 @@ with DAG(
sleep(1)
print("Finished")
- virtualenv_task = callable_virtualenv()
+ virtualenv_task = PythonVirtualenvOperator(
+ task_id="virtualenv_python",
+ python_callable=callable_virtualenv,
+ requirements=["colorama==0.4.0"],
+ system_site_packages=False,
+ )
# [END howto_operator_python_venv]
sleeping_task >> virtualenv_task
# [START howto_operator_external_python]
- @task.external_python(task_id="external_python",
python=PATH_TO_PYTHON_BINARY)
def callable_external_python():
"""
Example function that will be performed in a virtual environment.
@@ -132,23 +136,11 @@ with DAG(
sleep(1)
print("Finished")
- external_python_task = callable_external_python()
- # [END howto_operator_external_python]
-
- # [START howto_operator_external_python_classic]
- external_classic = ExternalPythonOperator(
- task_id="external_python_classic",
+ external_python_task = ExternalPythonOperator(
+ task_id="external_python",
+ python_callable=callable_external_python,
python=PATH_TO_PYTHON_BINARY,
- python_callable=x,
)
- # [END howto_operator_external_python_classic]
-
- # [START howto_operator_python_venv_classic]
- virtual_classic = PythonVirtualenvOperator(
- task_id="virtualenv_classic",
- requirements="colorama==0.4.0",
- python_callable=x,
- )
- # [END howto_operator_python_venv_classic]
+ # [END howto_operator_external_python]
- run_this >> external_classic >> external_python_task >> virtual_classic
+ run_this >> external_python_task >> virtualenv_task
diff --git a/airflow/example_dags/example_short_circuit_operator.py
b/airflow/example_dags/example_short_circuit_operator.py
index 18a0a8d850..23618d6dc4 100644
--- a/airflow/example_dags/example_short_circuit_operator.py
+++ b/airflow/example_dags/example_short_circuit_operator.py
@@ -31,7 +31,8 @@ with DAG(
start_date=pendulum.datetime(2021, 1, 1, tz="UTC"),
catchup=False,
tags=["example"],
-) as dag:
+):
+ # [START howto_operator_short_circuit]
cond_true = ShortCircuitOperator(
task_id="condition_is_True",
python_callable=lambda: True,
@@ -47,7 +48,9 @@ with DAG(
chain(cond_true, *ds_true)
chain(cond_false, *ds_false)
+ # [END howto_operator_short_circuit]
+ # [START howto_operator_short_circuit_trigger_rules]
[task_1, task_2, task_3, task_4, task_5, task_6] = [
EmptyOperator(task_id=f"task_{i}") for i in range(1, 7)
]
@@ -59,3 +62,4 @@ with DAG(
)
chain(task_1, [task_2, short_circuit], [task_3, task_4], [task_5, task_6],
task_7)
+ # [END howto_operator_short_circuit_trigger_rules]
diff --git a/docs/apache-airflow/howto/operator/python.rst
b/docs/apache-airflow/howto/operator/python.rst
index 6cfb5a335d..d785b3cdb2 100644
--- a/docs/apache-airflow/howto/operator/python.rst
+++ b/docs/apache-airflow/howto/operator/python.rst
@@ -22,28 +22,55 @@
PythonOperator
==============
-Use the ``@task`` decorator to execute Python callables.
+Use the :class:`~airflow.operators.python.PythonOperator` to execute Python
callables.
.. warning::
- The ``@task`` decorator is recommended over the classic
:class:`~airflow.operators.python.PythonOperator`
- to execute Python callables.
+ The ``@task`` decorator is recommended over the classic ``PythonOperator``
to execute Python callables.
-.. exampleinclude:: /../../airflow/example_dags/example_python_operator.py
- :language: python
- :dedent: 4
- :start-after: [START howto_operator_python]
- :end-before: [END howto_operator_python]
+.. tab-set::
+
+ .. tab-item:: @task
+ :sync: taskflow
+
+ .. exampleinclude::
/../../airflow/example_dags/example_python_decorator.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_python]
+ :end-before: [END howto_operator_python]
+
+ .. tab-item:: PythonOperator
+ :sync: operator
+
+ .. exampleinclude::
/../../airflow/example_dags/example_python_operator.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_python]
+ :end-before: [END howto_operator_python]
Passing in arguments
^^^^^^^^^^^^^^^^^^^^
Pass extra arguments to the ``@task`` decorated function as you would with a
normal Python function.
-.. exampleinclude:: /../../airflow/example_dags/example_python_operator.py
- :language: python
- :dedent: 4
- :start-after: [START howto_operator_python_kwargs]
- :end-before: [END howto_operator_python_kwargs]
+.. tab-set::
+
+ .. tab-item:: @task
+ :sync: taskflow
+
+ .. exampleinclude::
/../../airflow/example_dags/example_python_decorator.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_python_kwargs]
+ :end-before: [END howto_operator_python_kwargs]
+
+ .. tab-item:: PythonOperator
+ :sync: operator
+
+ .. exampleinclude::
/../../airflow/example_dags/example_python_operator.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_python_kwargs]
+ :end-before: [END howto_operator_python_kwargs]
Templating
^^^^^^^^^^
@@ -55,42 +82,58 @@ argument.
The ``templates_dict`` argument is templated, so each value in the dictionary
is evaluated as a :ref:`Jinja template <concepts:jinja-templating>`.
-.. exampleinclude:: /../../airflow/example_dags/example_python_operator.py
- :language: python
- :dedent: 4
- :start-after: [START howto_operator_python_render_sql]
- :end-before: [END howto_operator_python_render_sql]
+.. tab-set::
+
+ .. tab-item:: @task
+ :sync: taskflow
+ .. exampleinclude::
/../../airflow/example_dags/example_python_decorator.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_python_render_sql]
+ :end-before: [END howto_operator_python_render_sql]
+ .. tab-item:: PythonOperator
+ :sync: operator
+ .. exampleinclude::
/../../airflow/example_dags/example_python_operator.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_python_render_sql]
+ :end-before: [END howto_operator_python_render_sql]
.. _howto/operator:PythonVirtualenvOperator:
PythonVirtualenvOperator
========================
-Use the ``@task.virtualenv`` decorator to execute Python callables inside a
new Python virtual environment.
-The ``virtualenv`` package needs to be installed in the environment that runs
Airflow (as optional dependency ``pip install apache-airflow[virtualenv]
--constraint ...``).
+Use the :class:`~airflow.operators.python.PythonVirtualenvOperator` decorator
to execute Python callables
+inside a new Python virtual environment. The ``virtualenv`` package needs to
be installed in the environment
+that runs Airflow (as optional dependency ``pip install
apache-airflow[virtualenv] --constraint ...``).
.. warning::
- The ``@task.virtualenv`` decorator is recommended over the classic
:class:`~airflow.operators.python.PythonVirtualenvOperator`
+ The ``@task.virtualenv`` decorator is recommended over the classic
``PythonVirtualenvOperator``
to execute Python callables inside new Python virtual environments.
-TaskFlow example of using the PythonVirtualenvOperator:
+.. tab-set::
-.. exampleinclude:: /../../airflow/example_dags/example_python_operator.py
- :language: python
- :dedent: 4
- :start-after: [START howto_operator_python_venv]
- :end-before: [END howto_operator_python_venv]
+ .. tab-item:: @task.virtualenv
+ :sync: taskflow
-Classic example of using the PythonVirtualenvOperator:
+ .. exampleinclude::
/../../airflow/example_dags/example_python_decorator.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_python_venv]
+ :end-before: [END howto_operator_python_venv]
-.. exampleinclude:: /../../airflow/example_dags/example_python_operator.py
- :language: python
- :dedent: 4
- :start-after: [START howto_operator_python_venv_classic]
- :end-before: [END howto_operator_python_venv_classic]
+ .. tab-item:: PythonVirtualenvOperator
+ :sync: operator
+
+ .. exampleinclude::
/../../airflow/example_dags/example_python_operator.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_python_venv]
+ :end-before: [END howto_operator_python_venv]
Passing in arguments
^^^^^^^^^^^^^^^^^^^^
@@ -176,21 +219,29 @@ pre-defined environment. The virtualenv package should be
preinstalled in the en
In case ``dill`` is used, it has to be preinstalled in the environment (the
same version that is installed
in main Airflow environment).
-TaskFlow example of using the operator:
+.. warning::
+ The ``@task.external_python`` decorator is recommended over the classic
``ExternalPythonOperator``
+ to execute Python code in pre-defined Python environments.
-.. exampleinclude:: /../../airflow/example_dags/example_python_operator.py
- :language: python
- :dedent: 4
- :start-after: [START howto_operator_external_python]
- :end-before: [END howto_operator_external_python]
+.. tab-set::
-Classic example of using the operator:
+ .. tab-item:: @task.external_python
+ :sync: taskflow
-.. exampleinclude:: /../../airflow/example_dags/example_python_operator.py
- :language: python
- :dedent: 4
- :start-after: [START howto_operator_external_python_classic]
- :end-before: [END howto_operator_external_python_classic]
+ .. exampleinclude::
/../../airflow/example_dags/example_python_decorator.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_external_python]
+ :end-before: [END howto_operator_external_python]
+
+ .. tab-item:: ExternalPythonOperator
+ :sync: operator
+
+ .. exampleinclude::
/../../airflow/example_dags/example_python_operator.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_external_python]
+ :end-before: [END howto_operator_external_python]
Passing in arguments
@@ -216,27 +267,32 @@ If you want the context related to datetime objects like
``data_interval_start``
PythonBranchOperator
====================
-Use the ``@task.branch`` decorator to execute Python :ref:`branching
<concepts:branching>` tasks.
+Use the :class:`~airflow.operators.python.PythonBranchOperator` to execute
Python :ref:`branching <concepts:branching>`
+tasks.
.. warning::
- The ``@task.branch`` decorator is recommended over the classic
:class:`~airflow.operators.python.PythonBranchOperator`
+ The ``@task.branch`` decorator is recommended over the classic
``PythonBranchOperator``
to execute Python code.
-TaskFlow example of using the operator:
+.. tab-set::
+
+ .. tab-item:: @task.branch
+ :sync: taskflow
-.. exampleinclude::
/../../airflow/example_dags/example_branch_operator_decorator.py
- :language: python
- :dedent: 4
- :start-after: [START howto_operator_branch_python]
- :end-before: [END howto_operator_branch_python]
+ .. exampleinclude::
/../../airflow/example_dags/example_branch_operator_decorator.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_branch_python]
+ :end-before: [END howto_operator_branch_python]
-Classic example of using the operator:
+ .. tab-item:: PythonBranchOperator
+ :sync: operator
-.. exampleinclude:: /../../airflow/example_dags/example_branch_operator.py
- :language: python
- :dedent: 4
- :start-after: [START howto_operator_branch_python]
- :end-before: [END howto_operator_branch_python]
+ .. exampleinclude::
/../../airflow/example_dags/example_branch_operator.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_branch_python]
+ :end-before: [END howto_operator_branch_python]
Argument passing and templating options are the same like with
:ref:`howto/operator:PythonOperator`.
@@ -245,28 +301,32 @@ Argument passing and templating options are the same like
with :ref:`howto/opera
BranchPythonVirtualenvOperator
==============================
-Use the ``@task.branch_virtualenv`` decorator to execute Python
:ref:`branching <concepts:branching>` tasks and is a hybrid of
-the branch decorator with execution in a virtual environment.
+Use the :class:`~airflow.operators.python.BranchPythonVirtualenvOperator`
decorator to execute Python :ref:`branching <concepts:branching>`
+tasks and is a hybrid of the
:class:`~airflow.operators.python.PythonBranchOperator` with execution in a
virtual environment.
.. warning::
The ``@task.branch_virtualenv`` decorator is recommended over the classic
- :class:`~airflow.operators.python.BranchPythonVirtualenvOperator` to
execute Python code.
+ ``BranchPythonVirtualenvOperator`` to execute Python code.
-TaskFlow example of using the operator:
+.. tab-set::
-.. exampleinclude::
/../../airflow/example_dags/example_branch_operator_decorator.py
- :language: python
- :dedent: 4
- :start-after: [START howto_operator_branch_virtualenv]
- :end-before: [END howto_operator_branch_virtualenv]
+ .. tab-item:: @task.branch_virtualenv
+ :sync: taskflow
-Classic example of using the operator:
+ .. exampleinclude::
/../../airflow/example_dags/example_branch_operator_decorator.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_branch_virtualenv]
+ :end-before: [END howto_operator_branch_virtualenv]
-.. exampleinclude:: /../../airflow/example_dags/example_branch_operator.py
- :language: python
- :dedent: 4
- :start-after: [START howto_operator_branch_virtualenv]
- :end-before: [END howto_operator_branch_virtualenv]
+ .. tab-item:: BranchPythonVirtualenvOperator
+ :sync: operator
+
+ .. exampleinclude::
/../../airflow/example_dags/example_branch_operator.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_branch_virtualenv]
+ :end-before: [END howto_operator_branch_virtualenv]
Argument passing and templating options are the same like with
:ref:`howto/operator:PythonVirtualenvOperator`.
@@ -275,28 +335,33 @@ Argument passing and templating options are the same like
with :ref:`howto/opera
BranchExternalPythonOperator
============================
-Use the ``@task.branch_external_python`` decorator to execute Python
:ref:`branching <concepts:branching>` tasks and is a hybrid of
-the branch decorator with execution in an external Python environment.
+Use the :class:`~airflow.operators.python.BranchExternalPythonOperator` to
execute Python :ref:`branching <concepts:branching>`
+tasks and is a hybrid of the
:class:`~airflow.operators.python.PythonBranchOperator` with execution in an
+external Python environment.
.. warning::
The ``@task.branch_external_python`` decorator is recommended over the
classic
- :class:`~airflow.operators.python.BranchExternalPythonOperator` to execute
Python code.
+ ``BranchExternalPythonOperator`` to execute Python code.
+
+.. tab-set::
-TaskFlow example of using the operator:
+ .. tab-item:: @task.branch_external_python
+ :sync: taskflow
-.. exampleinclude::
/../../airflow/example_dags/example_branch_operator_decorator.py
- :language: python
- :dedent: 4
- :start-after: [START howto_operator_branch_ext_py]
- :end-before: [END howto_operator_branch_ext_py]
+ .. exampleinclude::
/../../airflow/example_dags/example_branch_operator_decorator.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_branch_ext_py]
+ :end-before: [END howto_operator_branch_ext_py]
-Classic example of using the operator:
+ .. tab-item:: BranchExternalPythonOperator
+ :sync: operator
-.. exampleinclude:: /../../airflow/example_dags/example_branch_operator.py
- :language: python
- :dedent: 4
- :start-after: [START howto_operator_branch_ext_py]
- :end-before: [END howto_operator_branch_ext_py]
+ .. exampleinclude::
/../../airflow/example_dags/example_branch_operator.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_branch_ext_py]
+ :end-before: [END howto_operator_branch_ext_py]
Argument passing and templating options are the same like with
:ref:`howto/operator:ExternalPythonOperator`.
@@ -305,26 +370,39 @@ Argument passing and templating options are the same like
with :ref:`howto/opera
ShortCircuitOperator
====================
-Use the ``@task.short_circuit`` decorator to control whether a pipeline
continues
+Use the :class:`~airflow.operators.python.ShortCircuitOperator` to control
whether a pipeline continues
if a condition is satisfied or a truthy value is obtained.
+The evaluation of this condition and truthy value is done via the output of a
callable. If the
+callable returns True or a truthy value, the pipeline is allowed to continue
and an :ref:`XCom <concepts:xcom>`
+of the output will be pushed. If the output is False or a falsy value, the
pipeline will be short-circuited
+based on the configured short-circuiting (more on this later). In the example
below, the tasks that follow the
+"condition_is_true" task will execute while the tasks downstream of the
"condition_is_false" task will be
+skipped.
+
.. warning::
- The ``@task.short_circuit`` decorator is recommended over the classic
:class:`~airflow.operators.python.ShortCircuitOperator`
+ The ``@task.short_circuit`` decorator is recommended over the classic
``ShortCircuitOperator``
to short-circuit pipelines via Python callables.
-The evaluation of this condition and truthy value
-is done via the output of the decorated function. If the decorated function
returns True or a truthy value,
-the pipeline is allowed to continue and an :ref:`XCom <concepts:xcom>` of the
output will be pushed. If the
-output is False or a falsy value, the pipeline will be short-circuited based
on the configured
-short-circuiting (more on this later). In the example below, the tasks that
follow the "condition_is_true"
-task will execute while the tasks downstream of the "condition_is_false" task
will be skipped.
+.. tab-set::
+
+ .. tab-item:: @task.short_circuit
+ :sync: taskflow
+ .. exampleinclude::
/../../airflow/example_dags/example_short_circuit_decorator.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_short_circuit]
+ :end-before: [END howto_operator_short_circuit]
-.. exampleinclude::
/../../airflow/example_dags/example_short_circuit_decorator.py
- :language: python
- :dedent: 4
- :start-after: [START howto_operator_short_circuit]
- :end-before: [END howto_operator_short_circuit]
+ .. tab-item:: ShortCircuitOperator
+ :sync: operator
+
+ .. exampleinclude::
/../../airflow/example_dags/example_short_circuit_operator.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_short_circuit]
+ :end-before: [END howto_operator_short_circuit]
The "short-circuiting" can be configured to either respect or ignore the
:ref:`trigger rule <concepts:trigger-rules>`
@@ -341,12 +419,25 @@ rules. This means while the tasks that follow the
"short_circuit" task will be s
since the decorated function returns False, "task_7" will still execute as its
set to execute when upstream
tasks have completed running regardless of status (i.e. the
``TriggerRule.ALL_DONE`` trigger rule).
-.. exampleinclude::
/../../airflow/example_dags/example_short_circuit_decorator.py
- :language: python
- :dedent: 4
- :start-after: [START howto_operator_short_circuit_trigger_rules]
- :end-before: [END howto_operator_short_circuit_trigger_rules]
+.. tab-set::
+
+ .. tab-item:: @task.short_circuit
+ :sync: taskflow
+
+ .. exampleinclude::
/../../airflow/example_dags/example_short_circuit_decorator.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_short_circuit_trigger_rules]
+ :end-before: [END howto_operator_short_circuit_trigger_rules]
+ .. tab-item:: ShortCircuitOperator
+ :sync: operator
+
+ .. exampleinclude::
/../../airflow/example_dags/example_short_circuit_operator.py
+ :language: python
+ :dedent: 4
+ :start-after: [START howto_operator_short_circuit_trigger_rules]
+ :end-before: [END howto_operator_short_circuit_trigger_rules]
Passing in arguments
^^^^^^^^^^^^^^^^^^^^
@@ -364,17 +455,29 @@ Jinja templating can be used in same way as described for
the PythonOperator.
PythonSensor
============
-Sensors can be used in two ways. One is to use the
:class:`~airflow.sensors.python.PythonSensor` to use arbitrary callable for
sensing. The callable
-should return True when it succeeds, False otherwise. The other uses the
Taskflow API utilizing the :class:`~airflow.decorators.task.sensor` as a
decorator on a function.
+The :class:`~airflow.sensors.python.PythonSensor` executes an arbitrary
callable and waits for its return
+value to be True.
+
+.. warning::
+ The ``@task.sensor`` decorator is recommended over the classic
``PythonSensor``
+ to execute Python callables to check for True condition.
+
+.. tab-set::
+
+ .. tab-item:: @task.sensor
+ :sync: taskflow
+
+ .. exampleinclude::
/../../airflow/example_dags/example_sensor_decorator.py
+ :language: python
+ :dedent: 4
+ :start-after: [START wait_function]
+ :end-before: [END wait_function]
-.. exampleinclude:: /../../airflow/example_dags/example_sensors.py
- :language: python
- :dedent: 4
- :start-after: [START example_python_sensors]
- :end-before: [END example_python_sensors]
+ .. tab-item:: PythonSensor
+ :sync: operator
-.. exampleinclude:: /../../airflow/example_dags/example_sensor_decorator.py
- :language: python
- :dedent: 4
- :start-after: [START wait_function]
- :end-before: [END wait_function]
+ .. exampleinclude:: /../../airflow/example_dags/example_sensors.py
+ :language: python
+ :dedent: 4
+ :start-after: [START example_python_sensors]
+ :end-before: [END example_python_sensors]
diff --git a/docs/conf.py b/docs/conf.py
index e832e1e012..f66e016ee9 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -151,6 +151,7 @@ extensions = [
"sphinx_airflow_theme",
"redirects",
"substitution_extensions",
+ "sphinx_design",
]
if PACKAGE_NAME == "apache-airflow":
extensions.extend(
@@ -317,6 +318,9 @@ if PACKAGE_NAME in ["apache-airflow", "helm-chart"]:
html_static_path = [f"{PACKAGE_NAME}/static"]
else:
html_static_path = []
+
+html_static_path.append("sphinx_design/static/") # Style overrides for the
sphinx-design extension.
+
# A list of JavaScript filename. The entry must be a filename string or a
# tuple containing the filename string and the attributes dictionary. The
# filename must be relative to the html_static_path, or a full URI with
@@ -344,6 +348,8 @@ if PACKAGE_NAME == "docker-stack":
# Substitute in links
manual_substitutions_in_generated_html = ["build.html"]
+html_css_files = ["custom.css"]
+
# -- Theme configuration
-------------------------------------------------------
# Custom sidebar templates, maps document names to template names.
html_sidebars = {
diff --git a/docs/sphinx_design/static/custom.css
b/docs/sphinx_design/static/custom.css
new file mode 100644
index 0000000000..b1cf49f37d
--- /dev/null
+++ b/docs/sphinx_design/static/custom.css
@@ -0,0 +1,33 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ This file contains style overrides for the sphinx-design extension.
+
+ The original defaults can be found here:
+ https://sphinx-design.readthedocs.io/en/alabaster-theme/css_variables.html
+ */
+
+:root {
+ --sd-color-tabs-label-active: #017cee;
+ --sd-color-tabs-label-hover: #68d1ff;
+ --sd-color-tabs-underline-active: #017cee;
+ --sd-color-tabs-underline-hover: #68d1ff;
+ --sd-color-tabs-underline: transparent;
+}
diff --git a/setup.py b/setup.py
index 96f3f4c703..ee60c88520 100644
--- a/setup.py
+++ b/setup.py
@@ -306,6 +306,7 @@ doc = [
"sphinx-argparse>=0.1.13",
"sphinx-autoapi>=2.0.0",
"sphinx-copybutton",
+ "sphinx-design>=0.5.0",
"sphinx-jinja>=2.0",
"sphinx-rtd-theme>=0.1.6",
"sphinx>=5.2.0",
diff --git a/tests/api_connexion/endpoints/test_task_instance_endpoint.py
b/tests/api_connexion/endpoints/test_task_instance_endpoint.py
index 3125031f56..16849d6cfa 100644
--- a/tests/api_connexion/endpoints/test_task_instance_endpoint.py
+++ b/tests/api_connexion/endpoints/test_task_instance_endpoint.py
@@ -234,7 +234,7 @@ class TestGetTaskInstance(TestTaskInstanceEndpoint):
"pid": 100,
"pool": "default_pool",
"pool_slots": 1,
- "priority_weight": 11,
+ "priority_weight": 9,
"queue": "default_queue",
"queued_when": None,
"sla_miss": None,
@@ -286,11 +286,11 @@ class TestGetTaskInstance(TestTaskInstanceEndpoint):
"map_index": -1,
"max_tries": 0,
"note": "placeholder-note",
- "operator": "_PythonDecoratedOperator",
+ "operator": "PythonOperator",
"pid": 100,
"pool": "default_pool",
"pool_slots": 1,
- "priority_weight": 11,
+ "priority_weight": 9,
"queue": "default_queue",
"queued_when": None,
"sla_miss": None,
@@ -331,11 +331,11 @@ class TestGetTaskInstance(TestTaskInstanceEndpoint):
"map_index": -1,
"max_tries": 0,
"note": "placeholder-note",
- "operator": "_PythonDecoratedOperator",
+ "operator": "PythonOperator",
"pid": 100,
"pool": "default_pool",
"pool_slots": 1,
- "priority_weight": 11,
+ "priority_weight": 9,
"queue": "default_queue",
"queued_when": None,
"sla_miss": None,
@@ -379,11 +379,11 @@ class TestGetTaskInstance(TestTaskInstanceEndpoint):
"map_index": -1,
"max_tries": 0,
"note": "placeholder-note",
- "operator": "_PythonDecoratedOperator",
+ "operator": "PythonOperator",
"pid": 100,
"pool": "default_pool",
"pool_slots": 1,
- "priority_weight": 11,
+ "priority_weight": 9,
"queue": "default_queue",
"queued_when": None,
"sla_miss": {
@@ -437,11 +437,11 @@ class TestGetTaskInstance(TestTaskInstanceEndpoint):
"map_index": map_index,
"max_tries": 0,
"note": "placeholder-note",
- "operator": "_PythonDecoratedOperator",
+ "operator": "PythonOperator",
"pid": 100,
"pool": "default_pool",
"pool_slots": 1,
- "priority_weight": 11,
+ "priority_weight": 9,
"queue": "default_queue",
"queued_when": None,
"sla_miss": None,
@@ -948,8 +948,8 @@ class TestGetTaskInstancesBatch(TestTaskInstanceEndpoint):
[
pytest.param(
{"dag_ids": ["example_python_operator", "example_skip_dag"]},
- 19,
- 19,
+ 17,
+ 17,
id="with dag filter",
),
],
@@ -2340,11 +2340,11 @@ class TestSetTaskInstanceNote(TestTaskInstanceEndpoint):
"map_index": -1,
"max_tries": 0,
"note": new_note_value,
- "operator": "_PythonDecoratedOperator",
+ "operator": "PythonOperator",
"pid": 100,
"pool": "default_pool",
"pool_slots": 1,
- "priority_weight": 11,
+ "priority_weight": 9,
"queue": "default_queue",
"queued_when": None,
"sla_miss": None,
@@ -2394,11 +2394,11 @@ class TestSetTaskInstanceNote(TestTaskInstanceEndpoint):
"map_index": map_index,
"max_tries": 0,
"note": new_note_value,
- "operator": "_PythonDecoratedOperator",
+ "operator": "PythonOperator",
"pid": 100,
"pool": "default_pool",
"pool_slots": 1,
- "priority_weight": 11,
+ "priority_weight": 9,
"queue": "default_queue",
"queued_when": None,
"sla_miss": None,
