This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new cd33c2a6a7 Migrate executor docs to respective providers (#37728)
cd33c2a6a7 is described below
commit cd33c2a6a73ca902daa234cf60dd0b7b9782bdc6
Author: Ryan Hatter <[email protected]>
AuthorDate: Tue Feb 27 14:02:57 2024 -0500
Migrate executor docs to respective providers (#37728)
---
.github/boring-cyborg.yml | 3 -
.../providers/celery/executors/celery_executor.py | 2 +-
.../kubernetes/executors/kubernetes_executor.py | 2 +-
.../celery_executor.rst} | 11 +--
.../celery_kubernetes_executor.rst} | 5 +-
.../img/run_task_on_celery_executor.png | Bin
docs/apache-airflow-providers-celery/index.rst | 10 +++
.../img/arch-diag-kubernetes.png | Bin
.../img/arch-diag-kubernetes2.png | Bin
.../img/k8s-failed-pod.png | Bin
.../img/k8s-happy-path.png | Bin
.../index.rst | 8 ++
.../kubernetes_executor.rst} | 10 +--
.../local_kubernetes_executor.rst} | 2 +-
.../operators.rst | 4 +-
.../administration-and-deployment/kubernetes.rst | 2 +-
.../logging-monitoring/check-health.rst | 2 +-
.../production-deployment.rst | 10 +--
docs/apache-airflow/best-practices.rst | 2 +-
.../core-concepts/{executor => }/debug.rst | 81 +++++++++++----------
.../core-concepts/executor/index.rst | 12 +--
docs/apache-airflow/core-concepts/index.rst | 7 ++
docs/apache-airflow/core-concepts/overview.rst | 4 +-
docs/apache-airflow/faq.rst | 2 +-
docs/apache-airflow/howto/docker-compose/index.rst | 4 +-
docs/apache-airflow/img/k8s-0-worker.jpeg | Bin 90008 -> 0 bytes
docs/apache-airflow/img/k8s-3-worker.jpeg | Bin 129900 -> 0 bytes
docs/apache-airflow/img/k8s-5-worker.jpeg | Bin 162253 -> 0 bytes
docs/apache-airflow/redirects.txt | 14 ++++
docs/exts/redirects.py | 10 +++
30 files changed, 124 insertions(+), 83 deletions(-)
diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml
index 0729cfb9ca..a2097e5ddc 100644
--- a/.github/boring-cyborg.yml
+++ b/.github/boring-cyborg.yml
@@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
---
-
# Details: https://github.com/kaxil/boring-cyborg
labelPRBasedOnFilePath:
@@ -154,8 +153,6 @@ labelPRBasedOnFilePath:
- airflow/example_dags/example_kubernetes_executor.py
- airflow/providers/cncf/kubernetes/**/*
- airflow/providers/celery/executors/celery_kubernetes_executor.py
- - docs/apache-airflow/core-concepts/executor/kubernetes.rst
- - docs/apache-airflow/core-concepts/executor/celery_kubernetes.rst
- docs/apache-airflow-providers-cncf-kubernetes/**/*
- kubernetes_tests/**/*
- tests/providers/cncf/kubernetes/**/*
diff --git a/airflow/providers/celery/executors/celery_executor.py
b/airflow/providers/celery/executors/celery_executor.py
index a8a4e320ec..ea17ded788 100644
--- a/airflow/providers/celery/executors/celery_executor.py
+++ b/airflow/providers/celery/executors/celery_executor.py
@@ -19,7 +19,7 @@
.. seealso::
For more information on how the CeleryExecutor works, take a look at the
guide:
- :ref:`executor:CeleryExecutor`
+ :doc:`/celery_executor`
"""
from __future__ import annotations
diff --git a/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py
b/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py
index f247897ec6..66fe9c2f50 100644
--- a/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py
+++ b/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py
@@ -19,7 +19,7 @@ KubernetesExecutor.
.. seealso::
For more information on how the KubernetesExecutor works, take a look at
the guide:
- :ref:`executor:KubernetesExecutor`
+ :doc:`/kubernetes_executor`
"""
from __future__ import annotations
diff --git a/docs/apache-airflow/core-concepts/executor/celery.rst
b/docs/apache-airflow-providers-celery/celery_executor.rst
similarity index 97%
rename from docs/apache-airflow/core-concepts/executor/celery.rst
rename to docs/apache-airflow-providers-celery/celery_executor.rst
index 1113bda530..16f454285f 100644
--- a/docs/apache-airflow/core-concepts/executor/celery.rst
+++ b/docs/apache-airflow-providers-celery/celery_executor.rst
@@ -15,9 +15,6 @@
specific language governing permissions and limitations
under the License.
-
-.. _executor:CeleryExecutor:
-
Celery Executor
===============
@@ -36,7 +33,7 @@ change your ``airflow.cfg`` to point the executor parameter to
For more information about setting up a Celery broker, refer to the
exhaustive `Celery documentation on the topic
<https://docs.celeryq.dev/en/latest/getting-started/>`_.
-The configuration parameters of the Celery Executor can be found in
:doc:`apache-airflow-providers-celery:configurations-ref`.
+The configuration parameters of the Celery Executor can be found in the Celery
provider's :doc:`configurations-ref`.
Here are a few imperative requirements for your workers:
@@ -97,7 +94,7 @@ Some caveats:
- Tasks can consume resources. Make sure your worker has enough resources to
run ``worker_concurrency`` tasks
- Queue names are limited to 256 characters, but each broker backend might
have its own restrictions
-See :doc:`/administration-and-deployment/modules_management` for details on
how Python and Airflow manage modules.
+See :doc:`apache-airflow:administration-and-deployment/modules_management` for
details on how Python and Airflow manage modules.
Architecture
------------
@@ -173,7 +170,7 @@ The components communicate with each other in many places
Task execution process
----------------------
-.. figure:: ../../img/run_task_on_celery_executor.png
+.. figure:: img/run_task_on_celery_executor.png
:scale: 50 %
Sequence diagram - task execution process
@@ -205,7 +202,7 @@ During this process, two 2 process are created:
| [11] **WorkerProcess** saves status information in **ResultBackend**.
| [13] When **SchedulerProcess** asks **ResultBackend** again about the
status, it will get information about the status of the task.
-.. _executor:CeleryExecutor:queue:
+.. _celery_executor:queue:
Queues
------
diff --git a/docs/apache-airflow/core-concepts/executor/celery_kubernetes.rst
b/docs/apache-airflow-providers-celery/celery_kubernetes_executor.rst
similarity index 95%
rename from docs/apache-airflow/core-concepts/executor/celery_kubernetes.rst
rename to docs/apache-airflow-providers-celery/celery_kubernetes_executor.rst
index 6416f5d2fa..77a6a81681 100644
--- a/docs/apache-airflow/core-concepts/executor/celery_kubernetes.rst
+++ b/docs/apache-airflow-providers-celery/celery_kubernetes_executor.rst
@@ -15,9 +15,6 @@
specific language governing permissions and limitations
under the License.
-
-.. _executor:CeleryKubernetesExecutor:
-
CeleryKubernetes Executor
=========================
@@ -36,7 +33,7 @@ An executor is chosen to run a task based on the task's queue.
``CeleryKubernetesExecutor`` inherits the scalability of the
``CeleryExecutor`` to
handle the high load at the peak time and runtime isolation of the
``KubernetesExecutor``.
-The configuration parameters of the Celery Executor can be found in
:doc:`apache-airflow-providers-celery:configurations-ref`.
+The configuration parameters of the Celery Executor can be found in the Celery
provider's :doc:`configurations-ref`.
When to use CeleryKubernetesExecutor
diff --git a/docs/apache-airflow/img/run_task_on_celery_executor.png
b/docs/apache-airflow-providers-celery/img/run_task_on_celery_executor.png
similarity index 100%
rename from docs/apache-airflow/img/run_task_on_celery_executor.png
rename to
docs/apache-airflow-providers-celery/img/run_task_on_celery_executor.png
diff --git a/docs/apache-airflow-providers-celery/index.rst
b/docs/apache-airflow-providers-celery/index.rst
index 342061d7fa..5effbe0fae 100644
--- a/docs/apache-airflow-providers-celery/index.rst
+++ b/docs/apache-airflow-providers-celery/index.rst
@@ -29,6 +29,16 @@
Changelog <changelog>
Security <security>
+
+.. toctree::
+ :hidden:
+ :maxdepth: 1
+ :caption: Executors
+
+ CeleryExecutor details <celery_executor>
+ CeleryKubernetesExecutor details <celery_kubernetes_executor>
+
+
.. toctree::
:hidden:
:maxdepth: 1
diff --git a/docs/apache-airflow/img/arch-diag-kubernetes.png
b/docs/apache-airflow-providers-cncf-kubernetes/img/arch-diag-kubernetes.png
similarity index 100%
rename from docs/apache-airflow/img/arch-diag-kubernetes.png
rename to
docs/apache-airflow-providers-cncf-kubernetes/img/arch-diag-kubernetes.png
diff --git a/docs/apache-airflow/img/arch-diag-kubernetes2.png
b/docs/apache-airflow-providers-cncf-kubernetes/img/arch-diag-kubernetes2.png
similarity index 100%
rename from docs/apache-airflow/img/arch-diag-kubernetes2.png
rename to
docs/apache-airflow-providers-cncf-kubernetes/img/arch-diag-kubernetes2.png
diff --git a/docs/apache-airflow/img/k8s-failed-pod.png
b/docs/apache-airflow-providers-cncf-kubernetes/img/k8s-failed-pod.png
similarity index 100%
rename from docs/apache-airflow/img/k8s-failed-pod.png
rename to docs/apache-airflow-providers-cncf-kubernetes/img/k8s-failed-pod.png
diff --git a/docs/apache-airflow/img/k8s-happy-path.png
b/docs/apache-airflow-providers-cncf-kubernetes/img/k8s-happy-path.png
similarity index 100%
rename from docs/apache-airflow/img/k8s-happy-path.png
rename to docs/apache-airflow-providers-cncf-kubernetes/img/k8s-happy-path.png
diff --git a/docs/apache-airflow-providers-cncf-kubernetes/index.rst
b/docs/apache-airflow-providers-cncf-kubernetes/index.rst
index d7c8586c08..5f597c2747 100644
--- a/docs/apache-airflow-providers-cncf-kubernetes/index.rst
+++ b/docs/apache-airflow-providers-cncf-kubernetes/index.rst
@@ -29,6 +29,14 @@
Changelog <changelog>
Security <security>
+.. toctree::
+ :hidden:
+ :maxdepth: 1
+ :caption: Executors
+
+ KubernetesExecutor details <kubernetes_executor>
+ LocalKubernetesExecutor details <local_kubernetes_executor>
+
.. toctree::
:hidden:
:maxdepth: 1
diff --git a/docs/apache-airflow/core-concepts/executor/kubernetes.rst
b/docs/apache-airflow-providers-cncf-kubernetes/kubernetes_executor.rst
similarity index 98%
rename from docs/apache-airflow/core-concepts/executor/kubernetes.rst
rename to docs/apache-airflow-providers-cncf-kubernetes/kubernetes_executor.rst
index e19264c96e..5662d7a0e1 100644
--- a/docs/apache-airflow/core-concepts/executor/kubernetes.rst
+++ b/docs/apache-airflow-providers-cncf-kubernetes/kubernetes_executor.rst
@@ -16,7 +16,7 @@
under the License.
-.. _executor:KubernetesExecutor:
+.. _KubernetesExecutor:
Kubernetes Executor
===================
@@ -38,12 +38,12 @@ KubernetesExecutor requires a non-sqlite database in the
backend.
When a DAG submits a task, the KubernetesExecutor requests a worker pod from
the Kubernetes API. The worker pod then runs the task, reports the result, and
terminates.
-.. image:: ../../img/arch-diag-kubernetes.png
+.. image:: img/arch-diag-kubernetes.png
One example of an Airflow deployment running on a distributed set of five
nodes in a Kubernetes cluster is shown below.
-.. image:: ../../img/arch-diag-kubernetes2.png
+.. image:: img/arch-diag-kubernetes2.png
Consistent with the regular Airflow architecture, the Workers need access to
the DAG files to execute the tasks within those DAGs and interact with the
Metadata repository. Also, configuration information specific to the Kubernetes
Executor, such as the worker namespace and image information, needs to be
specified in the Airflow Configuration file.
@@ -56,7 +56,7 @@ Additionally, the Kubernetes Executor enables specification
of additional featur
.. Airflow_Worker -> Kubernetes: Pod completes with state "Succeeded" and k8s
records in ETCD
.. Kubernetes -> Airflow_Scheduler: Airflow scheduler reads "Succeeded" from
k8s watcher thread
.. @enduml
-.. image:: ../../img/k8s-happy-path.png
+.. image:: img/k8s-happy-path.png
Configuration
-------------
@@ -272,7 +272,7 @@ In the case where a worker dies before it can report its
status to the backend D
..
.. @enduml
-.. image:: ../../img/k8s-failed-pod.png
+.. image:: img/k8s-failed-pod.png
A Kubernetes watcher is a thread that can subscribe to every change that
occurs in Kubernetes' database. It is alerted when pods start, run, end, and
fail.
diff --git a/docs/apache-airflow/core-concepts/executor/local_kubernetes.rst
b/docs/apache-airflow-providers-cncf-kubernetes/local_kubernetes_executor.rst
similarity index 97%
rename from docs/apache-airflow/core-concepts/executor/local_kubernetes.rst
rename to
docs/apache-airflow-providers-cncf-kubernetes/local_kubernetes_executor.rst
index d50c7fc878..daed3d4d1f 100644
--- a/docs/apache-airflow/core-concepts/executor/local_kubernetes.rst
+++
b/docs/apache-airflow-providers-cncf-kubernetes/local_kubernetes_executor.rst
@@ -16,7 +16,7 @@
under the License.
-.. _executor:LocalKubernetesExecutor:
+.. _LocalKubernetesExecutor:
LocalKubernetes Executor
=========================
diff --git a/docs/apache-airflow-providers-cncf-kubernetes/operators.rst
b/docs/apache-airflow-providers-cncf-kubernetes/operators.rst
index fcfec977e7..bfd629a49b 100644
--- a/docs/apache-airflow-providers-cncf-kubernetes/operators.rst
+++ b/docs/apache-airflow-providers-cncf-kubernetes/operators.rst
@@ -35,7 +35,7 @@ you to create and run Pods on a Kubernetes cluster.
- :ref:`EksPodOperator <howto/operator:EksPodOperator>` operator for `AWS
Elastic Kubernetes Engine <https://aws.amazon.com/eks/>`__.
.. note::
- The :doc:`Kubernetes executor
<apache-airflow:core-concepts/executor/kubernetes>` is **not** required to use
this operator.
+ The :doc:`Kubernetes executor <kubernetes_executor>` is **not** required to
use this operator.
How does this operator work?
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -602,7 +602,7 @@ you to create and run Jobs on a Kubernetes cluster.
- ``GKEStartJobOperator`` operator for `Google Kubernetes Engine
<https://cloud.google.com/kubernetes-engine/>`__.
.. note::
- The :doc:`Kubernetes executor
<apache-airflow:core-concepts/executor/kubernetes>` is **not** required to use
this operator.
+ The :doc:`Kubernetes executor <kubernetes_executor>` is **not** required to
use this operator.
How does this operator work?
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/docs/apache-airflow/administration-and-deployment/kubernetes.rst
b/docs/apache-airflow/administration-and-deployment/kubernetes.rst
index 160694fa11..77bf719e27 100644
--- a/docs/apache-airflow/administration-and-deployment/kubernetes.rst
+++ b/docs/apache-airflow/administration-and-deployment/kubernetes.rst
@@ -32,7 +32,7 @@ We maintain an :doc:`official Helm chart <helm-chart:index>`
for Airflow that he
Kubernetes Executor
^^^^^^^^^^^^^^^^^^^
-The :doc:`Kubernetes Executor </core-concepts/executor/kubernetes>` allows you
to run all the Airflow tasks on
+The :doc:`Kubernetes Executor
<apache-airflow-providers-cncf-kubernetes:kubernetes_executor>` allows you to
run all the Airflow tasks on
Kubernetes as separate Pods.
KubernetesPodOperator
diff --git
a/docs/apache-airflow/administration-and-deployment/logging-monitoring/check-health.rst
b/docs/apache-airflow/administration-and-deployment/logging-monitoring/check-health.rst
index 78cf0f3725..85fdedb225 100644
---
a/docs/apache-airflow/administration-and-deployment/logging-monitoring/check-health.rst
+++
b/docs/apache-airflow/administration-and-deployment/logging-monitoring/check-health.rst
@@ -135,7 +135,7 @@ HTTP monitoring for Celery Cluster
You can optionally use Flower to monitor the health of the Celery cluster. It
also provides an HTTP API that you can use to build a health check for your
environment.
-For details about installation, see: :ref:`executor:CeleryExecutor`. For
details about usage, see: `The Flower project documentation
<https://flower.readthedocs.io/>`__.
+For details about installation, see:
:doc:`apache-airflow-providers-celery:celery_executor`. For details about
usage, see: `The Flower project documentation
<https://flower.readthedocs.io/>`__.
CLI Check for Celery Workers
----------------------------
diff --git
a/docs/apache-airflow/administration-and-deployment/production-deployment.rst
b/docs/apache-airflow/administration-and-deployment/production-deployment.rst
index 281dc7f658..a341f7f178 100644
---
a/docs/apache-airflow/administration-and-deployment/production-deployment.rst
+++
b/docs/apache-airflow/administration-and-deployment/production-deployment.rst
@@ -60,8 +60,8 @@ Airflow uses
:class:`~airflow.executors.sequential_executor.SequentialExecutor`
nature, the user is limited to executing at most one task at a time.
``Sequential Executor`` also pauses
the scheduler when it runs a task, hence it is not recommended in a production
setup. You should use the
:class:`~airflow.executors.local_executor.LocalExecutor` for a single machine.
-For a multi-node setup, you should use the :doc:`Kubernetes executor
<../core-concepts/executor/kubernetes>` or
-the :doc:`Celery executor <../core-concepts/executor/celery>`.
+For a multi-node setup, you should use the :doc:`Kubernetes executor
<apache-airflow-providers-cncf-kubernetes:kubernetes_executor>` or
+the :doc:`Celery executor <apache-airflow-providers-celery:celery_executor>`.
Once you have configured the executor, it is necessary to make sure that every
node in the cluster contains
@@ -170,19 +170,19 @@ of the executor you use:
the tasks it runs - then you will need to clear and restart those tasks
manually after the upgrade
is completed (or rely on ``retry`` being set for stopped tasks).
-* For the :doc:`Celery executor <../core-concepts/executor/celery>`, you have
to first put your workers in
+* For the :doc:`Celery executor
<apache-airflow-providers-celery:celery_executor>`, you have to first put your
workers in
offline mode (usually by setting a single ``TERM`` signal to the workers),
wait until the workers
finish all the running tasks, and then upgrade the code (for example by
replacing the image the workers run
in and restart the workers). You can monitor your workers via ``flower``
monitoring tool and see the number
of running tasks going down to zero. Once the workers are upgraded, they
will be automatically put in online
mode and start picking up new tasks. You can then upgrade the ``Scheduler``
in a rolling restart mode.
-* For the :doc:`Kubernetes executor <../core-concepts/executor/kubernetes>`,
you can upgrade the scheduler
+* For the :doc:`Kubernetes executor
<apache-airflow-providers-cncf-kubernetes:kubernetes_executor>`, you can
upgrade the scheduler
triggerer, webserver in a rolling restart mode, and generally you should not
worry about the workers, as they
are managed by the Kubernetes cluster and will be automatically adopted by
``Schedulers`` when they are
upgraded and restarted.
-* For the :doc:``CeleryKubernetesExecutor
<../core-concepts/executor/celery-kubernetes>``, you follow the
+* For the :doc:``CeleryKubernetesExecutor
<apache-airflow-providers-celery:celery_kubernetes_executor>``, you follow the
same procedure as for the ``CeleryExecutor`` - you put the workers in
offline mode, wait for the running
tasks to complete, upgrade the workers, and then upgrade the scheduler,
triggerer and webserver in a
rolling restart mode - which should also adopt tasks run via the
``KubernetesExecutor`` part of the
diff --git a/docs/apache-airflow/best-practices.rst
b/docs/apache-airflow/best-practices.rst
index e166c10247..ff888a093e 100644
--- a/docs/apache-airflow/best-practices.rst
+++ b/docs/apache-airflow/best-practices.rst
@@ -82,7 +82,7 @@ it difficult to check the logs of that Task from the
Webserver. If that is not d
Communication
--------------
-Airflow executes tasks of a DAG on different servers in case you are using
:doc:`Kubernetes executor </core-concepts/executor/kubernetes>` or :doc:`Celery
executor </core-concepts/executor/celery>`.
+Airflow executes tasks of a DAG on different servers in case you are using
:doc:`Kubernetes executor
<apache-airflow-providers-cncf-kubernetes:kubernetes_executor>` or :doc:`Celery
executor <apache-airflow-providers-celery:celery_executor>`.
Therefore, you should not store any file or config in the local filesystem as
the next task is likely to run on a different server without access to it — for
example, a task that downloads the data file that the next task processes.
In the case of :class:`Local executor
<airflow.executors.local_executor.LocalExecutor>`,
storing a file on disk can make retries harder e.g., your task requires a
config file that is deleted by another task in DAG.
diff --git a/docs/apache-airflow/core-concepts/executor/debug.rst
b/docs/apache-airflow/core-concepts/debug.rst
similarity index 96%
rename from docs/apache-airflow/core-concepts/executor/debug.rst
rename to docs/apache-airflow/core-concepts/debug.rst
index 896f79ce1c..6dcdfa51a4 100644
--- a/docs/apache-airflow/core-concepts/executor/debug.rst
+++ b/docs/apache-airflow/core-concepts/debug.rst
@@ -15,44 +15,10 @@
specific language governing permissions and limitations
under the License.
-.. _executor:DebugExecutor:
-
-Debug Executor (deprecated)
-===========================
+.. _concepts:debugging:
-The :class:`~airflow.executors.debug_executor.DebugExecutor` is meant as
-a debug tool and can be used from IDE. It is a single process executor that
-queues :class:`~airflow.models.taskinstance.TaskInstance` and executes them by
running
-``_run_raw_task`` method.
-
-Due to its nature the executor can be used with SQLite database. When used
-with sensors the executor will change sensor mode to ``reschedule`` to avoid
-blocking the execution of DAG.
-
-Additionally ``DebugExecutor`` can be used in a fail-fast mode that will make
-all other running or scheduled tasks fail immediately. To enable this option
set
-``AIRFLOW__DEBUG__FAIL_FAST=True`` or adjust ``fail_fast`` option in your
``airflow.cfg``.
-For more information on setting the configuration, see
:doc:`../../howto/set-config`.
-
-**IDE setup steps:**
-
-1. Add ``main`` block at the end of your DAG file to make it runnable.
-
-It will run a backfill job:
-
-.. code-block:: python
-
- if __name__ == "__main__":
- from airflow.utils.state import State
-
- dag.clear()
- dag.run()
-
-
-2. Setup ``AIRFLOW__CORE__EXECUTOR=DebugExecutor`` in run configuration of
your IDE. In
- this step you should also setup all environment variables required by your
DAG.
-
-3. Run / debug the DAG file.
+Debugging Airflow DAGs
+======================
Testing DAGs with dag.test()
*****************************
@@ -74,7 +40,7 @@ and that's it! You can add argument such as
``execution_date`` if you want to te
you can run or debug DAGs as needed.
Comparison with DebugExecutor
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+-----------------------------
The ``dag.test`` command has the following benefits over the
:class:`~airflow.executors.debug_executor.DebugExecutor`
class, which is now deprecated:
@@ -102,3 +68,42 @@ Run ``python -m pdb <path to dag file>.py`` for an
interactive debugging experie
-> bash_command='echo 1',
(Pdb) run_this_last
<Task(EmptyOperator): run_this_last>
+
+.. _executor:DebugExecutor:
+
+Debug Executor (deprecated)
+***************************
+
+The :class:`~airflow.executors.debug_executor.DebugExecutor` is meant as
+a debug tool and can be used from IDE. It is a single process executor that
+queues :class:`~airflow.models.taskinstance.TaskInstance` and executes them by
running
+``_run_raw_task`` method.
+
+Due to its nature the executor can be used with SQLite database. When used
+with sensors the executor will change sensor mode to ``reschedule`` to avoid
+blocking the execution of DAG.
+
+Additionally ``DebugExecutor`` can be used in a fail-fast mode that will make
+all other running or scheduled tasks fail immediately. To enable this option
set
+``AIRFLOW__DEBUG__FAIL_FAST=True`` or adjust ``fail_fast`` option in your
``airflow.cfg``.
+For more information on setting the configuration, see
:doc:`../../howto/set-config`.
+
+**IDE setup steps:**
+
+1. Add ``main`` block at the end of your DAG file to make it runnable.
+
+It will run a backfill job:
+
+.. code-block:: python
+
+ if __name__ == "__main__":
+ from airflow.utils.state import State
+
+ dag.clear()
+ dag.run()
+
+
+2. Setup ``AIRFLOW__CORE__EXECUTOR=DebugExecutor`` in run configuration of
your IDE. In
+ this step you should also setup all environment variables required by your
DAG.
+
+3. Run / debug the DAG file.
diff --git a/docs/apache-airflow/core-concepts/executor/index.rst
b/docs/apache-airflow/core-concepts/executor/index.rst
index e1cc52cf6f..62bf73dd28 100644
--- a/docs/apache-airflow/core-concepts/executor/index.rst
+++ b/docs/apache-airflow/core-concepts/executor/index.rst
@@ -52,19 +52,15 @@ There are two types of executors - those that run tasks
*locally* (inside the ``
.. toctree::
:maxdepth: 1
- debug
local
sequential
**Remote Executors**
-.. toctree::
- :maxdepth: 1
-
- celery
- celery_kubernetes
- kubernetes
- local_kubernetes
+* :doc:`CeleryExecutor <apache-airflow-providers-celery:celery_executor>`
+* :doc:`CeleryKubernetesExecutor
<apache-airflow-providers-celery:celery_kubernetes_executor>`
+* :doc:`KubernetesExecutor
<apache-airflow-providers-cncf-kubernetes:kubernetes_executor>`
+* :doc:`KubernetesLocalExecutor
<apache-airflow-providers-cncf-kubernetes:local_kubernetes_executor>`
.. note::
diff --git a/docs/apache-airflow/core-concepts/index.rst
b/docs/apache-airflow/core-concepts/index.rst
index 9b24d4ea3f..dab97ea3ff 100644
--- a/docs/apache-airflow/core-concepts/index.rst
+++ b/docs/apache-airflow/core-concepts/index.rst
@@ -51,3 +51,10 @@ Here you can find detailed documentation about each one of
the core concepts of
xcoms
variables
params
+
+**Debugging**
+
+.. toctree::
+ :maxdepth: 1
+
+ debug
diff --git a/docs/apache-airflow/core-concepts/overview.rst
b/docs/apache-airflow/core-concepts/overview.rst
index c9d92677a2..b69fae9fb0 100644
--- a/docs/apache-airflow/core-concepts/overview.rst
+++ b/docs/apache-airflow/core-concepts/overview.rst
@@ -67,8 +67,8 @@ performance in your Airflow:
* Optional *worker*, which executes the tasks given to it by the scheduler. In
the basic installation
worker might be part of the scheduler not a separate component. It can be
run as a long running process
- in the :doc:`CeleryExecutor <executor/celery>`, or as a POD in the
- :doc:`KubernetesExecutor <executor/kubernetes>`.
+ in the :doc:`CeleryExecutor
<apache-airflow-providers-celery:celery_executor>`, or as a POD in the
+ :doc:`KubernetesExecutor
<apache-airflow-providers-cncf-kubernetes:kubernetes_executor>`.
* Optional *triggerer*, which executes deferred tasks in an asyncio event
loop. In basic installation
where deferred tasks are not used, a triggerer is not necessary. More about
deferring tasks can be
diff --git a/docs/apache-airflow/faq.rst b/docs/apache-airflow/faq.rst
index d95b4e42d3..f0e7469538 100644
--- a/docs/apache-airflow/faq.rst
+++ b/docs/apache-airflow/faq.rst
@@ -104,7 +104,7 @@ Operators or tasks also have configurations that improves
efficiency and schedul
per task.
- ``pool``: See :ref:`concepts:pool`.
- ``priority_weight``: See :ref:`concepts:priority-weight`.
-- ``queue``: See :ref:`executor:CeleryExecutor:queue` for CeleryExecutor
deployments only.
+- ``queue``: See :ref:`apache-airflow-providers-celery:celery_executor:queue`
for CeleryExecutor deployments only.
How to reduce DAG scheduling latency / task delay?
diff --git a/docs/apache-airflow/howto/docker-compose/index.rst
b/docs/apache-airflow/howto/docker-compose/index.rst
index 00f08915ea..2cd92fa048 100644
--- a/docs/apache-airflow/howto/docker-compose/index.rst
+++ b/docs/apache-airflow/howto/docker-compose/index.rst
@@ -20,7 +20,7 @@
Running Airflow in Docker
#########################
-This quick-start guide will allow you to quickly get Airflow up and running
with :doc:`CeleryExecutor </core-concepts/executor/celery>` in Docker.
+This quick-start guide will allow you to quickly get Airflow up and running
with the :doc:`CeleryExecutor
<apache-airflow-providers-celery:celery_executor>` in Docker.
.. caution::
This procedure can be useful for learning and exploration. However,
adapting it for use in real-world situations can be complicated. Making changes
to this procedure will require specialized expertise in Docker & Docker
Compose, and the Airflow community may not be able to help you.
@@ -99,7 +99,7 @@ Optionally, you can enable flower by adding ``--profile
flower`` option, e.g. ``
- ``flower`` - `The flower app <https://flower.readthedocs.io/en/latest/>`__
for monitoring the environment. It is available at ``http://localhost:5555``.
-All these services allow you to run Airflow with :doc:`CeleryExecutor
</core-concepts/executor/celery>`. For more information, see
:doc:`/core-concepts/overview`.
+All these services allow you to run Airflow with :doc:`CeleryExecutor
<apache-airflow-providers-celery:celery_executor>`. For more information, see
:doc:`/core-concepts/overview`.
Some directories in the container are mounted, which means that their contents
are synchronized between your computer and the container.
diff --git a/docs/apache-airflow/img/k8s-0-worker.jpeg
b/docs/apache-airflow/img/k8s-0-worker.jpeg
deleted file mode 100644
index 71b1aedf61..0000000000
Binary files a/docs/apache-airflow/img/k8s-0-worker.jpeg and /dev/null differ
diff --git a/docs/apache-airflow/img/k8s-3-worker.jpeg
b/docs/apache-airflow/img/k8s-3-worker.jpeg
deleted file mode 100644
index 0d6f5ca416..0000000000
Binary files a/docs/apache-airflow/img/k8s-3-worker.jpeg and /dev/null differ
diff --git a/docs/apache-airflow/img/k8s-5-worker.jpeg
b/docs/apache-airflow/img/k8s-5-worker.jpeg
deleted file mode 100644
index 30227840cf..0000000000
Binary files a/docs/apache-airflow/img/k8s-5-worker.jpeg and /dev/null differ
diff --git a/docs/apache-airflow/redirects.txt
b/docs/apache-airflow/redirects.txt
index ea2ccf3dff..5b2408c950 100644
--- a/docs/apache-airflow/redirects.txt
+++ b/docs/apache-airflow/redirects.txt
@@ -139,6 +139,20 @@ executor/local.rst core-concepts/executor/local.rst
executor/sequential.rst core-concepts/executor/sequential.rst
upgrading-from-1-10/upgrade-check.rst
howto/upgrading-from-1-10/upgrade-check.rst
core-concepts/dag-run.rst authoring-and-scheduling/cron.rst
+core-concepts/executor/debug.rst core-concepts/debug.rst
+
+# Moving provider executor docs to providers
+## The ``../`` indicates that it will move to the root of the docs directory,
unlike the rest of
+## the entries here which move to docs/apache-airflow.
+## It's okay to include ``/stable/``, because there's no relationship between
a version of
+## Airflow and the version of any provider package.
+### Kubernetes Executors
+core-concepts/executor/kubernetes.rst
../apache-airflow-providers-cncf-kubernetes/stable/kubernetes_executor.html
+core-concepts/executor/local_kubernetes.rst
../apache-airflow-providers-cncf-kubernetes/stable/local_kubernetes_executor.html
+
+### Celery Executors
+core-concepts/executor/celery_kubernetes.rst
../apache-airflow-providers-celery/stable/celery_kubernetes_executor.html
+core-concepts/executor/celery.rst
../apache-airflow-providers-celery/stable/celery_executor.html
# Python API
python-api-ref.rst public-airflow-interface.rst
diff --git a/docs/exts/redirects.py b/docs/exts/redirects.py
index b186da9994..373187f951 100644
--- a/docs/exts/redirects.py
+++ b/docs/exts/redirects.py
@@ -49,18 +49,28 @@ def generate_redirects(app):
if line.startswith("#"):
continue
+ # Split line into the original path `from_path` and where the URL
should redirect to `to_path`
from_path, _, to_path = line.rstrip().partition(" ")
log.debug("Redirecting '%s' to '%s'", from_path, to_path)
+ # in_suffix is often ".rst"
from_path = from_path.replace(in_suffix, ".html")
to_path = to_path.replace(in_suffix, ".html")
to_path_prefix = f"..{os.path.sep}" *
(len(from_path.split(os.path.sep)) - 1)
+ # The redirect path needs to move back to the root of the
apache-airflow docs directory
+ # or the root of the docs directory altogether for provider
packages.
+ if "../" and "providers" in to_path:
+ to_path_prefix = f"..{os.path.sep}" *
(len(from_path.split(os.path.sep)))
+ else:
+ to_path_prefix = f"..{os.path.sep}" *
(len(from_path.split(os.path.sep)) - 1)
+
to_path = to_path_prefix + to_path
log.debug("Resolved redirect '%s' to '%s'", from_path, to_path)
+ # This will be used to save an HTML file with `TEMPLATE` formatted
redirected_filename = os.path.join(app.builder.outdir, from_path)
redirected_directory = os.path.dirname(redirected_filename)
