This is an automated email from the ASF dual-hosted git repository.
onikolas pushed a commit to branch v2-10-stable
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v2-10-stable by this push:
new 12a124af3d0 Mark the hardcoded hybrid executors as deprecate and
multi-exec as stable (#46944)
12a124af3d0 is described below
commit 12a124af3d0483ae85c01874bcb45effa6834e26
Author: Niko Oliveira <[email protected]>
AuthorDate: Mon Feb 24 14:04:40 2025 -0800
Mark the hardcoded hybrid executors as deprecate and multi-exec as stable
(#46944)
* Mark Multiple Executor Configuration as stable
Multiple Executor Configuration (aka hybrid executors) has been released
for over half a year and can be marked stable. This gives users an
option to migrate to from the old hardcoded hybrid executors (e.g
CeleryKubernetesExecutor)
* Mark the statically-coded hybrid executors as deprecated
These executors will be removed in Airflow 3.0. Mark them as deprecated
so users are aware and have time to migrate.
- A deprecation warning will be printed when the executors are loaded
- Documentation has been updated to note that they are deprecated
- A news fragment added as well.
Co-authored-by: Jed Cunningham
<[email protected]>
---
airflow/executors/executor_loader.py | 20 +++++++++++++++++++-
docs/apache-airflow/core-concepts/executor/index.rst | 17 +++++++++++------
newsfragments/46944.significant.rst | 3 +++
tests/executors/test_executor_loader.py | 10 ++++++++--
tests/utils/test_log_handlers.py | 12 +++++++++---
5 files changed, 50 insertions(+), 12 deletions(-)
diff --git a/airflow/executors/executor_loader.py
b/airflow/executors/executor_loader.py
index 7ad42a2fb1b..c306d8f0004 100644
--- a/airflow/executors/executor_loader.py
+++ b/airflow/executors/executor_loader.py
@@ -21,11 +21,12 @@ from __future__ import annotations
import functools
import logging
import os
+import warnings
from contextlib import suppress
from typing import TYPE_CHECKING
from airflow.api_internal.internal_api_call import InternalApiConfig
-from airflow.exceptions import AirflowConfigException, UnknownExecutorException
+from airflow.exceptions import AirflowConfigException,
RemovedInAirflow3Warning, UnknownExecutorException
from airflow.executors.executor_constants import (
CELERY_EXECUTOR,
CELERY_KUBERNETES_EXECUTOR,
@@ -344,8 +345,24 @@ class ExecutorLoader:
if engine and engine.dialect.name == "sqlite":
raise AirflowConfigException(f"error: cannot use SQLite with the
{executor.__name__}")
+ @classmethod
+ def _warn_of_deprecated_executor(cls, executor_name: str) -> None:
+ """
+ Warn of deprecated executor.
+
+ :param executor_name: Name of the executor
+ """
+ warnings.warn(
+ f"\nThe use and support of the {executor_name} is deprecated and
will be removed in Airflow 3.0.\n"
+ "Please migrate to using Multiple Executor Configuration
instead:\n"
+
"https://airflow.apache.org/docs/apache-airflow/stable/core-concepts/executor/#using-multiple-executors-concurrently";,
+ RemovedInAirflow3Warning,
+ stacklevel=2,
+ )
+
@classmethod
def __load_celery_kubernetes_executor(cls) -> BaseExecutor:
+ cls._warn_of_deprecated_executor(CELERY_KUBERNETES_EXECUTOR)
celery_executor = import_string(cls.executors[CELERY_EXECUTOR])()
kubernetes_executor =
import_string(cls.executors[KUBERNETES_EXECUTOR])()
@@ -354,6 +371,7 @@ class ExecutorLoader:
@classmethod
def __load_local_kubernetes_executor(cls) -> BaseExecutor:
+ cls._warn_of_deprecated_executor(LOCAL_KUBERNETES_EXECUTOR)
local_executor = import_string(cls.executors[LOCAL_EXECUTOR])()
kubernetes_executor =
import_string(cls.executors[KUBERNETES_EXECUTOR])()
diff --git a/docs/apache-airflow/core-concepts/executor/index.rst
b/docs/apache-airflow/core-concepts/executor/index.rst
index 1bb11f2335a..2aa976ab86f 100644
--- a/docs/apache-airflow/core-concepts/executor/index.rst
+++ b/docs/apache-airflow/core-concepts/executor/index.rst
@@ -115,9 +115,6 @@ Airflow tasks are executed ad hoc inside containers/pods.
Each task is isolated
Using Multiple Executors Concurrently
-------------------------------------
-.. warning::
- Multiple executor configuration is an alpha/experimental feature at the
moment and may be subject to change without warning.
-
Starting with version 2.10.0, Airflow can now operate with a multi-executor
configuration. Each executor has its own set of pros and cons, often they are
trade-offs between latency, isolation and compute efficiency among other
properties (see :ref:`here <executor-types-comparison>` for comparisons of
executors). Running multiple executors allows you to make better use of the
strengths of all the available executors and avoid their weaknesses. In other
words, you can use a specific execut [...]
Configuration
@@ -208,15 +205,23 @@ When using a single executor, Airflow metrics will behave
as they were <2.9. But
Logging works the same as the single executor use case.
-Statically-coded Hybrid Executors
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Statically-coded Hybrid Executors (Deprecated)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
There are currently two "statically coded" executors, these executors are
hybrids of two different executors: the :doc:`LocalKubernetesExecutor
<apache-airflow-providers-cncf-kubernetes:local_kubernetes_executor>` and the
:doc:`CeleryKubernetesExecutor
<apache-airflow-providers-celery:celery_kubernetes_executor>`. Their
implementation is not native or intrinsic to core Airflow. These hybrid
executors instead make use of the ``queue`` field on Task Instances to indicate
and persist which [...]
Executors such as these also require hand crafting new "concrete" classes to
create each permutation of possible combinations of executors. This is
untenable as more executors are created and leads to more maintenance overhead.
Bespoke coding effort should not be required to use any combination of
executors.
-Therefore using these types of executors is no longer recommended.
+Therefore these types of executors are deprecated and using them is no longer
recommended.
+
+Migrating from Statically-Coded Hybrid Executors to Multi-Executor
Configuration
+""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+
+Here are some steps to migrate from statically-coded hybrid executors to the
new multi-executor configuration:
+1. **Update Airflow**: Ensure you are using Airflow 2.10.0 or later.
+2. **Update executor configuration**: Update the ``[core] executor``
configuration to replace the use of the statically-coded hybrid executor with
the new multi-executor configuration (i.e. a list of the executors you'd like
to use). For example, replace ``LocalKubernetesExecutor`` with
``LocalExecutor,KubernetesExecutor``.
+3. **Update Dags**: Update your dags to use the new multi-executor
configuration. This involves replacing the use of the ``queue`` field on your
tasks with the ``executor`` field to specify which executor you'd like to use.
Writing Your Own Executor
-------------------------
diff --git a/newsfragments/46944.significant.rst
b/newsfragments/46944.significant.rst
new file mode 100644
index 00000000000..8dabb9a3546
--- /dev/null
+++ b/newsfragments/46944.significant.rst
@@ -0,0 +1,3 @@
+Statically-coded executors are now deprecated; use Multiple Executor
Configuration
+
+The Statically-coded executors ``LocalKubernetesExecutor`` and
``CeleryKubernetesExecutor`` are now deprecated in favor of Multiple Executor
Configuration, which is now marked as stable. They will be removed in Airflow
3.0
diff --git a/tests/executors/test_executor_loader.py
b/tests/executors/test_executor_loader.py
index dc60b9cc507..7c9d21fb9e3 100644
--- a/tests/executors/test_executor_loader.py
+++ b/tests/executors/test_executor_loader.py
@@ -22,7 +22,7 @@ from unittest import mock
import pytest
from airflow import plugins_manager
-from airflow.exceptions import AirflowConfigException
+from airflow.exceptions import AirflowConfigException, RemovedInAirflow3Warning
from airflow.executors import executor_loader
from airflow.executors.executor_loader import ConnectorSource, ExecutorName
from airflow.executors.local_executor import LocalExecutor
@@ -68,7 +68,13 @@ class TestExecutorLoader:
)
def test_should_support_executor_from_core(self, executor_name):
with conf_vars({("core", "executor"): executor_name}):
- executor = executor_loader.ExecutorLoader.get_default_executor()
+ # These executors are deprecated and will be removed in Airflow
3.0 but we still need to support
+ # them in Airflow 2.10.X
+ if executor_name == "CeleryKubernetesExecutor":
+ with pytest.warns(RemovedInAirflow3Warning):
+ executor =
executor_loader.ExecutorLoader.get_default_executor()
+ else:
+ executor =
executor_loader.ExecutorLoader.get_default_executor()
assert executor is not None
assert executor_name == executor.__class__.__name__
assert executor.name is not None
diff --git a/tests/utils/test_log_handlers.py b/tests/utils/test_log_handlers.py
index 95483f2285f..2c1e2db1f49 100644
--- a/tests/utils/test_log_handlers.py
+++ b/tests/utils/test_log_handlers.py
@@ -206,7 +206,6 @@ class TestFileTaskLogHandler:
"executor_name",
[
(executor_constants.LOCAL_KUBERNETES_EXECUTOR),
- (executor_constants.CELERY_KUBERNETES_EXECUTOR),
(executor_constants.KUBERNETES_EXECUTOR),
(None),
],
@@ -216,7 +215,6 @@ class TestFileTaskLogHandler:
("core", "EXECUTOR"): ",".join(
[
executor_constants.LOCAL_KUBERNETES_EXECUTOR,
- executor_constants.CELERY_KUBERNETES_EXECUTOR,
executor_constants.KUBERNETES_EXECUTOR,
]
),
@@ -279,7 +277,15 @@ class TestFileTaskLogHandler:
file_handler.close()
assert hasattr(file_handler, "read")
- file_handler.read(ti)
+ # These executors are deprecated and will be removed in Airflow
3.0 but we still need to support
+ # them in Airflow 2.10.X
+ if executor_name in [
+ executor_constants.LOCAL_KUBERNETES_EXECUTOR,
+ ]:
+ with pytest.warns(RemovedInAirflow3Warning):
+ file_handler.read(ti)
+ else:
+ file_handler.read(ti)
os.remove(log_filename)
mock_get_task_log.assert_called_once()
