This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 0ef3225b1d Rmeove daskexcutor provider from Airflow code (#36269)
0ef3225b1d is described below
commit 0ef3225b1dc10f8706b9461d8669b768d462eb72
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun Dec 17 20:42:03 2023 +0100
Rmeove daskexcutor provider from Airflow code (#36269)
---
.github/boring-cyborg.yml | 11 +-
.../daskexecutor/.latest-doc-only-change.txt | 1 -
airflow/providers/daskexecutor/CHANGELOG.rst | 78 -------
airflow/providers/daskexecutor/__init__.py | 42 ----
.../providers/daskexecutor/executors/__init__.py | 16 --
.../daskexecutor/executors/dask_executor.py | 147 -------------
airflow/providers/daskexecutor/provider.yaml | 84 -------
dev/breeze/tests/test_packages.py | 6 +-
.../changelog.rst | 25 ---
.../commits.rst | 90 --------
.../configurations-ref.rst | 18 --
.../index.rst | 94 --------
.../installing-providers-from-sources.rst | 18 --
.../security.rst | 18 --
.../apache-airflow/core-concepts/executor/dask.rst | 56 -----
.../core-concepts/executor/index.rst | 1 -
docs/integration-logos/dask/dask.png | Bin 6134 -> 0 bytes
images/breeze/output_build-docs.svg | 64 +++---
images/breeze/output_build-docs.txt | 2 +-
...tput_release-management_add-back-references.svg | 14 +-
...tput_release-management_add-back-references.txt | 2 +-
.../output_release-management_publish-docs.svg | 14 +-
.../output_release-management_publish-docs.txt | 2 +-
scripts/ci/pre_commit/pre_commit_insert_extras.py | 7 -
tests/always/test_project_structure.py | 1 -
tests/providers/daskexecutor/__init__.py | 16 --
tests/providers/daskexecutor/test_dask_executor.py | 241 ---------------------
27 files changed, 59 insertions(+), 1009 deletions(-)
diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml
index 12125e382e..fd98dc5767 100644
--- a/.github/boring-cyborg.yml
+++ b/.github/boring-cyborg.yml
@@ -183,11 +183,6 @@ labelPRBasedOnFilePath:
- tests/providers/common/sql/**/*
- tests/system/providers/common/sql/**/*
- provider:daskexecutor:
- - airflow/providers/daskexecutor/**/*
- - docs/apache-airflow-providers-daskexecutor/**/*
- - tests/providers/daskexecutor/**/*
-
provider:databricks:
- airflow/providers/databricks/**/*
- docs/apache-airflow-providers-databricks/**/*
@@ -195,9 +190,9 @@ labelPRBasedOnFilePath:
- tests/system/providers/databricks/**/*
provider:datadog:
- - airflow/providers/daskexecutor/**/*
- - docs/apache-airflow-providers-daskexecutor/**/*
- - tests/providers/daskexecutor/**/*
+ - airflow/providers/datadog/**/*
+ - docs/apache-airflow-providers-datadog/**/*
+ - tests/providers/datadog/**/*
provider:dbt-cloud:
- airflow/providers/dbt/cloud/**/*
diff --git a/airflow/providers/daskexecutor/.latest-doc-only-change.txt
b/airflow/providers/daskexecutor/.latest-doc-only-change.txt
deleted file mode 100644
index 36cd4c153e..0000000000
--- a/airflow/providers/daskexecutor/.latest-doc-only-change.txt
+++ /dev/null
@@ -1 +0,0 @@
-99534e47f330ce0efb96402629dda5b2a4f16e8f
diff --git a/airflow/providers/daskexecutor/CHANGELOG.rst
b/airflow/providers/daskexecutor/CHANGELOG.rst
deleted file mode 100644
index 82b8a53b63..0000000000
--- a/airflow/providers/daskexecutor/CHANGELOG.rst
+++ /dev/null
@@ -1,78 +0,0 @@
- .. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- .. http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-
-.. NOTE TO CONTRIBUTORS:
- Please, only add notes to the Changelog just below the "Changelog" header
when there are some breaking changes
- and you want to add an explanation to the users on how they are supposed to
deal with them.
- The changelog is updated and maintained semi-automatically by release
manager.
-
-``apache-airflow-providers-daskexecutor``
-
-
-Changelog
----------
-
-1.1.1
-.....
-
-.. note::
- Mark daskexecutor provider as removed according to `consensus on the
[email protected]
<https://lists.apache.org/thread/fxv44cqqljrrhll3fdpdgc9h9fz5ghcy>`_
-
-.. note::
- This release of provider is only available for Airflow 2.6+ as explained in
the
- `Apache Airflow providers support policy
<https://github.com/apache/airflow/blob/main/PROVIDERS.rst#minimum-supported-version-of-airflow-for-community-managed-providers>`_.
-
-Misc
-~~~~
-
-* ``Mark daskexecutor provider as removed (#35965)``
-
-.. Review and move the new changes to one of the sections above:
- * ``Fix and reapply templates for provider documentation (#35686)``
- * ``Prepare docs 3rd wave of Providers October 2023 - FIX (#35233)``
- * ``Prepare docs 2nd wave of Providers November 2023 (#35836)``
- * ``Use reproducible builds for provider packages (#35693)``
- * ``Prepare docs 1st wave of Providers November 2023 (#35537)``
- * ``Prepare docs 3rd wave of Providers October 2023 (#35187)``
- * ``Pre-upgrade 'ruff==0.0.292' changes in providers (#35053)``
-
-1.1.0
-.....
-
-.. note::
- This release of provider is only available for Airflow 2.5+ as explained in
the
- `Apache Airflow providers support policy
<https://github.com/apache/airflow/blob/main/PROVIDERS.rst#minimum-supported-version-of-airflow-for-community-managed-providers>`_.
-
-Misc
-~~~~
-
-* ``Bump min airflow version of providers (#34728)``
-
-1.0.1
-.....
-
-Misc
-~~~~
-
-* ``Replace repr() with proper formatting (#33520)``
-
-
-1.0.0
-.....
-
-Initial version of the provider.
diff --git a/airflow/providers/daskexecutor/__init__.py
b/airflow/providers/daskexecutor/__init__.py
deleted file mode 100644
index b5f95ccfa8..0000000000
--- a/airflow/providers/daskexecutor/__init__.py
+++ /dev/null
@@ -1,42 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
-# OVERWRITTEN WHEN PREPARING DOCUMENTATION FOR THE PACKAGES.
-#
-# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
-# `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the
`dev/breeze/src/airflow_breeze/templates` DIRECTORY
-#
-from __future__ import annotations
-
-import packaging.version
-
-__all__ = ["__version__"]
-
-__version__ = "1.1.1"
-
-try:
- from airflow import __version__ as airflow_version
-except ImportError:
- from airflow.version import version as airflow_version
-
-if
packaging.version.parse(packaging.version.parse(airflow_version).base_version)
< packaging.version.parse(
- "2.5.0"
-):
- raise RuntimeError(
- f"The package `apache-airflow-providers-daskexecutor:{__version__}`
needs Apache Airflow 2.5.0+"
- )
diff --git a/airflow/providers/daskexecutor/executors/__init__.py
b/airflow/providers/daskexecutor/executors/__init__.py
deleted file mode 100644
index 13a83393a9..0000000000
--- a/airflow/providers/daskexecutor/executors/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/airflow/providers/daskexecutor/executors/dask_executor.py
b/airflow/providers/daskexecutor/executors/dask_executor.py
deleted file mode 100644
index 3e93f0161f..0000000000
--- a/airflow/providers/daskexecutor/executors/dask_executor.py
+++ /dev/null
@@ -1,147 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-DaskExecutor.
-
-.. seealso::
- For more information on how the DaskExecutor works, take a look at the
guide:
- :ref:`executor:DaskExecutor`
-"""
-from __future__ import annotations
-
-import subprocess
-from typing import TYPE_CHECKING, Any
-
-from distributed import Client, Future, as_completed
-from distributed.security import Security
-
-from airflow.configuration import conf
-from airflow.exceptions import AirflowException
-from airflow.executors.base_executor import BaseExecutor
-
-if TYPE_CHECKING:
- from airflow.executors.base_executor import CommandType
- from airflow.models.taskinstancekey import TaskInstanceKey
-
-
-# queue="default" is a special case since this is the base config default
queue name,
-# with respect to DaskExecutor, treat it as if no queue is provided
-_UNDEFINED_QUEUES = {None, "default"}
-
-
-class DaskExecutor(BaseExecutor):
- """DaskExecutor submits tasks to a Dask Distributed cluster."""
-
- supports_pickling: bool = False
-
- def __init__(self, cluster_address=None):
- super().__init__(parallelism=0)
- if cluster_address is None:
- cluster_address = conf.get("dask", "cluster_address")
- if not cluster_address:
- raise ValueError("Please provide a Dask cluster address in
airflow.cfg")
- self.cluster_address = cluster_address
- # ssl / tls parameters
- self.tls_ca = conf.get("dask", "tls_ca")
- self.tls_key = conf.get("dask", "tls_key")
- self.tls_cert = conf.get("dask", "tls_cert")
- self.client: Client | None = None
- self.futures: dict[Future, TaskInstanceKey] | None = None
-
- def start(self) -> None:
- if self.tls_ca or self.tls_key or self.tls_cert:
- security = Security(
- tls_client_key=self.tls_key,
- tls_client_cert=self.tls_cert,
- tls_ca_file=self.tls_ca,
- require_encryption=True,
- )
- else:
- security = None
-
- self.client = Client(self.cluster_address, security=security)
- self.futures = {}
-
- def execute_async(
- self,
- key: TaskInstanceKey,
- command: CommandType,
- queue: str | None = None,
- executor_config: Any | None = None,
- ) -> None:
- if TYPE_CHECKING:
- assert self.client
-
- self.validate_airflow_tasks_run_command(command)
-
- def airflow_run():
- return subprocess.check_call(command, close_fds=True)
-
- resources = None
- if queue not in _UNDEFINED_QUEUES:
- scheduler_info = self.client.scheduler_info()
- avail_queues = {
- resource for d in scheduler_info["workers"].values() for
resource in d["resources"]
- }
-
- if queue not in avail_queues:
- raise AirflowException(f"Attempted to submit task to an
unavailable queue: '{queue}'")
- resources = {queue: 1}
-
- future = self.client.submit(subprocess.check_call, command,
pure=False, resources=resources)
- self.futures[future] = key # type: ignore
-
- def _process_future(self, future: Future) -> None:
- if TYPE_CHECKING:
- assert self.futures
-
- if future.done():
- key = self.futures[future]
- if future.exception():
- self.log.error("Failed to execute task: %r",
future.exception())
- self.fail(key)
- elif future.cancelled():
- self.log.error("Failed to execute task")
- self.fail(key)
- else:
- self.success(key)
- self.futures.pop(future)
-
- def sync(self) -> None:
- if TYPE_CHECKING:
- assert self.futures
-
- # make a copy so futures can be popped during iteration
- for future in self.futures.copy():
- self._process_future(future)
-
- def end(self) -> None:
- if TYPE_CHECKING:
- assert self.client
- assert self.futures
-
- self.client.cancel(list(self.futures.keys()))
- for future in as_completed(self.futures.copy()):
- self._process_future(future)
-
- def terminate(self):
- if TYPE_CHECKING:
- assert self.futures
-
- self.client.cancel(self.futures.keys())
- self.end()
diff --git a/airflow/providers/daskexecutor/provider.yaml
b/airflow/providers/daskexecutor/provider.yaml
deleted file mode 100644
index 369143bcd9..0000000000
--- a/airflow/providers/daskexecutor/provider.yaml
+++ /dev/null
@@ -1,84 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
----
-package-name: apache-airflow-providers-daskexecutor
-name: Dask Executor
-description: |
- `Dask <https://www.dask.org/>`__
-
-removed: true
-suspended: true
-source-date-epoch: 1700827453
-versions:
- - 1.1.1
- - 1.1.0
- - 1.0.1
- - 1.0.0
-
-dependencies:
- - apache-airflow>=2.5.0
- # Dask support is limited, we need Dask team to upgrade support for dask if
we were to continue
- # Supporting it in the future
- - cloudpickle>=1.4.1
- # Dask and distributed in version 2023.5.0 break our test
- # See https://github.com/dask/dask/issues/10279
- - dask>=2.9.0,!=2022.10.1,!=2023.5.0
- - distributed>=2.11.1,!=2023.5.0
-
-integrations:
- - integration-name: Dask
- external-doc-url: https://www.dask.org/
- logo: /integration-logos/dask/dask.png
- tags: [service]
-
-executors:
- - airflow.providers.daskexecutor.executors.dask_executor.DaskExecutor
-
-config:
- dask:
- description: |
- This section only applies if you are using DaskExecutor.
- options:
- cluster_address:
- description: |
- The IP address and port of the Dask cluster's scheduler.
- version_added: ~
- type: string
- example: ~
- default: "127.0.0.1:8786"
- tls_ca:
- description: |
- Path to a CA certificate file encoded in PEM format to access a
secured Dask scheduler.
- version_added: ~
- type: string
- example: ~
- default: ""
- tls_cert:
- description: |
- Path to a certificate file for the client, encoded in PEM format.
- version_added: ~
- type: string
- example: ~
- default: ""
- tls_key:
- description: |
- Path to a key file for the client, encoded in PEM format.
- version_added: ~
- type: string
- example: ~
- default: ""
diff --git a/dev/breeze/tests/test_packages.py
b/dev/breeze/tests/test_packages.py
index f50b8e2c34..214b124f92 100644
--- a/dev/breeze/tests/test_packages.py
+++ b/dev/breeze/tests/test_packages.py
@@ -109,17 +109,17 @@ def test_get_provider_requirements():
def test_get_removed_providers():
# Modify it every time we schedule provider for removal or remove it
- assert ["apache.sqoop", "daskexecutor", "plexus"] ==
get_removed_provider_ids()
+ assert ["apache.sqoop", "plexus"] == get_removed_provider_ids()
def test_get_suspended_provider_ids():
# Modify it every time we suspend/resume provider
- assert ["apache.sqoop", "daskexecutor", "plexus"] ==
get_suspended_provider_ids()
+ assert ["apache.sqoop", "plexus"] == get_suspended_provider_ids()
def test_get_suspended_provider_folders():
# Modify it every time we suspend/resume provider
- assert ["apache/sqoop", "daskexecutor", "plexus"] ==
get_suspended_provider_folders()
+ assert ["apache/sqoop", "plexus"] == get_suspended_provider_folders()
@pytest.mark.parametrize(
diff --git a/docs/apache-airflow-providers-daskexecutor/changelog.rst
b/docs/apache-airflow-providers-daskexecutor/changelog.rst
deleted file mode 100644
index b825747cf1..0000000000
--- a/docs/apache-airflow-providers-daskexecutor/changelog.rst
+++ /dev/null
@@ -1,25 +0,0 @@
-
- .. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- .. http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
- .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
- OVERWRITTEN WHEN PREPARING PACKAGES.
-
- .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
- `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the
`dev/breeze/src/airflow_breeze/templates` DIRECTORY
-
-.. include:: ../../airflow/providers/daskexecutor/CHANGELOG.rst
diff --git a/docs/apache-airflow-providers-daskexecutor/commits.rst
b/docs/apache-airflow-providers-daskexecutor/commits.rst
deleted file mode 100644
index 5b21624429..0000000000
--- a/docs/apache-airflow-providers-daskexecutor/commits.rst
+++ /dev/null
@@ -1,90 +0,0 @@
-
- .. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- .. http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
- .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
- OVERWRITTEN WHEN PREPARING PACKAGES.
-
- .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
- `PROVIDER_COMMITS_TEMPLATE.rst.jinja2` IN the
`dev/breeze/src/airflow_breeze/templates` DIRECTORY
-
- .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE
OVERWRITTEN AT RELEASE TIME!
-
-Package apache-airflow-providers-daskexecutor
-------------------------------------------------------
-
-`Dask <https://www.dask.org/>`__
-
-
-This is detailed commit list of changes for versions provider package:
``daskexecutor``.
-For high-level changelog, see :doc:`package information including changelog
<index>`.
-
-
-
-1.1.1
-.....
-
-Latest change: 2023-11-30
-
-=================================================================================================
===========
==================================================================
-Commit
Committed Subject
-=================================================================================================
===========
==================================================================
-`9c1c9f450e
<https://github.com/apache/airflow/commit/9c1c9f450e289b40f94639db3f0686f592c8841e>`_
2023-11-30 ``Mark daskexecutor provider as removed (#35965)``
-`0b23d5601c
<https://github.com/apache/airflow/commit/0b23d5601c6f833392b0ea816e651dcb13a14685>`_
2023-11-24 ``Prepare docs 2nd wave of Providers November 2023 (#35836)``
-`99534e47f3
<https://github.com/apache/airflow/commit/99534e47f330ce0efb96402629dda5b2a4f16e8f>`_
2023-11-19 ``Use reproducible builds for provider packages (#35693)``
-`99df205f42
<https://github.com/apache/airflow/commit/99df205f42a754aa67f80b5983e1d228ff23267f>`_
2023-11-16 ``Fix and reapply templates for provider documentation (#35686)``
-`1b059c57d6
<https://github.com/apache/airflow/commit/1b059c57d6d57d198463e5388138bee8a08591b1>`_
2023-11-08 ``Prepare docs 1st wave of Providers November 2023 (#35537)``
-`d1c58d86de
<https://github.com/apache/airflow/commit/d1c58d86de1267d9268a1efe0a0c102633c051a1>`_
2023-10-28 ``Prepare docs 3rd wave of Providers October 2023 - FIX
(#35233)``
-`3592ff4046
<https://github.com/apache/airflow/commit/3592ff40465032fa041600be740ee6bc25e7c242>`_
2023-10-28 ``Prepare docs 3rd wave of Providers October 2023 (#35187)``
-`dd7ba3cae1
<https://github.com/apache/airflow/commit/dd7ba3cae139cb10d71c5ebc25fc496c67ee784e>`_
2023-10-19 ``Pre-upgrade 'ruff==0.0.292' changes in providers (#35053)``
-=================================================================================================
===========
==================================================================
-
-1.1.0
-.....
-
-Latest change: 2023-10-13
-
-=================================================================================================
=========== ===============================================================
-Commit
Committed Subject
-=================================================================================================
=========== ===============================================================
-`e9987d5059
<https://github.com/apache/airflow/commit/e9987d50598f70d84cbb2a5d964e21020e81c080>`_
2023-10-13 ``Prepare docs 1st wave of Providers in October 2023 (#34916)``
-`0c8e30e43b
<https://github.com/apache/airflow/commit/0c8e30e43b70e9d033e1686b327eb00aab82479c>`_
2023-10-05 ``Bump min airflow version of providers (#34728)``
-=================================================================================================
=========== ===============================================================
-
-1.0.1
-.....
-
-Latest change: 2023-08-26
-
-=================================================================================================
=========== ============================================================
-Commit
Committed Subject
-=================================================================================================
=========== ============================================================
-`c077d19060
<https://github.com/apache/airflow/commit/c077d190609f931387c1fcd7b8cc34f12e2372b9>`_
2023-08-26 ``Prepare docs for Aug 2023 3rd wave of Providers (#33730)``
-`abef61ff3d
<https://github.com/apache/airflow/commit/abef61ff3d6b9ae8dcb7f9dbbea78a9648a0c50b>`_
2023-08-20 ``Replace repr() with proper formatting (#33520)``
-=================================================================================================
=========== ============================================================
-
-1.0.0
-.....
-
-Latest change: 2023-07-29
-
-=================================================================================================
=========== ===============================================================
-Commit
Committed Subject
-=================================================================================================
=========== ===============================================================
-`d06b7af69a
<https://github.com/apache/airflow/commit/d06b7af69a65c50321ba2a9904551f3b8affc7f1>`_
2023-07-29 ``Prepare docs for July 2023 3rd wave of Providers (#32875)``
-`0c798252ef
<https://github.com/apache/airflow/commit/0c798252efb83844a2e0ba08d74b32eed2e317ad>`_
2023-07-25 ``Extract Dask executor to new daskexecutor provider (#32772)``
-=================================================================================================
=========== ===============================================================
diff --git a/docs/apache-airflow-providers-daskexecutor/configurations-ref.rst
b/docs/apache-airflow-providers-daskexecutor/configurations-ref.rst
deleted file mode 100644
index 5885c9d91b..0000000000
--- a/docs/apache-airflow-providers-daskexecutor/configurations-ref.rst
+++ /dev/null
@@ -1,18 +0,0 @@
- .. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- .. http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-.. include:: ../exts/includes/providers-configurations-ref.rst
diff --git a/docs/apache-airflow-providers-daskexecutor/index.rst
b/docs/apache-airflow-providers-daskexecutor/index.rst
deleted file mode 100644
index 1b7d7fa5c6..0000000000
--- a/docs/apache-airflow-providers-daskexecutor/index.rst
+++ /dev/null
@@ -1,94 +0,0 @@
-
- .. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- .. http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-``apache-airflow-providers-daskexecutor``
-=========================================
-
-
-.. toctree::
- :hidden:
- :maxdepth: 1
- :caption: Basics
-
- Home <self>
- Changelog <changelog>
- Security <security>
-
-.. toctree::
- :hidden:
- :maxdepth: 1
- :caption: References
-
- Configuration <configurations-ref>
- Python API <_api/airflow/providers/daskexecutor/index>
- PyPI Repository
<https://pypi.org/project/apache-airflow-providers-daskexecutor/>
- Installing from sources <installing-providers-from-sources>
-
-.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE
OVERWRITTEN AT RELEASE TIME!
-
-
-.. toctree::
- :hidden:
- :maxdepth: 1
- :caption: Commits
-
- Detailed list of commits <commits>
-
-
-Package apache-airflow-providers-daskexecutor
-------------------------------------------------------
-
-`Dask <https://www.dask.org/>`__
-
-
-Release: 1.1.1
-
-Provider package
-----------------
-
-This is a provider package for ``daskexecutor`` provider. All classes for this
provider package
-are in ``airflow.providers.daskexecutor`` python package.
-
- .. warning::
-
- This provider is not maintained anymore by the community. It has been
removed and is not going to be
- updated anymore. The removal was done according to the process
described in
- `Removing community providers
<https://github.com/apache/airflow/blob/main/PROVIDERS.rst#removing-community-providers>`_
-
- Feel free to contact Airflow Development Mailing List if you have any
questions.
-
-Installation
-------------
-
-You can install this package on top of an existing Airflow 2 installation (see
``Requirements`` below)
-for the minimum Airflow version supported) via
-``pip install apache-airflow-providers-daskexecutor``
-
-Requirements
-------------
-
-The minimum Apache Airflow version supported by this provider package is
``2.6.0``.
-
-================== ==================================
-PIP package Version required
-================== ==================================
-``apache-airflow`` ``>=2.5.0``
-``cloudpickle`` ``>=1.4.1``
-``dask`` ``>=2.9.0,!=2022.10.1,!=2023.5.0``
-``distributed`` ``>=2.11.1,!=2023.5.0``
-================== ==================================
diff --git
a/docs/apache-airflow-providers-daskexecutor/installing-providers-from-sources.rst
b/docs/apache-airflow-providers-daskexecutor/installing-providers-from-sources.rst
deleted file mode 100644
index b4e730f4ff..0000000000
---
a/docs/apache-airflow-providers-daskexecutor/installing-providers-from-sources.rst
+++ /dev/null
@@ -1,18 +0,0 @@
- .. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- .. http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-.. include:: ../exts/includes/installing-providers-from-sources.rst
diff --git a/docs/apache-airflow-providers-daskexecutor/security.rst
b/docs/apache-airflow-providers-daskexecutor/security.rst
deleted file mode 100644
index afa13dac6f..0000000000
--- a/docs/apache-airflow-providers-daskexecutor/security.rst
+++ /dev/null
@@ -1,18 +0,0 @@
- .. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- .. http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-.. include:: ../exts/includes/security.rst
diff --git a/docs/apache-airflow/core-concepts/executor/dask.rst
b/docs/apache-airflow/core-concepts/executor/dask.rst
deleted file mode 100644
index 117d396962..0000000000
--- a/docs/apache-airflow/core-concepts/executor/dask.rst
+++ /dev/null
@@ -1,56 +0,0 @@
- .. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- .. http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-
-.. _executor:DaskExecutor:
-
-Dask Executor
-=============
-
-:class:`airflow.providers.daskexecutor.executors.dask_executor.DaskExecutor`
allows you to run Airflow tasks in a Dask Distributed cluster.
-
-Dask clusters can be run on a single machine or on remote networks. For
complete
-details, consult the `Distributed documentation
<https://distributed.readthedocs.io/>`_.
-
-To create a cluster, first start a Scheduler:
-
-.. code-block:: bash
-
- # default settings for a local cluster
- DASK_HOST=127.0.0.1
- DASK_PORT=8786
-
- dask-scheduler --host $DASK_HOST --port $DASK_PORT
-
-Next start at least one Worker on any machine that can connect to the host:
-
-.. code-block:: bash
-
- dask-worker $DASK_HOST:$DASK_PORT
-
-Edit your ``airflow.cfg`` to set your executor to
:class:`airflow.providers.daskexecutor.executors.dask_executor.DaskExecutor`
and provide
-the Dask Scheduler address in the ``[dask]`` section. For more information on
setting the configuration,
-see :doc:`../../howto/set-config`.
-
-Please note:
-
-- Each Dask worker must be able to import Airflow and any dependencies you
- require.
-- The DaskExecutor implements queues using
- `Dask Worker Resources
<https://distributed.dask.org/en/latest/resources.html>`_ functionality. To
enable the use of
- queues, start your Dask workers with resources of the same name as the
desired queues and a limit of ``inf``.
- E.g. ``dask-worker <scheduler_address> --resources="QUEUE1=inf,QUEUE2=inf"``.
diff --git a/docs/apache-airflow/core-concepts/executor/index.rst
b/docs/apache-airflow/core-concepts/executor/index.rst
index 4a2761a0cf..e1cc52cf6f 100644
--- a/docs/apache-airflow/core-concepts/executor/index.rst
+++ b/docs/apache-airflow/core-concepts/executor/index.rst
@@ -63,7 +63,6 @@ There are two types of executors - those that run tasks
*locally* (inside the ``
celery
celery_kubernetes
- dask
kubernetes
local_kubernetes
diff --git a/docs/integration-logos/dask/dask.png
b/docs/integration-logos/dask/dask.png
deleted file mode 100644
index 1c2b27ee3d..0000000000
Binary files a/docs/integration-logos/dask/dask.png and /dev/null differ
diff --git a/images/breeze/output_build-docs.svg
b/images/breeze/output_build-docs.svg
index 9136e1258a..a63bbdbd32 100644
--- a/images/breeze/output_build-docs.svg
+++ b/images/breeze/output_build-docs.svg
@@ -1,4 +1,4 @@
-<svg class="rich-terminal" viewBox="0 0 1482 904.0"
xmlns="http://www.w3.org/2000/svg";>
+<svg class="rich-terminal" viewBox="0 0 1482 952.8"
xmlns="http://www.w3.org/2000/svg";>
<!-- Generated with Rich https://www.textualize.io -->
<style>
@@ -44,7 +44,7 @@
<defs>
<clipPath id="breeze-build-docs-clip-terminal">
- <rect x="0" y="0" width="1463.0" height="853.0" />
+ <rect x="0" y="0" width="1463.0" height="901.8" />
</clipPath>
<clipPath id="breeze-build-docs-line-0">
<rect x="0" y="1.5" width="1464" height="24.65"/>
@@ -148,9 +148,15 @@
<clipPath id="breeze-build-docs-line-33">
<rect x="0" y="806.7" width="1464" height="24.65"/>
</clipPath>
+<clipPath id="breeze-build-docs-line-34">
+ <rect x="0" y="831.1" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-build-docs-line-35">
+ <rect x="0" y="855.5" width="1464" height="24.65"/>
+ </clipPath>
</defs>
- <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1"
x="1" y="1" width="1480" height="902" rx="8"/><text
class="breeze-build-docs-title" fill="#c5c8c6" text-anchor="middle" x="740"
y="27">Command: build-docs</text>
+ <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1"
x="1" y="1" width="1480" height="950.8" rx="8"/><text
class="breeze-build-docs-title" fill="#c5c8c6" text-anchor="middle" x="740"
y="27">Command: build-docs</text>
<g transform="translate(26,22)">
<circle cx="0" cy="0" r="7" fill="#ff5f57"/>
<circle cx="22" cy="0" r="7" fill="#febc2e"/>
@@ -166,35 +172,37 @@
</text><text class="breeze-build-docs-r1" x="12.2" y="93.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-3)">[</text><text
class="breeze-build-docs-r4" x="24.4" y="93.2" textLength="85.4"
clip-path="url(#breeze-build-docs-line-3)">OPTIONS</text><text
class="breeze-build-docs-r1" x="109.8" y="93.2" textLength="36.6"
clip-path="url(#breeze-build-docs-line-3)">] [</text><text
class="breeze-build-docs-r4" x="146.4" y="93.2" textLength="85.4"
clip-path="url(#breeze-build-docs [...]
</text><text class="breeze-build-docs-r4" x="12.2" y="117.6" textLength="73.2"
clip-path="url(#breeze-build-docs-line-4)">apache</text><text
class="breeze-build-docs-r1" x="85.4" y="117.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-4)">.</text><text
class="breeze-build-docs-r4" x="97.6" y="117.6" textLength="109.8"
clip-path="url(#breeze-build-docs-line-4)">cassandra</text><text
class="breeze-build-docs-r1" x="207.4" y="117.6" textLength="36.6"
clip-path="url(#breeze-build- [...]
</text><text class="breeze-build-docs-r4" x="12.2" y="142" textLength="73.2"
clip-path="url(#breeze-build-docs-line-5)">apache</text><text
class="breeze-build-docs-r1" x="85.4" y="142" textLength="12.2"
clip-path="url(#breeze-build-docs-line-5)">.</text><text
class="breeze-build-docs-r4" x="97.6" y="142" textLength="61"
clip-path="url(#breeze-build-docs-line-5)">kafka</text><text
class="breeze-build-docs-r1" x="158.6" y="142" textLength="36.6"
clip-path="url(#breeze-build-docs-line-5)">& [...]
-</text><text class="breeze-build-docs-r4" x="12.2" y="166.4" textLength="97.6"
clip-path="url(#breeze-build-docs-line-6)">arangodb</text><text
class="breeze-build-docs-r1" x="109.8" y="166.4" textLength="36.6"
clip-path="url(#breeze-build-docs-line-6)"> | </text><text
class="breeze-build-docs-r4" x="146.4" y="166.4" textLength="61"
clip-path="url(#breeze-build-docs-line-6)">asana</text><text
class="breeze-build-docs-r1" x="207.4" y="166.4" textLength="36.6"
clip-path="url(#bree [...]
-</text><text class="breeze-build-docs-r4" x="12.2" y="190.8"
textLength="146.4"
clip-path="url(#breeze-build-docs-line-7)">daskexecutor</text><text
class="breeze-build-docs-r1" x="158.6" y="190.8" textLength="36.6"
clip-path="url(#breeze-build-docs-line-7)"> | </text><text
class="breeze-build-docs-r4" x="195.2" y="190.8" textLength="122"
clip-path="url(#breeze-build-docs-line-7)">databricks</text><text
class="breeze-build-docs-r1" x="317.2" y="190.8" textLength="36.6" clip-path
[...]
-</text><text class="breeze-build-docs-r1" x="12.2" y="215.2" textLength="24.4"
clip-path="url(#breeze-build-docs-line-8)">| </text><text
class="breeze-build-docs-r4" x="36.6" y="215.2" textLength="36.6"
clip-path="url(#breeze-build-docs-line-8)">fab</text><text
class="breeze-build-docs-r1" x="73.2" y="215.2" textLength="36.6"
clip-path="url(#breeze-build-docs-line-8)"> | </text><text
class="breeze-build-docs-r4" x="109.8" y="215.2" textLength="97.6"
clip-path="url(#breeze- [...]
-</text><text class="breeze-build-docs-r4" x="12.2" y="239.6"
textLength="109.8"
clip-path="url(#breeze-build-docs-line-9)">microsoft</text><text
class="breeze-build-docs-r1" x="122" y="239.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-9)">.</text><text
class="breeze-build-docs-r4" x="134.2" y="239.6" textLength="61"
clip-path="url(#breeze-build-docs-line-9)">azure</text><text
class="breeze-build-docs-r1" x="195.2" y="239.6" textLength="36.6"
clip-path="url(#breeze-build-doc [...]
-</text><text class="breeze-build-docs-r4" x="12.2" y="264" textLength="97.6"
clip-path="url(#breeze-build-docs-line-10)">openfaas</text><text
class="breeze-build-docs-r1" x="109.8" y="264" textLength="36.6"
clip-path="url(#breeze-build-docs-line-10)"> | </text><text
class="breeze-build-docs-r4" x="146.4" y="264" textLength="134.2"
clip-path="url(#breeze-build-docs-line-10)">openlineage</text><text
class="breeze-build-docs-r1" x="280.6" y="264" textLength="36.6"
clip-path="url(# [...]
-</text><text class="breeze-build-docs-r4" x="12.2" y="288.4" textLength="97.6"
clip-path="url(#breeze-build-docs-line-11)">postgres</text><text
class="breeze-build-docs-r1" x="109.8" y="288.4" textLength="36.6"
clip-path="url(#breeze-build-docs-line-11)"> | </text><text
class="breeze-build-docs-r4" x="146.4" y="288.4" textLength="73.2"
clip-path="url(#breeze-build-docs-line-11)">presto</text><text
class="breeze-build-docs-r1" x="219.6" y="288.4" textLength="36.6"
clip-path="url [...]
-</text><text class="breeze-build-docs-r4" x="12.2" y="312.8" textLength="73.2"
clip-path="url(#breeze-build-docs-line-12)">sqlite</text><text
class="breeze-build-docs-r1" x="85.4" y="312.8" textLength="36.6"
clip-path="url(#breeze-build-docs-line-12)"> | </text><text
class="breeze-build-docs-r4" x="122" y="312.8" textLength="36.6"
clip-path="url(#breeze-build-docs-line-12)">ssh</text><text
class="breeze-build-docs-r1" x="158.6" y="312.8" textLength="36.6"
clip-path="url(#breeze [...]
+</text><text class="breeze-build-docs-r4" x="12.2" y="166.4" textLength="97.6"
clip-path="url(#breeze-build-docs-line-6)">arangodb</text><text
class="breeze-build-docs-r1" x="109.8" y="166.4" textLength="36.6"
clip-path="url(#breeze-build-docs-line-6)"> | </text><text
class="breeze-build-docs-r4" x="146.4" y="166.4" textLength="61"
clip-path="url(#breeze-build-docs-line-6)">asana</text><text
class="breeze-build-docs-r1" x="207.4" y="166.4" textLength="36.6"
clip-path="url(#bree [...]
+</text><text class="breeze-build-docs-r1" x="12.2" y="190.8" textLength="24.4"
clip-path="url(#breeze-build-docs-line-7)">| </text><text
class="breeze-build-docs-r4" x="36.6" y="190.8" textLength="85.4"
clip-path="url(#breeze-build-docs-line-7)">datadog</text><text
class="breeze-build-docs-r1" x="122" y="190.8" textLength="36.6"
clip-path="url(#breeze-build-docs-line-7)"> | </text><text
class="breeze-build-docs-r4" x="158.6" y="190.8" textLength="36.6"
clip-path="url(#bree [...]
+</text><text class="breeze-build-docs-r4" x="12.2" y="215.2" textLength="73.2"
clip-path="url(#breeze-build-docs-line-8)">github</text><text
class="breeze-build-docs-r1" x="85.4" y="215.2" textLength="36.6"
clip-path="url(#breeze-build-docs-line-8)"> | </text><text
class="breeze-build-docs-r4" x="122" y="215.2" textLength="73.2"
clip-path="url(#breeze-build-docs-line-8)">google</text><text
class="breeze-build-docs-r1" x="195.2" y="215.2" textLength="36.6"
clip-path="url(#breeze [...]
+</text><text class="breeze-build-docs-r4" x="12.2" y="239.6"
textLength="109.8"
clip-path="url(#breeze-build-docs-line-9)">microsoft</text><text
class="breeze-build-docs-r1" x="122" y="239.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-9)">.</text><text
class="breeze-build-docs-r4" x="134.2" y="239.6" textLength="61"
clip-path="url(#breeze-build-docs-line-9)">mssql</text><text
class="breeze-build-docs-r1" x="195.2" y="239.6" textLength="36.6"
clip-path="url(#breeze-build-doc [...]
+</text><text class="breeze-build-docs-r4" x="12.2" y="264" textLength="122"
clip-path="url(#breeze-build-docs-line-10)">opensearch</text><text
class="breeze-build-docs-r1" x="134.2" y="264" textLength="36.6"
clip-path="url(#breeze-build-docs-line-10)"> | </text><text
class="breeze-build-docs-r4" x="170.8" y="264" textLength="97.6"
clip-path="url(#breeze-build-docs-line-10)">opsgenie</text><text
class="breeze-build-docs-r1" x="268.4" y="264" textLength="36.6"
clip-path="url(#bre [...]
+</text><text class="breeze-build-docs-r4" x="12.2" y="288.4" textLength="122"
clip-path="url(#breeze-build-docs-line-11)">salesforce</text><text
class="breeze-build-docs-r1" x="134.2" y="288.4" textLength="36.6"
clip-path="url(#breeze-build-docs-line-11)"> | </text><text
class="breeze-build-docs-r4" x="170.8" y="288.4" textLength="61"
clip-path="url(#breeze-build-docs-line-11)">samba</text><text
class="breeze-build-docs-r1" x="231.8" y="288.4" textLength="36.6"
clip-path="url(# [...]
+</text><text class="breeze-build-docs-r4" x="12.2" y="312.8" textLength="85.4"
clip-path="url(#breeze-build-docs-line-12)">tabular</text><text
class="breeze-build-docs-r1" x="97.6" y="312.8" textLength="36.6"
clip-path="url(#breeze-build-docs-line-12)"> | </text><text
class="breeze-build-docs-r4" x="134.2" y="312.8" textLength="97.6"
clip-path="url(#breeze-build-docs-line-12)">telegram</text><text
class="breeze-build-docs-r1" x="231.8" y="312.8" textLength="36.6"
clip-path="url [...]
</text><text class="breeze-build-docs-r1" x="1464" y="337.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-13)">
</text><text class="breeze-build-docs-r1" x="12.2" y="361.6"
textLength="195.2"
clip-path="url(#breeze-build-docs-line-14)">Build documents.</text><text
class="breeze-build-docs-r1" x="1464" y="361.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-14)">
</text><text class="breeze-build-docs-r1" x="1464" y="386" textLength="12.2"
clip-path="url(#breeze-build-docs-line-15)">
</text><text class="breeze-build-docs-r5" x="0" y="410.4" textLength="24.4"
clip-path="url(#breeze-build-docs-line-16)">╭─</text><text
class="breeze-build-docs-r5" x="24.4" y="410.4" textLength="134.2"
clip-path="url(#breeze-build-docs-line-16)"> Doc flags </text><text
class="breeze-build-docs-r5" x="158.6" y="410.4" textLength="1281"
clip-path="url(#breeze-build-docs-line-16)">────────────────────────────────────────────────────────────────────────────────────────────────
[...]
-</text><text class="breeze-build-docs-r5" x="0" y="434.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-17)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="434.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-17)">-</text><text
class="breeze-build-docs-r4" x="36.6" y="434.8" textLength="61"
clip-path="url(#breeze-build-docs-line-17)">-docs</text><text
class="breeze-build-docs-r4" x="97.6" y="434.8" textLength="61"
clip-path="url(#breeze-build-docs-line-17)"> [...]
-</text><text class="breeze-build-docs-r5" x="0" y="459.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-18)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="459.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-18)">-</text><text
class="breeze-build-docs-r4" x="36.6" y="459.2" textLength="134.2"
clip-path="url(#breeze-build-docs-line-18)">-spellcheck</text><text
class="breeze-build-docs-r4" x="170.8" y="459.2" textLength="61"
clip-path="url(#breeze-build-docs- [...]
-</text><text class="breeze-build-docs-r5" x="0" y="483.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-19)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="483.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-19)">-</text><text
class="breeze-build-docs-r4" x="36.6" y="483.6" textLength="73.2"
clip-path="url(#breeze-build-docs-line-19)">-clean</text><text
class="breeze-build-docs-r4" x="109.8" y="483.6" textLength="73.2"
clip-path="url(#breeze-build-docs-line [...]
-</text><text class="breeze-build-docs-r5" x="0" y="508" textLength="12.2"
clip-path="url(#breeze-build-docs-line-20)">│</text><text
class="breeze-build-docs-r1" x="329.4" y="508" textLength="1110.2"
clip-path="url(#breeze-build-docs-line-20)">before the build - useful for a clean build.                           
[...]
-</text><text class="breeze-build-docs-r5" x="0" y="532.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-21)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="532.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-21)">-</text><text
class="breeze-build-docs-r4" x="36.6" y="532.4" textLength="48.8"
clip-path="url(#breeze-build-docs-line-21)">-one</text><text
class="breeze-build-docs-r4" x="85.4" y="532.4" textLength="122"
clip-path="url(#breeze-build-docs-line-21) [...]
-</text><text class="breeze-build-docs-r5" x="0" y="556.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-22)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="556.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-22)">-</text><text
class="breeze-build-docs-r4" x="36.6" y="556.8" textLength="97.6"
clip-path="url(#breeze-build-docs-line-22)">-package</text><text
class="breeze-build-docs-r4" x="134.2" y="556.8" textLength="85.4"
clip-path="url(#breeze-build-docs-li [...]
-</text><text class="breeze-build-docs-r5" x="0" y="581.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-23)">│</text><text
class="breeze-build-docs-r1" x="329.4" y="581.2" textLength="305"
clip-path="url(#breeze-build-docs-line-23)">apache-airflow-providers-</text><text
class="breeze-build-docs-r7" x="634.4" y="581.2" textLength="122"
clip-path="url(#breeze-build-docs-line-23)"><provider></text><text
class="breeze-build-docs-r1" x="756.4" y="581.2" textLength="683.2" cli [...]
-</text><text class="breeze-build-docs-r5" x="0" y="605.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-24)">│</text><text
class="breeze-build-docs-r1" x="329.4" y="605.6" textLength="780.8"
clip-path="url(#breeze-build-docs-line-24)">full package name. The list of short hand names can be found in </text><text
class="breeze-build-docs-r4" x="1110.2" y="605.6" textLength="12.2"
clip-path="url(#breeze-build-docs-li [...]
-</text><text class="breeze-build-docs-r5" x="0" y="630" textLength="12.2"
clip-path="url(#breeze-build-docs-line-25)">│</text><text
class="breeze-build-docs-r8" x="329.4" y="630" textLength="1110.2"
clip-path="url(#breeze-build-docs-line-25)">(TEXT)                                        
[...]
-</text><text class="breeze-build-docs-r5" x="0" y="654.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-26)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="654.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-26)">-</text><text
class="breeze-build-docs-r4" x="36.6" y="654.4" textLength="85.4"
clip-path="url(#breeze-build-docs-line-26)">-github</text><text
class="breeze-build-docs-r4" x="122" y="654.4" textLength="134.2"
clip-path="url(#breeze-build-docs-line [...]
-</text><text class="breeze-build-docs-r5" x="0" y="678.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-27)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="678.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-27)">-</text><text
class="breeze-build-docs-r4" x="36.6" y="678.8" textLength="97.6"
clip-path="url(#breeze-build-docs-line-27)">-builder</text><text
class="breeze-build-docs-r1" x="329.4" y="678.8" textLength="756.4"
clip-path="url(#breeze-build-docs-l [...]
-</text><text class="breeze-build-docs-r5" x="0" y="703.2" textLength="1464"
clip-path="url(#breeze-build-docs-line-28)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-build-docs-r1" x="1464" y="703.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-28)">
-</text><text class="breeze-build-docs-r5" x="0" y="727.6" textLength="24.4"
clip-path="url(#breeze-build-docs-line-29)">╭─</text><text
class="breeze-build-docs-r5" x="24.4" y="727.6" textLength="195.2"
clip-path="url(#breeze-build-docs-line-29)"> Common options </text><text
class="breeze-build-docs-r5" x="219.6" y="727.6" textLength="1220"
clip-path="url(#breeze-build-docs-line-29)">───────────────────────────────────────────────────────────────────────────────────────────
[...]
-</text><text class="breeze-build-docs-r5" x="0" y="752" textLength="12.2"
clip-path="url(#breeze-build-docs-line-30)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="752" textLength="12.2"
clip-path="url(#breeze-build-docs-line-30)">-</text><text
class="breeze-build-docs-r4" x="36.6" y="752" textLength="48.8"
clip-path="url(#breeze-build-docs-line-30)">-dry</text><text
class="breeze-build-docs-r4" x="85.4" y="752" textLength="48.8"
clip-path="url(#breeze-build-docs-line-30)">-run< [...]
-</text><text class="breeze-build-docs-r5" x="0" y="776.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-31)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="776.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-31)">-</text><text
class="breeze-build-docs-r4" x="36.6" y="776.4" textLength="97.6"
clip-path="url(#breeze-build-docs-line-31)">-include</text><text
class="breeze-build-docs-r4" x="134.2" y="776.4" textLength="219.6"
clip-path="url(#breeze-build-docs-l [...]
-</text><text class="breeze-build-docs-r5" x="0" y="800.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-32)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="800.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-32)">-</text><text
class="breeze-build-docs-r4" x="36.6" y="800.8" textLength="97.6"
clip-path="url(#breeze-build-docs-line-32)">-verbose</text><text
class="breeze-build-docs-r6" x="378.2" y="800.8" textLength="24.4"
clip-path="url(#breeze-build-docs-li [...]
-</text><text class="breeze-build-docs-r5" x="0" y="825.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-33)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="825.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-33)">-</text><text
class="breeze-build-docs-r4" x="36.6" y="825.2" textLength="61"
clip-path="url(#breeze-build-docs-line-33)">-help</text><text
class="breeze-build-docs-r6" x="378.2" y="825.2" textLength="24.4"
clip-path="url(#breeze-build-docs-line-33 [...]
-</text><text class="breeze-build-docs-r5" x="0" y="849.6" textLength="1464"
clip-path="url(#breeze-build-docs-line-34)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-build-docs-r1" x="1464" y="849.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-34)">
+</text><text class="breeze-build-docs-r5" x="0" y="434.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-17)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="434.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-17)">-</text><text
class="breeze-build-docs-r4" x="36.6" y="434.8" textLength="61"
clip-path="url(#breeze-build-docs-line-17)">-docs</text><text
class="breeze-build-docs-r4" x="97.6" y="434.8" textLength="61"
clip-path="url(#breeze-build-docs-line-17)"> [...]
+</text><text class="breeze-build-docs-r5" x="0" y="459.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-18)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="459.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-18)">-</text><text
class="breeze-build-docs-r4" x="36.6" y="459.2" textLength="134.2"
clip-path="url(#breeze-build-docs-line-18)">-spellcheck</text><text
class="breeze-build-docs-r4" x="170.8" y="459.2" textLength="61"
clip-path="url(#breeze-build-docs- [...]
+</text><text class="breeze-build-docs-r5" x="0" y="483.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-19)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="483.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-19)">-</text><text
class="breeze-build-docs-r4" x="36.6" y="483.6" textLength="73.2"
clip-path="url(#breeze-build-docs-line-19)">-clean</text><text
class="breeze-build-docs-r4" x="109.8" y="483.6" textLength="73.2"
clip-path="url(#breeze-build-docs-line [...]
+</text><text class="breeze-build-docs-r5" x="0" y="508" textLength="12.2"
clip-path="url(#breeze-build-docs-line-20)">│</text><text
class="breeze-build-docs-r1" x="427" y="508" textLength="1012.6"
clip-path="url(#breeze-build-docs-line-20)">artifacts before the build - useful for a clean build.                         &#
[...]
+</text><text class="breeze-build-docs-r5" x="0" y="532.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-21)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="532.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-21)">-</text><text
class="breeze-build-docs-r4" x="36.6" y="532.4" textLength="48.8"
clip-path="url(#breeze-build-docs-line-21)">-one</text><text
class="breeze-build-docs-r4" x="85.4" y="532.4" textLength="122"
clip-path="url(#breeze-build-docs-line-21) [...]
+</text><text class="breeze-build-docs-r5" x="0" y="556.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-22)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="556.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-22)">-</text><text
class="breeze-build-docs-r4" x="36.6" y="556.8" textLength="97.6"
clip-path="url(#breeze-build-docs-line-22)">-package</text><text
class="breeze-build-docs-r4" x="134.2" y="556.8" textLength="85.4"
clip-path="url(#breeze-build-docs-li [...]
+</text><text class="breeze-build-docs-r5" x="0" y="581.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-23)">│</text><text
class="breeze-build-docs-r1" x="427" y="581.2" textLength="305"
clip-path="url(#breeze-build-docs-line-23)">apache-airflow-providers-</text><text
class="breeze-build-docs-r7" x="732" y="581.2" textLength="122"
clip-path="url(#breeze-build-docs-line-23)"><provider></text><text
class="breeze-build-docs-r1" x="854" y="581.2" textLength="585.6" clip-path
[...]
+</text><text class="breeze-build-docs-r5" x="0" y="605.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-24)">│</text><text
class="breeze-build-docs-r1" x="427" y="605.6" textLength="939.4"
clip-path="url(#breeze-build-docs-line-24)">matching the full package name. The list of short hand names can be found in </text><text
class="breeze-build-docs-r4" x="1366.4" y="605.6" textLength="12.2"
clip-path="url( [...]
+</text><text class="breeze-build-docs-r5" x="0" y="630" textLength="12.2"
clip-path="url(#breeze-build-docs-line-25)">│</text><text
class="breeze-build-docs-r1" x="427" y="630" textLength="1012.6"
clip-path="url(#breeze-build-docs-line-25)">output                                         
[...]
+</text><text class="breeze-build-docs-r5" x="0" y="654.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-26)">│</text><text
class="breeze-build-docs-r8" x="427" y="654.4" textLength="1012.6"
clip-path="url(#breeze-build-docs-line-26)">(TEXT)                                        &
[...]
+</text><text class="breeze-build-docs-r5" x="0" y="678.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-27)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="678.8" textLength="12.2"
clip-path="url(#breeze-build-docs-line-27)">-</text><text
class="breeze-build-docs-r4" x="36.6" y="678.8" textLength="97.6"
clip-path="url(#breeze-build-docs-line-27)">-include</text><text
class="breeze-build-docs-r4" x="134.2" y="678.8" textLength="219.6"
clip-path="url(#breeze-build-docs-l [...]
+</text><text class="breeze-build-docs-r5" x="0" y="703.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-28)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="703.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-28)">-</text><text
class="breeze-build-docs-r4" x="36.6" y="703.2" textLength="85.4"
clip-path="url(#breeze-build-docs-line-28)">-github</text><text
class="breeze-build-docs-r4" x="122" y="703.2" textLength="134.2"
clip-path="url(#breeze-build-docs-line [...]
+</text><text class="breeze-build-docs-r5" x="0" y="727.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-29)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="727.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-29)">-</text><text
class="breeze-build-docs-r4" x="36.6" y="727.6" textLength="97.6"
clip-path="url(#breeze-build-docs-line-29)">-builder</text><text
class="breeze-build-docs-r1" x="427" y="727.6" textLength="756.4"
clip-path="url(#breeze-build-docs-lin [...]
+</text><text class="breeze-build-docs-r5" x="0" y="752" textLength="12.2"
clip-path="url(#breeze-build-docs-line-30)">│</text><text
class="breeze-build-docs-r5" x="427" y="752" textLength="756.4"
clip-path="url(#breeze-build-docs-line-30)">[default: autodetect]                                     
[...]
+</text><text class="breeze-build-docs-r5" x="0" y="776.4" textLength="1464"
clip-path="url(#breeze-build-docs-line-31)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-build-docs-r1" x="1464" y="776.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-31)">
+</text><text class="breeze-build-docs-r5" x="0" y="800.8" textLength="24.4"
clip-path="url(#breeze-build-docs-line-32)">╭─</text><text
class="breeze-build-docs-r5" x="24.4" y="800.8" textLength="195.2"
clip-path="url(#breeze-build-docs-line-32)"> Common options </text><text
class="breeze-build-docs-r5" x="219.6" y="800.8" textLength="1220"
clip-path="url(#breeze-build-docs-line-32)">───────────────────────────────────────────────────────────────────────────────────────────
[...]
+</text><text class="breeze-build-docs-r5" x="0" y="825.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-33)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="825.2" textLength="12.2"
clip-path="url(#breeze-build-docs-line-33)">-</text><text
class="breeze-build-docs-r4" x="36.6" y="825.2" textLength="48.8"
clip-path="url(#breeze-build-docs-line-33)">-dry</text><text
class="breeze-build-docs-r4" x="85.4" y="825.2" textLength="48.8"
clip-path="url(#breeze-build-docs-line-33 [...]
+</text><text class="breeze-build-docs-r5" x="0" y="849.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-34)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="849.6" textLength="12.2"
clip-path="url(#breeze-build-docs-line-34)">-</text><text
class="breeze-build-docs-r4" x="36.6" y="849.6" textLength="97.6"
clip-path="url(#breeze-build-docs-line-34)">-verbose</text><text
class="breeze-build-docs-r6" x="158.6" y="849.6" textLength="24.4"
clip-path="url(#breeze-build-docs-li [...]
+</text><text class="breeze-build-docs-r5" x="0" y="874" textLength="12.2"
clip-path="url(#breeze-build-docs-line-35)">│</text><text
class="breeze-build-docs-r4" x="24.4" y="874" textLength="12.2"
clip-path="url(#breeze-build-docs-line-35)">-</text><text
class="breeze-build-docs-r4" x="36.6" y="874" textLength="61"
clip-path="url(#breeze-build-docs-line-35)">-help</text><text
class="breeze-build-docs-r6" x="158.6" y="874" textLength="24.4"
clip-path="url(#breeze-build-docs-line-35)">-h</t [...]
+</text><text class="breeze-build-docs-r5" x="0" y="898.4" textLength="1464"
clip-path="url(#breeze-build-docs-line-36)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-build-docs-r1" x="1464" y="898.4" textLength="12.2"
clip-path="url(#breeze-build-docs-line-36)">
</text>
</g>
</g>
diff --git a/images/breeze/output_build-docs.txt
b/images/breeze/output_build-docs.txt
index bf6cfd993c..880c1685d3 100644
--- a/images/breeze/output_build-docs.txt
+++ b/images/breeze/output_build-docs.txt
@@ -1 +1 @@
-335114f1080ed402060f67bc5c19a32e
+57cc85fdf8e35e5adb918676b18af7e8
diff --git a/images/breeze/output_release-management_add-back-references.svg
b/images/breeze/output_release-management_add-back-references.svg
index 506e7a43e8..d07c693926 100644
--- a/images/breeze/output_release-management_add-back-references.svg
+++ b/images/breeze/output_release-management_add-back-references.svg
@@ -137,13 +137,13 @@
</text><text class="breeze-release-management-add-back-references-r1" x="12.2"
y="93.2" textLength="12.2"
clip-path="url(#breeze-release-management-add-back-references-line-3)">[</text><text
class="breeze-release-management-add-back-references-r4" x="24.4" y="93.2"
textLength="85.4"
clip-path="url(#breeze-release-management-add-back-references-line-3)">OPTIONS</text><text
class="breeze-release-management-add-back-references-r1" x="109.8" y="93.2"
textLength="36.6" clip-path="url(#breeze- [...]
</text><text class="breeze-release-management-add-back-references-r4" x="12.2"
y="117.6" textLength="73.2"
clip-path="url(#breeze-release-management-add-back-references-line-4)">apache</text><text
class="breeze-release-management-add-back-references-r1" x="85.4" y="117.6"
textLength="12.2"
clip-path="url(#breeze-release-management-add-back-references-line-4)">.</text><text
class="breeze-release-management-add-back-references-r4" x="97.6" y="117.6"
textLength="109.8" clip-path="url(#breez [...]
</text><text class="breeze-release-management-add-back-references-r4" x="12.2"
y="142" textLength="73.2"
clip-path="url(#breeze-release-management-add-back-references-line-5)">apache</text><text
class="breeze-release-management-add-back-references-r1" x="85.4" y="142"
textLength="12.2"
clip-path="url(#breeze-release-management-add-back-references-line-5)">.</text><text
class="breeze-release-management-add-back-references-r4" x="97.6" y="142"
textLength="61" clip-path="url(#breeze-release [...]
-</text><text class="breeze-release-management-add-back-references-r4" x="12.2"
y="166.4" textLength="97.6"
clip-path="url(#breeze-release-management-add-back-references-line-6)">arangodb</text><text
class="breeze-release-management-add-back-references-r1" x="109.8" y="166.4"
textLength="36.6"
clip-path="url(#breeze-release-management-add-back-references-line-6)"> | </text><text
class="breeze-release-management-add-back-references-r4" x="146.4" y="166.4"
textLength="61" clip-pat [...]
-</text><text class="breeze-release-management-add-back-references-r4" x="12.2"
y="190.8" textLength="146.4"
clip-path="url(#breeze-release-management-add-back-references-line-7)">daskexecutor</text><text
class="breeze-release-management-add-back-references-r1" x="158.6" y="190.8"
textLength="36.6"
clip-path="url(#breeze-release-management-add-back-references-line-7)"> | </text><text
class="breeze-release-management-add-back-references-r4" x="195.2" y="190.8"
textLength="122" cl [...]
-</text><text class="breeze-release-management-add-back-references-r1" x="12.2"
y="215.2" textLength="24.4"
clip-path="url(#breeze-release-management-add-back-references-line-8)">| </text><text
class="breeze-release-management-add-back-references-r4" x="36.6" y="215.2"
textLength="36.6"
clip-path="url(#breeze-release-management-add-back-references-line-8)">fab</text><text
class="breeze-release-management-add-back-references-r1" x="73.2" y="215.2"
textLength="36.6" clip-path="url(#bre [...]
-</text><text class="breeze-release-management-add-back-references-r4" x="12.2"
y="239.6" textLength="109.8"
clip-path="url(#breeze-release-management-add-back-references-line-9)">microsoft</text><text
class="breeze-release-management-add-back-references-r1" x="122" y="239.6"
textLength="12.2"
clip-path="url(#breeze-release-management-add-back-references-line-9)">.</text><text
class="breeze-release-management-add-back-references-r4" x="134.2" y="239.6"
textLength="61" clip-path="url(#bree [...]
-</text><text class="breeze-release-management-add-back-references-r4" x="12.2"
y="264" textLength="97.6"
clip-path="url(#breeze-release-management-add-back-references-line-10)">openfaas</text><text
class="breeze-release-management-add-back-references-r1" x="109.8" y="264"
textLength="36.6"
clip-path="url(#breeze-release-management-add-back-references-line-10)"> | </text><text
class="breeze-release-management-add-back-references-r4" x="146.4" y="264"
textLength="134.2" clip-path [...]
-</text><text class="breeze-release-management-add-back-references-r4" x="12.2"
y="288.4" textLength="97.6"
clip-path="url(#breeze-release-management-add-back-references-line-11)">postgres</text><text
class="breeze-release-management-add-back-references-r1" x="109.8" y="288.4"
textLength="36.6"
clip-path="url(#breeze-release-management-add-back-references-line-11)"> | </text><text
class="breeze-release-management-add-back-references-r4" x="146.4" y="288.4"
textLength="73.2" clip [...]
-</text><text class="breeze-release-management-add-back-references-r4" x="12.2"
y="312.8" textLength="73.2"
clip-path="url(#breeze-release-management-add-back-references-line-12)">sqlite</text><text
class="breeze-release-management-add-back-references-r1" x="85.4" y="312.8"
textLength="36.6"
clip-path="url(#breeze-release-management-add-back-references-line-12)"> | </text><text
class="breeze-release-management-add-back-references-r4" x="122" y="312.8"
textLength="36.6" clip-path [...]
+</text><text class="breeze-release-management-add-back-references-r4" x="12.2"
y="166.4" textLength="97.6"
clip-path="url(#breeze-release-management-add-back-references-line-6)">arangodb</text><text
class="breeze-release-management-add-back-references-r1" x="109.8" y="166.4"
textLength="36.6"
clip-path="url(#breeze-release-management-add-back-references-line-6)"> | </text><text
class="breeze-release-management-add-back-references-r4" x="146.4" y="166.4"
textLength="61" clip-pat [...]
+</text><text class="breeze-release-management-add-back-references-r1" x="12.2"
y="190.8" textLength="24.4"
clip-path="url(#breeze-release-management-add-back-references-line-7)">| </text><text
class="breeze-release-management-add-back-references-r4" x="36.6" y="190.8"
textLength="85.4"
clip-path="url(#breeze-release-management-add-back-references-line-7)">datadog</text><text
class="breeze-release-management-add-back-references-r1" x="122" y="190.8"
textLength="36.6" clip-path="url(# [...]
+</text><text class="breeze-release-management-add-back-references-r4" x="12.2"
y="215.2" textLength="73.2"
clip-path="url(#breeze-release-management-add-back-references-line-8)">github</text><text
class="breeze-release-management-add-back-references-r1" x="85.4" y="215.2"
textLength="36.6"
clip-path="url(#breeze-release-management-add-back-references-line-8)"> | </text><text
class="breeze-release-management-add-back-references-r4" x="122" y="215.2"
textLength="73.2" clip-path=" [...]
+</text><text class="breeze-release-management-add-back-references-r4" x="12.2"
y="239.6" textLength="109.8"
clip-path="url(#breeze-release-management-add-back-references-line-9)">microsoft</text><text
class="breeze-release-management-add-back-references-r1" x="122" y="239.6"
textLength="12.2"
clip-path="url(#breeze-release-management-add-back-references-line-9)">.</text><text
class="breeze-release-management-add-back-references-r4" x="134.2" y="239.6"
textLength="61" clip-path="url(#bree [...]
+</text><text class="breeze-release-management-add-back-references-r4" x="12.2"
y="264" textLength="122"
clip-path="url(#breeze-release-management-add-back-references-line-10)">opensearch</text><text
class="breeze-release-management-add-back-references-r1" x="134.2" y="264"
textLength="36.6"
clip-path="url(#breeze-release-management-add-back-references-line-10)"> | </text><text
class="breeze-release-management-add-back-references-r4" x="170.8" y="264"
textLength="97.6" clip-path [...]
+</text><text class="breeze-release-management-add-back-references-r4" x="12.2"
y="288.4" textLength="122"
clip-path="url(#breeze-release-management-add-back-references-line-11)">salesforce</text><text
class="breeze-release-management-add-back-references-r1" x="134.2" y="288.4"
textLength="36.6"
clip-path="url(#breeze-release-management-add-back-references-line-11)"> | </text><text
class="breeze-release-management-add-back-references-r4" x="170.8" y="288.4"
textLength="61" clip- [...]
+</text><text class="breeze-release-management-add-back-references-r4" x="12.2"
y="312.8" textLength="85.4"
clip-path="url(#breeze-release-management-add-back-references-line-12)">tabular</text><text
class="breeze-release-management-add-back-references-r1" x="97.6" y="312.8"
textLength="36.6"
clip-path="url(#breeze-release-management-add-back-references-line-12)"> | </text><text
class="breeze-release-management-add-back-references-r4" x="134.2" y="312.8"
textLength="97.6" clip-p [...]
</text><text class="breeze-release-management-add-back-references-r1" x="1464"
y="337.2" textLength="12.2"
clip-path="url(#breeze-release-management-add-back-references-line-13)">
</text><text class="breeze-release-management-add-back-references-r1" x="12.2"
y="361.6" textLength="976"
clip-path="url(#breeze-release-management-add-back-references-line-14)">Command to add back references for documentation to make it backward compatible.</text><text
class="breeze-release-management-add-back-references-r1" x="1464" y="361.6"
textLength="12.2"
clip-path="url(#breeze-release-management-add-back-references-line-14)">
</text><text class="breeze-release-management-add-back-references-r1" x="1464"
y="386" textLength="12.2"
clip-path="url(#breeze-release-management-add-back-references-line-15)">
diff --git a/images/breeze/output_release-management_add-back-references.txt
b/images/breeze/output_release-management_add-back-references.txt
index 64adf9a576..801ac1f17f 100644
--- a/images/breeze/output_release-management_add-back-references.txt
+++ b/images/breeze/output_release-management_add-back-references.txt
@@ -1 +1 @@
-a98201199552183fd01ecf818651071e
+2c773f47446b3919f95ba5cc9a97bcdf
diff --git a/images/breeze/output_release-management_publish-docs.svg
b/images/breeze/output_release-management_publish-docs.svg
index bbe9f28789..d9eee5bacd 100644
--- a/images/breeze/output_release-management_publish-docs.svg
+++ b/images/breeze/output_release-management_publish-docs.svg
@@ -183,13 +183,13 @@
</text><text class="breeze-release-management-publish-docs-r1" x="12.2"
y="93.2" textLength="12.2"
clip-path="url(#breeze-release-management-publish-docs-line-3)">[</text><text
class="breeze-release-management-publish-docs-r4" x="24.4" y="93.2"
textLength="85.4"
clip-path="url(#breeze-release-management-publish-docs-line-3)">OPTIONS</text><text
class="breeze-release-management-publish-docs-r1" x="109.8" y="93.2"
textLength="36.6" clip-path="url(#breeze-release-management-publish-docs-lin
[...]
</text><text class="breeze-release-management-publish-docs-r4" x="12.2"
y="117.6" textLength="73.2"
clip-path="url(#breeze-release-management-publish-docs-line-4)">apache</text><text
class="breeze-release-management-publish-docs-r1" x="85.4" y="117.6"
textLength="12.2"
clip-path="url(#breeze-release-management-publish-docs-line-4)">.</text><text
class="breeze-release-management-publish-docs-r4" x="97.6" y="117.6"
textLength="109.8" clip-path="url(#breeze-release-management-publish-docs-l
[...]
</text><text class="breeze-release-management-publish-docs-r4" x="12.2"
y="142" textLength="73.2"
clip-path="url(#breeze-release-management-publish-docs-line-5)">apache</text><text
class="breeze-release-management-publish-docs-r1" x="85.4" y="142"
textLength="12.2"
clip-path="url(#breeze-release-management-publish-docs-line-5)">.</text><text
class="breeze-release-management-publish-docs-r4" x="97.6" y="142"
textLength="61"
clip-path="url(#breeze-release-management-publish-docs-line-5)">k [...]
-</text><text class="breeze-release-management-publish-docs-r4" x="12.2"
y="166.4" textLength="97.6"
clip-path="url(#breeze-release-management-publish-docs-line-6)">arangodb</text><text
class="breeze-release-management-publish-docs-r1" x="109.8" y="166.4"
textLength="36.6"
clip-path="url(#breeze-release-management-publish-docs-line-6)"> | </text><text
class="breeze-release-management-publish-docs-r4" x="146.4" y="166.4"
textLength="61" clip-path="url(#breeze-release-management-p [...]
-</text><text class="breeze-release-management-publish-docs-r4" x="12.2"
y="190.8" textLength="146.4"
clip-path="url(#breeze-release-management-publish-docs-line-7)">daskexecutor</text><text
class="breeze-release-management-publish-docs-r1" x="158.6" y="190.8"
textLength="36.6"
clip-path="url(#breeze-release-management-publish-docs-line-7)"> | </text><text
class="breeze-release-management-publish-docs-r4" x="195.2" y="190.8"
textLength="122" clip-path="url(#breeze-release-manage [...]
-</text><text class="breeze-release-management-publish-docs-r1" x="12.2"
y="215.2" textLength="24.4"
clip-path="url(#breeze-release-management-publish-docs-line-8)">| </text><text
class="breeze-release-management-publish-docs-r4" x="36.6" y="215.2"
textLength="36.6"
clip-path="url(#breeze-release-management-publish-docs-line-8)">fab</text><text
class="breeze-release-management-publish-docs-r1" x="73.2" y="215.2"
textLength="36.6" clip-path="url(#breeze-release-management-publish-docs [...]
-</text><text class="breeze-release-management-publish-docs-r4" x="12.2"
y="239.6" textLength="109.8"
clip-path="url(#breeze-release-management-publish-docs-line-9)">microsoft</text><text
class="breeze-release-management-publish-docs-r1" x="122" y="239.6"
textLength="12.2"
clip-path="url(#breeze-release-management-publish-docs-line-9)">.</text><text
class="breeze-release-management-publish-docs-r4" x="134.2" y="239.6"
textLength="61" clip-path="url(#breeze-release-management-publish-docs- [...]
-</text><text class="breeze-release-management-publish-docs-r4" x="12.2"
y="264" textLength="97.6"
clip-path="url(#breeze-release-management-publish-docs-line-10)">openfaas</text><text
class="breeze-release-management-publish-docs-r1" x="109.8" y="264"
textLength="36.6"
clip-path="url(#breeze-release-management-publish-docs-line-10)"> | </text><text
class="breeze-release-management-publish-docs-r4" x="146.4" y="264"
textLength="134.2" clip-path="url(#breeze-release-management-pu [...]
-</text><text class="breeze-release-management-publish-docs-r4" x="12.2"
y="288.4" textLength="97.6"
clip-path="url(#breeze-release-management-publish-docs-line-11)">postgres</text><text
class="breeze-release-management-publish-docs-r1" x="109.8" y="288.4"
textLength="36.6"
clip-path="url(#breeze-release-management-publish-docs-line-11)"> | </text><text
class="breeze-release-management-publish-docs-r4" x="146.4" y="288.4"
textLength="73.2" clip-path="url(#breeze-release-manageme [...]
-</text><text class="breeze-release-management-publish-docs-r4" x="12.2"
y="312.8" textLength="73.2"
clip-path="url(#breeze-release-management-publish-docs-line-12)">sqlite</text><text
class="breeze-release-management-publish-docs-r1" x="85.4" y="312.8"
textLength="36.6"
clip-path="url(#breeze-release-management-publish-docs-line-12)"> | </text><text
class="breeze-release-management-publish-docs-r4" x="122" y="312.8"
textLength="36.6" clip-path="url(#breeze-release-management-pu [...]
+</text><text class="breeze-release-management-publish-docs-r4" x="12.2"
y="166.4" textLength="97.6"
clip-path="url(#breeze-release-management-publish-docs-line-6)">arangodb</text><text
class="breeze-release-management-publish-docs-r1" x="109.8" y="166.4"
textLength="36.6"
clip-path="url(#breeze-release-management-publish-docs-line-6)"> | </text><text
class="breeze-release-management-publish-docs-r4" x="146.4" y="166.4"
textLength="61" clip-path="url(#breeze-release-management-p [...]
+</text><text class="breeze-release-management-publish-docs-r1" x="12.2"
y="190.8" textLength="24.4"
clip-path="url(#breeze-release-management-publish-docs-line-7)">| </text><text
class="breeze-release-management-publish-docs-r4" x="36.6" y="190.8"
textLength="85.4"
clip-path="url(#breeze-release-management-publish-docs-line-7)">datadog</text><text
class="breeze-release-management-publish-docs-r1" x="122" y="190.8"
textLength="36.6" clip-path="url(#breeze-release-management-publish-d [...]
+</text><text class="breeze-release-management-publish-docs-r4" x="12.2"
y="215.2" textLength="73.2"
clip-path="url(#breeze-release-management-publish-docs-line-8)">github</text><text
class="breeze-release-management-publish-docs-r1" x="85.4" y="215.2"
textLength="36.6"
clip-path="url(#breeze-release-management-publish-docs-line-8)"> | </text><text
class="breeze-release-management-publish-docs-r4" x="122" y="215.2"
textLength="73.2" clip-path="url(#breeze-release-management-publ [...]
+</text><text class="breeze-release-management-publish-docs-r4" x="12.2"
y="239.6" textLength="109.8"
clip-path="url(#breeze-release-management-publish-docs-line-9)">microsoft</text><text
class="breeze-release-management-publish-docs-r1" x="122" y="239.6"
textLength="12.2"
clip-path="url(#breeze-release-management-publish-docs-line-9)">.</text><text
class="breeze-release-management-publish-docs-r4" x="134.2" y="239.6"
textLength="61" clip-path="url(#breeze-release-management-publish-docs- [...]
+</text><text class="breeze-release-management-publish-docs-r4" x="12.2"
y="264" textLength="122"
clip-path="url(#breeze-release-management-publish-docs-line-10)">opensearch</text><text
class="breeze-release-management-publish-docs-r1" x="134.2" y="264"
textLength="36.6"
clip-path="url(#breeze-release-management-publish-docs-line-10)"> | </text><text
class="breeze-release-management-publish-docs-r4" x="170.8" y="264"
textLength="97.6" clip-path="url(#breeze-release-management-pu [...]
+</text><text class="breeze-release-management-publish-docs-r4" x="12.2"
y="288.4" textLength="122"
clip-path="url(#breeze-release-management-publish-docs-line-11)">salesforce</text><text
class="breeze-release-management-publish-docs-r1" x="134.2" y="288.4"
textLength="36.6"
clip-path="url(#breeze-release-management-publish-docs-line-11)"> | </text><text
class="breeze-release-management-publish-docs-r4" x="170.8" y="288.4"
textLength="61" clip-path="url(#breeze-release-managemen [...]
+</text><text class="breeze-release-management-publish-docs-r4" x="12.2"
y="312.8" textLength="85.4"
clip-path="url(#breeze-release-management-publish-docs-line-12)">tabular</text><text
class="breeze-release-management-publish-docs-r1" x="97.6" y="312.8"
textLength="36.6"
clip-path="url(#breeze-release-management-publish-docs-line-12)"> | </text><text
class="breeze-release-management-publish-docs-r4" x="134.2" y="312.8"
textLength="97.6" clip-path="url(#breeze-release-management [...]
</text><text class="breeze-release-management-publish-docs-r1" x="1464"
y="337.2" textLength="12.2"
clip-path="url(#breeze-release-management-publish-docs-line-13)">
</text><text class="breeze-release-management-publish-docs-r1" x="12.2"
y="361.6" textLength="707.6"
clip-path="url(#breeze-release-management-publish-docs-line-14)">Command to publish generated documentation to airflow-site</text><text
class="breeze-release-management-publish-docs-r1" x="1464" y="361.6"
textLength="12.2"
clip-path="url(#breeze-release-management-publish-docs-line-14)">
</text><text class="breeze-release-management-publish-docs-r1" x="1464"
y="386" textLength="12.2"
clip-path="url(#breeze-release-management-publish-docs-line-15)">
diff --git a/images/breeze/output_release-management_publish-docs.txt
b/images/breeze/output_release-management_publish-docs.txt
index 8f1db92578..d6d4ccf12d 100644
--- a/images/breeze/output_release-management_publish-docs.txt
+++ b/images/breeze/output_release-management_publish-docs.txt
@@ -1 +1 @@
-487a45ba549ea2572fb889c651e6d958
+f73c15c6c97cf17260b53abf3237c268
diff --git a/scripts/ci/pre_commit/pre_commit_insert_extras.py
b/scripts/ci/pre_commit/pre_commit_insert_extras.py
index 103cec230b..f1a96b0a22 100755
--- a/scripts/ci/pre_commit/pre_commit_insert_extras.py
+++ b/scripts/ci/pre_commit/pre_commit_insert_extras.py
@@ -45,13 +45,6 @@ INSTALL_FOOTER = "# END EXTRAS HERE"
CONSTANTS_HEADER = "# START EXTRAS HERE"
CONSTANTS_FOOTER = "# END EXTRAS HERE"
-DEFAULT_EXTRAS = (
-
"amazon,async,celery,cncf.kubernetes,daskexecutor,docker,elasticsearch,ftp,google,"
- "google_auth,grpc,hashicorp,http,ldap,microsoft.azure,mysql,odbc,pandas,"
- "postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv"
-)
-
-
if __name__ == "__main__":
install_file_path = AIRFLOW_SOURCES_DIR / "INSTALL"
contributing_file_path = AIRFLOW_SOURCES_DIR / "CONTRIBUTING.rst"
diff --git a/tests/always/test_project_structure.py
b/tests/always/test_project_structure.py
index bd5098c04f..3b1b8a1e97 100644
--- a/tests/always/test_project_structure.py
+++ b/tests/always/test_project_structure.py
@@ -106,7 +106,6 @@ class TestProjectStructure:
"tests/providers/cncf/kubernetes/utils/test_delete_from.py",
"tests/providers/cncf/kubernetes/utils/test_k8s_hashlib_wrapper.py",
"tests/providers/cncf/kubernetes/utils/test_xcom_sidecar.py",
- "tests/providers/daskexecutor/executors/test_dask_executor.py",
"tests/providers/databricks/hooks/test_databricks_base.py",
"tests/providers/google/cloud/fs/test_gcs.py",
"tests/providers/google/cloud/links/test_automl.py",
diff --git a/tests/providers/daskexecutor/__init__.py
b/tests/providers/daskexecutor/__init__.py
deleted file mode 100644
index 13a83393a9..0000000000
--- a/tests/providers/daskexecutor/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/tests/providers/daskexecutor/test_dask_executor.py
b/tests/providers/daskexecutor/test_dask_executor.py
deleted file mode 100644
index 69cd037716..0000000000
--- a/tests/providers/daskexecutor/test_dask_executor.py
+++ /dev/null
@@ -1,241 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-from datetime import timedelta
-from unittest import mock
-
-import pytest
-from distributed import LocalCluster
-
-from airflow.exceptions import AirflowException
-from airflow.jobs.backfill_job_runner import BackfillJobRunner
-from airflow.jobs.job import Job, run_job
-from airflow.models import DagBag
-from airflow.providers.daskexecutor.executors.dask_executor import DaskExecutor
-from airflow.utils import timezone
-from tests.test_utils.config import conf_vars
-
-pytestmark = pytest.mark.db_test
-
-try:
- # utility functions imported from the dask testing suite to instantiate a
test
- # cluster for tls tests
- from distributed import tests # noqa
- from distributed.utils_test import cluster as dask_testing_cluster,
get_cert, tls_security
-
- skip_tls_tests = False
-except ImportError:
- skip_tls_tests = True
-
- # In case the tests are skipped because of lacking test harness, get_cert
should be
- # overridden to avoid get_cert failing during test discovery as get_cert
is used
- # in conf_vars decorator
- def get_cert(x):
- return x
-
-
-DEFAULT_DATE = timezone.datetime(2017, 1, 1)
-SUCCESS_COMMAND = ["airflow", "tasks", "run", "--help"]
-FAIL_COMMAND = ["airflow", "tasks", "run", "false"]
-
-# For now we are temporarily removing Dask support until we get Dask Team help
us in making the
-# tests pass again
-skip_dask_tests = False
-
-
[email protected](skip_dask_tests, reason="The tests are skipped because it
needs testing from Dask team")
-class TestBaseDask:
- def assert_tasks_on_executor(self, executor, timeout_executor=120):
- # start the executor
- executor.start()
-
- executor.execute_async(key="success", command=SUCCESS_COMMAND)
- executor.execute_async(key="fail", command=FAIL_COMMAND)
-
- success_future = next(k for k, v in executor.futures.items() if v ==
"success")
- fail_future = next(k for k, v in executor.futures.items() if v ==
"fail")
-
- # wait for the futures to execute, with a timeout
- timeout = timezone.utcnow() + timedelta(seconds=timeout_executor)
- while not (success_future.done() and fail_future.done()):
- if timezone.utcnow() > timeout:
- raise ValueError(
- "The futures should have finished; there is probably "
- "an error communicating with the Dask cluster."
- )
-
- # both tasks should have finished
- assert success_future.done()
- assert fail_future.done()
-
- # check task exceptions
- assert success_future.exception() is None
- assert fail_future.exception() is not None
-
-
[email protected](skip_dask_tests, reason="The tests are skipped because it
needs testing from Dask team")
-class TestDaskExecutor(TestBaseDask):
- def setup_method(self):
- self.dagbag = DagBag(include_examples=True)
- self.cluster = LocalCluster()
-
- def test_supports_pickling(self):
- assert not DaskExecutor.supports_pickling
-
- def test_supports_sentry(self):
- assert not DaskExecutor.supports_sentry
-
- def test_dask_executor_functions(self):
- executor = DaskExecutor(cluster_address=self.cluster.scheduler_address)
- self.assert_tasks_on_executor(executor, timeout_executor=120)
-
- # This test is quarantined because it became rather flaky on our CI in
July 2023 and reason for this
- # is unknown. An issue for that was created:
https://github.com/apache/airflow/issues/32778 and the
- # marker should be removed while (possibly) the reason for flaky behaviour
is found and fixed.
- @pytest.mark.quarantined
- @pytest.mark.execution_timeout(180)
- def test_backfill_integration(self):
- """
- Test that DaskExecutor can be used to backfill example dags
- """
- dag = self.dagbag.get_dag("example_bash_operator")
-
- job = Job(
-
executor=DaskExecutor(cluster_address=self.cluster.scheduler_address),
- )
- job_runner = BackfillJobRunner(
- job=job,
- dag=dag,
- start_date=DEFAULT_DATE,
- end_date=DEFAULT_DATE,
- ignore_first_depends_on_past=True,
- )
- run_job(job=job, execute_callable=job_runner._execute)
-
- def teardown_method(self):
- self.cluster.close(timeout=5)
-
-
[email protected](
- skip_tls_tests, reason="The tests are skipped because distributed
framework could not be imported"
-)
-class TestDaskExecutorTLS(TestBaseDask):
- def setup_method(self):
- self.dagbag = DagBag(include_examples=True)
-
- @conf_vars(
- {
- ("dask", "tls_ca"): "certs/tls-ca-cert.pem",
- ("dask", "tls_cert"): "certs/tls-key-cert.pem",
- ("dask", "tls_key"): "certs/tls-key.pem",
- }
- )
- def test_tls(self):
- # These use test certs that ship with dask/distributed and should not
be
- # used in production
- with dask_testing_cluster(
- worker_kwargs={"security": tls_security(), "protocol": "tls"},
- scheduler_kwargs={"security": tls_security(), "protocol": "tls"},
- ) as (cluster, _):
- executor = DaskExecutor(cluster_address=cluster["address"])
-
- self.assert_tasks_on_executor(executor, timeout_executor=120)
-
- executor.end()
- # close the executor, the cluster context manager expects all
listeners
- # and tasks to have completed.
- executor.client.close()
-
-
@mock.patch("airflow.providers.daskexecutor.executors.dask_executor.DaskExecutor.sync")
- @mock.patch("airflow.executors.base_executor.BaseExecutor.trigger_tasks")
- @mock.patch("airflow.executors.base_executor.Stats.gauge")
- def test_gauge_executor_metrics(self, mock_stats_gauge,
mock_trigger_tasks, mock_sync):
- executor = DaskExecutor()
- executor.heartbeat()
- calls = [
- mock.call("executor.open_slots", mock.ANY),
- mock.call("executor.queued_tasks", mock.ANY),
- mock.call("executor.running_tasks", mock.ANY),
- ]
- mock_stats_gauge.assert_has_calls(calls)
-
-
[email protected](skip_dask_tests, reason="The tests are skipped because it
needs testing from Dask team")
-class TestDaskExecutorQueue:
- def test_dask_queues_no_resources(self):
- self.cluster = LocalCluster()
- executor = DaskExecutor(cluster_address=self.cluster.scheduler_address)
- executor.start()
-
- with pytest.raises(AirflowException):
- executor.execute_async(key="success", command=SUCCESS_COMMAND,
queue="queue1")
-
- def test_dask_queues_not_available(self):
- self.cluster = LocalCluster(resources={"queue1": 1})
- executor = DaskExecutor(cluster_address=self.cluster.scheduler_address)
- executor.start()
-
- with pytest.raises(AirflowException):
- # resource 'queue2' doesn't exist on cluster
- executor.execute_async(key="success", command=SUCCESS_COMMAND,
queue="queue2")
-
- def test_dask_queues(self):
- self.cluster = LocalCluster(resources={"queue1": 1})
- executor = DaskExecutor(cluster_address=self.cluster.scheduler_address)
- executor.start()
-
- executor.execute_async(key="success", command=SUCCESS_COMMAND,
queue="queue1")
- success_future = next(k for k, v in executor.futures.items() if v ==
"success")
-
- # wait for the futures to execute, with a timeout
- timeout = timezone.utcnow() + timedelta(seconds=120)
- while not success_future.done():
- if timezone.utcnow() > timeout:
- raise ValueError(
- "The futures should have finished; there is probably "
- "an error communicating with the Dask cluster."
- )
-
- assert success_future.done()
- assert success_future.exception() is None
-
- @pytest.mark.execution_timeout(120)
- def test_dask_queues_no_queue_specified(self):
- self.cluster = LocalCluster(resources={"queue1": 1})
- executor = DaskExecutor(cluster_address=self.cluster.scheduler_address)
- executor.start()
-
- # no queue specified for executing task
- executor.execute_async(key="success", command=SUCCESS_COMMAND)
- success_future = next(k for k, v in executor.futures.items() if v ==
"success")
-
- # wait for the futures to execute, with a timeout
- timeout = timezone.utcnow() + timedelta(seconds=100)
- while not success_future.done():
- if timezone.utcnow() > timeout:
- raise ValueError(
- "The futures should have finished; there is probably "
- "an error communicating with the Dask cluster."
- )
-
- assert success_future.done()
- assert success_future.exception() is None
-
- def teardown_method(self):
- self.cluster.close(timeout=5)
