This is an automated email from the ASF dual-hosted git repository.
vincbeck pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 8312e91569 Make docker decorator's type annotation consistent with
operator (#35568)
8312e91569 is described below
commit 8312e915691fd7d59a9a2372d8adcd4bfa4a6733
Author: Andrey Anshin <[email protected]>
AuthorDate: Fri Nov 10 22:31:01 2023 +0400
Make docker decorator's type annotation consistent with operator (#35568)
---
airflow/decorators/__init__.pyi | 78 +++++++++++++++++++++++++++++++++--------
1 file changed, 64 insertions(+), 14 deletions(-)
diff --git a/airflow/decorators/__init__.pyi b/airflow/decorators/__init__.pyi
index f718e35777..48cdf61946 100644
--- a/airflow/decorators/__init__.pyi
+++ b/airflow/decorators/__init__.pyi
@@ -37,6 +37,7 @@ from airflow.decorators.short_circuit import
short_circuit_task
from airflow.decorators.task_group import task_group
from airflow.models.dag import dag
from airflow.providers.cncf.kubernetes.secret import Secret
+from airflow.typing_compat import Literal
# Please keep this in sync with __init__.py's __all__.
__all__ = [
@@ -325,6 +326,7 @@ class TaskDecoratorCollection:
docker_url: str = "unix://var/run/docker.sock",
environment: dict[str, str] | None = None,
private_environment: dict[str, str] | None = None,
+ env_file: str | None = None,
force_pull: bool = False,
mem_limit: float | str | None = None,
host_tmp_dir: str | None = None,
@@ -332,22 +334,36 @@ class TaskDecoratorCollection:
tls_ca_cert: str | None = None,
tls_client_cert: str | None = None,
tls_client_key: str | None = None,
+ tls_verify: bool = True,
tls_hostname: str | bool | None = None,
tls_ssl_version: str | None = None,
+ mount_tmp_dir: bool = True,
tmp_dir: str = "/tmp/airflow",
user: str | int | None = None,
mounts: list[str] | None = None,
+ entrypoint: str | list[str] | None = None,
working_dir: str | None = None,
xcom_all: bool = False,
docker_conn_id: str | None = None,
dns: list[str] | None = None,
dns_search: list[str] | None = None,
- auto_remove: bool = False,
+ auto_remove: Literal["never", "success", "force"] = "never",
shm_size: int | None = None,
tty: bool = False,
+ hostname: str | None = None,
privileged: bool = False,
cap_add: str | None = None,
extra_hosts: dict[str, str] | None = None,
+ retrieve_output: bool = False,
+ retrieve_output_path: str | None = None,
+ timeout: int = 60,
+ device_requests: list[dict] | None = None,
+ log_opts_max_size: str | None = None,
+ log_opts_max_file: str | None = None,
+ ipc_mode: str | None = None,
+ skip_on_exit_code: int | Container[int] | None = None,
+ port_bindings: dict | None = None,
+ ulimits: list[dict] | None = None,
**kwargs,
) -> TaskDecorator:
"""Create a decorator to convert the decorated callable to a Docker
task.
@@ -361,34 +377,41 @@ class TaskDecoratorCollection:
:param api_version: Remote API version. Set to ``auto`` to
automatically
detect the server's version.
:param container_name: Name of the container. Optional (templated)
- :param cpus: Number of CPUs to assign to the container. This value
gets multiplied with 1024.
+ :param cpus: Number of CPUs to assign to the container.
+ This value gets multiplied with 1024. See
+ https://docs.docker.com/engine/reference/run/#cpu-share-constraint
:param docker_url: URL of the host running the docker daemon.
Default is unix://var/run/docker.sock
:param environment: Environment variables to set in the container.
(templated)
:param private_environment: Private environment variables to set in
the container.
These are not templated, and hidden from the website.
+ :param env_file: Relative path to the ``.env`` file with environment
variables to set in the container.
+ Overridden by variables in the environment parameter.
:param force_pull: Pull the docker image on every run. Default is
False.
:param mem_limit: Maximum amount of memory the container can use.
Either a float value, which represents the limit in bytes,
or a string like ``128m`` or ``1g``.
:param host_tmp_dir: Specify the location of the temporary directory
on the host which will
be mapped to tmp_dir. If not provided defaults to using the
standard system temp directory.
- :param network_mode: Network mode for the container.
- It can be one of the following:
- bridge - Create new network stack for the container with default
docker bridge network
- None - No networking for this container
- container:<name|id> - Use the network stack of another container
specified via <name|id>
- host - Use the host network stack. Incompatible with
`port_bindings`
- '<network-name>|<network-id>' - Connects the container to user
created network(using `docker
- network create` command)
+ :param network_mode: Network mode for the container. It can be one of
the following:
+
+ - ``"bridge"``: Create new network stack for the container with
default docker bridge network
+ - ``"none"``: No networking for this container
+ - ``"container:<name|id>"``: Use the network stack of another
container specified via <name|id>
+ - ``"host"``: Use the host network stack. Incompatible with
`port_bindings`
+ - ``"<network-name>|<network-id>"``: Connects the container to
user created network
+ (using ``docker network create`` command)
:param tls_ca_cert: Path to a PEM-encoded certificate authority
to secure the docker connection.
:param tls_client_cert: Path to the PEM-encoded certificate
used to authenticate docker client.
:param tls_client_key: Path to the PEM-encoded key used to
authenticate docker client.
+ :param tls_verify: Set ``True`` to verify the validity of the provided
certificate.
:param tls_hostname: Hostname to match against
the docker server certificate or False to disable the check.
:param tls_ssl_version: Version of SSL to use when communicating with
docker daemon.
+ :param mount_tmp_dir: Specify whether the temporary directory should
be bind-mounted
+ from the host to the container. Defaults to True
:param tmp_dir: Mount point inside the container to
a temporary directory created on the host by the operator.
The path is also made available via the environment variable
@@ -396,22 +419,49 @@ class TaskDecoratorCollection:
:param user: Default user inside the docker container.
:param mounts: List of mounts to mount into the container, e.g.
``['/host/path:/container/path',
'/host/path2:/container/path2:ro']``.
+ :param entrypoint: Overwrite the default ENTRYPOINT of the image
:param working_dir: Working directory to
set on the container (equivalent to the -w switch the docker
client)
:param xcom_all: Push all the stdout or just the last line.
The default is False (last line).
- :param docker_conn_id: ID of the Airflow connection to use
+ :param docker_conn_id: The :ref:`Docker connection id
<howto/connection:docker>`
:param dns: Docker custom DNS servers
:param dns_search: Docker custom DNS search domain
- :param auto_remove: Auto-removal of the container on daemon side when
the
- container's process exits.
- The default is False.
+ :param auto_remove: Enable removal of the container when the
container's process exits. Possible values:
+
+ - ``never``: (default) do not remove container
+ - ``success``: remove on success
+ - ``force``: always remove container
:param shm_size: Size of ``/dev/shm`` in bytes. The size must be
greater than 0. If omitted uses system default.
:param tty: Allocate pseudo-TTY to the container
This needs to be set see logs of the Docker container.
+ :param hostname: Optional hostname for the container.
:param privileged: Give extended privileges to this container.
:param cap_add: Include container capabilities
+ :param extra_hosts: Additional hostnames to resolve inside the
container,
+ as a mapping of hostname to IP address.
+ :param retrieve_output: Should this docker image consistently attempt
to pull from and output
+ file before manually shutting down the image. Useful for cases
where users want a pickle serialized
+ output that is not posted to logs
+ :param retrieve_output_path: path for output file that will be
retrieved and passed to xcom
+ :param device_requests: Expose host resources such as GPUs to the
container.
+ :param log_opts_max_size: The maximum size of the log before it is
rolled.
+ A positive integer plus a modifier representing the unit of
measure (k, m, or g).
+ Eg: 10m or 1g Defaults to -1 (unlimited).
+ :param log_opts_max_file: The maximum number of log files that can be
present.
+ If rolling the logs creates excess files, the oldest file is
removed.
+ Only effective when max-size is also set. A positive integer.
Defaults to 1.
+ :param ipc_mode: Set the IPC mode for the container.
+ :param skip_on_exit_code: If task exits with this exit code, leave the
task
+ in ``skipped`` state (default: None). If set to ``None``, any
non-zero
+ exit code will be treated as a failure.
+ :param port_bindings: Publish a container's port(s) to the host. It is
a
+ dictionary of value where the key indicates the port to open
inside the container
+ and value indicates the host port that binds to the container port.
+ Incompatible with ``"host"`` in ``network_mode``.
+ :param ulimits: List of ulimit options to set for the container. Each
item should
+ be a :py:class:`docker.types.Ulimit` instance.
"""
# [END decorator_signature]
def kubernetes(
