This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new fac6aa4870 Use `python-on-whales` in docker tests (#38421)
fac6aa4870 is described below
commit fac6aa4870fc794a8be9908b59fd32f692201e98
Author: Andrey Anshin <[email protected]>
AuthorDate: Sat Mar 23 14:10:40 2024 +0400
Use `python-on-whales` in docker tests (#38421)
---
.../commands/release_management_commands.py | 35 +----
dev/breeze/src/airflow_breeze/utils/run_tests.py | 35 +++--
.../src/airflow_breeze/utils/virtualenv_utils.py | 100 +++++++++++++++
docker_tests/{constants.py => conftest.py} | 12 +-
docker_tests/constants.py | 5 +
docker_tests/docker_tests_utils.py | 105 ---------------
docker_tests/docker_utils.py | 141 +++++++++++++++++++++
docker_tests/requirements.txt | 4 +-
docker_tests/test_ci_image.py | 30 ++---
docker_tests/test_docker_compose_quick_start.py | 122 ++++++------------
.../test_examples_of_prod_image_building.py | 19 ++-
docker_tests/test_prod_image.py | 120 +++++++-----------
12 files changed, 392 insertions(+), 336 deletions(-)
diff --git
a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
index 16bdac52b5..5d725ffa32 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -155,6 +155,7 @@ from airflow_breeze.utils.run_utils import (
)
from airflow_breeze.utils.shared_options import get_dry_run, get_verbose
from airflow_breeze.utils.versions import is_pre_release
+from airflow_breeze.utils.virtualenv_utils import create_pip_command,
create_venv
argument_provider_packages = click.argument(
"provider_packages",
@@ -449,36 +450,8 @@ def _check_sdist_to_wheel_dists(dists_info:
tuple[DistributionPackageInfo, ...])
continue
if not venv_created:
- venv_path = (Path(tmp_dir_name) / ".venv").resolve().absolute()
- venv_command_result = run_command(
- [sys.executable, "-m", "venv", venv_path.as_posix()],
- check=False,
- capture_output=True,
- )
- if venv_command_result.returncode != 0:
- get_console().print(
- f"[error]Error when initializing virtualenv in
{venv_path.as_posix()}:[/]\n"
-
f"{venv_command_result.stdout}\n{venv_command_result.stderr}"
- )
- python_path = venv_path / "bin" / "python"
- if not python_path.exists():
- get_console().print(
- f"\n[errors]Python interpreter is not exist in path
{python_path}. Exiting!\n"
- )
- sys.exit(1)
- pip_command = (python_path.as_posix(), "-m", "pip")
- result = run_command(
- [*pip_command, "install", f"pip=={AIRFLOW_PIP_VERSION}"],
- check=False,
- capture_output=True,
- text=True,
- )
- if result.returncode != 0:
- get_console().print(
- f"[error]Error when installing pip in
{venv_path.as_posix()}[/]\n"
- f"{result.stdout}\n{result.stderr}"
- )
- sys.exit(1)
+ python_path = create_venv(Path(tmp_dir_name) / ".venv",
pip_version=AIRFLOW_PIP_VERSION)
+ pip_command = create_pip_command(python_path)
venv_created = True
returncode = _check_sdist_to_wheel(di, pip_command,
str(tmp_dir_name))
@@ -492,7 +465,7 @@ def _check_sdist_to_wheel_dists(dists_info:
tuple[DistributionPackageInfo, ...])
sys.exit(1)
-def _check_sdist_to_wheel(dist_info: DistributionPackageInfo, pip_command:
tuple[str, ...], cwd: str) -> int:
+def _check_sdist_to_wheel(dist_info: DistributionPackageInfo, pip_command:
list[str], cwd: str) -> int:
get_console().print(
f"[info]Validate build wheel from sdist distribution for package
{dist_info.package!r}.[/]"
)
diff --git a/dev/breeze/src/airflow_breeze/utils/run_tests.py
b/dev/breeze/src/airflow_breeze/utils/run_tests.py
index 69aac4abc5..3abd58aec7 100644
--- a/dev/breeze/src/airflow_breeze/utils/run_tests.py
+++ b/dev/breeze/src/airflow_breeze/utils/run_tests.py
@@ -22,10 +22,15 @@ import sys
from itertools import chain
from subprocess import DEVNULL
+from airflow_breeze.global_constants import PIP_VERSION
from airflow_breeze.utils.console import Output, get_console
from airflow_breeze.utils.packages import get_excluded_provider_folders,
get_suspended_provider_folders
from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT
from airflow_breeze.utils.run_utils import run_command
+from airflow_breeze.utils.virtualenv_utils import create_temp_venv
+
+DOCKER_TESTS_ROOT = AIRFLOW_SOURCES_ROOT / "docker_tests"
+DOCKER_TESTS_REQUIREMENTS = DOCKER_TESTS_ROOT / "requirements.txt"
def verify_an_image(
@@ -47,19 +52,20 @@ def verify_an_image(
return command_result.returncode, f"Testing {image_type} python
{image_name}"
pytest_args = ("-n", str(os.cpu_count()), "--color=yes")
if image_type == "PROD":
- test_path = AIRFLOW_SOURCES_ROOT / "docker_tests" /
"test_prod_image.py"
+ test_path = DOCKER_TESTS_ROOT / "test_prod_image.py"
else:
- test_path = AIRFLOW_SOURCES_ROOT / "docker_tests" / "test_ci_image.py"
+ test_path = DOCKER_TESTS_ROOT / "test_ci_image.py"
env = os.environ.copy()
env["DOCKER_IMAGE"] = image_name
if slim_image:
env["TEST_SLIM_IMAGE"] = "true"
- command_result = run_command(
- [sys.executable, "-m", "pytest", str(test_path), *pytest_args,
*extra_pytest_args],
- env=env,
- output=output,
- check=False,
- )
+ with create_temp_venv(pip_version=PIP_VERSION,
requirements_file=DOCKER_TESTS_REQUIREMENTS) as py_exe:
+ command_result = run_command(
+ [py_exe, "-m", "pytest", str(test_path), *pytest_args,
*extra_pytest_args],
+ env=env,
+ output=output,
+ check=False,
+ )
return command_result.returncode, f"Testing {image_type} python
{image_name}"
@@ -73,16 +79,17 @@ def run_docker_compose_tests(
get_console().print(f"[error]Error when inspecting PROD image:
{command_result.returncode}[/]")
return command_result.returncode, f"Testing docker-compose python with
{image_name}"
pytest_args = ("--color=yes",)
- test_path = AIRFLOW_SOURCES_ROOT / "docker_tests" /
"test_docker_compose_quick_start.py"
+ test_path = DOCKER_TESTS_ROOT / "test_docker_compose_quick_start.py"
env = os.environ.copy()
env["DOCKER_IMAGE"] = image_name
if skip_docker_compose_deletion:
env["SKIP_DOCKER_COMPOSE_DELETION"] = "true"
- command_result = run_command(
- [sys.executable, "-m", "pytest", str(test_path), *pytest_args,
*extra_pytest_args],
- env=env,
- check=False,
- )
+ with create_temp_venv(pip_version=PIP_VERSION,
requirements_file=DOCKER_TESTS_REQUIREMENTS) as py_exe:
+ command_result = run_command(
+ [py_exe, "-m", "pytest", str(test_path), *pytest_args,
*extra_pytest_args],
+ env=env,
+ check=False,
+ )
return command_result.returncode, f"Testing docker-compose python with
{image_name}"
diff --git a/dev/breeze/src/airflow_breeze/utils/virtualenv_utils.py
b/dev/breeze/src/airflow_breeze/utils/virtualenv_utils.py
new file mode 100644
index 0000000000..0288e49b90
--- /dev/null
+++ b/dev/breeze/src/airflow_breeze/utils/virtualenv_utils.py
@@ -0,0 +1,100 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+import contextlib
+import sys
+import tempfile
+from pathlib import Path
+from typing import Generator
+
+from airflow_breeze.utils.console import get_console
+from airflow_breeze.utils.run_utils import run_command
+
+
+def create_pip_command(python: str | Path) -> list[str]:
+ return [python.as_posix() if hasattr(python, "as_posix") else str(python),
"-m", "pip"]
+
+
+def create_venv(
+ venv_path: str | Path,
+ python: str | None = None,
+ pip_version: str | None = None,
+ requirements_file: str | Path | None = None,
+) -> str:
+ venv_path = Path(venv_path).resolve().absolute()
+ venv_command_result = run_command(
+ [python or sys.executable, "-m", "venv", venv_path.as_posix()],
+ check=False,
+ capture_output=True,
+ )
+ if venv_command_result.returncode != 0:
+ get_console().print(
+ f"[error]Error when initializing virtualenv in
{venv_path.as_posix()}:[/]\n"
+ f"{venv_command_result.stdout}\n{venv_command_result.stderr}"
+ )
+ sys.exit(venv_command_result.returncode)
+ python_path = venv_path / "bin" / "python"
+ if not python_path.exists():
+ get_console().print(f"\n[errors]Python interpreter is not exist in
path {python_path}. Exiting!\n")
+ sys.exit(1)
+ pip_command = create_pip_command(python_path)
+ if pip_version:
+ result = run_command(
+ [*pip_command, "install", f"pip=={pip_version}", "-q"],
+ check=False,
+ capture_output=False,
+ text=True,
+ )
+ if result.returncode != 0:
+ get_console().print(
+ f"[error]Error when installing pip in
{venv_path.as_posix()}[/]\n"
+ f"{result.stdout}\n{result.stderr}"
+ )
+ sys.exit(result.returncode)
+ if requirements_file:
+ requirements_file = Path(requirements_file).absolute().as_posix()
+ result = run_command(
+ [*pip_command, "install", "-r", requirements_file, "-q"],
+ check=True,
+ capture_output=False,
+ text=True,
+ )
+ if result.returncode != 0:
+ get_console().print(
+ f"[error]Error when installing packages from
{requirements_file}[/]\n"
+ f"{result.stdout}\n{result.stderr}"
+ )
+ sys.exit(result.returncode)
+ return python_path.as_posix()
+
+
[email protected]
+def create_temp_venv(
+ python: str | None = None,
+ pip_version: str | None = None,
+ requirements_file: str | Path | None = None,
+ prefix: str | None = None,
+) -> Generator[str, None, None]:
+ with tempfile.TemporaryDirectory(prefix=prefix) as tmp_dir_name:
+ yield create_venv(
+ Path(tmp_dir_name) / ".venv",
+ python=python,
+ pip_version=pip_version,
+ requirements_file=requirements_file,
+ )
diff --git a/docker_tests/constants.py b/docker_tests/conftest.py
similarity index 80%
copy from docker_tests/constants.py
copy to docker_tests/conftest.py
index db79d1d862..c570469506 100644
--- a/docker_tests/constants.py
+++ b/docker_tests/conftest.py
@@ -14,8 +14,16 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+
from __future__ import annotations
-from pathlib import Path
+import os
+
+import pytest
+
+from docker_tests.constants import DEFAULT_DOCKER_IMAGE
+
-SOURCE_ROOT = Path(__file__).resolve().parents[1]
[email protected]
+def default_docker_image() -> str:
+ return os.environ.get("DOCKER_IMAGE") or DEFAULT_DOCKER_IMAGE
diff --git a/docker_tests/constants.py b/docker_tests/constants.py
index db79d1d862..83c77e2fb4 100644
--- a/docker_tests/constants.py
+++ b/docker_tests/constants.py
@@ -16,6 +16,11 @@
# under the License.
from __future__ import annotations
+import os
from pathlib import Path
SOURCE_ROOT = Path(__file__).resolve().parents[1]
+
+DEFAULT_PYTHON_MAJOR_MINOR_VERSION = "3.8"
+DEFAULT_DOCKER_IMAGE =
f"ghcr.io/apache/airflow/main/prod/python{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}:latest"
+DOCKER_IMAGE = os.environ.get("DOCKER_IMAGE") or DEFAULT_DOCKER_IMAGE
diff --git a/docker_tests/docker_tests_utils.py
b/docker_tests/docker_tests_utils.py
deleted file mode 100644
index 7eea98e9bd..0000000000
--- a/docker_tests/docker_tests_utils.py
+++ /dev/null
@@ -1,105 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-import os
-
-from docker_tests.command_utils import run_command
-
-DEFAULT_PYTHON_MAJOR_MINOR_VERSION = "3.8"
-
-docker_image = os.environ.get(
- "DOCKER_IMAGE",
f"ghcr.io/apache/airflow/main/prod/python{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}:latest"
-)
-
-print("Using docker image: ", docker_image)
-
-
-def run_bash_in_docker(bash_script, **kwargs):
- docker_command = [
- "docker",
- "run",
- "--rm",
- "-e",
- "COLUMNS=180",
- "--entrypoint",
- "/bin/bash",
- docker_image,
- "-c",
- bash_script,
- ]
- return run_command(docker_command, **kwargs)
-
-
-def run_python_in_docker(python_script, **kwargs):
- docker_command = [
- "docker",
- "run",
- "--rm",
- "-e",
- "COLUMNS=180",
- "-e",
- "PYTHONDONTWRITEBYTECODE=true",
- docker_image,
- "python",
- "-c",
- python_script,
- ]
- return run_command(docker_command, **kwargs)
-
-
-def display_dependency_conflict_message():
- print(
- """
-***** Beginning of the instructions ****
-
-The image did not pass 'pip check' verification. This means that there are
some conflicting dependencies
-in the image.
-
-It can mean one of those:
-
-1) The main is currently broken (other PRs will fail with the same error)
-2) You changed some dependencies in pyproject.toml (either manually or
automatically by pre-commit)
- and they are conflicting.
-
-
-
-In case 1) - apologies for the trouble.Please let committers know and they
will fix it. You might
-be asked to rebase to the latest main after the problem is fixed.
-
-In case 2) - Follow the steps below:
-
-* try to build CI and then PROD image locally with breeze, adding
--upgrade-to-newer-dependencies flag
- (repeat it for all python versions)
-
-CI image:
-
- breeze ci-image build --upgrade-to-newer-dependencies --python 3.8
-
-Production image:
-
- breeze ci-image build --production-image --upgrade-to-newer-dependencies
--python 3.8
-
-* You will see error messages there telling which requirements are conflicting
and which packages caused the
- conflict. Add the limitation that caused the conflict to
EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS
- variable in Dockerfile.ci. Note that the limitations might be different for
Dockerfile.ci and Dockerfile
- because not all packages are installed by default in the PROD Dockerfile. So
you might find that you
- only need to add the limitation to the Dockerfile.ci
-
-***** End of the instructions ****
-"""
- )
diff --git a/docker_tests/docker_utils.py b/docker_tests/docker_utils.py
new file mode 100644
index 0000000000..1c9aea8a42
--- /dev/null
+++ b/docker_tests/docker_utils.py
@@ -0,0 +1,141 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import os
+from time import monotonic, sleep
+
+from python_on_whales import docker
+from python_on_whales.exceptions import NoSuchContainer
+
+from docker_tests.constants import DEFAULT_DOCKER_IMAGE
+
+
+def run_cmd_in_docker(
+ cmd: list[str] | None = None,
+ image: str | None = None,
+ entrypoint: str | None = None,
+ envs: dict[str, str] | None = None,
+ remove: bool = True,
+ **kwargs,
+):
+ cmd = cmd or []
+ envs = envs or {}
+ return docker.run(
+ image=image or os.environ.get("DOCKER_IMAGE") or DEFAULT_DOCKER_IMAGE,
+ entrypoint=entrypoint,
+ command=cmd,
+ remove=remove,
+ envs={"COLUMNS": "180", **envs},
+ **kwargs,
+ )
+
+
+def run_bash_in_docker(bash_script: str, **kwargs):
+ kwargs.pop("entrypoint", None)
+ return run_cmd_in_docker(cmd=["-c", bash_script], entrypoint="/bin/bash",
**kwargs)
+
+
+def run_python_in_docker(python_script, **kwargs):
+ kwargs.pop("entrypoint", None)
+ envs = {"PYTHONDONTWRITEBYTECODE": "true", **kwargs.pop("envs", {})}
+ return run_cmd_in_docker(cmd=["python", "-c", python_script], envs=envs,
**kwargs)
+
+
+def run_airflow_cmd_in_docker(cmd: list[str] | None = None, **kwargs):
+ kwargs.pop("entrypoint", None)
+ return run_cmd_in_docker(cmd=["airflow", *(cmd or [])], **kwargs)
+
+
+def display_dependency_conflict_message():
+ print(
+ """
+***** Beginning of the instructions ****
+
+The image did not pass 'pip check' verification. This means that there are
some conflicting dependencies
+in the image.
+
+It can mean one of those:
+
+1) The main is currently broken (other PRs will fail with the same error)
+2) You changed some dependencies in pyproject.toml (either manually or
automatically by pre-commit)
+ and they are conflicting.
+
+
+
+In case 1) - apologies for the trouble.Please let committers know and they
will fix it. You might
+be asked to rebase to the latest main after the problem is fixed.
+
+In case 2) - Follow the steps below:
+
+* try to build CI and then PROD image locally with breeze, adding
--upgrade-to-newer-dependencies flag
+ (repeat it for all python versions)
+
+CI image:
+
+ breeze ci-image build --upgrade-to-newer-dependencies --python 3.8
+
+Production image:
+
+ breeze ci-image build --production-image --upgrade-to-newer-dependencies
--python 3.8
+
+* You will see error messages there telling which requirements are conflicting
and which packages caused the
+ conflict. Add the limitation that caused the conflict to
EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS
+ variable in Dockerfile.ci. Note that the limitations might be different for
Dockerfile.ci and Dockerfile
+ because not all packages are installed by default in the PROD Dockerfile. So
you might find that you
+ only need to add the limitation to the Dockerfile.ci
+
+***** End of the instructions ****
+"""
+ )
+
+
+def wait_for_container(container_id: str, timeout: int = 300):
+ print(f"Waiting for container: [{container_id}] for {timeout} more
seconds.")
+ start_time = monotonic()
+ while True:
+ if timeout != 0 and monotonic() - start_time > timeout:
+ err_msg = f"Timeout. The operation takes longer than the maximum
waiting time ({timeout}s)"
+ raise TimeoutError(err_msg)
+
+ try:
+ container = docker.container.inspect(container_id)
+ except NoSuchContainer:
+ msg = f"Container ID {container_id!r} not found."
+ if timeout != 0:
+ msg += f"\nWaiting for {int(timeout - (monotonic() -
start_time))} more seconds"
+ print(msg)
+ sleep(5)
+ continue
+
+ container_msg = f"Container {container.name}[{container_id}]"
+ if (state := container.state).status in ("running", "restarting"):
+ if state.health is None or state.health.status == "healthy":
+ print(
+ f"{container_msg}. Status: {state.status!r}. "
+ f"Healthcheck: {state.health.status if state.health else
'not set'!r}"
+ )
+ break
+ elif state.status == "exited":
+ print(f"{container_msg}. Status: {state.status!r}. Exit Code:
{state.exit_code}")
+ break
+
+ msg = f"{container_msg} has state:\n {state}"
+ if timeout != 0:
+ msg += f"\nWaiting for {int(timeout - (monotonic() - start_time))}
more seconds"
+ print(msg)
+ sleep(1)
diff --git a/docker_tests/requirements.txt b/docker_tests/requirements.txt
index 60b4005325..61a70cded4 100644
--- a/docker_tests/requirements.txt
+++ b/docker_tests/requirements.txt
@@ -2,4 +2,6 @@
# Internal meta-task for track https://github.com/apache/airflow/issues/37156
pytest>=7.4.4,<8.0
pytest-xdist
-requests
+# Requests 3 if it will be released, will be heavily breaking.
+requests>=2.27.0,<3
+python-on-whales>=0.70.0
diff --git a/docker_tests/test_ci_image.py b/docker_tests/test_ci_image.py
index 7ec65c425c..393813ad16 100644
--- a/docker_tests/test_ci_image.py
+++ b/docker_tests/test_ci_image.py
@@ -16,34 +16,22 @@
# under the License.
from __future__ import annotations
-import subprocess
+from python_on_whales import DockerException
-from docker_tests.command_utils import run_command
-from docker_tests.docker_tests_utils import
display_dependency_conflict_message, docker_image
+from docker_tests.docker_utils import display_dependency_conflict_message,
run_bash_in_docker
-def test_pip_dependencies_conflict():
+def test_pip_dependencies_conflict(default_docker_image):
try:
- run_command(["docker", "run", "--rm", "--entrypoint", "/bin/bash",
docker_image, "-c", "pip check"])
- except subprocess.CalledProcessError as ex:
+ run_bash_in_docker("pip check", image=default_docker_image)
+ except DockerException:
display_dependency_conflict_message()
- raise ex
+ raise
def test_providers_present():
try:
- run_command(
- [
- "docker",
- "run",
- "--rm",
- "--entrypoint",
- "/bin/bash",
- docker_image,
- "-c",
- "airflow providers list",
- ],
- )
- except subprocess.CalledProcessError as ex:
+ run_bash_in_docker("airflow providers list")
+ except DockerException:
display_dependency_conflict_message()
- raise ex
+ raise
diff --git a/docker_tests/test_docker_compose_quick_start.py
b/docker_tests/test_docker_compose_quick_start.py
index 7c70c39f05..eab0a7a62e 100644
--- a/docker_tests/test_docker_compose_quick_start.py
+++ b/docker_tests/test_docker_compose_quick_start.py
@@ -19,18 +19,18 @@ from __future__ import annotations
import json
import os
import shlex
-import subprocess
-import sys
from pprint import pprint
from shutil import copyfile
-from time import monotonic, sleep
+from time import sleep
+import pytest
import requests
+from python_on_whales import DockerClient, docker
+from python_on_whales.exceptions import DockerException
# isort:off (needed to workaround isort bug)
from docker_tests.command_utils import run_command
from docker_tests.constants import SOURCE_ROOT
-from docker_tests.docker_tests_utils import docker_image
# isort:on (needed to workaround isort bug)
@@ -52,56 +52,11 @@ def api_request(method: str, path: str, base_url: str =
"http://localhost:8080/a
return response.json()
-def wait_for_container(container_id: str, timeout: int = 300):
- container_name = (
- subprocess.check_output(["docker", "inspect", container_id,
"--format", "{{ .Name }}"])
- .decode()
- .strip()
- )
- print(f"Waiting for container: {container_name} [{container_id}] for
{timeout} more seconds.")
- waiting_done = False
- start_time = monotonic()
- while not waiting_done:
- container_state = (
- subprocess.check_output(["docker", "inspect", container_id,
"--format", "{{ .State.Status }}"])
- .decode()
- .strip()
- )
- if container_state in ("running", "restarting"):
- health_status = (
- subprocess.check_output(
- [
- "docker",
- "inspect",
- container_id,
- "--format",
- "{{ if .State.Health }}{{ .State.Health.Status }}{{
else }}no-check{{ end }}",
- ]
- )
- .decode()
- .strip()
- )
- current_time = monotonic()
- print(
- f"{container_name}: container_state={container_state},
health_status={health_status}. "
- f"Waiting for {int(timeout - (current_time - start_time))}
more seconds"
- )
-
- if health_status == "healthy" or health_status == "no-check":
- waiting_done = True
- else:
- print(f"{container_name}: container_state={container_state}")
- waiting_done = True
- if timeout != 0 and monotonic() - start_time > timeout:
- raise Exception(f"Timeout. The operation takes longer than the
maximum waiting time ({timeout}s)")
- sleep(1)
-
-
def wait_for_terminal_dag_state(dag_id, dag_run_id):
print(f" Simplified representation of DAG {dag_id} ".center(72, "="))
pprint(api_request("GET", f"dags/{DAG_ID}/details"))
- # Wait 80 seconds
+ # Wait 400 seconds
for _ in range(400):
dag_state = api_request("GET",
f"dags/{dag_id}/dagRuns/{dag_run_id}").get("state")
print(f"Waiting for DAG Run: dag_state={dag_state}")
@@ -110,11 +65,10 @@ def wait_for_terminal_dag_state(dag_id, dag_run_id):
break
-def test_trigger_dag_and_wait_for_result(tmp_path_factory, monkeypatch):
+def test_trigger_dag_and_wait_for_result(default_docker_image,
tmp_path_factory, monkeypatch):
"""Simple test which reproduce setup docker-compose environment and
trigger example dag."""
tmp_dir = tmp_path_factory.mktemp("airflow-quick-start")
- monkeypatch.chdir(tmp_dir)
- monkeypatch.setenv("AIRFLOW_IMAGE_NAME", docker_image)
+ monkeypatch.setenv("AIRFLOW_IMAGE_NAME", default_docker_image)
compose_file_path = (
SOURCE_ROOT / "docs" / "apache-airflow" / "howto" / "docker-compose" /
"docker-compose.yaml"
@@ -130,19 +84,24 @@ def test_trigger_dag_and_wait_for_result(tmp_path_factory,
monkeypatch):
print(" .env file content ".center(72, "="))
print(dot_env_file.read_text())
- # check if docker-compose is available
- compose_command = ["docker", "compose"]
- success = run_command([*compose_command, "version"], check=False)
- if not success:
- print("ERROR: `docker compose` not available. Make sure compose plugin
is installed")
- sys.exit(1)
- compose_command.extend(["--project-name", "quick-start"])
- run_command([*compose_command, "config"])
- run_command([*compose_command, "down", "--volumes", "--remove-orphans"])
- run_command([*compose_command, "up", "-d", "--wait"])
- api_request("PATCH", path=f"dags/{DAG_ID}", json={"is_paused": False})
- api_request("POST", path=f"dags/{DAG_ID}/dagRuns", json={"dag_run_id":
DAG_RUN_ID})
+ compose_version = None
try:
+ compose_version = docker.compose.version()
+ except DockerException:
+ pytest.fail("`docker compose` not available. Make sure compose plugin
is installed")
+ try:
+ docker_version = docker.version()
+ except NotImplementedError:
+ docker_version = run_command(["docker", "version"], return_output=True)
+
+ compose = DockerClient(compose_project_name="quick-start",
compose_project_directory=tmp_dir).compose
+ compose.down(remove_orphans=True, volumes=True, quiet=True)
+ try:
+ compose.up(detach=True, wait=True, color=not
os.environ.get("NO_COLOR"))
+
+ api_request("PATCH", path=f"dags/{DAG_ID}", json={"is_paused": False})
+ api_request("POST", path=f"dags/{DAG_ID}/dagRuns", json={"dag_run_id":
DAG_RUN_ID})
+
wait_for_terminal_dag_state(dag_id=DAG_ID, dag_run_id=DAG_RUN_ID)
dag_state = api_request("GET",
f"dags/{DAG_ID}/dagRuns/{DAG_RUN_ID}").get("state")
assert dag_state == "success"
@@ -153,26 +112,29 @@ def
test_trigger_dag_and_wait_for_result(tmp_path_factory, monkeypatch):
pprint(api_request("GET", f"dags/{DAG_ID}/dagRuns"))
print(f"HTTP: GET dags/{DAG_ID}/dagRuns/{DAG_RUN_ID}/taskInstances")
pprint(api_request("GET",
f"dags/{DAG_ID}/dagRuns/{DAG_RUN_ID}/taskInstances"))
- print(f"Current working directory: {os.getcwd()}")
- run_command(["docker", "version"])
- run_command([*compose_command, "version"])
- run_command(["docker", "ps"])
- run_command([*compose_command, "logs"])
- ps_output = run_command([*compose_command, "ps", "--format", "json"],
return_output=True)
- container_names = [container["Name"] for container in
json.loads(ps_output)]
- for container in container_names:
- print(f"Health check for {container}")
- result = run_command(
- ["docker", "inspect", "--format", "{{json .State}}",
container], return_output=True
- )
- pprint(json.loads(result))
+ print(" Docker Version ".center(72, "="))
+ print(docker_version)
+ print(" Docker Compose Version ".center(72, "="))
+ print(compose_version)
+ print(" Compose Config ".center(72, "="))
+ print(json.dumps(compose.config(return_json=True), indent=4))
+
+ for service in compose.ps(all=True):
+ print(f" Service: {service.name} ".center(72, "-"))
+ print(" Service State ".center(72, "."))
+ pprint(service.state)
+ print(" Service Config ".center(72, "."))
+ pprint(service.config)
+ print(" Service Logs ".center(72, "."))
+ print(service.logs())
raise
finally:
if not os.environ.get("SKIP_DOCKER_COMPOSE_DELETION"):
- run_command([*compose_command, "down", "--volumes"])
+ compose.down(remove_orphans=True, volumes=True, quiet=True)
print("Docker compose instance deleted")
else:
print("Skipping docker-compose deletion")
print()
print("You can run inspect your docker-compose by running commands
starting with:")
- print(" ".join([shlex.quote(arg) for arg in compose_command]))
+ quoted_command = map(shlex.quote, map(str,
compose.docker_compose_cmd))
+ print(" ".join(quoted_command))
diff --git a/docker_tests/test_examples_of_prod_image_building.py
b/docker_tests/test_examples_of_prod_image_building.py
index 991f9c8e2b..6931a78b79 100644
--- a/docker_tests/test_examples_of_prod_image_building.py
+++ b/docker_tests/test_examples_of_prod_image_building.py
@@ -24,6 +24,7 @@ from pathlib import Path
import pytest
import requests
+from python_on_whales import docker
# isort:off (needed to workaround isort bug)
from docker_tests.command_utils import run_command
@@ -52,17 +53,21 @@ def test_shell_script_example(script_file):
@pytest.mark.parametrize("dockerfile",
glob.glob(f"{DOCKER_EXAMPLES_DIR}/**/Dockerfile", recursive=True))
-def test_dockerfile_example(dockerfile):
+def test_dockerfile_example(dockerfile, tmp_path):
rel_dockerfile_path = Path(dockerfile).relative_to(DOCKER_EXAMPLES_DIR)
image_name = str(rel_dockerfile_path).lower().replace("/", "-")
content = Path(dockerfile).read_text()
test_image = os.environ.get("TEST_IMAGE", get_latest_airflow_image())
- new_content = re.sub(r"FROM apache/airflow:.*", rf"FROM {test_image}",
content)
+
+ test_image_file = tmp_path / image_name
+ test_image_file.write_text(re.sub(r"FROM apache/airflow:.*", rf"FROM
{test_image}", content))
try:
- run_command(
- ["docker", "build", ".", "--tag", image_name, "-f", "-"],
- cwd=str(Path(dockerfile).parent),
- input=new_content.encode(),
+ image = docker.build(
+ context_path=Path(dockerfile).parent,
+ tags=image_name,
+ file=test_image_file,
+ load=True, # Load image to docker daemon
)
+ assert image
finally:
- run_command(["docker", "rmi", "--force", image_name])
+ docker.image.remove(image_name, force=True)
diff --git a/docker_tests/test_prod_image.py b/docker_tests/test_prod_image.py
index ac56cc6eff..3e74dc4836 100644
--- a/docker_tests/test_prod_image.py
+++ b/docker_tests/test_prod_image.py
@@ -18,18 +18,18 @@ from __future__ import annotations
import json
import os
-import subprocess
from importlib.util import find_spec
from pathlib import Path
import pytest
+from python_on_whales import DockerException
-from docker_tests.command_utils import run_command
from docker_tests.constants import SOURCE_ROOT
-from docker_tests.docker_tests_utils import (
+from docker_tests.docker_utils import (
display_dependency_conflict_message,
- docker_image,
+ run_airflow_cmd_in_docker,
run_bash_in_docker,
+ run_cmd_in_docker,
run_python_in_docker,
)
@@ -49,45 +49,36 @@ REGULAR_IMAGE_PROVIDERS = [
class TestCommands:
- def test_without_command(self):
+ def test_without_command(self, default_docker_image):
"""Checking the image without a command. It should return non-zero
exit code."""
- with pytest.raises(subprocess.CalledProcessError) as ctx:
- run_command(["docker", "run", "--rm", "-e", "COLUMNS=180",
docker_image])
- assert 2 == ctx.value.returncode
-
- def test_airflow_command(self):
- """Checking 'airflow' command It should return non-zero exit code."""
- with pytest.raises(subprocess.CalledProcessError) as ctx:
- run_command(["docker", "run", "--rm", "-e", "COLUMNS=180",
docker_image, "airflow"])
- assert 2 == ctx.value.returncode
-
- def test_airflow_version(self):
- """Checking 'airflow version' command It should return zero exit
code."""
- output = run_command(
- ["docker", "run", "--rm", "-e", "COLUMNS=180", docker_image,
"airflow", "version"],
- return_output=True,
- )
+ with pytest.raises(DockerException) as ctx:
+ run_cmd_in_docker(image=default_docker_image)
+ assert 2 == ctx.value.return_code
+
+ def test_airflow_command(self, default_docker_image):
+ """Checking 'airflow' command. It should return non-zero exit code."""
+ with pytest.raises(DockerException) as ctx:
+ run_airflow_cmd_in_docker(image=default_docker_image)
+ assert 2 == ctx.value.return_code
+
+ def test_airflow_version(self, default_docker_image):
+ """Checking 'airflow version' command. It should return zero exit
code."""
+ output = run_airflow_cmd_in_docker(["version"],
image=default_docker_image)
assert "2." in output
- def test_python_version(self):
- """Checking 'python --version' command It should return zero exit
code."""
- output = run_command(
- ["docker", "run", "--rm", "-e", "COLUMNS=180", docker_image,
"python", "--version"],
- return_output=True,
- )
+ def test_python_version(self, default_docker_image):
+ """Checking 'python --version' command. It should return zero exit
code."""
+ output = run_cmd_in_docker(cmd=["python", "--version"],
image=default_docker_image)
assert "Python 3." in output
- def test_bash_version(self):
+ def test_bash_version(self, default_docker_image):
"""Checking 'bash --version' command It should return zero exit
code."""
- output = run_command(
- ["docker", "run", "--rm", "-e", "COLUMNS=180", docker_image,
"bash", "--version"],
- return_output=True,
- )
+ output = run_cmd_in_docker(cmd=["bash", "--version"],
image=default_docker_image)
assert "GNU bash," in output
class TestPythonPackages:
- def test_required_providers_are_installed(self):
+ def test_required_providers_are_installed(self, default_docker_image):
if os.environ.get("TEST_SLIM_IMAGE"):
packages_to_install = set(SLIM_IMAGE_PROVIDERS)
package_file = AIRFLOW_PRE_INSTALLED_PROVIDERS_FILE_PATH
@@ -95,9 +86,7 @@ class TestPythonPackages:
packages_to_install = set(REGULAR_IMAGE_PROVIDERS)
package_file = PROD_IMAGE_PROVIDERS_FILE_PATH
assert len(packages_to_install) != 0
- output = run_bash_in_docker(
- "airflow providers list --output json", stderr=subprocess.DEVNULL,
return_output=True
- )
+ output = run_bash_in_docker("airflow providers list --output json",
image=default_docker_image)
providers = json.loads(output)
packages_installed = set(d["package_name"] for d in providers)
assert len(packages_installed) != 0
@@ -106,12 +95,12 @@ class TestPythonPackages:
packages_to_install == packages_installed
), f"List of expected installed packages and image content mismatch.
Check {package_file} file."
- def test_pip_dependencies_conflict(self):
+ def test_pip_dependencies_conflict(self, default_docker_image):
try:
- run_bash_in_docker("pip check")
- except subprocess.CalledProcessError as ex:
+ run_bash_in_docker("pip check", image=default_docker_image)
+ except DockerException:
display_dependency_conflict_message()
- raise ex
+ raise
PACKAGE_IMPORTS = {
"amazon": ["boto3", "botocore", "watchtower"],
@@ -179,47 +168,28 @@ class TestPythonPackages:
@pytest.mark.skipif(os.environ.get("TEST_SLIM_IMAGE") == "true",
reason="Skipped with slim image")
@pytest.mark.parametrize("package_name,import_names",
PACKAGE_IMPORTS.items())
- def test_check_dependencies_imports(self, package_name, import_names):
- run_python_in_docker(f"import {','.join(import_names)}")
+ def test_check_dependencies_imports(self, package_name, import_names,
default_docker_image):
+ run_python_in_docker(f"import {','.join(import_names)}",
image=default_docker_image)
class TestExecuteAsRoot:
- def test_execute_airflow_as_root(self):
- run_command(
- [
- "docker",
- "run",
- "--rm",
- "--user",
- "0",
- "-e",
- "PYTHONDONTWRITEBYTECODE=true",
- docker_image,
- "airflow",
- "info",
- ]
+ def test_execute_airflow_as_root(self, default_docker_image):
+ run_cmd_in_docker(
+ cmd=["airflow", "info"],
+ user=0,
+ envs={"PYTHONDONTWRITEBYTECODE": "true"},
+ image=default_docker_image,
)
- def test_run_custom_python_packages_as_root(self, tmp_path):
+ def test_run_custom_python_packages_as_root(self, tmp_path,
default_docker_image):
(tmp_path / "__init__.py").write_text("")
(tmp_path / "awesome.py").write_text('print("Awesome")')
- run_command(
- [
- "docker",
- "run",
- "--rm",
- "-e",
- f"PYTHONPATH={tmp_path}",
- "-e",
- "PYTHONDONTWRITEBYTECODE=true",
- "-v",
- f"{tmp_path}:{tmp_path}",
- "--user",
- "0",
- docker_image,
- "python",
- "-c",
- "import awesome",
- ]
+ output = run_cmd_in_docker(
+ envs={"PYTHONPATH": "/custom/mount", "PYTHONDONTWRITEBYTECODE":
"true"},
+ volumes=[(tmp_path.as_posix(), "/custom/mount")],
+ user=0,
+ cmd=["python", "-c", "import awesome"],
+ image=default_docker_image,
)
+ assert output.strip() == "Awesome"
