This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 73a3733391 Bump min-version for docker and docker-compose (#33572)
73a3733391 is described below
commit 73a37333918abe0612120d95169b9e377274810b
Author: Jarek Potiuk <[email protected]>
AuthorDate: Mon Aug 21 17:43:44 2023 +0200
Bump min-version for docker and docker-compose (#33572)
It's been quite some time since we bumped min versions for docker
and docker-compose for Breeze and there are already a few cases where
it held us back from more efficiently using some newer docker or
docker-compose features (for example in #33547). Also docker-compose
v1 is sufficiently old an unmaintained, that it's about time to
get rid of it and ask the users to migrate to docker-compose v2.
The new min versions propose:
* Docker: 23.0.0 released 2023-02-01 (6 months ago)
* Docker Compose 2.14.0 release 2022-12-15 (8 months ago)
Docker v1 is not supported any more and the users are directed
to migrate to v2 to continue using Breeze.
---
.../airflow_breeze/commands/developer_commands.py | 7 ++-
.../airflow_breeze/commands/testing_commands.py | 12 +++--
dev/breeze/src/airflow_breeze/global_constants.py | 4 +-
.../airflow_breeze/utils/docker_command_utils.py | 51 +++++++++----------
dev/breeze/tests/test_docker_command_utils.py | 58 ++++++++++++----------
5 files changed, 67 insertions(+), 65 deletions(-)
diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py
b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
index 59671a273f..40fc095478 100644
--- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
@@ -77,7 +77,6 @@ from airflow_breeze.utils.common_options import (
from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.custom_param_types import BetterChoice,
NotVerifiedBetterChoice
from airflow_breeze.utils.docker_command_utils import (
- DOCKER_COMPOSE_COMMAND,
check_docker_resources,
get_env_variables_for_docker_commands,
get_extra_docker_flags,
@@ -652,7 +651,7 @@ def compile_www_assets(dev: bool):
@option_dry_run
def down(preserve_volumes: bool, cleanup_mypy_cache: bool):
perform_environment_checks()
- command_to_execute = [*DOCKER_COMPOSE_COMMAND, "down", "--remove-orphans"]
+ command_to_execute = ["docker", "compose", "down", "--remove-orphans"]
if not preserve_volumes:
command_to_execute.append("--volumes")
shell_params = ShellParams(backend="all", include_mypy_volume=True)
@@ -723,7 +722,7 @@ def enter_shell(**kwargs) -> RunCommandResult:
if shell_params.include_mypy_volume:
create_mypy_volume_if_needed()
shell_params.print_badge_info()
- cmd = [*DOCKER_COMPOSE_COMMAND, "run", "--service-ports", "-e", "BREEZE",
"--rm", "airflow"]
+ cmd = ["docker", "compose", "run", "--service-ports", "-e", "BREEZE",
"--rm", "airflow"]
cmd_added = shell_params.command_passed
env_variables = get_env_variables_for_docker_commands(shell_params)
if cmd_added is not None:
@@ -763,7 +762,7 @@ def find_airflow_container() -> str | None:
check_docker_resources(exec_shell_params.airflow_image_name)
exec_shell_params.print_badge_info()
env_variables = get_env_variables_for_docker_commands(exec_shell_params)
- cmd = [*DOCKER_COMPOSE_COMMAND, "ps", "--all", "--filter",
"status=running", "airflow"]
+ cmd = ["docker", "compose", "ps", "--all", "--filter", "status=running",
"airflow"]
docker_compose_ps_command = run_command(
cmd, text=True, capture_output=True, env=env_variables, check=False
)
diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py
b/dev/breeze/src/airflow_breeze/commands/testing_commands.py
index 68e3c688c6..c7c018a0da 100644
--- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py
@@ -58,7 +58,6 @@ from airflow_breeze.utils.common_options import (
from airflow_breeze.utils.console import Output, get_console
from airflow_breeze.utils.custom_param_types import BetterChoice,
NotVerifiedBetterChoice
from airflow_breeze.utils.docker_command_utils import (
- DOCKER_COMPOSE_COMMAND,
get_env_variables_for_docker_commands,
perform_environment_checks,
remove_docker_networks,
@@ -170,7 +169,8 @@ def _run_test(
# This is needed for Docker-compose 1 compatibility
env_variables["COMPOSE_PROJECT_NAME"] = compose_project_name
down_cmd = [
- *DOCKER_COMPOSE_COMMAND,
+ "docker",
+ "compose",
"--project-name",
compose_project_name,
"down",
@@ -178,7 +178,8 @@ def _run_test(
]
run_command(down_cmd, env=env_variables, output=output, check=False)
run_cmd = [
- *DOCKER_COMPOSE_COMMAND,
+ "docker",
+ "compose",
"--project-name",
compose_project_name,
"run",
@@ -226,7 +227,8 @@ def _run_test(
if not skip_docker_compose_down:
run_command(
[
- *DOCKER_COMPOSE_COMMAND,
+ "docker",
+ "compose",
"--project-name",
compose_project_name,
"rm",
@@ -587,6 +589,6 @@ def helm_tests(
env_variables["HELM_TEST_PACKAGE"] = helm_test_package
perform_environment_checks()
cleanup_python_generated_files()
- cmd = [*DOCKER_COMPOSE_COMMAND, "run", "--service-ports", "--rm",
"airflow", *extra_pytest_args]
+ cmd = ["docker", "compose", "run", "--service-ports", "--rm", "airflow",
*extra_pytest_args]
result = run_command(cmd, env=env_variables, check=False,
output_outside_the_group=True)
sys.exit(result.returncode)
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py
b/dev/breeze/src/airflow_breeze/global_constants.py
index ceafd6c43f..c0740801d5 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -350,8 +350,8 @@ GITHUB_ACTIONS = ""
ISSUE_ID = ""
NUM_RUNS = ""
-MIN_DOCKER_VERSION = "20.10.0"
-MIN_DOCKER_COMPOSE_VERSION = "1.29.0"
+MIN_DOCKER_VERSION = "23.0.0"
+MIN_DOCKER_COMPOSE_VERSION = "2.14.0"
AIRFLOW_SOURCES_FROM = "."
AIRFLOW_SOURCES_TO = "/opt/airflow"
diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
index 5332b149ed..d0a12e965a 100644
--- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
@@ -250,6 +250,7 @@ def check_docker_version():
[warning]install docker at least: {MIN_DOCKER_VERSION} version.[/]
"""
)
+ sys.exit(1)
else:
good_version = compare_version(docker_version, MIN_DOCKER_VERSION)
if good_version:
@@ -257,10 +258,12 @@ def check_docker_version():
else:
get_console().print(
f"""
-[warning]Your version of docker is too old:{docker_version}.
-Please upgrade to at least {MIN_DOCKER_VERSION}[/]
+[error]Your version of docker is too old: {docker_version}.\n[/]
+[warning]Please upgrade to at least {MIN_DOCKER_VERSION}.\n[/]
+You can find installation instructions here:
https://docs.docker.com/engine/install/
"""
)
+ sys.exit(1)
def check_remote_ghcr_io_commands():
@@ -306,9 +309,6 @@ def check_remote_ghcr_io_commands():
sys.exit(1)
-DOCKER_COMPOSE_COMMAND = ["docker", "compose"]
-
-
def check_docker_compose_version():
"""Checks if the docker compose version is as expected.
@@ -328,43 +328,40 @@ def check_docker_compose_version():
dry_run_override=False,
)
except Exception:
- docker_compose_version_command = ["docker-compose", "--version"]
- docker_compose_version_result = run_command(
- docker_compose_version_command,
- no_output_dump_on_exception=True,
- capture_output=True,
- text=True,
- dry_run_override=False,
+ get_console().print(
+ "[error]You either do not have docker-composer or have
docker-compose v1 installed.[/]\n"
+ "[warning]Breeze does not support docker-compose v1 any more as it
has been replaced by v2.[/]\n"
+ "Follow https://docs.docker.com/compose/migrate/ to migrate to v2"
)
- DOCKER_COMPOSE_COMMAND.clear()
- DOCKER_COMPOSE_COMMAND.append("docker-compose")
+ sys.exit(1)
if docker_compose_version_result.returncode == 0:
docker_compose_version = docker_compose_version_result.stdout
version_extracted = version_pattern.search(docker_compose_version)
if version_extracted is not None:
- docker_version = ".".join(version_extracted.groups())
- good_version = compare_version(docker_version,
MIN_DOCKER_COMPOSE_VERSION)
+ docker_compose_version = ".".join(version_extracted.groups())
+ good_version = compare_version(docker_compose_version,
MIN_DOCKER_COMPOSE_VERSION)
if good_version:
- get_console().print(f"[success]Good version of docker-compose:
{docker_version}[/]")
+ get_console().print(f"[success]Good version of docker-compose:
{docker_compose_version}[/]")
else:
get_console().print(
f"""
-[warning]You have too old version of docker-compose: {docker_version}! At
least 1.29 needed! Please upgrade!
-"""
- )
- get_console().print(
- """
-See https://docs.docker.com/compose/install/ for instructions.
-Make sure docker-compose you install is first on the PATH variable of yours.
+[error]You have too old version of docker-compose:
{docker_compose_version}!\n[/]
+[warning]At least {MIN_DOCKER_COMPOSE_VERSION} needed! Please upgrade!\n[/]
+See https://docs.docker.com/compose/install/ for installation instructions.\n
+Make sure docker-compose you install is first on the PATH variable of yours.\n
"""
)
+ sys.exit(1)
else:
get_console().print(
- """
-[warning]Unknown docker-compose version. At least 1.29 is needed![/]
-[warning]If Breeze fails upgrade to latest available docker-compose version.[/]
+ f"""
+[error]Unknown docker-compose version.[/]
+[warning]At least {MIN_DOCKER_COMPOSE_VERSION} needed! Please upgrade!\n[/]
+See https://docs.docker.com/compose/install/ for installation instructions.\n
+Make sure docker-compose you install is first on the PATH variable of yours.\n
"""
)
+ sys.exit(1)
def get_env_variable_value(arg_name: str, params: CommonBuildParams |
ShellParams):
diff --git a/dev/breeze/tests/test_docker_command_utils.py
b/dev/breeze/tests/test_docker_command_utils.py
index 83081ad770..4b1513e30f 100644
--- a/dev/breeze/tests/test_docker_command_utils.py
+++ b/dev/breeze/tests/test_docker_command_utils.py
@@ -36,7 +36,9 @@ def test_check_docker_version_unknown(
mock_get_console, mock_run_command, mock_check_docker_permission_denied
):
mock_check_docker_permission_denied.return_value = False
- check_docker_version()
+ with pytest.raises(SystemExit) as e:
+ check_docker_version()
+ assert e.value.code == 1
expected_run_command_calls = [
call(
["docker", "version", "--format", "{{.Client.Version}}"],
@@ -51,7 +53,7 @@ def test_check_docker_version_unknown(
mock_get_console.return_value.print.assert_called_with(
"""
[warning]Your version of docker is unknown. If the scripts fail, please make
sure to[/]
-[warning]install docker at least: 20.10.0 version.[/]
+[warning]install docker at least: 23.0.0 version.[/]
"""
)
@@ -65,7 +67,9 @@ def test_check_docker_version_too_low(
mock_check_docker_permission_denied.return_value = False
mock_run_command.return_value.returncode = 0
mock_run_command.return_value.stdout = "0.9"
- check_docker_version()
+ with pytest.raises(SystemExit) as e:
+ check_docker_version()
+ assert e.value.code == 1
mock_check_docker_permission_denied.assert_called()
mock_run_command.assert_called_with(
["docker", "version", "--format", "{{.Client.Version}}"],
@@ -77,7 +81,8 @@ def test_check_docker_version_too_low(
)
mock_get_console.return_value.print.assert_called_with(
"""
-[warning]Your version of docker is too old:0.9.\nPlease upgrade to at least
20.10.0[/]
+[error]Your version of docker is too old: 0.9.\n[/]\n[warning]Please upgrade
to at least 23.0.0.\n[/]\n\
+You can find installation instructions here:
https://docs.docker.com/engine/install/
"""
)
@@ -88,7 +93,7 @@ def test_check_docker_version_too_low(
def test_check_docker_version_ok(mock_get_console, mock_run_command,
mock_check_docker_permission_denied):
mock_check_docker_permission_denied.return_value = False
mock_run_command.return_value.returncode = 0
- mock_run_command.return_value.stdout = "20.10.0"
+ mock_run_command.return_value.stdout = "23.0.0"
check_docker_version()
mock_check_docker_permission_denied.assert_called()
mock_run_command.assert_called_with(
@@ -99,7 +104,7 @@ def test_check_docker_version_ok(mock_get_console,
mock_run_command, mock_check_
check=False,
dry_run_override=False,
)
- mock_get_console.return_value.print.assert_called_with("[success]Good
version of Docker: 20.10.0.[/]")
+ mock_get_console.return_value.print.assert_called_with("[success]Good
version of Docker: 23.0.0.[/]")
@mock.patch("airflow_breeze.utils.docker_command_utils.check_docker_permission_denied")
@@ -108,7 +113,7 @@ def test_check_docker_version_ok(mock_get_console,
mock_run_command, mock_check_
def test_check_docker_version_higher(mock_get_console, mock_run_command,
mock_check_docker_permission_denied):
mock_check_docker_permission_denied.return_value = False
mock_run_command.return_value.returncode = 0
- mock_run_command.return_value.stdout = "21.10.0"
+ mock_run_command.return_value.stdout = "24.0.0"
check_docker_version()
mock_check_docker_permission_denied.assert_called()
mock_run_command.assert_called_with(
@@ -119,13 +124,15 @@ def test_check_docker_version_higher(mock_get_console,
mock_run_command, mock_ch
check=False,
dry_run_override=False,
)
- mock_get_console.return_value.print.assert_called_with("[success]Good
version of Docker: 21.10.0.[/]")
+ mock_get_console.return_value.print.assert_called_with("[success]Good
version of Docker: 24.0.0.[/]")
@mock.patch("airflow_breeze.utils.docker_command_utils.run_command")
@mock.patch("airflow_breeze.utils.docker_command_utils.get_console")
def test_check_docker_compose_version_unknown(mock_get_console,
mock_run_command):
- check_docker_compose_version()
+ with pytest.raises(SystemExit) as e:
+ check_docker_compose_version()
+ assert e.value.code == 1
expected_run_command_calls = [
call(
["docker", "compose", "version"],
@@ -138,8 +145,9 @@ def
test_check_docker_compose_version_unknown(mock_get_console, mock_run_command
mock_run_command.assert_has_calls(expected_run_command_calls)
mock_get_console.return_value.print.assert_called_with(
"""
-[warning]Unknown docker-compose version. At least 1.29 is needed![/]
-[warning]If Breeze fails upgrade to latest available docker-compose version.[/]
+[error]Unknown docker-compose version.[/]\n[warning]At least 2.14.0 needed!
Please upgrade!\n[/]
+See https://docs.docker.com/compose/install/ for installation instructions.\n
+Make sure docker-compose you install is first on the PATH variable of yours.\n
"""
)
@@ -149,7 +157,9 @@ def
test_check_docker_compose_version_unknown(mock_get_console, mock_run_command
def test_check_docker_compose_version_low(mock_get_console, mock_run_command):
mock_run_command.return_value.returncode = 0
mock_run_command.return_value.stdout = "1.28.5"
- check_docker_compose_version()
+ with pytest.raises(SystemExit) as e:
+ check_docker_compose_version()
+ assert e.value.code == 1
mock_run_command.assert_called_with(
["docker", "compose", "version"],
no_output_dump_on_exception=True,
@@ -157,27 +167,21 @@ def
test_check_docker_compose_version_low(mock_get_console, mock_run_command):
text=True,
dry_run_override=False,
)
- expected_print_calls = [
- call(
- """
-[warning]You have too old version of docker-compose: 1.28.5! At least 1.29
needed! Please upgrade!
-"""
- ),
- call(
- """
-See https://docs.docker.com/compose/install/ for instructions.
-Make sure docker-compose you install is first on the PATH variable of yours.
+ mock_get_console.return_value.print.assert_called_with(
+ """
+[error]You have too old version of docker-compose: 1.28.5!\n[/]
+[warning]At least 2.14.0 needed! Please upgrade!\n[/]
+See https://docs.docker.com/compose/install/ for installation instructions.\n
+Make sure docker-compose you install is first on the PATH variable of yours.\n
"""
- ),
- ]
- mock_get_console.return_value.print.assert_has_calls(expected_print_calls)
+ )
@mock.patch("airflow_breeze.utils.docker_command_utils.run_command")
@mock.patch("airflow_breeze.utils.docker_command_utils.get_console")
def test_check_docker_compose_version_ok(mock_get_console, mock_run_command):
mock_run_command.return_value.returncode = 0
- mock_run_command.return_value.stdout = "1.29.0"
+ mock_run_command.return_value.stdout = "2.14.0"
check_docker_compose_version()
mock_run_command.assert_called_with(
["docker", "compose", "version"],
@@ -187,7 +191,7 @@ def test_check_docker_compose_version_ok(mock_get_console,
mock_run_command):
dry_run_override=False,
)
mock_get_console.return_value.print.assert_called_with(
- "[success]Good version of docker-compose: 1.29.0[/]"
+ "[success]Good version of docker-compose: 2.14.0[/]"
)
