This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v3-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-3-test by this push:
new 8f25ac3bd48 [v3-3-test] Stabilize testcontainers provider tests on
slow (ARM) CI runners (#68963) (#69054)
8f25ac3bd48 is described below
commit 8f25ac3bd480fb972cb2b265f0c8d35b0e503ef9
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Fri Jun 26 16:05:48 2026 -0400
[v3-3-test] Stabilize testcontainers provider tests on slow (ARM) CI
runners (#68963) (#69054)
The mongo provider tests boot a real mongo:8.0 container via testcontainers
in the setup phase of the first test that uses the session fixture. That
bring-up -- image pull, container start, readiness ping -- has to fit inside
the per-test pytest setup-timeout (60s). On the GitHub-hosted ARM canary,
where the image cache is cold and registry/disk throughput is lower, the
pull alone can eat most of that budget, so setup times out and cascades into
"ERROR at setup" across every TestMongoHook test. AMD does not hit this
because it runs on every PR with warm caches; ARM only runs on the cron
canary.
Two complementary changes:
- Pre-pull the testcontainers images (mongo:8.0 + ryuk) on the shared host
daemon before the timed run, but only in CI and only when the owning
provider's tests are actually in the selection -- so the slow cold-cache
pull happens outside the per-test setup timeout. Locally it stays a no-op.
- Give pytest setup/teardown timeouts a higher floor (180s) for container-
backed fixtures, while keeping execution-timeout at the per-test value so
genuinely hung tests are still caught quickly.
(cherry picked from commit 9b0fe283845e749428f134bd5a53655ea6702783)
Co-authored-by: Jarek Potiuk <[email protected]>
---
.../airflow_breeze/commands/testing_commands.py | 25 +++++++
dev/breeze/src/airflow_breeze/global_constants.py | 11 +++
dev/breeze/src/airflow_breeze/utils/run_tests.py | 36 +++++++++-
dev/breeze/tests/test_run_test_args.py | 79 +++++++++++++++++++++-
4 files changed, 147 insertions(+), 4 deletions(-)
diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py
b/dev/breeze/src/airflow_breeze/commands/testing_commands.py
index cc48ee0fb99..c3c210d9e8c 100644
--- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py
@@ -89,6 +89,7 @@ from airflow_breeze.global_constants import (
ALL_TEST_TYPE,
ALLOWED_KUBERNETES_VERSIONS,
ALLOWED_TEST_TYPE_CHOICES,
+ TESTCONTAINERS_IMAGES_BY_PROVIDER,
GroupOfTests,
all_selective_core_test_types,
providers_test_type,
@@ -105,6 +106,7 @@ from airflow_breeze.utils.docker_command_utils import (
perform_environment_checks,
remove_docker_networks,
)
+from airflow_breeze.utils.environment_check import is_ci_environment
from airflow_breeze.utils.parallel import (
GenericRegexpProgressMatcher,
SummarizeAfter,
@@ -117,6 +119,7 @@ from airflow_breeze.utils.run_tests import (
are_all_test_paths_excluded,
file_name_from_test_type,
generate_args_for_pytest,
+ is_provider_selected_in_test_types,
run_docker_compose_tests,
)
from airflow_breeze.utils.run_utils import RunCommandResult, run_command
@@ -429,6 +432,28 @@ def pull_images_for_docker_compose(shell_params:
ShellParams):
"pull",
]
run_command(pull_cmd, output=None, check=False, env=env)
+ pull_testcontainers_images(shell_params)
+
+
+def pull_testcontainers_images(shell_params: ShellParams):
+ """Pre-pull images that specific provider tests start directly via
testcontainers.
+
+ These bypass docker compose, so ``docker compose pull`` does not warm
them. Only in CI, and only
+ when the owning provider's tests are actually in the run, pull the image
on the shared host daemon
+ before the timed run -- keeping the (cold-cache, ARM-slow) pull out of the
per-test setup timeout.
+ Locally this is a no-op: testcontainers pulls on demand as usual.
+ """
+ if not is_ci_environment() or shell_params.test_group !=
GroupOfTests.PROVIDERS:
+ return
+ test_types = shell_params.parallel_test_types_list or (
+ [shell_params.test_type] if shell_params.test_type else []
+ )
+ env = shell_params.env_variables_for_docker_commands
+ for provider_id, images in TESTCONTAINERS_IMAGES_BY_PROVIDER.items():
+ if not is_provider_selected_in_test_types(provider_id, test_types):
+ continue
+ for image in images:
+ run_command(["docker", "pull", image], output=None, check=False,
env=env)
def run_tests_in_parallel(
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py
b/dev/breeze/src/airflow_breeze/global_constants.py
index 4d952341891..202f98f0fca 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -72,6 +72,17 @@ CUSTOM_BACKEND = "custom"
ALLOWED_BACKENDS = [SQLITE_BACKEND, MYSQL_BACKEND, POSTGRES_BACKEND,
NONE_BACKEND, CUSTOM_BACKEND]
ALLOWED_PROD_BACKENDS = [MYSQL_BACKEND, POSTGRES_BACKEND]
DEFAULT_BACKEND = ALLOWED_BACKENDS[0]
+# Docker images that specific provider tests pull directly via testcontainers
(bypassing docker
+# compose, so the "docker compose pull" pre-pull does not cover them). Keyed
by the provider
+# distribution (dotted id) whose tests pull them, so an image is only
pre-pulled when that provider's
+# tests are actually in the run. Pre-pulling (in CI only, before the timed
run) keeps the pull -- slow
+# on cold caches, notably the GitHub-hosted ARM canary -- out of the per-test
setup timeout. Keep the
+# tags in sync with the provider test conftests that use them (currently
+# providers/mongo/tests/conftest.py: MONGO_IMAGE + the testcontainers ryuk
reaper).
+TESTCONTAINERS_IMAGES_BY_PROVIDER: dict[str, list[str]] = {
+ "mongo": ["mongo:8.0", "testcontainers/ryuk:0.8.1"],
+}
+
TESTABLE_CORE_INTEGRATIONS = ["kerberos", "otel", "redis"]
TESTABLE_PROVIDERS_INTEGRATIONS = [
"celery",
diff --git a/dev/breeze/src/airflow_breeze/utils/run_tests.py
b/dev/breeze/src/airflow_breeze/utils/run_tests.py
index 3bc2a2e0f7b..ef87dd1f6e3 100644
--- a/dev/breeze/src/airflow_breeze/utils/run_tests.py
+++ b/dev/breeze/src/airflow_breeze/utils/run_tests.py
@@ -62,6 +62,11 @@ IGNORE_WARNING_OUTPUT_FOR_TEST_GROUPS = [
GroupOfTests.PYTHON_API_CLIENT,
]
+# Floor for pytest setup/teardown timeouts. Container-backed fixtures
(testcontainers) pull an image
+# and boot a container during setup; on slower runners this can exceed the
regular per-test timeout.
+# Execution-timeout is kept at the per-test value so hung tests are still
caught quickly.
+MIN_SETUP_TEARDOWN_TIMEOUT = 180
+
def verify_an_image(
image_name: str,
@@ -365,6 +370,27 @@ PROVIDERS_LIST_PREFIX = "Providers["
PROVIDERS_LIST_EXCLUDE_PREFIX = "Providers[-"
+def is_provider_selected_in_test_types(provider_id: str, test_types:
list[str]) -> bool:
+ """Whether a provider's tests are included by any of the given pytest
test-type selectors.
+
+ A bare selector such as ``Providers`` (no bracket) runs every provider;
``Providers[a,b]`` runs
+ only the listed ones; ``Providers[-a,b]`` runs all except the listed ones.
+ """
+ for test_type in test_types:
+ if test_type.startswith(PROVIDERS_LIST_EXCLUDE_PREFIX):
+ excluded = test_type[len(PROVIDERS_LIST_EXCLUDE_PREFIX) :
-1].split(",")
+ if provider_id not in excluded:
+ return True
+ elif test_type.startswith(PROVIDERS_LIST_PREFIX):
+ included = test_type[len(PROVIDERS_LIST_PREFIX) : -1].split(",")
+ if provider_id in included:
+ return True
+ else:
+ # A non-bracketed selector (e.g. "Providers", "All") runs every
provider.
+ return True
+ return False
+
+
def convert_test_type_to_pytest_args(
*,
test_group: GroupOfTests,
@@ -475,6 +501,12 @@ def generate_args_for_pytest(
integration: tuple | None = None,
):
result_log_file, warnings_file, coverage_file = test_paths(test_type,
backend)
+ # Container-backed fixtures (e.g. the testcontainers MongoDB used by
provider tests) pull their
+ # image and boot the container in the *setup* phase of the first test that
uses them. On slower
+ # runners -- notably the GitHub-hosted ARM canary, where the image cache
is cold -- that can exceed
+ # the regular per-test timeout. Give setup/teardown a higher floor so the
bring-up fits, while
+ # keeping execution-timeout tight so a genuinely hung test is still caught
quickly.
+ setup_teardown_timeout = max(test_timeout, MIN_SETUP_TEARDOWN_TIMEOUT)
if skip_db_tests and parallel_test_types_list:
args = convert_parallel_types_to_folders(
test_group=test_group,
@@ -496,9 +528,9 @@ def generate_args_for_pytest(
f"--junitxml={result_log_file}",
# timeouts in seconds for individual tests
"--timeouts-order=moi",
- f"--setup-timeout={test_timeout}",
+ f"--setup-timeout={setup_teardown_timeout}",
f"--execution-timeout={test_timeout}",
- f"--teardown-timeout={test_timeout}",
+ f"--teardown-timeout={setup_teardown_timeout}",
"--disable-warnings",
# Only display summary for non-expected cases
#
diff --git a/dev/breeze/tests/test_run_test_args.py
b/dev/breeze/tests/test_run_test_args.py
index 454787eb74b..bac6d67288a 100644
--- a/dev/breeze/tests/test_run_test_args.py
+++ b/dev/breeze/tests/test_run_test_args.py
@@ -20,8 +20,8 @@ from unittest.mock import patch
import pytest
-from airflow_breeze.commands.testing_commands import _run_test
-from airflow_breeze.global_constants import GroupOfTests
+from airflow_breeze.commands.testing_commands import _run_test,
pull_testcontainers_images
+from airflow_breeze.global_constants import TESTCONTAINERS_IMAGES_BY_PROVIDER,
GroupOfTests
from airflow_breeze.params.shell_params import ShellParams
@@ -152,3 +152,78 @@ def
test_none_test_type_with_extra_args_does_not_skip(mock_run_command):
skip_docker_compose_down=True,
)
assert mock_run_command.call_count >= 2 # compose down + compose run
+
+
[email protected](
+ ("test_timeout", "expected_setup_teardown", "expected_execution"),
+ [
+ # Below the floor: setup/teardown are raised to the floor, execution
stays tight.
+ (60, 180, 60),
+ # At/above the floor: all three follow the requested per-test timeout.
+ (300, 300, 300),
+ ],
+)
+def test_setup_teardown_timeout_has_minimum_floor(
+ mock_run_command, test_timeout, expected_setup_teardown, expected_execution
+):
+ """Container-backed fixtures boot in the setup phase, so setup/teardown
get a higher floor while
+ execution-timeout stays at the per-test value to keep catching hung tests
quickly."""
+ _run_test(
+ shell_params=ShellParams(test_group=GroupOfTests.PROVIDERS,
test_type="Providers"),
+ extra_pytest_args=(),
+ python_version="3.9",
+ output=None,
+ test_timeout=test_timeout,
+ skip_docker_compose_down=True,
+ )
+ arg_str = " ".join(mock_run_command.call_args_list[1].args[0])
+ assert f"--setup-timeout={expected_setup_teardown}" in arg_str
+ assert f"--teardown-timeout={expected_setup_teardown}" in arg_str
+ assert f"--execution-timeout={expected_execution}" in arg_str
+
+
[email protected]
+def mock_is_ci_environment():
+ with patch("airflow_breeze.commands.testing_commands.is_ci_environment")
as mck:
+ mck.return_value = True
+ yield mck
+
+
+def _pulled_images(mock_run_command):
+ return {
+ call.args[0][-1]
+ for call in mock_run_command.call_args_list
+ if call.args and call.args[0][:2] == ["docker", "pull"]
+ }
+
+
[email protected](
+ ("test_group", "test_types", "expected"),
+ [
+ # mongo provider in the selection -> its testcontainers images are
pre-pulled
+ (GroupOfTests.PROVIDERS, ["Providers[mongo]"],
set(TESTCONTAINERS_IMAGES_BY_PROVIDER["mongo"])),
+ # a bare "Providers" selector runs every provider -> mongo included
+ (GroupOfTests.PROVIDERS, ["Providers"],
set(TESTCONTAINERS_IMAGES_BY_PROVIDER["mongo"])),
+ # mongo excluded by name -> nothing pulled
+ (GroupOfTests.PROVIDERS, ["Providers[mysql,neo4j]"], set()),
+ (GroupOfTests.PROVIDERS, ["Providers[-mongo]"], set()),
+ # not the providers group -> nothing pulled
+ (GroupOfTests.CORE, ["Providers[mongo]"], set()),
+ ],
+ ids=["mongo-listed", "all-providers", "mongo-not-listed",
"mongo-excluded", "core-group"],
+)
+def test_pull_testcontainers_images_scoped_to_selected_providers(
+ mock_run_command, mock_is_ci_environment, test_group, test_types, expected
+):
+ """In CI, a provider's testcontainers images are pulled only when that
provider is in the run."""
+ pull_testcontainers_images(ShellParams(test_group=test_group,
parallel_test_types_list=test_types))
+ assert _pulled_images(mock_run_command) == expected
+
+
+def test_pull_testcontainers_images_noop_outside_ci(mock_run_command,
mock_is_ci_environment):
+ """Outside CI it is a no-op -- testcontainers pulls on demand as usual."""
+ mock_is_ci_environment.return_value = False
+ pull_testcontainers_images(
+ ShellParams(test_group=GroupOfTests.PROVIDERS,
parallel_test_types_list=["Providers[mongo]"])
+ )
+ assert _pulled_images(mock_run_command) == set()
