This is an automated email from the ASF dual-hosted git repository.
gopidesu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 66d5e72a711 Add e2e test for remote logging (#56191)
66d5e72a711 is described below
commit 66d5e72a7117d3f7fb8b32a61f8215016facee57
Author: GPK <[email protected]>
AuthorDate: Wed Oct 15 19:09:57 2025 +0100
Add e2e test for remote logging (#56191)
* Prefetch remote log connection id for api server in order to read remote
logs
* fix docker compose file path
* Fixup tests
* Add test with mock_aws
* Fixup test
* Extend quick start docker with localstack
* remove comment
* add test connection
* fix static checks
* Add only e2e tests for remote logging
---
.github/workflows/additional-prod-image-tests.yml | 12 +++-
.github/workflows/airflow-e2e-tests.yml | 19 ++---
.../docs/howto/docker-compose/docker-compose.yaml | 2 +
airflow-core/tests/unit/utils/test_log_handlers.py | 76 ++++++++++++++++++++
airflow-e2e-tests/docker/localstack.yml | 42 +++++++++++
airflow-e2e-tests/pyproject.toml | 1 +
airflow-e2e-tests/scripts/init-aws.sh | 20 ++++++
.../tests/airflow_e2e_tests/conftest.py | 42 ++++++++---
.../tests/airflow_e2e_tests/constants.py | 7 +-
.../airflow_e2e_tests/e2e_test_utils/clients.py | 7 ++
.../airflow_e2e_tests/remote_log_tests/__init__.py | 16 +++++
.../remote_log_tests/test_remote_logging.py | 83 ++++++++++++++++++++++
.../images/output_testing_airflow-e2e-tests.svg | 32 +++++----
.../images/output_testing_airflow-e2e-tests.txt | 2 +-
.../airflow_breeze/commands/testing_commands.py | 13 ++++
.../commands/testing_commands_config.py | 1 +
dev/breeze/src/airflow_breeze/utils/run_tests.py | 4 +-
17 files changed, 343 insertions(+), 36 deletions(-)
diff --git a/.github/workflows/additional-prod-image-tests.yml
b/.github/workflows/additional-prod-image-tests.yml
index a6d0f890d11..ec98cbb2d26 100644
--- a/.github/workflows/additional-prod-image-tests.yml
+++ b/.github/workflows/additional-prod-image-tests.yml
@@ -192,7 +192,7 @@ jobs:
- name: "Run Task SDK integration tests"
run: breeze testing task-sdk-integration-tests
- test-e2e-integration-tests:
+ test-e2e-integration-tests-basic:
name: "Test e2e integration tests with PROD image"
uses: ./.github/workflows/airflow-e2e-tests.yml
with:
@@ -200,3 +200,13 @@ jobs:
platform: ${{ inputs.platform }}
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
+
+ test-e2e-integration-tests-remote-log:
+ name: "Remote logging tests with PROD image"
+ uses: ./.github/workflows/airflow-e2e-tests.yml
+ with:
+ runners: ${{ inputs.runners }}
+ platform: ${{ inputs.platform }}
+ default-python-version: "${{ inputs.default-python-version }}"
+ use-uv: ${{ inputs.use-uv }}
+ e2e_test_mode: "remote_log"
diff --git a/.github/workflows/airflow-e2e-tests.yml
b/.github/workflows/airflow-e2e-tests.yml
index 31e6758fdaa..a540fba4d6f 100644
--- a/.github/workflows/airflow-e2e-tests.yml
+++ b/.github/workflows/airflow-e2e-tests.yml
@@ -44,6 +44,10 @@ on: # yamllint disable-line rule:truthy
description: "Tag of the Docker image to test"
type: string
required: true
+ e2e_test_mode:
+ description: "Test mode - basic or remote_log"
+ type: string
+ default: "basic"
workflow_call:
inputs:
@@ -67,6 +71,10 @@ on: # yamllint disable-line rule:truthy
description: "Tag of the Docker image to test"
type: string
default: ""
+ e2e_test_mode:
+ description: "Test mode - quick or full"
+ type: string
+ default: "basic"
jobs:
test-e2e-integration-tests:
@@ -101,14 +109,7 @@ jobs:
run: breeze testing airflow-e2e-tests
env:
DOCKER_IMAGE: "${{ inputs.docker-image-tag }}"
- - name: "Upload e2e test report"
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
# v4.6.2
- with:
- name: e2e-test-report
- path: './airflow-e2e-tests/_e2e_test_report.json'
- retention-days: '7'
- if-no-files-found: 'error'
- if: always()
+ E2E_TEST_MODE: "${{ inputs.e2e_test_mode }}"
- name: Zip logs
run: |
cd ./airflow-e2e-tests && zip -r logs.zip logs
@@ -116,7 +117,7 @@ jobs:
- name: "Upload logs"
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
# v4.6.2
with:
- name: e2e-test-logs
+ name: "e2e-test-logs-${{ inputs.e2e_test_mode }}"
path: './airflow-e2e-tests/logs.zip'
retention-days: '7'
if-no-files-found: 'error'
diff --git a/airflow-core/docs/howto/docker-compose/docker-compose.yaml
b/airflow-core/docs/howto/docker-compose/docker-compose.yaml
index 2c2a614c9ef..3892e044143 100644
--- a/airflow-core/docs/howto/docker-compose/docker-compose.yaml
+++ b/airflow-core/docs/howto/docker-compose/docker-compose.yaml
@@ -51,6 +51,8 @@ x-airflow-common:
# and uncomment the "build" line below, Then run `docker-compose build` to
build the images.
image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:|version|}
# build: .
+ env_file:
+ - ${ENV_FILE_PATH:-.env}
environment:
&airflow-common-env
AIRFLOW__CORE__EXECUTOR: CeleryExecutor
diff --git a/airflow-core/tests/unit/utils/test_log_handlers.py
b/airflow-core/tests/unit/utils/test_log_handlers.py
index 6d2caaa8a43..0b85a31a117 100644
--- a/airflow-core/tests/unit/utils/test_log_handlers.py
+++ b/airflow-core/tests/unit/utils/test_log_handlers.py
@@ -72,6 +72,7 @@ from airflow.utils.state import State, TaskInstanceState
from airflow.utils.types import DagRunType
from tests_common.test_utils.config import conf_vars
+from tests_common.test_utils.db import clear_db_connections, clear_db_runs
from tests_common.test_utils.file_task_handler import (
convert_list_to_stream,
extract_events,
@@ -86,6 +87,15 @@ TASK_LOGGER = "airflow.task"
FILE_TASK_HANDLER = "task"
[email protected](autouse=True)
+def cleanup_tables():
+ clear_db_runs()
+ clear_db_connections()
+ yield
+ clear_db_runs()
+ clear_db_connections()
+
+
class TestFileTaskLogHandler:
def clean_up(self):
with create_session() as session:
@@ -603,6 +613,72 @@ class TestFileTaskLogHandler:
actual = h.handler.baseFilename
assert actual == os.fspath(tmp_path / expected)
+ @skip_if_force_lowest_dependencies_marker
+ def test_read_remote_logs_with_real_s3_remote_log_io(self,
create_task_instance, session):
+ """Test _read_remote_logs method using real S3RemoteLogIO with mock
AWS"""
+ import tempfile
+
+ import boto3
+ from moto import mock_aws
+
+ from airflow.models.connection import Connection
+ from airflow.providers.amazon.aws.log.s3_task_handler import
S3RemoteLogIO
+
+ def setup_mock_aws():
+ """Set up mock AWS S3 bucket and connection."""
+ s3_client = boto3.client("s3", region_name="us-east-1")
+ s3_client.create_bucket(Bucket="test-airflow-logs")
+ return s3_client
+
+ with mock_aws():
+ aws_conn = Connection(
+ conn_id="aws_s3_conn",
+ conn_type="aws",
+ login="test_access_key",
+ password="test_secret_key",
+ extra='{"region_name": "us-east-1"}',
+ )
+ session.add(aws_conn)
+ session.commit()
+ s3_client = setup_mock_aws()
+
+ ti = create_task_instance(
+ dag_id="test_dag_s3_remote_logs",
+ task_id="test_task_s3_remote_logs",
+ run_type=DagRunType.SCHEDULED,
+ logical_date=DEFAULT_DATE,
+ )
+ ti.try_number = 1
+
+ with tempfile.TemporaryDirectory() as temp_dir:
+ s3_remote_log_io = S3RemoteLogIO(
+ remote_base="s3://test-airflow-logs/logs",
+ base_log_folder=temp_dir,
+ delete_local_copy=False,
+ )
+
+ with conf_vars({("logging", "REMOTE_LOG_CONN_ID"):
"aws_s3_conn"}):
+ fth = FileTaskHandler("")
+ log_relative_path = fth._render_filename(ti, 1)
+
+ log_content = "Log line 1 from S3\nLog line 2 from S3\nLog
line 3 from S3"
+ s3_client.put_object(
+ Bucket="test-airflow-logs",
+ Key=f"logs/{log_relative_path}",
+ Body=log_content.encode("utf-8"),
+ )
+
+ import airflow.logging_config
+
+ airflow.logging_config.REMOTE_TASK_LOG = s3_remote_log_io
+
+ sources, logs = fth._read_remote_logs(ti, try_number=1)
+
+ assert len(sources) > 0, f"Expected sources but got:
{sources}"
+ assert len(logs) > 0, f"Expected logs but got: {logs}"
+ assert logs[0] == log_content
+ assert f"s3://test-airflow-logs/logs/{log_relative_path}"
in sources[0]
+
@pytest.mark.parametrize("logical_date", ((None), (DEFAULT_DATE)))
class TestFilenameRendering:
diff --git a/airflow-e2e-tests/docker/localstack.yml
b/airflow-e2e-tests/docker/localstack.yml
new file mode 100644
index 00000000000..4a3f87d37d7
--- /dev/null
+++ b/airflow-e2e-tests/docker/localstack.yml
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+services:
+ localstack:
+ container_name: "${LOCALSTACK_DOCKER_NAME:-localstack-main}"
+ image: localstack/localstack:4.7
+ labels:
+ breeze.description: "Integration that emulates AWS services locally."
+ ports:
+ - "4566:4566" # LocalStack Gateway
+ - "4510-4559:4510-4559" # external services port range
+ environment:
+ # LocalStack configuration:
https://docs.localstack.cloud/references/configuration/
+ - DEBUG=${DEBUG:-0}
+ - SERVICES=s3
+ - AWS_ACCESS_KEY_ID=test
+ - AWS_SECRET_ACCESS_KEY=test
+ - AWS_DEFAULT_REGION=us-east-1
+ volumes:
+ - "./init-aws.sh:/etc/localstack/init/ready.d/init-aws.sh"
+ - "/var/run/docker.sock:/var/run/docker.sock"
+ healthcheck:
+ test: ["CMD", "curl", "-f", "http://localhost:4566/_localstack/health";]
+ interval: 10s
+ timeout: 5s
+ retries: 10
+ start_period: 10s
diff --git a/airflow-e2e-tests/pyproject.toml b/airflow-e2e-tests/pyproject.toml
index 950b1545e9d..67bac50b858 100644
--- a/airflow-e2e-tests/pyproject.toml
+++ b/airflow-e2e-tests/pyproject.toml
@@ -39,6 +39,7 @@ dependencies = [
"apache-airflow-devel-common",
"python-on-whales>=0.70.0",
"testcontainers>=4.12.0",
+ "boto3",
]
[tool.pytest.ini_options]
diff --git a/airflow-e2e-tests/scripts/init-aws.sh
b/airflow-e2e-tests/scripts/init-aws.sh
new file mode 100755
index 00000000000..ca5a1cfe078
--- /dev/null
+++ b/airflow-e2e-tests/scripts/init-aws.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+aws --endpoint-url=http://localstack:4566 s3 mb s3://test-airflow-logs
+aws --endpoint-url=http://localstack:4566 s3 ls
diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py
b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py
index d63c17e87a9..c639bffa80f 100644
--- a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py
+++ b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py
@@ -26,10 +26,13 @@ from rich.console import Console
from testcontainers.compose import DockerCompose
from airflow_e2e_tests.constants import (
- AIRFLOW_ROOT_PATH,
+ AWS_INIT_PATH,
DOCKER_COMPOSE_HOST_PORT,
+ DOCKER_COMPOSE_PATH,
DOCKER_IMAGE,
E2E_DAGS_FOLDER,
+ E2E_TEST_MODE,
+ LOCALSTACK_PATH,
LOGS_FOLDER,
TEST_REPORT_FILE,
)
@@ -39,16 +42,33 @@ compose_instance = None
airflow_logs_path = None
+def _setup_s3_integration(dot_env_file, tmp_dir):
+ copyfile(LOCALSTACK_PATH, tmp_dir / "localstack.yml")
+
+ copyfile(AWS_INIT_PATH, tmp_dir / "init-aws.sh")
+ current_permissions = os.stat(tmp_dir / "init-aws.sh").st_mode
+ os.chmod(tmp_dir / "init-aws.sh", current_permissions | 0o111)
+
+ dot_env_file.write_text(
+ f"AIRFLOW_UID={os.getuid()}\n"
+ "AWS_DEFAULT_REGION=us-east-1\n"
+ "AWS_ENDPOINT_URL_S3=http://localstack:4566\n";
+ "AIRFLOW__LOGGING__REMOTE_LOGGING=true\n"
+ "AIRFLOW_CONN_AWS_S3_LOGS=aws://test:test@\n"
+ "AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID=aws_s3_logs\n"
+ "AIRFLOW__LOGGING__DELETE_LOCAL_LOGS=true\n"
+ "AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER=s3://test-airflow-logs\n"
+ )
+ os.environ["ENV_FILE_PATH"] = str(dot_env_file)
+
+
def spin_up_airflow_environment(tmp_path_factory):
global compose_instance
global airflow_logs_path
tmp_dir = tmp_path_factory.mktemp("airflow-e2e-tests")
- compose_file_path = (
- AIRFLOW_ROOT_PATH / "airflow-core" / "docs" / "howto" /
"docker-compose" / "docker-compose.yaml"
- )
-
- copyfile(compose_file_path, tmp_dir / "docker-compose.yaml")
+ console.print(f"[yellow]Using docker compose file: {DOCKER_COMPOSE_PATH}")
+ copyfile(DOCKER_COMPOSE_PATH, tmp_dir / "docker-compose.yaml")
subfolders = ("dags", "logs", "plugins", "config")
@@ -63,19 +83,23 @@ def spin_up_airflow_environment(tmp_path_factory):
copytree(E2E_DAGS_FOLDER, tmp_dir / "dags", dirs_exist_ok=True)
dot_env_file = tmp_dir / ".env"
+ dot_env_file.write_text(f"AIRFLOW_UID={os.getuid()}\n")
console.print(f"[yellow]Creating .env file :[/ {dot_env_file}")
- dot_env_file.write_text(f"AIRFLOW_UID={os.getuid()}\n")
os.environ["AIRFLOW_IMAGE_NAME"] = DOCKER_IMAGE
+ compose_file_names = ["docker-compose.yaml"]
+
+ if E2E_TEST_MODE == "remote_log":
+ compose_file_names.append("localstack.yml")
+ _setup_s3_integration(dot_env_file, tmp_dir)
# If we are using the image from ghcr.io/apache/airflow/main we do not pull
# as it is already available and loaded using prepare_breeze_and_image
step in workflow
pull = False if DOCKER_IMAGE.startswith("ghcr.io/apache/airflow/main/")
else True
console.print(f"[blue]Spinning up airflow environment using
{DOCKER_IMAGE}")
-
- compose_instance = DockerCompose(tmp_dir,
compose_file_name=["docker-compose.yaml"], pull=pull)
+ compose_instance = DockerCompose(tmp_dir,
compose_file_name=compose_file_names, pull=pull)
compose_instance.start()
diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py
b/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py
index 54225059e0f..a208487da8b 100644
--- a/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py
+++ b/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py
@@ -27,7 +27,9 @@ DEFAULT_DOCKER_IMAGE =
f"ghcr.io/apache/airflow/main/prod/python{DEFAULT_PYTHON_
DOCKER_IMAGE = os.environ.get("DOCKER_IMAGE") or DEFAULT_DOCKER_IMAGE
os.environ["AIRFLOW_UID"] = str(os.getuid())
-DOCKER_COMPOSE_PATH = AIRFLOW_ROOT_PATH / "airflow-core" / "docs" / "howto" /
"docker-compose"
+DOCKER_COMPOSE_PATH = (
+ AIRFLOW_ROOT_PATH / "airflow-core" / "docs" / "howto" / "docker-compose" /
"docker-compose.yaml"
+)
AIRFLOW_WWW_USER_USERNAME = os.environ.get("_AIRFLOW_WWW_USER_USERNAME",
"airflow")
AIRFLOW_WWW_USER_PASSWORD = os.environ.get("_AIRFLOW_WWW_USER_PASSWORD",
"airflow")
@@ -37,3 +39,6 @@ E2E_DAGS_FOLDER = AIRFLOW_ROOT_PATH / "airflow-e2e-tests" /
"tests" / "airflow_e
# The logs folder where the Airflow logs will be copied to and uploaded to
github artifacts
LOGS_FOLDER = AIRFLOW_ROOT_PATH / "airflow-e2e-tests" / "logs"
TEST_REPORT_FILE = AIRFLOW_ROOT_PATH / "airflow-e2e-tests" /
"_e2e_test_report.json"
+LOCALSTACK_PATH = AIRFLOW_ROOT_PATH / "airflow-e2e-tests" / "docker" /
"localstack.yml"
+E2E_TEST_MODE = os.environ.get("E2E_TEST_MODE", "basic")
+AWS_INIT_PATH = AIRFLOW_ROOT_PATH / "airflow-e2e-tests" / "scripts" /
"init-aws.sh"
diff --git
a/airflow-e2e-tests/tests/airflow_e2e_tests/e2e_test_utils/clients.py
b/airflow-e2e-tests/tests/airflow_e2e_tests/e2e_test_utils/clients.py
index 6d775726347..32abeb5331d 100644
--- a/airflow-e2e-tests/tests/airflow_e2e_tests/e2e_test_utils/clients.py
+++ b/airflow-e2e-tests/tests/airflow_e2e_tests/e2e_test_utils/clients.py
@@ -121,6 +121,13 @@ class AirflowClient:
run_id=resp["dag_run_id"],
)
+ def get_task_logs(self, dag_id: str, run_id: str, task_id: str,
try_number: int = 1):
+ """Get task logs via API."""
+ return self._make_request(
+ method="GET",
+
endpoint=f"dags/{dag_id}/dagRuns/{run_id}/taskInstances/{task_id}/logs/{try_number}",
+ )
+
class TaskSDKClient:
"""Client for interacting with the Task SDK API."""
diff --git
a/airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_tests/__init__.py
b/airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_tests/__init__.py
new file mode 100644
index 00000000000..13a83393a91
--- /dev/null
+++ b/airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_tests/__init__.py
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git
a/airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_tests/test_remote_logging.py
b/airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_tests/test_remote_logging.py
new file mode 100644
index 00000000000..265bfca6bbe
--- /dev/null
+++
b/airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_tests/test_remote_logging.py
@@ -0,0 +1,83 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+import boto3
+import pytest
+
+from airflow_e2e_tests.e2e_test_utils.clients import AirflowClient
+
+
+class TestRemoteLogging:
+ airflow_client = AirflowClient()
+ dag_id = "example_xcom_test"
+
+ def test_dag_unpause(self):
+ self.airflow_client.un_pause_dag(
+ TestRemoteLogging.dag_id,
+ )
+
+ def test_remote_logging_s3(self):
+ """Test that a DAG using remote logging to S3 completes
successfully."""
+
+ self.airflow_client.un_pause_dag(TestRemoteLogging.dag_id)
+
+ resp = self.airflow_client.trigger_dag(
+ TestRemoteLogging.dag_id, json={"logical_date":
datetime.now(timezone.utc).isoformat()}
+ )
+ state = self.airflow_client.wait_for_dag_run(
+ dag_id=TestRemoteLogging.dag_id,
+ run_id=resp["dag_run_id"],
+ )
+
+ assert state == "success", (
+ f"DAG {TestRemoteLogging.dag_id} did not complete successfully.
Final state: {state}"
+ )
+
+ task_logs = self.airflow_client.get_task_logs(
+ dag_id=TestRemoteLogging.dag_id,
+ task_id="bash_pull",
+ run_id=resp["dag_run_id"],
+ )
+
+ task_log_sources = [
+ source for content in task_logs.get("content", [{}]) for source in
content.get("sources", [])
+ ]
+
+ s3_client = boto3.client(
+ "s3",
+ endpoint_url="http://localhost:4566";,
+ aws_access_key_id="test",
+ aws_secret_access_key="test",
+ region_name="us-east-1",
+ )
+
+ # This bucket will be created part of the docker-compose setup in
+ bucket_name = "test-airflow-logs"
+ response = s3_client.list_objects_v2(Bucket=bucket_name)
+
+ if "Contents" not in response:
+ pytest.fail("No objects found in S3 bucket %s", bucket_name)
+
+ # s3 key format:
dag_id=example_xcom/run_id=manual__2025-09-29T23:32:09.457215+00:00/task_id=bash_pull/attempt=1.log
+
+ log_files = [f"s3://{bucket_name}/{obj['Key']}" for obj in
response["Contents"]]
+ assert any(source in log_files for source in task_log_sources), (
+ f"None of the log sources {task_log_sources} were found in S3
bucket logs {log_files}"
+ )
diff --git a/dev/breeze/doc/images/output_testing_airflow-e2e-tests.svg
b/dev/breeze/doc/images/output_testing_airflow-e2e-tests.svg
index 9abcda1d332..7fc70e474b9 100644
--- a/dev/breeze/doc/images/output_testing_airflow-e2e-tests.svg
+++ b/dev/breeze/doc/images/output_testing_airflow-e2e-tests.svg
@@ -1,4 +1,4 @@
-<svg class="rich-terminal" viewBox="0 0 1482 538.0"
xmlns="http://www.w3.org/2000/svg";>
+<svg class="rich-terminal" viewBox="0 0 1482 562.4"
xmlns="http://www.w3.org/2000/svg";>
<!-- Generated with Rich https://www.textualize.io -->
<style>
@@ -43,7 +43,7 @@
<defs>
<clipPath id="breeze-testing-airflow-e2e-tests-clip-terminal">
- <rect x="0" y="0" width="1463.0" height="487.0" />
+ <rect x="0" y="0" width="1463.0" height="511.4" />
</clipPath>
<clipPath id="breeze-testing-airflow-e2e-tests-line-0">
<rect x="0" y="1.5" width="1464" height="24.65"/>
@@ -102,9 +102,12 @@
<clipPath id="breeze-testing-airflow-e2e-tests-line-18">
<rect x="0" y="440.7" width="1464" height="24.65"/>
</clipPath>
+<clipPath id="breeze-testing-airflow-e2e-tests-line-19">
+ <rect x="0" y="465.1" width="1464" height="24.65"/>
+ </clipPath>
</defs>
- <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1"
x="1" y="1" width="1480" height="536" rx="8"/><text
class="breeze-testing-airflow-e2e-tests-title" fill="#c5c8c6"
text-anchor="middle" x="740"
y="27">Command: testing airflow-e2e-tests</text>
+ <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1"
x="1" y="1" width="1480" height="560.4" rx="8"/><text
class="breeze-testing-airflow-e2e-tests-title" fill="#c5c8c6"
text-anchor="middle" x="740"
y="27">Command: testing airflow-e2e-tests</text>
<g transform="translate(26,22)">
<circle cx="0" cy="0" r="7" fill="#ff5f57"/>
<circle cx="22" cy="0" r="7" fill="#febc2e"/>
@@ -120,20 +123,21 @@
</text><text class="breeze-testing-airflow-e2e-tests-r1" x="12.2" y="93.2"
textLength="268.4"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-3)">Run Airflow E2E tests.</text><text
class="breeze-testing-airflow-e2e-tests-r1" x="1464" y="93.2"
textLength="12.2" clip-path="url(#breeze-testing-airflow-e2e-tests-line-3)">
</text><text class="breeze-testing-airflow-e2e-tests-r1" x="1464" y="117.6"
textLength="12.2" clip-path="url(#breeze-testing-airflow-e2e-tests-line-4)">
</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="142"
textLength="24.4"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-5)">╭─</text><text
class="breeze-testing-airflow-e2e-tests-r5" x="24.4" y="142" textLength="305"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-5)"> Airflow E2E tests flags </text><text
class="breeze-testing-airflow-e2e-tests-r5" x="329.4" y="142"
textLength="1110.2" clip-path="url(#breeze-testing-airflow-e2e-tests- [...]
-</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="166.4"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-6)">│</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="24.4" y="166.4"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-6)">-</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="36.6" y="166.4"
textLength="73.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-6)">-image</text><text
class="breeze-test [...]
-</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="190.8"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-7)">│</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="24.4" y="190.8"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-7)">-</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="36.6" y="190.8"
textLength="85.4"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-7)">-python</text><text
class="breeze-tes [...]
+</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="166.4"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-6)">│</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="24.4" y="166.4"
textLength="146.4"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-6)">--image-name</text><text
class="breeze-testing-airflow-e2e-tests-r6" x="414.8" y="166.4"
textLength="24.4"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-6)">-n</text><text
class="br [...]
+</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="190.8"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-7)">│</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="24.4" y="190.8"
textLength="97.6"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-7)">--python</text><text
class="breeze-testing-airflow-e2e-tests-r6" x="414.8" y="190.8"
textLength="24.4"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-7)">-p</text><text
class="breeze- [...]
</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="215.2"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-8)">│</text><text
class="breeze-testing-airflow-e2e-tests-r7" x="463.6" y="215.2"
textLength="732"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-8)">(>3.10< | 3.11 | 3.12 | 3.13)                     
[...]
</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="239.6"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-9)">│</text><text
class="breeze-testing-airflow-e2e-tests-r5" x="463.6" y="239.6"
textLength="732"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-9)">[default: 3.10]                            &#
[...]
-</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="264"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-10)">│</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="24.4" y="264" textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-10)">-</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="36.6" y="264" textLength="61"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-10)">-skip</text><text
class="breeze-testing-ai [...]
-</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="288.4"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-11)">│</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="24.4" y="288.4"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-11)">-</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="36.6" y="288.4"
textLength="97.6"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-11)">-include</text><text
class="breeze [...]
-</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="312.8"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-12)">│</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="24.4" y="312.8"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-12)">-</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="36.6" y="312.8"
textLength="85.4"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-12)">-github</text><text
class="breeze- [...]
+</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="264"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-10)">│</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="24.4" y="264" textLength="366"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-10)">--skip-docker-compose-deletion</text><text
class="breeze-testing-airflow-e2e-tests-r1" x="463.6" y="264" textLength="671"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-10)">Skip de [...]
+</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="288.4"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-11)">│</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="24.4" y="288.4" textLength="305"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-11)">--include-success-outputs</text><text
class="breeze-testing-airflow-e2e-tests-r1" x="463.6" y="288.4"
textLength="841.8"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-11)">Whether&# [...]
+</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="312.8"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-12)">│</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="24.4" y="312.8"
textLength="231.8"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-12)">--github-repository</text><text
class="breeze-testing-airflow-e2e-tests-r6" x="414.8" y="312.8"
textLength="24.4"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-12)">-g</text><text [...]
</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="337.2"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-13)">│</text><text
class="breeze-testing-airflow-e2e-tests-r5" x="463.6" y="337.2"
textLength="585.6"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-13)">[default: apache/airflow]                       </text><text
class [...]
-</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="361.6"
textLength="1464"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-14)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-testing-airflow-e2e-tests-r1" x="1464" y="361.6"
textLength="12.2" clip-path="url(#breeze-testing-airflow-e2e-tests-line-14)">
-</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="386"
textLength="24.4"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-15)">╭─</text><text
class="breeze-testing-airflow-e2e-tests-r5" x="24.4" y="386" textLength="195.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-15)"> Common options </text><text
class="breeze-testing-airflow-e2e-tests-r5" x="219.6" y="386"
textLength="1220"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-15)">─────── [...]
-</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="410.4"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-16)">│</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="24.4" y="410.4"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-16)">-</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="36.6" y="410.4"
textLength="97.6"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-16)">-verbose</text><text
class="breeze [...]
-</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="434.8"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-17)">│</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="24.4" y="434.8"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-17)">-</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="36.6" y="434.8"
textLength="48.8"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-17)">-dry</text><text
class="breeze-tes [...]
-</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="459.2"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-18)">│</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="24.4" y="459.2"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-18)">-</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="36.6" y="459.2" textLength="61"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-18)">-help</text><text
class="breeze-test [...]
-</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="483.6"
textLength="1464"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-19)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-testing-airflow-e2e-tests-r1" x="1464" y="483.6"
textLength="12.2" clip-path="url(#breeze-testing-airflow-e2e-tests-line-19)">
+</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="361.6"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-14)">│</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="24.4" y="361.6" textLength="183"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-14)">--e2e-test-mode</text><text
class="breeze-testing-airflow-e2e-tests-r1" x="463.6" y="361.6"
textLength="463.6"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-14)">Specify the
[...]
+</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="386"
textLength="1464"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-15)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-testing-airflow-e2e-tests-r1" x="1464" y="386" textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-15)">
+</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="410.4"
textLength="24.4"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-16)">╭─</text><text
class="breeze-testing-airflow-e2e-tests-r5" x="24.4" y="410.4"
textLength="195.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-16)"> Common options </text><text
class="breeze-testing-airflow-e2e-tests-r5" x="219.6" y="410.4"
textLength="1220" clip-path="url(#breeze-testing-airflow-e2e-tests-line-16)">─
[...]
+</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="434.8"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-17)">│</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="24.4" y="434.8"
textLength="109.8"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-17)">--verbose</text><text
class="breeze-testing-airflow-e2e-tests-r6" x="158.6" y="434.8"
textLength="24.4"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-17)">-v</text><text
class="br [...]
+</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="459.2"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-18)">│</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="24.4" y="459.2"
textLength="109.8"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-18)">--dry-run</text><text
class="breeze-testing-airflow-e2e-tests-r6" x="158.6" y="459.2"
textLength="24.4"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-18)">-D</text><text
class="br [...]
+</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="483.6"
textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-19)">│</text><text
class="breeze-testing-airflow-e2e-tests-r4" x="24.4" y="483.6"
textLength="73.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-19)">--help</text><text
class="breeze-testing-airflow-e2e-tests-r6" x="158.6" y="483.6"
textLength="24.4"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-19)">-h</text><text
class="breeze [...]
+</text><text class="breeze-testing-airflow-e2e-tests-r5" x="0" y="508"
textLength="1464"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-20)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text
class="breeze-testing-airflow-e2e-tests-r1" x="1464" y="508" textLength="12.2"
clip-path="url(#breeze-testing-airflow-e2e-tests-line-20)">
</text>
</g>
</g>
diff --git a/dev/breeze/doc/images/output_testing_airflow-e2e-tests.txt
b/dev/breeze/doc/images/output_testing_airflow-e2e-tests.txt
index b770f1f4d2d..e7095bc0b2d 100644
--- a/dev/breeze/doc/images/output_testing_airflow-e2e-tests.txt
+++ b/dev/breeze/doc/images/output_testing_airflow-e2e-tests.txt
@@ -1 +1 @@
-4816cb6514217c64134f80453bc8e9de
+34b0e3acf4185caf7b76b05037e91c06
diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py
b/dev/breeze/src/airflow_breeze/commands/testing_commands.py
index a10c823705d..efcd2a70f97 100644
--- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py
@@ -1267,6 +1267,16 @@ def python_api_client_tests(
sys.exit(returncode)
+option_e2e_test_mode = click.option(
+ "--e2e-test-mode",
+ help="Specify the mode to use for E2E tests.",
+ default="basic",
+ show_default=True,
+ envvar="E2E_TEST_MODE",
+ type=click.Choice(["basic", "remote_log"], case_sensitive=False),
+)
+
+
@group_for_testing.command(
name="airflow-e2e-tests",
context_settings=dict(
@@ -1281,6 +1291,7 @@ def python_api_client_tests(
@option_include_success_outputs
@option_verbose
@option_dry_run
+@option_e2e_test_mode
@click.argument("extra_pytest_args", nargs=-1, type=click.Path(path_type=str))
def airflow_e2e_tests(
python: str,
@@ -1288,6 +1299,7 @@ def airflow_e2e_tests(
skip_docker_compose_deletion: bool,
github_repository: str,
include_success_outputs: bool,
+ e2e_test_mode: str,
extra_pytest_args: tuple,
):
"""Run Airflow E2E tests."""
@@ -1308,6 +1320,7 @@ def airflow_e2e_tests(
skip_docker_compose_deletion=skip_docker_compose_deletion,
test_type="airflow-e2e-tests",
skip_image_check=skip_image_check,
+ test_mode=e2e_test_mode,
)
sys.exit(return_code)
diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py
b/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py
index e671be482bb..e8116d47ecc 100644
--- a/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py
+++ b/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py
@@ -281,6 +281,7 @@ TESTING_PARAMETERS: dict[str, list[dict[str, str |
list[str]]]] = {
"--skip-docker-compose-deletion",
"--include-success-outputs",
"--github-repository",
+ "--e2e-test-mode",
],
}
],
diff --git a/dev/breeze/src/airflow_breeze/utils/run_tests.py
b/dev/breeze/src/airflow_breeze/utils/run_tests.py
index 55a9c1ec6cf..0ac96c9c663 100644
--- a/dev/breeze/src/airflow_breeze/utils/run_tests.py
+++ b/dev/breeze/src/airflow_breeze/utils/run_tests.py
@@ -104,6 +104,7 @@ def run_docker_compose_tests(
include_success_outputs: bool,
test_type: str = "docker-compose",
skip_image_check: bool = False,
+ test_mode: str = "basic",
) -> tuple[int, str]:
if not skip_image_check:
command_result = run_command(["docker", "inspect", image_name],
check=False, stdout=DEVNULL)
@@ -116,7 +117,7 @@ def run_docker_compose_tests(
test_path = Path("tests") / "task_sdk_tests" /
"test_task_sdk_health.py"
cwd = TASK_SDK_TESTS_ROOT_PATH.as_posix()
elif test_type == "airflow-e2e-tests":
- test_path = Path("tests") / "airflow_e2e_tests"
+ test_path = Path("tests") / "airflow_e2e_tests" / f"{test_mode}_tests"
cwd = AIRFLOW_E2E_TESTS_ROOT_PATH.as_posix()
else:
test_path = Path("tests") / "docker_tests" /
"test_docker_compose_quick_start.py"
@@ -124,6 +125,7 @@ def run_docker_compose_tests(
env = os.environ.copy()
env["DOCKER_IMAGE"] = image_name
+ env["E2E_TEST_MODE"] = test_mode
if skip_docker_compose_deletion:
env["SKIP_DOCKER_COMPOSE_DELETION"] = "true"
if include_success_outputs:
