This is an automated email from the ASF dual-hosted git repository.
vincbeck pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 47839275ca9 Log the HTTP response in HttpToS3Operator when
log_response is set (#72951)
47839275ca9 is described below
commit 47839275ca95c9fd61484e856020de136fdf06cf
Author: Bingqin Wang <[email protected]>
AuthorDate: Wed Sep 16 11:16:06 2026 -0500
Log the HTTP response in HttpToS3Operator when log_response is set (#72951)
---
.../providers/amazon/aws/transfers/http_to_s3.py | 2 ++
.../unit/amazon/aws/transfers/test_http_to_s3.py | 22 ++++++++++++++++++++++
2 files changed, 24 insertions(+)
diff --git
a/providers/amazon/src/airflow/providers/amazon/aws/transfers/http_to_s3.py
b/providers/amazon/src/airflow/providers/amazon/aws/transfers/http_to_s3.py
index 363212239a0..607e18072de 100644
--- a/providers/amazon/src/airflow/providers/amazon/aws/transfers/http_to_s3.py
+++ b/providers/amazon/src/airflow/providers/amazon/aws/transfers/http_to_s3.py
@@ -164,6 +164,8 @@ class HttpToS3Operator(BaseOperator):
def execute(self, context: Context):
self.log.info("Calling HTTP method")
response = self.http_hook.run(self.endpoint, self.data, self.headers,
self.extra_options)
+ if self.log_response:
+ self.log.info(response.text)
self.s3_hook.load_bytes(
response.content,
diff --git
a/providers/amazon/tests/unit/amazon/aws/transfers/test_http_to_s3.py
b/providers/amazon/tests/unit/amazon/aws/transfers/test_http_to_s3.py
index aa95b9ec8ef..d596bdec3f4 100644
--- a/providers/amazon/tests/unit/amazon/aws/transfers/test_http_to_s3.py
+++ b/providers/amazon/tests/unit/amazon/aws/transfers/test_http_to_s3.py
@@ -18,9 +18,11 @@
from __future__ import annotations
import datetime
+import logging
from unittest import mock
import boto3
+import pytest
from moto import mock_aws
from airflow.models.dag import DAG
@@ -74,3 +76,23 @@ class TestHttpToS3Operator:
assert len(objects_in_bucket["Contents"]) == 1
# the object found should be consistent with dest_key specified earlier
assert objects_in_bucket["Contents"][0]["Key"] == self.s3_key
+
+ @pytest.mark.parametrize("log_response", [True, False])
+ @mock_aws
+ def test_execute_logs_response_only_when_requested(self, requests_mock,
caplog, log_response):
+ requests_mock.register_uri("GET", EXAMPLE_URL, content=self.response)
+ boto3.client("s3").create_bucket(Bucket=self.s3_bucket)
+ operator = HttpToS3Operator(
+ task_id="http_to_s3_operator",
+ http_conn_id=self.http_conn_id,
+ endpoint=self.endpoint,
+ s3_key=self.s3_key,
+ s3_bucket=self.s3_bucket,
+ log_response=log_response,
+ dag=self.dag,
+ )
+
+ with caplog.at_level(logging.INFO, logger=operator.log.name):
+ operator.execute(None)
+
+ assert (self.response.decode() in caplog.text) is log_response
