This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 79f6383c38 filename template arg in providers file task handlers
backward compitability support (#41633)
79f6383c38 is described below
commit 79f6383c38d1ed7409684b9475a903bcc77b2e4b
Author: Gopal Dirisala <[email protected]>
AuthorDate: Wed Aug 21 15:07:12 2024 +0530
filename template arg in providers file task handlers backward
compitability support (#41633)
---
airflow/providers/amazon/aws/log/cloudwatch_task_handler.py | 2 +-
airflow/providers/elasticsearch/log/es_task_handler.py | 1 +
tests/providers/alibaba/cloud/log/test_oss_task_handler.py | 4 ++++
.../providers/amazon/aws/log/test_cloudwatch_task_handler.py | 8 ++++++++
tests/providers/amazon/aws/log/test_s3_task_handler.py | 4 ++++
tests/providers/elasticsearch/log/test_es_task_handler.py | 11 +++++++++++
tests/providers/google/cloud/log/test_gcs_task_handler.py | 9 +++++++++
tests/providers/microsoft/azure/log/test_wasb_task_handler.py | 10 ++++++++++
8 files changed, 48 insertions(+), 1 deletion(-)
diff --git a/airflow/providers/amazon/aws/log/cloudwatch_task_handler.py
b/airflow/providers/amazon/aws/log/cloudwatch_task_handler.py
index 69efacfad3..b7383c8f46 100644
--- a/airflow/providers/amazon/aws/log/cloudwatch_task_handler.py
+++ b/airflow/providers/amazon/aws/log/cloudwatch_task_handler.py
@@ -75,7 +75,7 @@ class CloudwatchTaskHandler(FileTaskHandler, LoggingMixin):
trigger_should_wrap = True
- def __init__(self, base_log_folder: str, log_group_arn: str):
+ def __init__(self, base_log_folder: str, log_group_arn: str, **kwargs):
super().__init__(base_log_folder)
split_arn = log_group_arn.split(":")
diff --git a/airflow/providers/elasticsearch/log/es_task_handler.py
b/airflow/providers/elasticsearch/log/es_task_handler.py
index 6473b8ddd7..7eda572c54 100644
--- a/airflow/providers/elasticsearch/log/es_task_handler.py
+++ b/airflow/providers/elasticsearch/log/es_task_handler.py
@@ -159,6 +159,7 @@ class ElasticsearchTaskHandler(FileTaskHandler,
ExternalLoggingMixin, LoggingMix
es_kwargs: dict | None | Literal["default_es_kwargs"] =
"default_es_kwargs",
*,
log_id_template: str | None = None,
+ **kwargs,
):
es_kwargs = es_kwargs or {}
if es_kwargs == "default_es_kwargs":
diff --git a/tests/providers/alibaba/cloud/log/test_oss_task_handler.py
b/tests/providers/alibaba/cloud/log/test_oss_task_handler.py
index 2c00c4d022..b17d953410 100644
--- a/tests/providers/alibaba/cloud/log/test_oss_task_handler.py
+++ b/tests/providers/alibaba/cloud/log/test_oss_task_handler.py
@@ -178,3 +178,7 @@ class TestOSSTaskHandler:
handler.close()
assert os.path.exists(handler.handler.baseFilename) ==
expected_existence_of_local_copy
+
+ def test_filename_template_for_backward_compatibility(self):
+ # filename_template arg support for running the latest provider on
airflow 2
+ OSSTaskHandler(self.base_log_folder, self.oss_log_folder,
filename_template=None)
diff --git a/tests/providers/amazon/aws/log/test_cloudwatch_task_handler.py
b/tests/providers/amazon/aws/log/test_cloudwatch_task_handler.py
index f1870c4d76..801935a549 100644
--- a/tests/providers/amazon/aws/log/test_cloudwatch_task_handler.py
+++ b/tests/providers/amazon/aws/log/test_cloudwatch_task_handler.py
@@ -228,6 +228,14 @@ class TestCloudwatchTaskHandler:
mock_log_handler_close.assert_called_once()
+ def test_filename_template_for_backward_compatibility(self):
+ # filename_template arg support for running the latest provider on
airflow 2
+ CloudwatchTaskHandler(
+ self.local_log_location,
+
f"arn:aws:logs:{self.region_name}:11111111:log-group:{self.remote_log_group}",
+ filename_template=None,
+ )
+
def generate_log_events(conn, log_group_name, log_stream_name, log_events):
conn.create_log_group(logGroupName=log_group_name)
diff --git a/tests/providers/amazon/aws/log/test_s3_task_handler.py
b/tests/providers/amazon/aws/log/test_s3_task_handler.py
index 3412011eb4..fcb5d3c783 100644
--- a/tests/providers/amazon/aws/log/test_s3_task_handler.py
+++ b/tests/providers/amazon/aws/log/test_s3_task_handler.py
@@ -221,3 +221,7 @@ class TestS3TaskHandler:
handler.close()
assert os.path.exists(handler.handler.baseFilename) ==
expected_existence_of_local_copy
+
+ def test_filename_template_for_backward_compatibility(self):
+ # filename_template arg support for running the latest provider on
airflow 2
+ S3TaskHandler(self.local_log_location, self.remote_log_base,
filename_template=None)
diff --git a/tests/providers/elasticsearch/log/test_es_task_handler.py
b/tests/providers/elasticsearch/log/test_es_task_handler.py
index 6469a636ff..9321f49d77 100644
--- a/tests/providers/elasticsearch/log/test_es_task_handler.py
+++ b/tests/providers/elasticsearch/log/test_es_task_handler.py
@@ -655,6 +655,17 @@ class TestElasticsearchTaskHandler:
mock_callable.assert_called_once_with({})
assert result == "callable_index_pattern"
+ def test_filename_template_for_backward_compatibility(self):
+ # filename_template arg support for running the latest provider on
airflow 2
+ ElasticsearchTaskHandler(
+ base_log_folder="local/log/location",
+ end_of_log_mark="end_of_log\n",
+ write_stdout=False,
+ json_format=False,
+ json_fields="asctime,filename,lineno,levelname,message,exc_text",
+ filename_template=None,
+ )
+
def test_safe_attrgetter():
class A: ...
diff --git a/tests/providers/google/cloud/log/test_gcs_task_handler.py
b/tests/providers/google/cloud/log/test_gcs_task_handler.py
index a860e52e15..7653c4d837 100644
--- a/tests/providers/google/cloud/log/test_gcs_task_handler.py
+++ b/tests/providers/google/cloud/log/test_gcs_task_handler.py
@@ -283,3 +283,12 @@ class TestGCSTaskHandler:
handler.close()
assert os.path.exists(handler.handler.baseFilename) ==
expected_existence_of_local_copy
+
+ @pytest.fixture(autouse=True)
+ def test_filename_template_for_backward_compatibility(self,
local_log_location):
+ # filename_template arg support for running the latest provider on
airflow 2
+ GCSTaskHandler(
+ base_log_folder=local_log_location,
+ gcs_log_folder="gs://bucket/remote/log/location",
+ filename_template=None,
+ )
diff --git a/tests/providers/microsoft/azure/log/test_wasb_task_handler.py
b/tests/providers/microsoft/azure/log/test_wasb_task_handler.py
index e74efe89e9..7de68ec63a 100644
--- a/tests/providers/microsoft/azure/log/test_wasb_task_handler.py
+++ b/tests/providers/microsoft/azure/log/test_wasb_task_handler.py
@@ -204,3 +204,13 @@ class TestWasbTaskHandler:
handler.close()
assert os.path.exists(handler.handler.baseFilename) ==
expected_existence_of_local_copy
+
+ def test_filename_template_for_backward_compatibility(self):
+ # filename_template arg support for running the latest provider on
airflow 2
+ WasbTaskHandler(
+ base_log_folder=self.local_log_location,
+ wasb_log_folder=self.wasb_log_folder,
+ wasb_container=self.container_name,
+ delete_local_copy=True,
+ filename_template=None,
+ )
