This is an automated email from the ASF dual-hosted git repository.
shahar1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new e8ae8502ad7 Reject empty match_glob in GCSToS3Operator on old google
provider (#71591)
e8ae8502ad7 is described below
commit e8ae8502ad7542c2653fe01c554d232e9eba280f
Author: DhanushAnegondi <[email protected]>
AuthorDate: Sat Sep 12 08:39:35 2026 -0700
Reject empty match_glob in GCSToS3Operator on old google provider (#71591)
match_glob is a template field, so its value is not available until
Jinja rendering happens on the worker, well after __init__ has run. The
guard that rejects match_glob on an unsupported google provider tested
it for truthiness, which the validate-operators-init prek hook flags.
The check only asks whether the argument was supplied; it never
inspects the value. Per the false-positive guidance on #70296 that
makes it a provision check, which is fixed in place rather than moved
to execute(). Moving it would break the check under
render_template_as_native_obj=True, where a supplied field can render
to None and so becomes indistinguishable from an omitted one, and would
defer a static authoring mistake from Dag parse time to every task
instance and retry.
Switching to an explicit is not None comparison changes one case:
match_glob="" is now rejected on a google provider older than 10.3.0,
where truthiness previously read the supplied empty string as absent.
An empty glob is not a valid pattern and was never honoured on those
versions.
Narrow the raised exception from AirflowException to ValueError, which
is what an invalid argument warrants, and drop the file from
generated/known_airflow_exceptions.txt accordingly. This follows the
merged #70359 precedent for S3DeleteObjectsOperator, which made the
same one-for-one change to the same four kinds of file.
Also remove the class from the prek exemption list, since the hook
fails on stale entries, and add tests covering supplied, empty-string
and omitted match_glob.
Related: #70296
---
generated/known_airflow_exceptions.txt | 1 -
.../providers/amazon/aws/transfers/gcs_to_s3.py | 8 ++----
.../unit/amazon/aws/transfers/test_gcs_to_s3.py | 33 ++++++++++++++++++++++
.../ci/prek/validate_operators_init_exemptions.txt | 1 -
4 files changed, 36 insertions(+), 7 deletions(-)
diff --git a/generated/known_airflow_exceptions.txt
b/generated/known_airflow_exceptions.txt
index 4269e6eb801..c60aa6a73bd 100644
--- a/generated/known_airflow_exceptions.txt
+++ b/generated/known_airflow_exceptions.txt
@@ -112,7 +112,6 @@
providers/amazon/src/airflow/providers/amazon/aws/sensors/sagemaker.py::8
providers/amazon/src/airflow/providers/amazon/aws/sensors/sagemaker_unified_studio.py::1
providers/amazon/src/airflow/providers/amazon/aws/sensors/sqs.py::2
providers/amazon/src/airflow/providers/amazon/aws/sensors/step_function.py::1
-providers/amazon/src/airflow/providers/amazon/aws/transfers/gcs_to_s3.py::1
providers/amazon/src/airflow/providers/amazon/aws/transfers/redshift_to_s3.py::1
providers/amazon/src/airflow/providers/amazon/aws/transfers/s3_to_dynamodb.py::3
providers/amazon/src/airflow/providers/amazon/aws/transfers/s3_to_redshift.py::3
diff --git
a/providers/amazon/src/airflow/providers/amazon/aws/transfers/gcs_to_s3.py
b/providers/amazon/src/airflow/providers/amazon/aws/transfers/gcs_to_s3.py
index 0eeb0f35b5d..84e0816048a 100644
--- a/providers/amazon/src/airflow/providers/amazon/aws/transfers/gcs_to_s3.py
+++ b/providers/amazon/src/airflow/providers/amazon/aws/transfers/gcs_to_s3.py
@@ -26,7 +26,7 @@ from typing import TYPE_CHECKING
from packaging.version import Version
from airflow.providers.amazon.aws.hooks.s3 import S3Hook
-from airflow.providers.common.compat.sdk import AirflowException, BaseOperator
+from airflow.providers.common.compat.sdk import BaseOperator
from airflow.providers.google.cloud.hooks.gcs import GCSHook
if TYPE_CHECKING:
@@ -146,10 +146,8 @@ class GCSToS3Operator(BaseOperator):
self.__is_match_glob_supported = False
except ImportError: # __version__ was added in 10.1.0, so this means
it's < 10.3.0
self.__is_match_glob_supported = False
- if not self.__is_match_glob_supported and match_glob:
- raise AirflowException(
- "The 'match_glob' parameter requires
'apache-airflow-providers-google>=10.3.0'."
- )
+ if not self.__is_match_glob_supported and match_glob is not None:
+ raise ValueError("The 'match_glob' parameter requires
'apache-airflow-providers-google>=10.3.0'.")
self.match_glob = match_glob
self.gcp_user_project = gcp_user_project
diff --git a/providers/amazon/tests/unit/amazon/aws/transfers/test_gcs_to_s3.py
b/providers/amazon/tests/unit/amazon/aws/transfers/test_gcs_to_s3.py
index 108eb0debff..feaca455f6b 100644
--- a/providers/amazon/tests/unit/amazon/aws/transfers/test_gcs_to_s3.py
+++ b/providers/amazon/tests/unit/amazon/aws/transfers/test_gcs_to_s3.py
@@ -75,6 +75,39 @@ class TestGCSToS3Operator:
user_project=None,
)
+ @pytest.mark.parametrize(
+ "match_glob",
+ [
+ pytest.param("**/*.csv", id="value_supplied"),
+ pytest.param("", id="empty_string_is_still_supplied"),
+ ],
+ )
+ @mock.patch("airflow.providers.google.__version__", "10.2.0")
+ def test_init__match_glob_rejected_on_older_google_provider(self,
match_glob):
+ """Supplying match_glob requires
apache-airflow-providers-google>=10.3.0.
+
+ The check asks whether the argument was supplied, not whether its
value is truthy,
+ so an empty string is rejected too - it was supplied.
+ """
+ with pytest.raises(ValueError, match=r"requires
'apache-airflow-providers-google>=10\.3\.0'"):
+ GCSToS3Operator(
+ task_id=TASK_ID,
+ gcs_bucket=GCS_BUCKET,
+ dest_s3_key=S3_BUCKET,
+ match_glob=match_glob,
+ )
+
+ @mock.patch("airflow.providers.google.__version__", "10.2.0")
+ def
test_init__match_glob_omitted_is_accepted_on_older_google_provider(self):
+ """Omitting match_glob leaves nothing to reject, whatever the google
provider version."""
+ operator = GCSToS3Operator(
+ task_id=TASK_ID,
+ gcs_bucket=GCS_BUCKET,
+ dest_s3_key=S3_BUCKET,
+ )
+
+ assert operator.match_glob is None
+
@mock.patch("airflow.providers.amazon.aws.transfers.gcs_to_s3.GCSHook")
def test_execute_incremental(self, mock_hook):
mock_hook.return_value.list.return_value = MOCK_FILES
diff --git a/scripts/ci/prek/validate_operators_init_exemptions.txt
b/scripts/ci/prek/validate_operators_init_exemptions.txt
index 803a2ad7495..bbb80b5fec7 100644
--- a/scripts/ci/prek/validate_operators_init_exemptions.txt
+++ b/scripts/ci/prek/validate_operators_init_exemptions.txt
@@ -7,7 +7,6 @@
# Burn-down tracked at https://github.com/apache/airflow/issues/70296
providers/amazon/src/airflow/providers/amazon/aws/operators/neptune.py::NeptuneStartDbClusterOperator
providers/amazon/src/airflow/providers/amazon/aws/operators/neptune.py::NeptuneStopDbClusterOperator
-providers/amazon/src/airflow/providers/amazon/aws/transfers/gcs_to_s3.py::GCSToS3Operator
providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/pod.py::KubernetesPodOperator
providers/google/src/airflow/providers/google/cloud/operators/cloud_build.py::CloudBuildCreateBuildOperator
providers/google/src/airflow/providers/google/cloud/operators/cloud_storage_transfer_service.py::CloudDataTransferServiceCreateJobOperator
