This is an automated email from the ASF dual-hosted git repository.
shahar1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new e2372bbf0aa Read the legacy extra__google_cloud_platform__ prefix for
GCS credentials (#73613)
e2372bbf0aa is described below
commit e2372bbf0aa5d0e4a370734bb1d8ba808f6e8c00
Author: Pankaj Singh <[email protected]>
AuthorDate: Thu Sep 24 01:29:45 2026 +0530
Read the legacy extra__google_cloud_platform__ prefix for GCS credentials
(#73613)
GoogleBaseHook still falls back to fields written as
extra__google_cloud_platform__<name> -- the spelling older Airflow
connection UIs used for custom extra fields. This code only looked at the
bare key, so a connection created that way had its key_path silently
invisible and the unsupported-field guard never saw a prefixed
impersonation_chain.
---
.../providers/common/sql/datafusion/engine.py | 14 +++++++++++---
.../unit/common/sql/datafusion/test_engine.py | 22 ++++++++++++++++++++++
2 files changed, 33 insertions(+), 3 deletions(-)
diff --git
a/providers/common/sql/src/airflow/providers/common/sql/datafusion/engine.py
b/providers/common/sql/src/airflow/providers/common/sql/datafusion/engine.py
index f35bd5786ab..1b8a3998ef6 100644
--- a/providers/common/sql/src/airflow/providers/common/sql/datafusion/engine.py
+++ b/providers/common/sql/src/airflow/providers/common/sql/datafusion/engine.py
@@ -153,6 +153,14 @@ class DataFusionEngine(LoggingMixin):
conf[key] = conn.extra_dejson[key]
return conf
+ def _get_gcp_extra_field(extra_dejson: dict[str, Any], field_name:
str) -> Any:
+ # Older Airflow connection UIs wrote custom extra fields as
+ # extra__google_cloud_platform__<field_name> instead of the bare
key; GoogleBaseHook
+ # still reads that legacy spelling as a fallback, so this must too.
+ if field_name in extra_dejson:
+ return extra_dejson[field_name]
+ return
extra_dejson.get(f"extra__google_cloud_platform__{field_name}")
+
match conn.conn_type:
case "aws":
try:
@@ -179,14 +187,14 @@ class DataFusionEngine(LoggingMixin):
case "google_cloud_platform":
extra_dejson = conn.extra_dejson
for unsupported_field in ("key_secret_name",
"credential_config_file", "impersonation_chain"):
- if extra_dejson.get(unsupported_field):
+ if _get_gcp_extra_field(extra_dejson, unsupported_field):
raise ValueError(
f"Connection field {unsupported_field!r} is not
supported for DataFusion "
"GCS access; only key_path, keyfile_dict,
GOOGLE_APPLICATION_CREDENTIALS, or "
"ambient credentials (gcloud ADC file / metadata
server) are used."
)
- key_path = extra_dejson.get("key_path") or None
- keyfile_dict = extra_dejson.get("keyfile_dict") or None
+ key_path = _get_gcp_extra_field(extra_dejson, "key_path") or
None
+ keyfile_dict = _get_gcp_extra_field(extra_dejson,
"keyfile_dict") or None
if key_path and keyfile_dict:
raise ValueError(
"The `keyfile_dict` and `key_path` fields are mutually
exclusive. "
diff --git
a/providers/common/sql/tests/unit/common/sql/datafusion/test_engine.py
b/providers/common/sql/tests/unit/common/sql/datafusion/test_engine.py
index 5e78e0c7761..10f01b43cb9 100644
--- a/providers/common/sql/tests/unit/common/sql/datafusion/test_engine.py
+++ b/providers/common/sql/tests/unit/common/sql/datafusion/test_engine.py
@@ -372,6 +372,28 @@ class TestDataFusionEngine:
with pytest.raises(ValueError, match=f"{unsupported_field!r} is not
supported"):
engine._get_credentials(mock_conn)
+ def test_get_credentials_gcs_reads_legacy_extra_prefixed_key_path(self):
+ """Older Airflow connection UIs wrote custom extra fields as
+ extra__google_cloud_platform__<field>; GoogleBaseHook still reads that
spelling."""
+ mock_conn = MagicMock()
+ mock_conn.conn_type = "google_cloud_platform"
+ mock_conn.extra_dejson = {"extra__google_cloud_platform__key_path":
"/path/to/key.json"}
+ engine = DataFusionEngine()
+
+ credentials, extra_config = engine._get_credentials(mock_conn)
+
+ assert credentials == {"key_path": "/path/to/key.json"}
+ assert extra_config == {}
+
+ def
test_get_credentials_gcs_rejects_legacy_extra_prefixed_unsupported_field(self):
+ mock_conn = MagicMock()
+ mock_conn.conn_type = "google_cloud_platform"
+ mock_conn.extra_dejson =
{"extra__google_cloud_platform__impersonation_chain": "some-chain"}
+ engine = DataFusionEngine()
+
+ with pytest.raises(ValueError, match="'impersonation_chain' is not
supported"):
+ engine._get_credentials(mock_conn)
+
def test_get_credentials_unknown_type(self):
mock_conn = MagicMock()
mock_conn.conn_type = "dummy"
