This is an automated email from the ASF dual-hosted git repository.
dabla pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 58b5a806456 Flag conn-fields in hook but absent from provider.yaml in
static checks (#69655)
58b5a806456 is described below
commit 58b5a8064567afb54517ad523f751898ac39b18c
Author: David Blain <[email protected]>
AuthorDate: Thu Jul 16 21:43:55 2026 +0200
Flag conn-fields in hook but absent from provider.yaml in static checks
(#69655)
* Flag conn-fields in hook but absent from provider.yaml
* Added missing conn-fields in provider.yaml of Hive, Google, Azure and
Snowflake providers so those are in sync with defined fields in
get_connection_form_widgets method of corresponding hook
* Added cloud_environment in provider.yaml for AzureBaseHook
* Fixed addition of cloud_environment in get_provider_info
---------
Co-authored-by: BKD7702 <[email protected]>
---
providers/apache/hive/provider.yaml | 14 +++++++
.../providers/apache/hive/get_provider_info.py | 5 +++
providers/google/provider.yaml | 8 ++++
.../airflow/providers/google/get_provider_info.py | 2 +
providers/microsoft/azure/provider.yaml | 5 ++-
.../providers/microsoft/azure/get_provider_info.py | 4 ++
providers/snowflake/provider.yaml | 6 +++
.../providers/snowflake/get_provider_info.py | 4 ++
scripts/ci/prek/check_provider_conn_fields.py | 44 +++++++++++++---------
.../in_container/run_provider_yaml_files_check.py | 13 +++----
.../test_check_conn_fields_match_form_widgets.py | 23 +++++++----
11 files changed, 93 insertions(+), 35 deletions(-)
diff --git a/providers/apache/hive/provider.yaml
b/providers/apache/hive/provider.yaml
index db2e4006942..5af0916987e 100644
--- a/providers/apache/hive/provider.yaml
+++ b/providers/apache/hive/provider.yaml
@@ -195,6 +195,20 @@ connection-types:
- boolean
- 'null'
default: false
+ ssl:
+ label: Ssl
+ schema:
+ type:
+ - boolean
+ - 'null'
+ default: true
+ zoo_keeper_namespace:
+ label: Zoo Keeper Namespace
+ schema:
+ type:
+ - string
+ - 'null'
+ default: 'hiveserver2'
- hook-class-name: airflow.providers.apache.hive.hooks.hive.HiveServer2Hook
hook-name: "Hive Server 2 Thrift"
connection-type: hiveserver2
diff --git
a/providers/apache/hive/src/airflow/providers/apache/hive/get_provider_info.py
b/providers/apache/hive/src/airflow/providers/apache/hive/get_provider_info.py
index f9d7cd5b0d7..654afa27467 100644
---
a/providers/apache/hive/src/airflow/providers/apache/hive/get_provider_info.py
+++
b/providers/apache/hive/src/airflow/providers/apache/hive/get_provider_info.py
@@ -131,6 +131,11 @@ def get_provider_info():
"label": "High Availability mode",
"schema": {"type": ["boolean", "null"], "default":
False},
},
+ "ssl": {"label": "Ssl", "schema": {"type": ["boolean",
"null"], "default": True}},
+ "zoo_keeper_namespace": {
+ "label": "Zoo Keeper Namespace",
+ "schema": {"type": ["string", "null"], "default":
"hiveserver2"},
+ },
},
},
{
diff --git a/providers/google/provider.yaml b/providers/google/provider.yaml
index 3dbed0b0485..3a9d2b7ad1a 100644
--- a/providers/google/provider.yaml
+++ b/providers/google/provider.yaml
@@ -1202,6 +1202,10 @@ connection-types:
schema:
type: ["boolean", "null"]
default: false
+ quota_project_id:
+ label: "Quota Project ID"
+ schema:
+ type: ["string", "null"]
- hook-class-name: airflow.providers.google.cloud.hooks.spanner.SpannerHook
connection-type: gcpspanner
- hook-class-name:
airflow.providers.google.cloud.hooks.dataprep.GoogleDataprepHook
@@ -1281,6 +1285,10 @@ connection-types:
schema:
type: ["boolean", "null"]
default: false
+ quota_project_id:
+ label: "Quota Project ID"
+ schema:
+ type: ["string", "null"]
use_legacy_sql:
label: "Use Legacy SQL"
schema:
diff --git a/providers/google/src/airflow/providers/google/get_provider_info.py
b/providers/google/src/airflow/providers/google/get_provider_info.py
index 92966a4b367..13feaa8d1e7 100644
--- a/providers/google/src/airflow/providers/google/get_provider_info.py
+++ b/providers/google/src/airflow/providers/google/get_provider_info.py
@@ -1434,6 +1434,7 @@ def get_provider_info():
"label": "Anonymous credentials (ignores all other
settings)",
"schema": {"type": ["boolean", "null"], "default":
False},
},
+ "quota_project_id": {"label": "Quota Project ID",
"schema": {"type": ["string", "null"]}},
},
},
{
@@ -1512,6 +1513,7 @@ def get_provider_info():
"label": "Anonymous credentials (ignores all other
settings)",
"schema": {"type": ["boolean", "null"], "default":
False},
},
+ "quota_project_id": {"label": "Quota Project ID",
"schema": {"type": ["string", "null"]}},
"use_legacy_sql": {"label": "Use Legacy SQL", "schema":
{"type": ["boolean", "null"]}},
"location": {"label": "Location", "schema": {"type":
["string", "null"]}},
"priority": {
diff --git a/providers/microsoft/azure/provider.yaml
b/providers/microsoft/azure/provider.yaml
index fe6366023e7..cbcc049aa09 100644
--- a/providers/microsoft/azure/provider.yaml
+++ b/providers/microsoft/azure/provider.yaml
@@ -384,7 +384,6 @@ transfers:
how-to-guide:
/docs/apache-airflow-providers-microsoft-azure/transfer/gcs_to_wasb.rst
python-module: airflow.providers.microsoft.azure.transfers.gcs_to_wasb
-
connection-types:
- hook-class-name:
airflow.providers.microsoft.azure.hooks.base_azure.AzureBaseHook
hook-name: "Azure"
@@ -417,6 +416,10 @@ connection-types:
label: Workload Identity Tenant ID
schema:
type: ["string", "null"]
+ cloud_environment:
+ label: Azure Cloud Environment
+ schema:
+ type: ["string", "null"]
- hook-class-name:
airflow.providers.microsoft.azure.hooks.ai_agents.AzureAIAgentsHook
hook-name: "Azure AI Foundry Hosted Agents"
connection-type: azure_ai_agents
diff --git
a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/get_provider_info.py
b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/get_provider_info.py
index e6a6be7c87b..bf7ab5e96a8 100644
---
a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/get_provider_info.py
+++
b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/get_provider_info.py
@@ -408,6 +408,10 @@ def get_provider_info():
"label": "Workload Identity Tenant ID",
"schema": {"type": ["string", "null"]},
},
+ "cloud_environment": {
+ "label": "Azure Cloud Environment",
+ "schema": {"type": ["string", "null"]},
+ },
},
},
{
diff --git a/providers/snowflake/provider.yaml
b/providers/snowflake/provider.yaml
index ecb3451a274..2189d5a1ef7 100644
--- a/providers/snowflake/provider.yaml
+++ b/providers/snowflake/provider.yaml
@@ -252,6 +252,12 @@ connection-types:
- string
- 'null'
format: password
+ workload_identity_provider:
+ label: Workload Identity Provider
+ schema:
+ type:
+ - string
+ - 'null'
insecure_mode:
label: Insecure Mode
schema:
diff --git
a/providers/snowflake/src/airflow/providers/snowflake/get_provider_info.py
b/providers/snowflake/src/airflow/providers/snowflake/get_provider_info.py
index e5b52603876..7e20af8004e 100644
--- a/providers/snowflake/src/airflow/providers/snowflake/get_provider_info.py
+++ b/providers/snowflake/src/airflow/providers/snowflake/get_provider_info.py
@@ -142,6 +142,10 @@ def get_provider_info():
"label": "Private key (Text)",
"schema": {"type": ["string", "null"], "format":
"password"},
},
+ "workload_identity_provider": {
+ "label": "Workload Identity Provider",
+ "schema": {"type": ["string", "null"]},
+ },
"insecure_mode": {
"label": "Insecure Mode",
"schema": {"type": ["boolean", "null"]},
diff --git a/scripts/ci/prek/check_provider_conn_fields.py
b/scripts/ci/prek/check_provider_conn_fields.py
index ed4faf157c5..8d50c16335b 100644
--- a/scripts/ci/prek/check_provider_conn_fields.py
+++ b/scripts/ci/prek/check_provider_conn_fields.py
@@ -81,22 +81,23 @@ def build_mismatch_error(
hook_class_name: str,
) -> str | None:
"""
- Check that every key declared in ``conn-fields`` exists in
- ``get_connection_form_widgets()``.
-
- ``conn-fields`` is the new React UI's view of a connection type and is
- intentionally a *subset* of the Flask form widgets — fields can be omitted
- from ``conn-fields`` on purpose. We therefore only flag keys that appear
in
- ``conn-fields`` but are absent from the hook's form (invalid / stale
- declarations). The reverse direction (hook fields not listed in
- ``conn-fields``) is not an error.
-
- Return an error string when stale keys are found, or ``None`` when the
- declared keys are all valid.
+ Check that ``conn-fields`` and ``get_connection_form_widgets()`` are in
sync.
+
+ Once a provider declares ``conn-fields``, those declarations must exactly
+ match the hook's form widgets — we flag both directions:
+
+ - Keys present in ``conn-fields`` but absent from the hook → stale /
invalid
+ declarations that should be removed.
+ - Keys present in the hook but absent from ``conn-fields`` → fields that
will
+ be invisible in the new React connection UI, almost certainly
unintentional.
+
+ Return an error string when any mismatch is found, or ``None`` when the
sets
+ are identical.
"""
only_in_yaml = yaml_keys - hook_keys
+ only_in_hook = hook_keys - yaml_keys
- if not only_in_yaml:
+ if not only_in_yaml and not only_in_hook:
return None
lines = [
@@ -104,9 +105,16 @@ def build_mismatch_error(
f"`{hook_class_name}.get_connection_form_widgets()` "
f"for connection-type '{connection_type}':"
]
- lines.append(
- " Fields in provider.yaml conn-fields but NOT in
get_connection_form_widgets(): "
- + ", ".join(sorted(only_in_yaml))
- )
- lines.append("[yellow]How to fix it[/]: Remove the stale key(s) from
conn-fields in provider.yaml.")
+ if only_in_yaml:
+ lines.append(
+ " Fields in provider.yaml conn-fields but NOT in
get_connection_form_widgets(): "
+ + ", ".join(sorted(only_in_yaml))
+ )
+ lines.append("[yellow]How to fix it[/]: Remove the stale key(s) from
conn-fields in provider.yaml.")
+ if only_in_hook:
+ lines.append(
+ " Fields in get_connection_form_widgets() but NOT in
provider.yaml conn-fields: "
+ + ", ".join(sorted(only_in_hook))
+ )
+ lines.append("[yellow]How to fix it[/]: Add the missing key(s) to
conn-fields in provider.yaml.")
return "\n".join(lines)
diff --git a/scripts/in_container/run_provider_yaml_files_check.py
b/scripts/in_container/run_provider_yaml_files_check.py
index 922e84b710b..9518e75ad65 100755
--- a/scripts/in_container/run_provider_yaml_files_check.py
+++ b/scripts/in_container/run_provider_yaml_files_check.py
@@ -481,14 +481,11 @@ def
check_hook_class_name_entries_in_connection_types(yaml_files: dict[str, dict
@run_check("Checking that conn-fields in provider.yaml match
get_connection_form_widgets() of the hook class")
def check_conn_fields_match_form_widgets(yaml_files: dict[str, dict]) ->
tuple[int, int]:
"""
- For every connection-type entry whose hook declares ``conn-fields``,
- verify that every key in ``conn-fields`` also exists in the hook's
- ``get_connection_form_widgets()``.
-
- ``conn-fields`` is optional and is intentionally allowed to be a *subset*
- of the hook's form widgets (the new React UI may expose fewer fields than
- the legacy Flask form), so extra hook widgets are not flagged — only
- ``conn-fields`` keys absent from the hook are reported as stale.
+ For every connection-type entry that both declares ``conn-fields`` and
whose
+ hook overrides ``get_connection_form_widgets()``, verify the two sets are
+ identical — stale YAML keys (in ``conn-fields`` but not in the hook) and
+ missing YAML keys (in the hook but absent from ``conn-fields``) are both
+ reported as errors.
"""
num_checks = 0
num_errors = 0
diff --git a/scripts/tests/ci/prek/test_check_conn_fields_match_form_widgets.py
b/scripts/tests/ci/prek/test_check_conn_fields_match_form_widgets.py
index caa2b2fc352..db63f6783aa 100644
--- a/scripts/tests/ci/prek/test_check_conn_fields_match_form_widgets.py
+++ b/scripts/tests/ci/prek/test_check_conn_fields_match_form_widgets.py
@@ -54,8 +54,6 @@ class TestBuildMismatchError:
[
pytest.param({"a", "b"}, {"a", "b"}, id="matching-keys"),
pytest.param(set(), set(), id="empty-sets"),
- # Hook may have more keys than conn-fields — that is intentional
(subset allowed).
- pytest.param({"a"}, {"a", "extra_hook"},
id="hook-has-extra-keys-no-error"),
],
)
def test_no_mismatch_returns_none(self, yaml_keys, hook_keys):
@@ -69,18 +67,26 @@ class TestBuildMismatchError:
{"a"},
["extra_yaml", "NOT in get_connection_form_widgets"],
["NOT in provider.yaml conn-fields"],
- id="extra-in-yaml",
+ id="extra-in-yaml-only",
+ ),
+ pytest.param(
+ {"a"},
+ {"a", "extra_hook"},
+ ["extra_hook", "NOT in provider.yaml conn-fields"],
+ ["NOT in get_connection_form_widgets"],
+ id="extra-in-hook-only",
),
- # only_in_hook no longer triggers an error — only only_in_yaml does
pytest.param(
{"a", "only_yaml"},
{"a", "only_hook"},
[
"only_yaml",
"NOT in get_connection_form_widgets",
+ "only_hook",
+ "NOT in provider.yaml conn-fields",
],
- ["only_hook", "NOT in provider.yaml conn-fields"],
- id="both-sides-only-yaml-reported",
+ [],
+ id="both-sides-both-reported",
),
],
)
@@ -103,8 +109,6 @@ class TestCheckConnFieldsForEntry:
pytest.param(None, _skip,
id="skip-missing-conn-fields-when-hook-has-no-widgets"),
# Hook with widgets but no conn-fields is allowed: new UI
intentionally omits custom fields.
pytest.param(None, _get_keys("field_a"),
id="no-conn-fields-with-hook-widgets-is-ok"),
- # Hook has extra keys not in conn-fields — allowed (conn-fields is
a valid subset).
- pytest.param(["a"], _get_keys("a", "extra_hook"),
id="hook-extra-keys-no-error"),
],
)
def test_no_errors(self, conn_fields, get_keys):
@@ -114,6 +118,9 @@ class TestCheckConnFieldsForEntry:
"conn_fields, get_keys, expected_in_error",
[
pytest.param(["a", "extra"], _get_keys("a"), "extra",
id="extra-key-in-yaml"),
+ pytest.param(
+ ["a"], _get_keys("a", "extra_hook"), "extra_hook",
id="hook-has-extra-key-not-in-yaml"
+ ),
pytest.param(["a"], _raise, "boom",
id="unexpected-exception-message"),
pytest.param(["a"], _raise, HOOK_CLASS,
id="unexpected-exception-mentions-hook-class"),
],
