jason810496 commented on code in PR #60410:
URL: https://github.com/apache/airflow/pull/60410#discussion_r2744967510
##########
airflow-core/src/airflow/providers_manager.py:
##########
@@ -902,6 +903,117 @@ def _get_attr(obj: Any, attr_name: str):
return None
return getattr(obj, attr_name)
+ def _get_connection_type_config_from_yaml(
Review Comment:
Same here.
##########
airflow-core/src/airflow/providers_manager.py:
##########
@@ -902,6 +903,117 @@ def _get_attr(obj: Any, attr_name: str):
return None
return getattr(obj, attr_name)
+ def _get_connection_type_config_from_yaml(
+ self, provider_info: ProviderInfo, connection_type: str
+ ) -> dict | None:
+ """Get connection type config from provider.yaml if it exists."""
+ connection_types = provider_info.data.get("connection-types", [])
+ for conn_config in connection_types:
+ if conn_config.get("connection-type") == connection_type:
+ return conn_config
+ return None
+
+ @staticmethod
+ def _get_mock_field_for_field_type(field_type, field_format):
Review Comment:
It seems not use anymore.
##########
airflow-core/src/airflow/providers_manager.py:
##########
@@ -525,6 +525,7 @@ def initialize_providers_hooks(self):
self._init_airflow_core_hooks()
self.initialize_providers_list()
self._discover_hooks()
+ self._load_ui_metadata_from_yaml()
Review Comment:
Naming is hard. I'm not sure would it be better to remove all the `_yaml` or
any `yaml` here?
Since, in the Provider Manger context, we will load `def get_provider_info`
entry point via `importlib_metadata` instead of reading from `.yaml` directly.
The `provider.yaml` will only be treated as source of truth at codebase
(prek static check level via `update-providers-build-files` hook). For the
runtime, `def get_provider_info` will be our source of truth for provider
metadata instead.
##########
airflow-core/src/airflow/providers_manager.py:
##########
@@ -902,6 +903,117 @@ def _get_attr(obj: Any, attr_name: str):
return None
return getattr(obj, attr_name)
+ def _get_connection_type_config_from_yaml(
+ self, provider_info: ProviderInfo, connection_type: str
+ ) -> dict | None:
+ """Get connection type config from provider.yaml if it exists."""
+ connection_types = provider_info.data.get("connection-types", [])
+ for conn_config in connection_types:
+ if conn_config.get("connection-type") == connection_type:
+ return conn_config
+ return None
+
+ @staticmethod
+ def _get_mock_field_for_field_type(field_type, field_format):
+ from airflow.api_fastapi.core_api.services.ui.connections import
HookMetaService
+
+ field_type_to_mock_field_map = {
+ ("string", "password"): HookMetaService.MockPasswordField,
+ ("string", None): HookMetaService.MockStringField,
+ ("integer", None): HookMetaService.MockIntegerField,
+ ("boolean", None): HookMetaService.MockBooleanField,
+ ("number", None): HookMetaService.MockIntegerField,
+ }
+
+ field_class = field_type_to_mock_field_map.get((field_type,
field_format))
+
+ return field_class
+
+ def _yaml_to_api_format(self, field_name: str, field_def: dict) -> dict:
+ """Convert yaml conn-fields to format expected by the API."""
+ schema_def = field_def.get("schema", {})
+
+ # build schema dict with label moved to `title` per jsonschema
convention
+ schema = schema_def.copy()
+ if "label" in field_def:
+ schema["title"] = field_def.get("label")
+
+ return {
+ "value": schema_def.get("default"),
+ "schema": schema,
+ "description": field_def.get("description"),
+ "source": None,
+ }
+
+ def _add_widgets_from_yaml(
+ self, package_name: str, hook_class_name: str, connection_type: str,
conn_fields_yaml: dict
+ ) -> None:
+ """Parse conn-fields from yaml and add to connection_form_widgets."""
+ for field_name, field_def in conn_fields_yaml.items():
+ field_data = self._yaml_to_api_format(field_name, field_def)
+
+ prefixed_name = f"extra__{connection_type}__{field_name}"
+ if prefixed_name in self._connection_form_widgets:
+ log.warning(
+ "Field %s for connection type %s already added, skipping",
+ field_name,
+ connection_type,
+ )
+ continue
+
+ self._connection_form_widgets[prefixed_name] =
ConnectionFormWidgetInfo(
+ hook_class_name=hook_class_name,
+ package_name=package_name,
+ field=field_data,
+ field_name=field_name,
+ is_sensitive=field_def.get("sensitive", False),
+ )
+
+ def _add_customized_fields_from_yaml(
+ self, package_name: str, connection_type: str, behaviour_yaml: dict
+ ) -> None:
+ """Process ui-field-behaviour from yaml and add to field_behaviours."""
+ if connection_type in self._field_behaviours:
+ log.warning(
+ "Field behaviour for connection type %s already exists,
skipping yaml",
+ connection_type,
+ )
+ return
+
+ # convert yaml keys to python style
+ customized_fields = {
+ "hidden_fields": behaviour_yaml.get("hidden-fields", []),
+ "relabeling": behaviour_yaml.get("relabeling", {}),
+ "placeholders": behaviour_yaml.get("placeholders", {}),
+ }
+
+ try:
+
self._customized_form_fields_schema_validator.validate(customized_fields)
+ customized_fields =
_ensure_prefix_for_placeholders(customized_fields, connection_type)
+ self._field_behaviours[connection_type] = customized_fields
+ except Exception as e:
+ log.warning(
+ "Failed to add field behaviour from yaml for %s in package %s:
%s",
+ connection_type,
+ package_name,
+ e,
+ )
+
+ def _load_ui_metadata_from_yaml(self) -> None:
Review Comment:
Since we will call `_init_airflow_core_hooks` -> `_import_hook` anyway.
How about adding early return before the constructing `wtforms` in
`_import_hook` to reduce loading time? (avoid setting `_field_behaviours` field
by `_import_hook` first time and setting by `_load_ui_metadata` second time)
```python
if conn_config.get("conn-fields") or conn_config.get("ui-field-behaviour"):
return
```
https://github.com/apache/airflow/blob/c93cb32db735eb7c16c2ebce044a5795ff9c5ed7/airflow-core/src/airflow/providers_manager.py#L942-L946
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
