kacpermuda commented on code in PR #38982:
URL: https://github.com/apache/airflow/pull/38982#discussion_r1607990291
##########
airflow/providers/openlineage/utils/utils.py:
##########
@@ -61,11 +61,31 @@ def get_job_name(task: TaskInstance) -> str:
def get_custom_facets(task_instance: TaskInstance | None = None) -> dict[str,
Any]:
+ from airflow.providers.openlineage.extractors.manager import
try_import_from_string
+
custom_facets = {}
# check for -1 comes from SmartSensor compatibility with dynamic task
mapping
# this comes from Airflow code
if hasattr(task_instance, "map_index") and getattr(task_instance,
"map_index") != -1:
custom_facets["airflow_mappedTask"] =
AirflowMappedTaskRunFacet.from_task_instance(task_instance)
+
+ # Append custom run facets by executing the custom_facet_functions.
+ for custom_facet_func in conf.custom_facet_functions():
+ func: type[function] | None = try_import_from_string(custom_facet_func)
+ if not func:
+ log.warning("Unable to import `%s`, will ignore it.",
custom_facet_func)
+ facet = func(task_instance) if func else None
+ if facet and isinstance(facet, dict):
+ duplicate_facet_keys = [facet_key for facet_key in facet.keys() if
facet_key in custom_facets]
+ if duplicate_facet_keys:
+ log.warning(
+ "Got duplicate facets key(s), `%s` from `%s`, will ignore
it.",
+ ", ".join(duplicate_facet_keys),
+ custom_facet_func,
+ )
+ else:
+ log.info(f"Appending custom facets from {custom_facet_func}.")
+ custom_facets.update(facet)
Review Comment:
```suggestion
duplicate_facet_keys = [facet_key for facet_key in facet.keys()
if facet_key in custom_facets]
if duplicate_facet_keys:
log.warning(
"Duplicate OpenLineage custom facets key(s) found: `%s`
from function `%s`.",
", ".join(duplicate_facet_keys),
custom_facet_func,
)
log.debug("Adding OpenLineage custom facet with key(s): `%s`
from function `%s`.", tuple(facet), custom_facet_func)
custom_facets.update(facet)
```
##########
airflow/providers/openlineage/utils/utils.py:
##########
@@ -61,11 +61,31 @@ def get_job_name(task: TaskInstance) -> str:
def get_custom_facets(task_instance: TaskInstance | None = None) -> dict[str,
Any]:
+ from airflow.providers.openlineage.extractors.manager import
try_import_from_string
+
custom_facets = {}
# check for -1 comes from SmartSensor compatibility with dynamic task
mapping
# this comes from Airflow code
if hasattr(task_instance, "map_index") and getattr(task_instance,
"map_index") != -1:
custom_facets["airflow_mappedTask"] =
AirflowMappedTaskRunFacet.from_task_instance(task_instance)
+
+ # Append custom run facets by executing the custom_facet_functions.
+ for custom_facet_func in conf.custom_facet_functions():
+ func: type[function] | None = try_import_from_string(custom_facet_func)
+ if not func:
+ log.warning("Unable to import `%s`, will ignore it.",
custom_facet_func)
+ facet = func(task_instance) if func else None
Review Comment:
```suggestion
if not func:
log.warning("OpenLineage is unable to import custom facet
function `%s`; will ignore it.", custom_facet_func)
continue
facets: dict[str, BaseFacet] = func(task_instance)
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
