This is an automated email from the ASF dual-hosted git repository.
mobuchowski pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new c7c65474810 OpenLineage: accept whole config when instantiating
OpenLineageClient. (#43740)
c7c65474810 is described below
commit c7c65474810a1d4a22320b9064fa1374e38129c9
Author: Jakub Dardzinski <[email protected]>
AuthorDate: Wed Nov 6 15:45:22 2024 +0100
OpenLineage: accept whole config when instantiating OpenLineageClient.
(#43740)
Signed-off-by: Jakub Dardzinski <[email protected]>
---
generated/provider_dependencies.json | 4 ++--
.../src/airflow/providers/openlineage/plugins/adapter.py | 8 +++-----
providers/src/airflow/providers/openlineage/provider.yaml | 6 ++----
providers/tests/openlineage/plugins/test_adapter.py | 11 -----------
4 files changed, 7 insertions(+), 22 deletions(-)
diff --git a/generated/provider_dependencies.json
b/generated/provider_dependencies.json
index 371b4fad8eb..865ce0931ea 100644
--- a/generated/provider_dependencies.json
+++ b/generated/provider_dependencies.json
@@ -976,8 +976,8 @@
"apache-airflow-providers-common-sql>=1.6.0",
"apache-airflow>=2.8.0",
"attrs>=22.2",
- "openlineage-integration-common>=1.22.0,<1.24.0",
- "openlineage-python>=1.22.0,<1.24.0"
+ "openlineage-integration-common>=1.24.2",
+ "openlineage-python>=1.24.2"
],
"devel-deps": [],
"plugins": [
diff --git a/providers/src/airflow/providers/openlineage/plugins/adapter.py
b/providers/src/airflow/providers/openlineage/plugins/adapter.py
index 51eb649adbf..fb58cc5dc02 100644
--- a/providers/src/airflow/providers/openlineage/plugins/adapter.py
+++ b/providers/src/airflow/providers/openlineage/plugins/adapter.py
@@ -84,7 +84,7 @@ class OpenLineageAdapter(LoggingMixin):
"OpenLineage configuration found. Transport type: `%s`",
config.get("type", "no type provided"),
)
- self._client = OpenLineageClient.from_dict(config=config)
+ self._client = OpenLineageClient(config=config)
else:
self.log.debug(
"OpenLineage configuration not found directly in Airflow. "
@@ -98,9 +98,7 @@ class OpenLineageAdapter(LoggingMixin):
openlineage_config_path = conf.config_path(check_legacy_env_var=False)
if openlineage_config_path:
config = self._read_yaml_config(openlineage_config_path)
- if config:
- return config.get("transport", None)
- self.log.debug("OpenLineage config file is empty: `%s`",
openlineage_config_path)
+ return config
else:
self.log.debug("OpenLineage config_path configuration not found.")
@@ -109,7 +107,7 @@ class OpenLineageAdapter(LoggingMixin):
if not transport_config:
self.log.debug("OpenLineage transport configuration not found.")
return None
- return transport_config
+ return {"transport": transport_config}
@staticmethod
def _read_yaml_config(path: str) -> dict | None:
diff --git a/providers/src/airflow/providers/openlineage/provider.yaml
b/providers/src/airflow/providers/openlineage/provider.yaml
index bfe7963a508..e638daba78e 100644
--- a/providers/src/airflow/providers/openlineage/provider.yaml
+++ b/providers/src/airflow/providers/openlineage/provider.yaml
@@ -54,10 +54,8 @@ dependencies:
- apache-airflow-providers-common-sql>=1.6.0
- apache-airflow-providers-common-compat>=1.2.1
- attrs>=22.2
- # Temporary limiting Open Lineage version to 1.23.0 due to compatibility
issues with 1.24.0
- # Causing test failures, waiting for 1.24.1 to be released to fix the issue
- - openlineage-integration-common>=1.22.0,<1.24.0
- - openlineage-python>=1.22.0,<1.24.0
+ - openlineage-integration-common>=1.24.2
+ - openlineage-python>=1.24.2
integrations:
- integration-name: OpenLineage
diff --git a/providers/tests/openlineage/plugins/test_adapter.py
b/providers/tests/openlineage/plugins/test_adapter.py
index a2af9f393da..f0928dd70db 100644
--- a/providers/tests/openlineage/plugins/test_adapter.py
+++ b/providers/tests/openlineage/plugins/test_adapter.py
@@ -81,17 +81,6 @@ def test_create_client_from_config_with_options():
assert client.transport.url == "http://ol-api:5000";
-@conf_vars(
- {
- ("openlineage", "transport"): '{"url": "http://ol-api:5000";,'
- ' "auth": {"type": "api_key", "apiKey": "api-key"}}'
- }
-)
-def test_fails_to_create_client_without_type():
- with pytest.raises(KeyError):
- OpenLineageAdapter().get_or_create_openlineage_client()
-
-
def test_create_client_from_yaml_config():
current_folder = pathlib.Path(__file__).parent.resolve()
yaml_config = str((current_folder / "openlineage_configs" /
"http.yaml").resolve())
