This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 0afaa5d8165 fix: add explicit requirement for OpenLineage version on
DBT function (#47999)
0afaa5d8165 is described below
commit 0afaa5d8165d9ad5c001510661505161ec1bbd94
Author: Kacper Muda <[email protected]>
AuthorDate: Mon Mar 31 03:44:20 2025 +0200
fix: add explicit requirement for OpenLineage version on DBT function
(#47999)
---
generated/provider_dependencies.json | 1 +
providers/dbt/cloud/README.rst | 15 +++++++-------
providers/dbt/cloud/pyproject.toml | 6 ++----
.../providers/dbt/cloud/get_provider_info.py | 6 ++----
.../providers/dbt/cloud/utils/openlineage.py | 11 +++--------
.../tests/unit/dbt/cloud/utils/test_openlineage.py | 23 +++++++++++-----------
6 files changed, 28 insertions(+), 34 deletions(-)
diff --git a/generated/provider_dependencies.json
b/generated/provider_dependencies.json
index 41126faeaae..2c5fc991291 100644
--- a/generated/provider_dependencies.json
+++ b/generated/provider_dependencies.json
@@ -495,6 +495,7 @@
"dbt.cloud": {
"deps": [
"aiohttp>=3.9.2",
+ "apache-airflow-providers-common-compat>=1.6.0",
"apache-airflow-providers-http",
"apache-airflow>=2.9.0",
"asgiref>=2.3.0"
diff --git a/providers/dbt/cloud/README.rst b/providers/dbt/cloud/README.rst
index 31d55c2dd18..bbf2ab3079b 100644
--- a/providers/dbt/cloud/README.rst
+++ b/providers/dbt/cloud/README.rst
@@ -50,14 +50,15 @@ The package supports the following python versions:
3.9,3.10,3.11,3.12
Requirements
------------
-================================= ==================
-PIP package Version required
-================================= ==================
-``apache-airflow`` ``>=2.9.0``
+========================================== ==================
+PIP package Version required
+========================================== ==================
+``apache-airflow`` ``>=2.9.0``
+``apache-airflow-providers-common-compat`` ``>=1.6.0``
``apache-airflow-providers-http``
-``asgiref`` ``>=2.3.0``
-``aiohttp`` ``>=3.9.2``
-================================= ==================
+``asgiref`` ``>=2.3.0``
+``aiohttp`` ``>=3.9.2``
+========================================== ==================
Cross provider package dependencies
-----------------------------------
diff --git a/providers/dbt/cloud/pyproject.toml
b/providers/dbt/cloud/pyproject.toml
index ba0f1680221..45eea06e4ae 100644
--- a/providers/dbt/cloud/pyproject.toml
+++ b/providers/dbt/cloud/pyproject.toml
@@ -58,6 +58,7 @@ requires-python = "~=3.9"
# After you modify the dependencies, and rebuild your Breeze CI image with
``breeze ci-image build``
dependencies = [
"apache-airflow>=2.9.0",
+ "apache-airflow-providers-common-compat>=1.6.0",
"apache-airflow-providers-http",
"asgiref>=2.3.0",
"aiohttp>=3.9.2",
@@ -68,10 +69,7 @@ dependencies = [
[project.optional-dependencies]
# pip install apache-airflow-providers-dbt-cloud[openlineage]
"openlineage" = [
- "apache-airflow-providers-openlineage>=1.7.0",
-]
-"common.compat" = [
- "apache-airflow-providers-common-compat"
+ "apache-airflow-providers-openlineage>=2.0.0",
]
[dependency-groups]
diff --git
a/providers/dbt/cloud/src/airflow/providers/dbt/cloud/get_provider_info.py
b/providers/dbt/cloud/src/airflow/providers/dbt/cloud/get_provider_info.py
index 450c58410f4..6d0813d9648 100644
--- a/providers/dbt/cloud/src/airflow/providers/dbt/cloud/get_provider_info.py
+++ b/providers/dbt/cloud/src/airflow/providers/dbt/cloud/get_provider_info.py
@@ -96,13 +96,11 @@ def get_provider_info():
"extra-links":
["airflow.providers.dbt.cloud.operators.dbt.DbtCloudRunJobOperatorLink"],
"dependencies": [
"apache-airflow>=2.9.0",
+ "apache-airflow-providers-common-compat>=1.6.0",
"apache-airflow-providers-http",
"asgiref>=2.3.0",
"aiohttp>=3.9.2",
],
- "optional-dependencies": {
- "openlineage": ["apache-airflow-providers-openlineage>=1.7.0"],
- "common.compat": ["apache-airflow-providers-common-compat"],
- },
+ "optional-dependencies": {"openlineage":
["apache-airflow-providers-openlineage>=2.0.0"]},
"devel-dependencies": [],
}
diff --git
a/providers/dbt/cloud/src/airflow/providers/dbt/cloud/utils/openlineage.py
b/providers/dbt/cloud/src/airflow/providers/dbt/cloud/utils/openlineage.py
index cd3fd927162..983ee48e63c 100644
--- a/providers/dbt/cloud/src/airflow/providers/dbt/cloud/utils/openlineage.py
+++ b/providers/dbt/cloud/src/airflow/providers/dbt/cloud/utils/openlineage.py
@@ -21,6 +21,7 @@ import logging
import re
from typing import TYPE_CHECKING
+from airflow.providers.common.compat.openlineage.check import
require_openlineage_version
from airflow.providers.dbt.cloud.version_compat import AIRFLOW_V_2_10_PLUS,
AIRFLOW_V_3_0_PLUS
if TYPE_CHECKING:
@@ -54,6 +55,7 @@ def _get_try_number(val):
return val.try_number - 1
+@require_openlineage_version(provider_min_version="2.0.0")
def generate_openlineage_events_from_dbt_cloud_run(
operator: DbtCloudRunJobOperator | DbtCloudJobRunSensor, task_instance:
TaskInstance
) -> OperatorLineage:
@@ -73,14 +75,7 @@ def generate_openlineage_events_from_dbt_cloud_run(
"""
from openlineage.common.provider.dbt import DbtCloudArtifactProcessor,
ParentRunMetadata
- try:
- from airflow.providers.openlineage.conf import namespace
- except ModuleNotFoundError as e:
- from airflow.exceptions import AirflowOptionalProviderFeatureException
-
- msg = "Please install `apache-airflow-providers-openlineage>=1.7.0`"
- raise AirflowOptionalProviderFeatureException(e, msg)
-
+ from airflow.providers.openlineage.conf import namespace
from airflow.providers.openlineage.extractors import OperatorLineage
from airflow.providers.openlineage.plugins.adapter import (
_PRODUCER,
diff --git a/providers/dbt/cloud/tests/unit/dbt/cloud/utils/test_openlineage.py
b/providers/dbt/cloud/tests/unit/dbt/cloud/utils/test_openlineage.py
index d7d753cc2a4..b96cb549eb7 100644
--- a/providers/dbt/cloud/tests/unit/dbt/cloud/utils/test_openlineage.py
+++ b/providers/dbt/cloud/tests/unit/dbt/cloud/utils/test_openlineage.py
@@ -89,23 +89,24 @@ def get_dbt_artifact(*args, **kwargs):
def test_previous_version_openlineage_provider():
- """When using OpenLineage, the dbt-cloud provider now depends on
openlineage provider >= 1.7"""
- original_import = __import__
+ """When using OpenLineage, the dbt-cloud provider now depends on
openlineage provider >= 2.0"""
- def custom_import(name, *args, **kwargs):
- if name == "airflow.providers.openlineage.conf":
- raise ModuleNotFoundError("No module named
'airflow.providers.openlineage.conf")
- else:
- return original_import(name, *args, **kwargs)
+ def _mock_version(package):
+ if package == "apache-airflow-providers-openlineage":
+ return "1.99.0"
+ raise Exception("Unexpected package")
mock_operator = MagicMock()
mock_task_instance = MagicMock()
- with patch("builtins.__import__", side_effect=custom_import):
- with pytest.raises(AirflowOptionalProviderFeatureException) as exc:
+ expected_err = (
+ "OpenLineage provider version `1.99.0` is lower than required `2.0.0`,
"
+ "skipping function `generate_openlineage_events_from_dbt_cloud_run`
execution"
+ )
+
+ with patch("importlib.metadata.version", side_effect=_mock_version):
+ with pytest.raises(AirflowOptionalProviderFeatureException,
match=expected_err):
generate_openlineage_events_from_dbt_cloud_run(mock_operator,
mock_task_instance)
- assert str(exc.value.args[0]) == "No module named
'airflow.providers.openlineage.conf"
- assert str(exc.value.args[1]) == "Please install
`apache-airflow-providers-openlineage>=1.7.0`"
class TestGenerateOpenLineageEventsFromDbtCloudRun:
