This is an automated email from the ASF dual-hosted git repository.
jason810496 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 43038e01182 Fix Azure Data Factory hook broken with
azure-mgmt-datafactory 10 (#69800)
43038e01182 is described below
commit 43038e011829eae8082f25a233dade5deef0c145
Author: PoAn Yang <[email protected]>
AuthorDate: Wed Jul 15 20:26:26 2026 +0900
Fix Azure Data Factory hook broken with azure-mgmt-datafactory 10 (#69800)
* Fix Azure Data Factory hook broken by azure-mgmt-datafactory 10
Signed-off-by: PoAn Yang <[email protected]>
* Fix Azure Data Factory hook broken by azure-mgmt-datafactory 10
Signed-off-by: PoAn Yang <[email protected]>
---------
Signed-off-by: PoAn Yang <[email protected]>
---
providers/microsoft/azure/README.rst | 2 +-
providers/microsoft/azure/docs/index.rst | 2 +-
providers/microsoft/azure/pyproject.toml | 2 +-
.../microsoft/azure/hooks/data_factory.py | 46 ++++++++++++--
.../microsoft/azure/hooks/test_data_factory.py | 73 ++++++++++++++++------
uv.lock | 8 +--
6 files changed, 102 insertions(+), 31 deletions(-)
diff --git a/providers/microsoft/azure/README.rst
b/providers/microsoft/azure/README.rst
index 63f0e97e181..be404f636ee 100644
--- a/providers/microsoft/azure/README.rst
+++ b/providers/microsoft/azure/README.rst
@@ -74,7 +74,7 @@ PIP package Version required
``azure-synapse-artifacts`` ``>=0.17.0``
``azure-storage-file-datalake`` ``>=12.9.1``
``azure-kusto-data`` ``>=4.1.0,!=5.0.0``
-``azure-mgmt-datafactory`` ``>=2.0.0``
+``azure-mgmt-datafactory`` ``>=10.0.0``
``azure-mgmt-containerregistry`` ``>=8.0.0``
``azure-mgmt-compute`` ``>=33.0.0``
``azure-mgmt-containerinstance`` ``>=10.1.0``
diff --git a/providers/microsoft/azure/docs/index.rst
b/providers/microsoft/azure/docs/index.rst
index 0ad9ca9512f..b326188e24e 100644
--- a/providers/microsoft/azure/docs/index.rst
+++ b/providers/microsoft/azure/docs/index.rst
@@ -128,7 +128,7 @@ PIP package Version required
``azure-synapse-artifacts`` ``>=0.17.0``
``azure-storage-file-datalake`` ``>=12.9.1``
``azure-kusto-data`` ``>=4.1.0,!=5.0.0``
-``azure-mgmt-datafactory`` ``>=2.0.0``
+``azure-mgmt-datafactory`` ``>=10.0.0``
``azure-mgmt-containerregistry`` ``>=8.0.0``
``azure-mgmt-compute`` ``>=33.0.0``
``azure-mgmt-containerinstance`` ``>=10.1.0``
diff --git a/providers/microsoft/azure/pyproject.toml
b/providers/microsoft/azure/pyproject.toml
index 9b2d117593b..91327c2c74d 100644
--- a/providers/microsoft/azure/pyproject.toml
+++ b/providers/microsoft/azure/pyproject.toml
@@ -84,7 +84,7 @@ dependencies = [
"azure-storage-file-datalake>=12.9.1",
# azure-kusto-data 5.0.0 pins requests to a specific version which makes
resolving dependencies harder
"azure-kusto-data>=4.1.0,!=5.0.0",
- "azure-mgmt-datafactory>=2.0.0",
+ "azure-mgmt-datafactory>=10.0.0",
"azure-mgmt-containerregistry>=8.0.0",
"azure-mgmt-compute>=33.0.0",
"azure-mgmt-containerinstance>=10.1.0",
diff --git
a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/data_factory.py
b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/data_factory.py
index dd408bed515..61ae3c8848b 100644
---
a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/data_factory.py
+++
b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/data_factory.py
@@ -39,6 +39,7 @@ from collections.abc import Callable
from functools import wraps
from typing import IO, TYPE_CHECKING, Any, TypeVar, cast
+from azure.core import MatchConditions
from azure.identity import ClientSecretCredential, DefaultAzureCredential
from azure.identity.aio import (
ClientSecretCredential as AsyncClientSecretCredential,
@@ -278,7 +279,12 @@ class AzureDataFactoryHook(BaseHook):
raise AirflowException(f"Factory {factory!r} does not exist.")
return self.get_conn().factories.create_or_update(
- resource_group_name, factory_name, factory, if_match, **config
+ resource_group_name,
+ factory_name,
+ factory,
+ etag=if_match,
+ match_condition=MatchConditions.IfNotModified if if_match else
None,
+ **config,
)
@provide_targeted_factory
@@ -339,7 +345,12 @@ class AzureDataFactoryHook(BaseHook):
:return: The linked service.
"""
return self.get_conn().linked_services.get(
- resource_group_name, factory_name, linked_service_name,
if_none_match, **config
+ resource_group_name,
+ factory_name,
+ linked_service_name,
+ etag=if_none_match,
+ match_condition=MatchConditions.IfModified if if_none_match else
None,
+ **config,
)
def _linked_service_exists(self, resource_group_name, factory_name,
linked_service_name) -> bool:
@@ -549,7 +560,12 @@ class AzureDataFactoryHook(BaseHook):
:return: The DataFlowResource.
"""
return self.get_conn().data_flows.get(
- resource_group_name, factory_name, dataflow_name, if_none_match,
**config
+ resource_group_name,
+ factory_name,
+ dataflow_name,
+ etag=if_none_match,
+ match_condition=MatchConditions.IfModified if if_none_match else
None,
+ **config,
)
def _dataflow_exists(
@@ -597,7 +613,13 @@ class AzureDataFactoryHook(BaseHook):
raise AirflowException(f"Dataflow {dataflow_name!r} does not
exist.")
return self.get_conn().data_flows.create_or_update(
- resource_group_name, factory_name, dataflow_name, dataflow,
if_match, **config
+ resource_group_name,
+ factory_name,
+ dataflow_name,
+ dataflow,
+ etag=if_match,
+ match_condition=MatchConditions.IfNotModified if if_match else
None,
+ **config,
)
@provide_targeted_factory
@@ -627,7 +649,13 @@ class AzureDataFactoryHook(BaseHook):
raise AirflowException(f"Dataflow {dataflow_name!r} already
exists.")
return self.get_conn().data_flows.create_or_update(
- resource_group_name, factory_name, dataflow_name, dataflow,
if_match, **config
+ resource_group_name,
+ factory_name,
+ dataflow_name,
+ dataflow,
+ etag=if_match,
+ match_condition=MatchConditions.IfNotModified if if_match else
None,
+ **config,
)
@provide_targeted_factory
@@ -923,7 +951,13 @@ class AzureDataFactoryHook(BaseHook):
raise AirflowException(f"Trigger {trigger_name!r} does not exist.")
return self.get_conn().triggers.create_or_update(
- resource_group_name, factory_name, trigger_name, trigger,
if_match, **config
+ resource_group_name,
+ factory_name,
+ trigger_name,
+ trigger,
+ etag=if_match,
+ match_condition=MatchConditions.IfNotModified if if_match else
None,
+ **config,
)
@provide_targeted_factory
diff --git
a/providers/microsoft/azure/tests/unit/microsoft/azure/hooks/test_data_factory.py
b/providers/microsoft/azure/tests/unit/microsoft/azure/hooks/test_data_factory.py
index 92608f6101a..1b64d29c3ce 100644
---
a/providers/microsoft/azure/tests/unit/microsoft/azure/hooks/test_data_factory.py
+++
b/providers/microsoft/azure/tests/unit/microsoft/azure/hooks/test_data_factory.py
@@ -21,6 +21,7 @@ from unittest import mock
from unittest.mock import MagicMock, PropertyMock, patch
import pytest
+from azure.core import MatchConditions
from azure.mgmt.datafactory.aio import DataFactoryManagementClient
from airflow.models.connection import Connection
@@ -242,12 +243,18 @@ def test_create_factory(hook: AzureDataFactoryHook):
hook._conn.factories.create_or_update.assert_called_with(RESOURCE_GROUP,
FACTORY, MODEL)
-def test_update_factory(hook: AzureDataFactoryHook):
[email protected](
+ ("if_match", "expected_match_condition"),
+ [(None, None), ("etag-value", MatchConditions.IfNotModified)],
+)
+def test_update_factory(hook: AzureDataFactoryHook, if_match,
expected_match_condition):
with patch.object(hook, "_factory_exists") as mock_factory_exists:
mock_factory_exists.return_value = True
- hook.update_factory(MODEL, RESOURCE_GROUP, FACTORY)
+ hook.update_factory(MODEL, RESOURCE_GROUP, FACTORY, if_match)
- hook._conn.factories.create_or_update.assert_called_with(RESOURCE_GROUP,
FACTORY, MODEL, None)
+ hook._conn.factories.create_or_update.assert_called_with(
+ RESOURCE_GROUP, FACTORY, MODEL, etag=if_match,
match_condition=expected_match_condition
+ )
def test_update_factory_non_existent(hook: AzureDataFactoryHook):
@@ -264,10 +271,16 @@ def test_delete_factory(hook: AzureDataFactoryHook):
hook._conn.factories.delete.assert_called_with(RESOURCE_GROUP, FACTORY)
-def test_get_linked_service(hook: AzureDataFactoryHook):
- hook.get_linked_service(NAME, RESOURCE_GROUP, FACTORY)
[email protected](
+ ("if_none_match", "expected_match_condition"),
+ [(None, None), ("etag-value", MatchConditions.IfModified)],
+)
+def test_get_linked_service(hook: AzureDataFactoryHook, if_none_match,
expected_match_condition):
+ hook.get_linked_service(NAME, RESOURCE_GROUP, FACTORY, if_none_match)
- hook._conn.linked_services.get.assert_called_with(RESOURCE_GROUP, FACTORY,
NAME, None)
+ hook._conn.linked_services.get.assert_called_with(
+ RESOURCE_GROUP, FACTORY, NAME, etag=if_none_match,
match_condition=expected_match_condition
+ )
def test_create_linked_service(hook: AzureDataFactoryHook):
@@ -332,24 +345,42 @@ def test_delete_dataset(hook: AzureDataFactoryHook):
hook._conn.datasets.delete.assert_called_with(RESOURCE_GROUP, FACTORY,
NAME)
-def test_get_dataflow(hook: AzureDataFactoryHook):
- hook.get_dataflow(NAME, RESOURCE_GROUP, FACTORY)
[email protected](
+ ("if_none_match", "expected_match_condition"),
+ [(None, None), ("etag-value", MatchConditions.IfModified)],
+)
+def test_get_dataflow(hook: AzureDataFactoryHook, if_none_match,
expected_match_condition):
+ hook.get_dataflow(NAME, RESOURCE_GROUP, FACTORY, if_none_match)
- hook._conn.data_flows.get.assert_called_with(RESOURCE_GROUP, FACTORY,
NAME, None)
+ hook._conn.data_flows.get.assert_called_with(
+ RESOURCE_GROUP, FACTORY, NAME, etag=if_none_match,
match_condition=expected_match_condition
+ )
-def test_create_dataflow(hook: AzureDataFactoryHook):
- hook.create_dataflow(NAME, MODEL, RESOURCE_GROUP, FACTORY)
[email protected](
+ ("if_match", "expected_match_condition"),
+ [(None, None), ("etag-value", MatchConditions.IfNotModified)],
+)
+def test_create_dataflow(hook: AzureDataFactoryHook, if_match,
expected_match_condition):
+ hook.create_dataflow(NAME, MODEL, RESOURCE_GROUP, FACTORY, if_match)
- hook._conn.data_flows.create_or_update.assert_called_with(RESOURCE_GROUP,
FACTORY, NAME, MODEL, None)
+ hook._conn.data_flows.create_or_update.assert_called_with(
+ RESOURCE_GROUP, FACTORY, NAME, MODEL, etag=if_match,
match_condition=expected_match_condition
+ )
-def test_update_dataflow(hook: AzureDataFactoryHook):
[email protected](
+ ("if_match", "expected_match_condition"),
+ [(None, None), ("etag-value", MatchConditions.IfNotModified)],
+)
+def test_update_dataflow(hook: AzureDataFactoryHook, if_match,
expected_match_condition):
with patch.object(hook, "_dataflow_exists") as mock_dataflow_exists:
mock_dataflow_exists.return_value = True
- hook.update_dataflow(NAME, MODEL, RESOURCE_GROUP, FACTORY)
+ hook.update_dataflow(NAME, MODEL, RESOURCE_GROUP, FACTORY, if_match)
- hook._conn.data_flows.create_or_update.assert_called_with(RESOURCE_GROUP,
FACTORY, NAME, MODEL, None)
+ hook._conn.data_flows.create_or_update.assert_called_with(
+ RESOURCE_GROUP, FACTORY, NAME, MODEL, etag=if_match,
match_condition=expected_match_condition
+ )
def test_update_dataflow_non_existent(hook: AzureDataFactoryHook):
@@ -473,12 +504,18 @@ def test_create_trigger(hook: AzureDataFactoryHook):
hook._conn.triggers.create_or_update.assert_called_with(RESOURCE_GROUP,
FACTORY, NAME, MODEL)
-def test_update_trigger(hook: AzureDataFactoryHook):
[email protected](
+ ("if_match", "expected_match_condition"),
+ [(None, None), ("etag-value", MatchConditions.IfNotModified)],
+)
+def test_update_trigger(hook: AzureDataFactoryHook, if_match,
expected_match_condition):
with patch.object(hook, "_trigger_exists") as mock_trigger_exists:
mock_trigger_exists.return_value = True
- hook.update_trigger(NAME, MODEL, RESOURCE_GROUP, FACTORY)
+ hook.update_trigger(NAME, MODEL, RESOURCE_GROUP, FACTORY, if_match)
- hook._conn.triggers.create_or_update.assert_called_with(RESOURCE_GROUP,
FACTORY, NAME, MODEL, None)
+ hook._conn.triggers.create_or_update.assert_called_with(
+ RESOURCE_GROUP, FACTORY, NAME, MODEL, etag=if_match,
match_condition=expected_match_condition
+ )
def test_update_trigger_non_existent(hook: AzureDataFactoryHook):
diff --git a/uv.lock b/uv.lock
index aeb4d3f9de6..b5e8f242a27 100644
--- a/uv.lock
+++ b/uv.lock
@@ -6364,7 +6364,7 @@ requires-dist = [
{ name = "azure-mgmt-containerinstance", specifier = ">=10.1.0" },
{ name = "azure-mgmt-containerregistry", specifier = ">=8.0.0" },
{ name = "azure-mgmt-cosmosdb", specifier = ">=3.0.0" },
- { name = "azure-mgmt-datafactory", specifier = ">=2.0.0" },
+ { name = "azure-mgmt-datafactory", specifier = ">=10.0.0" },
{ name = "azure-mgmt-datalake-store", specifier = ">=0.5.0" },
{ name = "azure-mgmt-resource", specifier = ">=2.2.0" },
{ name = "azure-mgmt-storage", specifier = ">=16.0.0" },
@@ -9726,16 +9726,16 @@ wheels = [
[[package]]
name = "azure-mgmt-datafactory"
-version = "9.3.0"
+version = "10.0.0"
source = { registry = "https://pypi.org/simple"; }
dependencies = [
{ name = "azure-mgmt-core" },
{ name = "isodate" },
{ name = "typing-extensions" },
]
-sdist = { url =
"https://files.pythonhosted.org/packages/e8/5e/c28a8a50885185a9a06ff659594fec24f1a840622f21883b376358dd1188/azure_mgmt_datafactory-9.3.0.tar.gz";,
hash =
"sha256:f5fdd5cd416f0ed71dfedf05dc7677b8f0e52f3428fd5b17b04c9200dd8d36b3", size
= 494270, upload-time = "2026-03-11T07:52:35.206Z" }
+sdist = { url =
"https://files.pythonhosted.org/packages/56/f9/81397c6afd0fed926ffe3093d108d0392745bddc05c5dfc69880dfb70bb2/azure_mgmt_datafactory-10.0.0.tar.gz";,
hash =
"sha256:c83eeb2b092c1f1e5bb2499ffbba6fd7809a1274739930f9db1e336273405587", size
= 717334, upload-time = "2026-07-08T09:40:00.573Z" }
wheels = [
- { url =
"https://files.pythonhosted.org/packages/33/35/ea6a784652119a7373a730d0eb87d5c0b7d153721d7e23c8e9b76547aca4/azure_mgmt_datafactory-9.3.0-py3-none-any.whl";,
hash =
"sha256:fddb855a27e3f7b78328f184df146d71e433e1dfb9cc4923ea503c53813f0504", size
= 570055, upload-time = "2026-03-11T07:52:37.337Z" },
+ { url =
"https://files.pythonhosted.org/packages/bb/41/b5a74df49ed17aa27361fc55fd88a07bfe7c7a55bcb5ed210c0e0e712d57/azure_mgmt_datafactory-10.0.0-py3-none-any.whl";,
hash =
"sha256:d0e93194c9e1cd820d2ed4f92813902af89cf4421c02e72a729ae32d87171198", size
= 601269, upload-time = "2026-07-08T09:40:02.559Z" },
]
[[package]]
