This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 251cd91bb24 Migrate google + pagerduty + anthropic providers to
`httpx2` (#72111)
251cd91bb24 is described below
commit 251cd91bb24a335f9ec320ec1b3f2929f9e4577c
Author: Zach <[email protected]>
AuthorDate: Wed Sep 9 15:10:21 2026 -0400
Migrate google + pagerduty + anthropic providers to `httpx2` (#72111)
* Migrate google + pagerduty providers to `httpx2`
* Add a prek check that keeps providers on `httpx2`
The `httpx` package is unmaintained and Airflow is moving to the `httpx2`
fork,
so providers already migrated must not drift back.
The check exempts `airbyte`, `common/ai`, `weaviate` and `microsoft/azure`,
whose third-party SDKs are still built on `httpx`. Their objects cross that
boundary, and the two packages' classes are distinct, so switching those
providers would break at runtime rather than fail loudly.
* Address feedback on ast-grep pre-commit hook
---
providers/.ast-grep/rules/no-httpx-import.yml | 43 ++++++++++++++++++++++
providers/.ast-grep/sgconfig.yml | 28 ++++++++++++++
providers/.pre-commit-config.yaml | 9 +++++
.../src/airflow/providers/airbyte/hooks/airbyte.py | 3 ++
providers/google/README.rst | 2 +-
providers/google/docs/changelog.rst | 7 ++++
providers/google/docs/index.rst | 2 +-
providers/google/pyproject.toml | 2 +-
.../providers/google/cloud/hooks/cloud_sql.py | 10 +----
.../unit/google/cloud/hooks/test_cloud_sql.py | 25 +++++++++++++
.../providers/microsoft/azure/hooks/msgraph.py | 3 ++
providers/pagerduty/README.rst | 2 +-
providers/pagerduty/docs/changelog.rst | 5 +++
providers/pagerduty/docs/index.rst | 2 +-
providers/pagerduty/pyproject.toml | 2 +-
.../unit/pagerduty/hooks/test_pagerduty_events.py | 10 ++---
.../airflow/providers/weaviate/hooks/weaviate.py | 3 ++
uv.lock | 6 +--
18 files changed, 142 insertions(+), 22 deletions(-)
diff --git a/providers/.ast-grep/rules/no-httpx-import.yml
b/providers/.ast-grep/rules/no-httpx-import.yml
new file mode 100644
index 00000000000..a40a96131d1
--- /dev/null
+++ b/providers/.ast-grep/rules/no-httpx-import.yml
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+id: no-httpx-import
+language: python
+severity: error
+message: Import 'httpx2' instead of 'httpx' -- see
https://github.com/apache/airflow/issues/70522
+note: >-
+ The 'airbyte', 'common/ai', 'weaviate' and 'microsoft/azure' providers are
exempt
+ because the third-party SDKs they hand these objects to are themselves built
on
+ 'httpx', and 'httpx2' classes are distinct from their 'httpx' counterparts.
+files:
+ - "**/*.py"
+ignores:
+ - "airbyte/**"
+ - "common/ai/**"
+ - "weaviate/**"
+ - "microsoft/azure/**"
+rule:
+ all:
+ - kind: dotted_name
+ - regex: ^httpx($|\.)
+ - any:
+ - inside:
+ kind: import_statement
+ stopBy: end
+ - inside:
+ kind: import_from_statement
+ field: module_name
diff --git a/providers/.ast-grep/sgconfig.yml b/providers/.ast-grep/sgconfig.yml
new file mode 100644
index 00000000000..ff99acc8c13
--- /dev/null
+++ b/providers/.ast-grep/sgconfig.yml
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+# NOTE: this file configures `ast-grep`, which is used for enforcing custom
lint / auto-formatting
+# across a number of different languages. Currently, we use this in
pre-commit hook to avoid regressions of
+# reintroducing `httpx` in provider packages which currently don't use `httpx`.
+#
+# If new ast-grep rules are required, they may be added in the rules directory
listed below.
+#
+# See also:
+# https://github.com/ast-grep/ast-grep
+# https://ast-grep.github.io/guide/introduction
+ruleDirs:
+ - rules
diff --git a/providers/.pre-commit-config.yaml
b/providers/.pre-commit-config.yaml
index 4c624358aae..c8ff25f217d 100644
--- a/providers/.pre-commit-config.yaml
+++ b/providers/.pre-commit-config.yaml
@@ -280,3 +280,12 @@ repos:
types: [python]
files: ^.*/src/airflow/providers/.*\.py$
require_serial: true
+ - repo: https://github.com/boidolr/ast-grep-pre-commit
+ rev: 0.45.2
+ hooks:
+ - id: ast-grep
+ name: Check providers import httpx2 rather than httpx
+ types: [python]
+ # Pinned to the providers config rather than relying on ast-grep
walking up
+ # from the working directory, so a future root sgconfig.yml cannot
shadow it.
+ args: ["--config", "./.ast-grep/sgconfig.yml"]
diff --git a/providers/airbyte/src/airflow/providers/airbyte/hooks/airbyte.py
b/providers/airbyte/src/airflow/providers/airbyte/hooks/airbyte.py
index dd7cc045626..6f4fbe3ae55 100644
--- a/providers/airbyte/src/airflow/providers/airbyte/hooks/airbyte.py
+++ b/providers/airbyte/src/airflow/providers/airbyte/hooks/airbyte.py
@@ -20,6 +20,9 @@ from __future__ import annotations
import time
from typing import Any, TypeVar
+# Stays on httpx (not httpx2): the client built below is handed to
airbyte-api, which builds
+# and sends httpx.Request objects through it, and the two packages' classes
are distinct.
+# Migrate once airbyte-api moves to httpx2; tracked at
https://github.com/apache/airflow/issues/70522
import httpx
from airbyte_api import AirbyteAPI
from airbyte_api.api import CancelJobRequest, GetJobRequest
diff --git a/providers/google/README.rst b/providers/google/README.rst
index 40545c6684f..288c25c0e52 100644
--- a/providers/google/README.rst
+++ b/providers/google/README.rst
@@ -122,7 +122,7 @@ PIP package Version required
``google-cloud-run`` ``>=0.10.0``
``google-cloud-batch`` ``>=0.13.0``
``grpcio-gcp`` ``>=0.2.2``
-``httpx`` ``>=0.25.0``
+``httpx2`` ``>=2.0.0``
``looker-sdk`` ``>=22.4.0,!=24.18.0``
``pandas-gbq`` ``>=0.7.0``
``pandas`` ``>=2.1.2; python_version <
"3.13"``
diff --git a/providers/google/docs/changelog.rst
b/providers/google/docs/changelog.rst
index c1e4de295eb..fc11e72def1 100644
--- a/providers/google/docs/changelog.rst
+++ b/providers/google/docs/changelog.rst
@@ -27,6 +27,13 @@
Changelog
---------
+.. note::
+ This provider now depends on ``httpx2`` instead of ``httpx``, and
``httpx2`` verifies TLS
+ against the operating system trust store rather than the ``certifi``
bundle. If
+ ``CloudSqlProxyRunner`` downloads the ``cloud-sql-proxy`` binary through a
proxy presenting
+ a private CA, install that CA into the OS trust store or point
``SSL_CERT_FILE`` (or
+ ``SSL_CERT_DIR``) at it -- adding it to ``certifi`` alone no longer has
any effect.
+
22.4.0
......
diff --git a/providers/google/docs/index.rst b/providers/google/docs/index.rst
index db9c84efeae..424f9ec2858 100644
--- a/providers/google/docs/index.rst
+++ b/providers/google/docs/index.rst
@@ -175,7 +175,7 @@ PIP package Version required
``google-cloud-run`` ``>=0.10.0``
``google-cloud-batch`` ``>=0.13.0``
``grpcio-gcp`` ``>=0.2.2``
-``httpx`` ``>=0.25.0``
+``httpx2`` ``>=2.0.0``
``looker-sdk`` ``>=22.4.0,!=24.18.0``
``pandas-gbq`` ``>=0.7.0``
``pandas`` ``>=2.1.2; python_version <
"3.13"``
diff --git a/providers/google/pyproject.toml b/providers/google/pyproject.toml
index 07f2b310091..4a4386de52f 100644
--- a/providers/google/pyproject.toml
+++ b/providers/google/pyproject.toml
@@ -132,7 +132,7 @@ dependencies = [
"google-cloud-run>=0.10.0",
"google-cloud-batch>=0.13.0",
"grpcio-gcp>=0.2.2",
- "httpx>=0.25.0",
+ "httpx2>=2.0.0",
# looker-sdk 24.18.0 has issues in import looker_sdk.rtl, No module named
looker_sdk.rtl
# See https://github.com/looker-open-source/sdk-codegen/issues/1518
"looker-sdk>=22.4.0,!=24.18.0",
diff --git
a/providers/google/src/airflow/providers/google/cloud/hooks/cloud_sql.py
b/providers/google/src/airflow/providers/google/cloud/hooks/cloud_sql.py
index 57889e96c2b..0f672570f33 100644
--- a/providers/google/src/airflow/providers/google/cloud/hooks/cloud_sql.py
+++ b/providers/google/src/airflow/providers/google/cloud/hooks/cloud_sql.py
@@ -34,14 +34,13 @@ import subprocess
import time
import uuid
from collections.abc import Sequence
-from inspect import signature
from pathlib import Path
from subprocess import PIPE, Popen
from tempfile import NamedTemporaryFile, _TemporaryFileWrapper, gettempdir
from typing import TYPE_CHECKING, Any, cast
from urllib.parse import quote_plus
-import httpx
+import httpx2
from aiohttp import ClientSession
from gcloud.aio.auth import AioSession, Token
from googleapiclient.discovery import Resource, build
@@ -608,12 +607,7 @@ class CloudSqlProxyRunner(LoggingMixin):
download_url = self._get_sql_proxy_download_url()
proxy_path_tmp = self.sql_proxy_path + ".tmp"
self.log.info("Downloading cloud_sql_proxy from %s to %s",
download_url, proxy_path_tmp)
- # httpx has a breaking API change (follow_redirects vs allow_redirects)
- # and this should work with both versions (cf. issue #20088)
- if "follow_redirects" in signature(httpx.get).parameters.keys():
- response = httpx.get(download_url, follow_redirects=True)
- else:
- response = httpx.get(download_url, allow_redirects=True) # type:
ignore[call-arg]
+ response = httpx2.get(download_url, follow_redirects=True)
# Downloading to .tmp file first to avoid case where partially
downloaded
# binary is used by parallel operator which uses the same fixed binary
path
with open(proxy_path_tmp, "wb") as file:
diff --git a/providers/google/tests/unit/google/cloud/hooks/test_cloud_sql.py
b/providers/google/tests/unit/google/cloud/hooks/test_cloud_sql.py
index 518f509e522..955add00379 100644
--- a/providers/google/tests/unit/google/cloud/hooks/test_cloud_sql.py
+++ b/providers/google/tests/unit/google/cloud/hooks/test_cloud_sql.py
@@ -1918,6 +1918,31 @@ class TestCloudSqlProxyRunner:
with pytest.raises(ValueError, match="The sql_proxy_version should
match the regular expression"):
runner._get_sql_proxy_download_url()
+ @mock.patch("airflow.providers.google.cloud.hooks.cloud_sql.httpx2.get")
+ def test_download_sql_proxy_follows_redirects(self, mock_get, tmp_path):
+ """The download URL redirects, so the request must opt in to following
them."""
+ mock_get.return_value = mock.Mock(status_code=200, content=b"binary")
+ runner = CloudSqlProxyRunner(
+ path_prefix=str(tmp_path / "12345678"),
+ instance_specification="project:us-east-1:instance",
+ )
+
+ runner._download_sql_proxy_if_needed()
+
+ mock_get.assert_called_once_with(runner._get_sql_proxy_download_url(),
follow_redirects=True)
+ assert runner.sql_proxy_was_downloaded is True
+
+ @mock.patch("airflow.providers.google.cloud.hooks.cloud_sql.httpx2.get")
+ def test_download_sql_proxy_raises_on_error_status(self, mock_get,
tmp_path):
+ mock_get.return_value = mock.Mock(status_code=404, content=b"",
reason_phrase="Not Found")
+ runner = CloudSqlProxyRunner(
+ path_prefix=str(tmp_path / "12345678"),
+ instance_specification="project:us-east-1:instance",
+ )
+
+ with pytest.raises(AirflowException, match="Status code = 404. Reason
= Not Found"):
+ runner._download_sql_proxy_if_needed()
+
def test_cloud_sql_proxy_runner_adds_enable_iam_login_flag(self):
runner = CloudSqlProxyRunner(
path_prefix="12345678",
diff --git
a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/msgraph.py
b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/msgraph.py
index a389c8725be..b9a74f0f97b 100644
---
a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/msgraph.py
+++
b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/msgraph.py
@@ -44,6 +44,9 @@ from types import TracebackType
from typing import TYPE_CHECKING, Any, cast
from urllib.parse import quote, urljoin, urlparse
+# Stays on httpx (not httpx2): the client below is handed to kiota_http, which
builds and
+# consumes httpx objects, and the two packages' classes are distinct.
+# Migrate once msgraph-core/kiota_http move to httpx2; tracked at
https://github.com/apache/airflow/issues/70522
import httpx
from azure.core.credentials_async import AsyncTokenCredential
from azure.identity.aio import CertificateCredential, ClientSecretCredential
diff --git a/providers/pagerduty/README.rst b/providers/pagerduty/README.rst
index 0165433426b..ac7d5fe4d8a 100644
--- a/providers/pagerduty/README.rst
+++ b/providers/pagerduty/README.rst
@@ -56,7 +56,7 @@ PIP package Version required
``apache-airflow`` ``>=2.11.0``
``apache-airflow-providers-common-compat`` ``>=1.10.1``
``apache-airflow-providers-http``
-``pagerduty`` ``>=2.3.0``
+``pagerduty`` ``>=7.0.0``
========================================== ==================
The changelog for the provider package can be found in the
diff --git a/providers/pagerduty/docs/changelog.rst
b/providers/pagerduty/docs/changelog.rst
index eb94a468257..64070e03eba 100644
--- a/providers/pagerduty/docs/changelog.rst
+++ b/providers/pagerduty/docs/changelog.rst
@@ -27,6 +27,11 @@
Changelog
---------
+.. note::
+ The minimum required version of the ``pagerduty`` SDK is now ``7.0.0``,
which replaced
+ ``httpx`` with ``httpx2``. Deployments pinning an older SDK must upgrade
it alongside this
+ provider. The provider's own hooks and notifier are unchanged.
+
5.2.6
.....
diff --git a/providers/pagerduty/docs/index.rst
b/providers/pagerduty/docs/index.rst
index 0f6ea4906db..e25dfb369e8 100644
--- a/providers/pagerduty/docs/index.rst
+++ b/providers/pagerduty/docs/index.rst
@@ -95,7 +95,7 @@ PIP package Version required
``apache-airflow`` ``>=2.11.0``
``apache-airflow-providers-common-compat`` ``>=1.10.1``
``apache-airflow-providers-http``
-``pagerduty`` ``>=2.3.0``
+``pagerduty`` ``>=7.0.0``
========================================== ==================
Downloading official packages
diff --git a/providers/pagerduty/pyproject.toml
b/providers/pagerduty/pyproject.toml
index 47d4fcb3337..c61f8e073df 100644
--- a/providers/pagerduty/pyproject.toml
+++ b/providers/pagerduty/pyproject.toml
@@ -62,7 +62,7 @@ dependencies = [
"apache-airflow>=2.11.0",
"apache-airflow-providers-common-compat>=1.10.1",
"apache-airflow-providers-http",
- "pagerduty>=2.3.0",
+ "pagerduty>=7.0.0",
]
[dependency-groups]
diff --git
a/providers/pagerduty/tests/unit/pagerduty/hooks/test_pagerduty_events.py
b/providers/pagerduty/tests/unit/pagerduty/hooks/test_pagerduty_events.py
index 6e86fb489d4..c139878d886 100644
--- a/providers/pagerduty/tests/unit/pagerduty/hooks/test_pagerduty_events.py
+++ b/providers/pagerduty/tests/unit/pagerduty/hooks/test_pagerduty_events.py
@@ -20,7 +20,7 @@ from __future__ import annotations
from unittest import mock
from unittest.mock import patch
-import httpx
+import httpx2
import pagerduty
import pytest
from pagerduty import EventsApiV2Client
@@ -91,10 +91,10 @@ class TestPagerdutyEventsHook:
"message": "Change event processed",
"status": "success",
}
- mock_response = httpx.Response(
+ mock_response = httpx2.Response(
status_code=202,
json=mock_response_body,
- request=httpx.Request("POST",
"https://events.pagerduty.com/v2/change/enqueue";),
+ request=httpx2.Request("POST",
"https://events.pagerduty.com/v2/change/enqueue";),
)
mock_response.ok = True
@@ -114,10 +114,10 @@ class TestPagerdutyEventsHook:
"message": "Event processed",
"dedup_key": dedup_key,
}
- mock_response = httpx.Response(
+ mock_response = httpx2.Response(
status_code=202,
json=mock_response_body,
- request=httpx.Request("POST",
"https://events.pagerduty.com/v2/enqueue";),
+ request=httpx2.Request("POST",
"https://events.pagerduty.com/v2/enqueue";),
)
mock_response.ok = True
mock_request.return_value = mock_response
diff --git
a/providers/weaviate/src/airflow/providers/weaviate/hooks/weaviate.py
b/providers/weaviate/src/airflow/providers/weaviate/hooks/weaviate.py
index 10dce9bed09..6757f616854 100644
--- a/providers/weaviate/src/airflow/providers/weaviate/hooks/weaviate.py
+++ b/providers/weaviate/src/airflow/providers/weaviate/hooks/weaviate.py
@@ -69,6 +69,9 @@ REQUESTS_EXCEPTIONS_TYPES = (
def check_http_error_is_retryable(exc: BaseException):
try:
+ # Stays on httpx (not httpx2): ``exc`` is raised inside
weaviate-client, which is built
+ # on httpx, so an httpx2.ConnectError would never match and retries
would silently stop.
+ # Migrate once weaviate-client moves to httpx2; tracked at
https://github.com/apache/airflow/issues/70522
import httpx
if isinstance(exc, httpx.ConnectError):
diff --git a/uv.lock b/uv.lock
index a39119a8d0f..a609f7ac284 100644
--- a/uv.lock
+++ b/uv.lock
@@ -5892,7 +5892,7 @@ dependencies = [
{ name = "google-cloud-workflows" },
{ name = "google-genai" },
{ name = "grpcio-gcp" },
- { name = "httpx" },
+ { name = "httpx2" },
{ name = "immutabledict" },
{ name = "looker-sdk" },
{ name = "pandas", version = "2.3.3", source = { registry =
"https://pypi.org/simple"; }, marker = "python_full_version < '3.11'" },
@@ -6092,7 +6092,7 @@ requires-dist = [
{ name = "google-cloud-workflows", specifier = ">=1.10.0" },
{ name = "google-genai", specifier = ">=2.8.0" },
{ name = "grpcio-gcp", specifier = ">=0.2.2" },
- { name = "httpx", specifier = ">=0.25.0" },
+ { name = "httpx2", specifier = ">=2.0.0" },
{ name = "immutabledict", specifier = ">=4.2.0" },
{ name = "looker-sdk", specifier = ">=22.4.0,!=24.18.0" },
{ name = "pandas", marker = "python_full_version < '3.13'", specifier =
">=2.1.2" },
@@ -7432,7 +7432,7 @@ requires-dist = [
{ name = "apache-airflow", editable = "." },
{ name = "apache-airflow-providers-common-compat", editable =
"providers/common/compat" },
{ name = "apache-airflow-providers-http", editable = "providers/http" },
- { name = "pagerduty", specifier = ">=2.3.0" },
+ { name = "pagerduty", specifier = ">=7.0.0" },
]
[package.metadata.requires-dev]
