This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-release.git
The following commit(s) were added to refs/heads/main by this push:
new 629b32c Migrate httpx to aiohttp, and remove httpx
629b32c is described below
commit 629b32c0417b3a64b98fb528edd5ce3d7e2b190a
Author: Sean B. Palmer <[email protected]>
AuthorDate: Thu Jun 26 19:59:46 2025 +0100
Migrate httpx to aiohttp, and remove httpx
---
atr/blueprints/admin/admin.py | 4 ++--
atr/datasources/apache.py | 50 +++++++++++++++++++++----------------------
atr/routes/keys.py | 16 +++++++-------
atr/routes/vote.py | 12 +++++------
atr/util.py | 39 ++++++++++++++++-----------------
poetry.lock | 49 +-----------------------------------------
pyproject.toml | 1 -
uv.lock | 38 +++++++-------------------------
8 files changed, 69 insertions(+), 140 deletions(-)
diff --git a/atr/blueprints/admin/admin.py b/atr/blueprints/admin/admin.py
index 3322f3a..c4ee69f 100644
--- a/atr/blueprints/admin/admin.py
+++ b/atr/blueprints/admin/admin.py
@@ -27,11 +27,11 @@ from collections.abc import Callable, Mapping
from typing import Any, Final
import aiofiles.os
+import aiohttp
import aioshutil
import asfquart
import asfquart.base as base
import asfquart.session as session
-import httpx
import quart
import sqlalchemy.orm as orm
import sqlmodel
@@ -577,7 +577,7 @@ async def admin_projects_update() -> str |
response.Response | tuple[Mapping[str
f"(PMCs and PPMCs) with membership data",
"category": "success",
}, 200
- except httpx.RequestError as e:
+ except aiohttp.ClientError as e:
return {
"message": f"Failed to fetch data: {e!s}",
"category": "error",
diff --git a/atr/datasources/apache.py b/atr/datasources/apache.py
index daa87e1..c3b2da1 100644
--- a/atr/datasources/apache.py
+++ b/atr/datasources/apache.py
@@ -22,7 +22,7 @@ from __future__ import annotations
import datetime
from typing import Annotated, Any, Final
-import httpx
+import aiohttp
import atr.schema as schema
@@ -215,10 +215,10 @@ class ProjectsData(schema.DictRoot[ProjectStatus]):
async def get_active_committee_data() -> CommitteeData:
"""Returns the list of currently active committees."""
- async with httpx.AsyncClient() as client:
- response = await client.get(_WHIMSY_COMMITTEE_INFO_URL)
- response.raise_for_status()
- data = response.json()
+ async with aiohttp.ClientSession() as session:
+ async with session.get(_WHIMSY_COMMITTEE_INFO_URL) as response:
+ response.raise_for_status()
+ data = await response.json()
return CommitteeData.model_validate(data)
@@ -226,28 +226,28 @@ async def get_active_committee_data() -> CommitteeData:
async def get_current_podlings_data() -> PodlingsData:
"""Returns the list of current podlings."""
- async with httpx.AsyncClient() as client:
- response = await client.get(_PROJECTS_PODLINGS_URL)
- response.raise_for_status()
- data = response.json()
+ async with aiohttp.ClientSession() as session:
+ async with session.get(_PROJECTS_PODLINGS_URL) as response:
+ response.raise_for_status()
+ data = await response.json()
return PodlingsData.model_validate(data)
async def get_groups_data() -> GroupsData:
"""Returns LDAP Groups with their members."""
- async with httpx.AsyncClient() as client:
- response = await client.get(_PROJECTS_GROUPS_URL)
- response.raise_for_status()
- data = response.json()
+ async with aiohttp.ClientSession() as session:
+ async with session.get(_PROJECTS_GROUPS_URL) as response:
+ response.raise_for_status()
+ data = await response.json()
return GroupsData.model_validate(data)
async def get_ldap_projects_data() -> LDAPProjectsData:
- async with httpx.AsyncClient() as client:
- response = await client.get(_WHIMSY_PROJECTS_URL)
- response.raise_for_status()
- data = response.json()
+ async with aiohttp.ClientSession() as session:
+ async with session.get(_WHIMSY_PROJECTS_URL) as response:
+ response.raise_for_status()
+ data = await response.json()
return LDAPProjectsData.model_validate(data)
@@ -255,19 +255,19 @@ async def get_ldap_projects_data() -> LDAPProjectsData:
async def get_projects_data() -> ProjectsData:
"""Returns the list of projects."""
- async with httpx.AsyncClient() as client:
- response = await client.get(_PROJECTS_PROJECTS_URL)
- response.raise_for_status()
- data = response.json()
+ async with aiohttp.ClientSession() as session:
+ async with session.get(_PROJECTS_PROJECTS_URL) as response:
+ response.raise_for_status()
+ data = await response.json()
return ProjectsData.model_validate(data)
async def get_retired_committee_data() -> RetiredCommitteeData:
"""Returns the list of retired committees."""
- async with httpx.AsyncClient() as client:
- response = await client.get(_WHIMSY_COMMITTEE_RETIRED_URL)
- response.raise_for_status()
- data = response.json()
+ async with aiohttp.ClientSession() as session:
+ async with session.get(_WHIMSY_COMMITTEE_RETIRED_URL) as response:
+ response.raise_for_status()
+ data = await response.json()
return RetiredCommitteeData.model_validate(data)
diff --git a/atr/routes/keys.py b/atr/routes/keys.py
index da2dbe9..c875f86 100644
--- a/atr/routes/keys.py
+++ b/atr/routes/keys.py
@@ -30,9 +30,9 @@ import textwrap
from collections.abc import Awaitable, Callable, Sequence
import aiofiles.os
+import aiohttp
import asfquart as asfquart
import asfquart.base as base
-import httpx
import quart
import werkzeug.datastructures as datastructures
import werkzeug.wrappers.response as response
@@ -636,13 +636,13 @@ async def _format_keys_file(
async def _get_keys_text(keys_url: str, render: Callable[[str],
Awaitable[str]]) -> str:
try:
- async with httpx.AsyncClient() as client:
- response = await client.get(keys_url, follow_redirects=True)
- response.raise_for_status()
- return response.text
- except httpx.HTTPStatusError as e:
- raise base.ASFQuartException(f"Error fetching URL:
{e.response.status_code} {e.response.reason_phrase}")
- except httpx.RequestError as e:
+ async with aiohttp.ClientSession() as session:
+ async with session.get(keys_url, allow_redirects=True) as response:
+ response.raise_for_status()
+ return await response.text()
+ except aiohttp.ClientResponseError as e:
+ raise base.ASFQuartException(f"Error fetching URL: {e.status}
{e.message}")
+ except aiohttp.ClientError as e:
raise base.ASFQuartException(f"Error fetching URL: {e}")
diff --git a/atr/routes/vote.py b/atr/routes/vote.py
index e90ffac..f23fc15 100644
--- a/atr/routes/vote.py
+++ b/atr/routes/vote.py
@@ -19,7 +19,7 @@ import json
import logging
import os
-import httpx
+import aiohttp
import quart
import werkzeug.wrappers.response as response
import wtforms
@@ -164,11 +164,11 @@ async def _task_archive_url(task_mid: str) -> str | None:
lid = "user-tests.tooling.apache.org"
url =
f"https://lists.apache.org/api/email.lua?id=%3C{task_mid}%3E&listid=%3C{lid}%3E";
try:
- async with httpx.AsyncClient() as client:
- response = await client.get(url)
- response.raise_for_status()
- # TODO: Check whether this blocks from network
- email_data = response.json()
+ async with aiohttp.ClientSession() as session:
+ async with session.get(url) as response:
+ response.raise_for_status()
+ # TODO: Check whether this blocks from network
+ email_data = await response.json()
mid = email_data["mid"]
if not isinstance(mid, str):
return None
diff --git a/atr/util.py b/atr/util.py
index 0aa3549..e554ea2 100644
--- a/atr/util.py
+++ b/atr/util.py
@@ -33,12 +33,12 @@ from collections.abc import AsyncGenerator, Callable,
Iterable, Sequence
from typing import Any, Final, TypeVar
import aiofiles.os
+import aiohttp
import aioshutil
import asfquart
import asfquart.base as base
import asfquart.session as session
import gitignore_parser
-import httpx
import jinja2
import quart
import quart_wtf
@@ -463,26 +463,25 @@ def get_unfinished_dir() -> pathlib.Path:
async def get_urls_as_completed(urls: Sequence[str]) ->
AsyncGenerator[tuple[str, int | str | None, bytes]]:
"""GET a list of URLs in parallel and yield (url, status, content_bytes)
as they become available."""
- async with httpx.AsyncClient() as client:
- tasks = [asyncio.create_task(client.get(url)) for url in urls]
- for future in asyncio.as_completed(tasks):
- try:
- response = await future
- except Exception as e:
- yield ("", str(e), b"")
- continue
- url = str(response.url)
+ async with aiohttp.ClientSession() as session:
+
+ async def _fetch(one_url: str) -> tuple[str, int | str | None, bytes]:
try:
- response.raise_for_status()
- yield (url, response.status_code, await response.aread())
- except httpx.HTTPStatusError as e:
- if e.response.status_code == 200:
- # This should not happen
- yield (url, str(e), b"")
- else:
- yield (url, e.response.status_code, b"")
- except Exception as e:
- yield (url, str(e), b"")
+ async with session.get(one_url) as resp:
+ try:
+ resp.raise_for_status()
+ return (str(resp.url), resp.status, await resp.read())
+ except aiohttp.ClientResponseError as e:
+ url = str(e.request_info.real_url)
+ if e.status == 200:
+ return (url, str(e), b"")
+ return (url, e.status, b"")
+ except Exception as exc:
+ return ("", str(exc), b"")
+
+ tasks = [asyncio.create_task(_fetch(u)) for u in urls]
+ for future in asyncio.as_completed(tasks):
+ yield await future
async def has_files(release: models.Release) -> bool:
diff --git a/poetry.lock b/poetry.lock
index e217940..3588a71 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1164,53 +1164,6 @@ files = [
{file = "hpack-4.1.0.tar.gz", hash =
"sha256:ec5eca154f7056aa06f196a557655c5b009b382873ac8d1e66e79e87535f1dca"},
]
-[[package]]
-name = "httpcore"
-version = "1.0.9"
-description = "A minimal low-level HTTP client."
-optional = false
-python-versions = ">=3.8"
-groups = ["main"]
-files = [
- {file = "httpcore-1.0.9-py3-none-any.whl", hash =
"sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"},
- {file = "httpcore-1.0.9.tar.gz", hash =
"sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"},
-]
-
-[package.dependencies]
-certifi = "*"
-h11 = ">=0.16"
-
-[package.extras]
-asyncio = ["anyio (>=4.0,<5.0)"]
-http2 = ["h2 (>=3,<5)"]
-socks = ["socksio (==1.*)"]
-trio = ["trio (>=0.22.0,<1.0)"]
-
-[[package]]
-name = "httpx"
-version = "0.28.1"
-description = "The next generation HTTP client."
-optional = false
-python-versions = ">=3.8"
-groups = ["main"]
-files = [
- {file = "httpx-0.28.1-py3-none-any.whl", hash =
"sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"},
- {file = "httpx-0.28.1.tar.gz", hash =
"sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"},
-]
-
-[package.dependencies]
-anyio = "*"
-certifi = "*"
-httpcore = "==1.*"
-idna = "*"
-
-[package.extras]
-brotli = ["brotli ; platform_python_implementation == \"CPython\"",
"brotlicffi ; platform_python_implementation != \"CPython\""]
-cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"]
-http2 = ["h2 (>=3,<5)"]
-socks = ["socksio (==1.*)"]
-zstd = ["zstandard (>=0.18.0)"]
-
[[package]]
name = "hypercorn"
version = "0.17.3"
@@ -3195,4 +3148,4 @@ propcache = ">=0.2.1"
[metadata]
lock-version = "2.1"
python-versions = "~=3.13"
-content-hash =
"54eafdba2809654d5c799509841e7fb6e3cb67635d88f7770de3217dc5c68c44"
+content-hash =
"3cf10d16cd21f3d39e21f7cb26cfbede7ecb0da0a0b893b9a49fb7b73e2aab3a"
diff --git a/pyproject.toml b/pyproject.toml
index 2153c78..2b1e1d8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -27,7 +27,6 @@ dependencies = [
"email-validator~=2.2.0",
"gitignore-parser (>=0.1.12,<0.2.0)",
"greenlet>=3.1.1,<4.0.0",
- "httpx~=0.27",
"hypercorn~=0.17",
"ldap3 (==2.10.2rc2)",
"python-decouple~=3.8",
diff --git a/uv.lock b/uv.lock
index 6b3b278..cc93d65 100644
--- a/uv.lock
+++ b/uv.lock
@@ -531,6 +531,12 @@ wheels = [
{ url =
"https://files.pythonhosted.org/packages/ee/45/b82e3c16be2182bff01179db177fe144d58b5dc787a7d4492c6ed8b9317f/frozenlist-1.7.0-py3-none-any.whl";,
hash =
"sha256:9a5af342e34f7e97caf8c995864c7a396418ae2859cc6fdf1b1073020d516a7e", size
= 13106 },
]
+[[package]]
+name = "gitignore-parser"
+version = "0.1.12"
+source = { registry = "https://pypi.org/simple"; }
+sdist = { url =
"https://files.pythonhosted.org/packages/86/a8/faf07759672973362e3f1f9742281a90aec7846e8a4043c4df5652990054/gitignore_parser-0.1.12.tar.gz";,
hash =
"sha256:78b22243adc0f02102c56c5e8c9a1d9121394142ca6143a90daa7f8d7a07a17e", size
= 5407 }
+
[[package]]
name = "greenlet"
version = "3.2.3"
@@ -586,34 +592,6 @@ wheels = [
{ url =
"https://files.pythonhosted.org/packages/07/c6/80c95b1b2b94682a72cbdbfb85b81ae2daffa4291fbfa1b1464502ede10d/hpack-4.1.0-py3-none-any.whl";,
hash =
"sha256:157ac792668d995c657d93111f46b4535ed114f0c9c8d672271bbec7eae1b496", size
= 34357 },
]
-[[package]]
-name = "httpcore"
-version = "1.0.9"
-source = { registry = "https://pypi.org/simple"; }
-dependencies = [
- { name = "certifi" },
- { name = "h11" },
-]
-sdist = { url =
"https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz";,
hash =
"sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size
= 85484 }
-wheels = [
- { url =
"https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl";,
hash =
"sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size
= 78784 },
-]
-
-[[package]]
-name = "httpx"
-version = "0.28.1"
-source = { registry = "https://pypi.org/simple"; }
-dependencies = [
- { name = "anyio" },
- { name = "certifi" },
- { name = "httpcore" },
- { name = "idna" },
-]
-sdist = { url =
"https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz";,
hash =
"sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size
= 141406 }
-wheels = [
- { url =
"https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl";,
hash =
"sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size
= 73517 },
-]
-
[[package]]
name = "hypercorn"
version = "0.17.3"
@@ -1401,8 +1379,8 @@ dependencies = [
{ name = "dnspython" },
{ name = "dunamai" },
{ name = "email-validator" },
+ { name = "gitignore-parser" },
{ name = "greenlet" },
- { name = "httpx" },
{ name = "hypercorn" },
{ name = "ldap3" },
{ name = "python-decouple" },
@@ -1447,8 +1425,8 @@ requires-dist = [
{ name = "dnspython", specifier = ">=2.7.0,<3.0.0" },
{ name = "dunamai", specifier = ">=1.23.0" },
{ name = "email-validator", specifier = "~=2.2.0" },
+ { name = "gitignore-parser", specifier = ">=0.1.12,<0.2.0" },
{ name = "greenlet", specifier = ">=3.1.1,<4.0.0" },
- { name = "httpx", specifier = "~=0.27" },
{ name = "hypercorn", specifier = "~=0.17" },
{ name = "ldap3", specifier = "==2.10.2rc2" },
{ name = "python-decouple", specifier = "~=3.8" },
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
