This is an automated email from the ASF dual-hosted git repository.
tn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-atr-experiments.git
The following commit(s) were added to refs/heads/main by this push:
new e0bb1d2 add methods to retrieve committee data from whimsy
e0bb1d2 is described below
commit e0bb1d2104a9f556735a758273b6ec66f051e973
Author: Thomas Neidhart <t...@apache.org>
AuthorDate: Thu Feb 20 16:38:36 2025 +0100
add methods to retrieve committee data from whimsy
---
atr/apache.py | 80 ++++++++++++++++++++++++++++++++++++----
atr/blueprints/secret/secret.py | 6 +--
atr/util.py | 4 +-
tests/test_apache.py | 82 +++++++++++++++++++++++++++++++++++++++--
4 files changed, 156 insertions(+), 16 deletions(-)
diff --git a/atr/apache.py b/atr/apache.py
index aad9e7c..96f1c72 100644
--- a/atr/apache.py
+++ b/atr/apache.py
@@ -17,24 +17,30 @@
from __future__ import annotations
+from datetime import datetime
from typing import Annotated
import httpx
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
from atr.util import DictToList
-_WHIMSY_COMMITTEE_URL = "https://whimsy.apache.org/public/committee-info.json";
+_WHIMSY_COMMITTEE_INFO_URL =
"https://whimsy.apache.org/public/committee-info.json";
+_WHIMSY_COMMITTEE_RETIRED_URL =
"https://whimsy.apache.org/public/committee-retired.json";
_WHIMSY_PROJECTS_URL =
"https://whimsy.apache.org/public/public_ldap_projects.json";
-class ApacheProjects(BaseModel):
- lastTimestamp: str
+class LDAPProjects(BaseModel):
+ last_timestamp: str = Field(alias="lastTimestamp")
project_count: int
- projects: Annotated[list[ApacheProject], DictToList(key="name")]
+ projects: Annotated[list[Project], DictToList(key="name")]
+ @property
+ def last_time(self) -> datetime:
+ return datetime.strptime(self.last_timestamp, "%Y%m%d%H%M%S%z")
-class ApacheProject(BaseModel):
+
+class Project(BaseModel):
name: str
createTimestamp: str
modifyTimestamp: str
@@ -46,10 +52,68 @@ class ApacheProject(BaseModel):
podling: str | None = None
-async def get_apache_project_data() -> ApacheProjects:
+class CommitteeInfo(BaseModel):
+ last_updated: str
+ committee_count: int
+ pmc_count: int
+ committees: Annotated[list[Committee], DictToList(key="name")]
+
+
+class CommitteeRetired(BaseModel):
+ last_updated: str
+ retired_count: int
+ retired: Annotated[list[RetiredCommittee], DictToList(key="name")]
+
+
+class Committee(BaseModel):
+ name: str
+ display_name: str
+ site: str
+ description: str
+ mail_list: str
+ established: str
+ report: list[str]
+ # chair: Annotated[list[User], DictToList(key="id")]
+ roster_count: int
+ roster: Annotated[list[User], DictToList(key="id")]
+ pmc: bool
+
+
+class User(BaseModel):
+ id: str
+ name: str
+ date: str
+
+
+class RetiredCommittee(BaseModel):
+ name: str
+ display_name: str
+ retired: str
+ description: str
+
+
+async def get_ldap_projects_data() -> LDAPProjects:
async with httpx.AsyncClient() as client:
response = await client.get(_WHIMSY_PROJECTS_URL)
response.raise_for_status()
data = response.json()
- return ApacheProjects.model_validate(data)
+ return LDAPProjects.model_validate(data)
+
+
+async def get_committee_info_data() -> CommitteeInfo:
+ async with httpx.AsyncClient() as client:
+ response = await client.get(_WHIMSY_COMMITTEE_INFO_URL)
+ response.raise_for_status()
+ data = response.json()
+
+ return CommitteeInfo.model_validate(data)
+
+
+async def get_committee_retired_data() -> CommitteeRetired:
+ async with httpx.AsyncClient() as client:
+ response = await client.get(_WHIMSY_COMMITTEE_RETIRED_URL)
+ response.raise_for_status()
+ data = response.json()
+
+ return CommitteeRetired.model_validate(data)
diff --git a/atr/blueprints/secret/secret.py b/atr/blueprints/secret/secret.py
index ae7b791..1d8a2e7 100644
--- a/atr/blueprints/secret/secret.py
+++ b/atr/blueprints/secret/secret.py
@@ -26,7 +26,7 @@ from werkzeug.wrappers.response import Response
from asfquart.base import ASFQuartException
from asfquart.session import read as session_read
-from atr.apache import ApacheProjects, get_apache_project_data
+from atr.apache import LDAPProjects, get_ldap_projects_data
from atr.db import get_session
from atr.db.models import (
PMC,
@@ -185,11 +185,11 @@ async def secret_pmcs_update() -> str | Response:
return await render_template("secret/update-pmcs.html")
-async def secret_pmcs_update_data() -> tuple[ApacheProjects, dict, dict]:
+async def secret_pmcs_update_data() -> tuple[LDAPProjects, dict, dict]:
"""Fetch and update PMCs and podlings from remote data."""
# Fetch committee-info.json from whimsy.apache.org
try:
- apache_projects = await get_apache_project_data()
+ apache_projects = await get_ldap_projects_data()
except (httpx.RequestError, json.JSONDecodeError) as e:
raise FlashError(f"Failed to fetch committee data: {e!s}")
diff --git a/atr/util.py b/atr/util.py
index 1e44012..9346c62 100644
--- a/atr/util.py
+++ b/atr/util.py
@@ -71,8 +71,8 @@ def _get_dict_to_list_inner_type_adapter(source_type: Any,
key: str) -> TypeAdap
assert (other_fields := {k: v for k, v in fields.items() if k != key}) #
noqa: RUF018
model_name = f"{cls.__name__}Inner"
- inner_model = create_model(model_name, **{k: (Any, v) for k, v in
other_fields.items()}) # type: ignore
- return TypeAdapter(dict[Annotated[Any, key_field], inner_model]) # type:
ignore
+ inner_model = create_model(model_name, **{k: (v.annotation, v) for k, v in
other_fields.items()}) # type: ignore
+ return TypeAdapter(dict[Annotated[str, key_field], inner_model]) # type:
ignore
def _get_dict_to_list_validator(inner_adapter: TypeAdapter[dict[Any, Any]],
key: str) -> Any:
diff --git a/tests/test_apache.py b/tests/test_apache.py
index 57bbbcc..8741d5a 100644
--- a/tests/test_apache.py
+++ b/tests/test_apache.py
@@ -17,10 +17,10 @@
import json
-from atr.apache import ApacheProjects
+from atr.apache import CommitteeInfo, CommitteeRetired, LDAPProjects
-def test_model():
+def test_ldap_projects_model():
json_data = """
{
"lastTimestamp": "20250219115218Z",
@@ -44,8 +44,84 @@ def test_model():
}
}
}"""
- projects = ApacheProjects.model_validate(json.loads(json_data))
+ projects = LDAPProjects.model_validate(json.loads(json_data))
assert projects is not None
assert projects.project_count == 1
assert projects.projects[0].name == "tooling"
+
+
+def test_committee_info_model():
+ json_data = """
+{
+ "last_updated": "2025-02-19 21:57:21 UTC",
+ "committee_count": 1,
+ "pmc_count": 1,
+ "committees": {
+ "tooling": {
+ "display_name": "Tooling",
+ "site": "http://tooling.apache.org/";,
+ "description": "tools, tools, tools",
+ "mail_list": "tooling",
+ "established": "01/2025",
+ "report": [
+ "January",
+ "April",
+ "July",
+ "October"
+ ],
+ "chair": {
+ "wave": {
+ "name": "Dave Fisher"
+ }
+ },
+ "roster_count": 3,
+ "roster": {
+ "wave": {
+ "name": "Dave Fisher",
+ "date": "2025-01-01"
+ },
+ "sbp": {
+ "name": "Sean B. Palmer",
+ "date": "2025-02-01"
+ },
+ "tn": {
+ "name": "Thomas Neidhart",
+ "date": "2025-03-01"
+ }
+ },
+ "pmc": true
+ }
+ }
+}"""
+ committees = CommitteeInfo.model_validate(json.loads(json_data))
+
+ assert committees is not None
+ assert committees.pmc_count == 1
+
+ tooling = committees.committees[0]
+ assert tooling.name == "tooling"
+ assert len(tooling.roster) == 3
+ assert "tn" in map(lambda x: x.id, tooling.roster)
+
+
+def test_committee_retired_model():
+ json_data = """
+{
+ "last_updated": "2025-02-19 21:57:21 UTC",
+ "retired_count": 1,
+ "retired": {
+ "abdera": {
+ "display_name": "Abdera",
+ "description": "blablabla",
+ "retired": "2017-03"
+ }
+ }
+}"""
+ retired_committees = CommitteeRetired.model_validate(json.loads(json_data))
+
+ assert retired_committees is not None
+ assert retired_committees.retired_count == 1
+
+ pmc = retired_committees.retired[0]
+ assert pmc.name == "abdera"
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@tooling.apache.org
For additional commands, e-mail: dev-h...@tooling.apache.org
