This is an automated email from the ASF dual-hosted git repository.
tn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-atr-experiments.git
commit ca1bf870c4d78fcb8b33f75d88c5eaa1029a3277
Author: Thomas Neidhart <t...@apache.org>
AuthorDate: Thu Feb 20 12:10:12 2025 +0100
add apache module for retrieving apache specific data from whimsy
---
.pre-commit-config.yaml | 2 ++
atr/apache.py | 55 +++++++++++++++++++++++++++++++++++++++++
atr/blueprints/secret/secret.py | 37 ++++++++++++---------------
atr/util.py | 52 ++++++++++++++++++++++++++++++++++++++
pyproject.toml | 12 ++++++++-
tests/__init__.py | 0
tests/test_apache.py | 34 +++++++++++++++++++++++++
7 files changed, 170 insertions(+), 22 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 35483d4..ef043df 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -63,6 +63,7 @@ repos:
entry: mypy
language: system
types: [python]
+ exclude: "tests" # see
https://github.com/pre-commit/pre-commit/issues/2967
args:
- --config-file=pyproject.toml
- id: pyright
@@ -70,3 +71,4 @@ repos:
entry: pyright
language: system
types: [python]
+ exclude: "tests"
diff --git a/atr/apache.py b/atr/apache.py
new file mode 100644
index 0000000..aad9e7c
--- /dev/null
+++ b/atr/apache.py
@@ -0,0 +1,55 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+from typing import Annotated
+
+import httpx
+from pydantic import BaseModel
+
+from atr.util import DictToList
+
+_WHIMSY_COMMITTEE_URL = "https://whimsy.apache.org/public/committee-info.json";
+_WHIMSY_PROJECTS_URL =
"https://whimsy.apache.org/public/public_ldap_projects.json";
+
+
+class ApacheProjects(BaseModel):
+ lastTimestamp: str
+ project_count: int
+ projects: Annotated[list[ApacheProject], DictToList(key="name")]
+
+
+class ApacheProject(BaseModel):
+ name: str
+ createTimestamp: str
+ modifyTimestamp: str
+ member_count: int
+ owner_count: int
+ members: list[str]
+ owners: list[str]
+ pmc: bool = False
+ podling: str | None = None
+
+
+async def get_apache_project_data() -> ApacheProjects:
+ async with httpx.AsyncClient() as client:
+ response = await client.get(_WHIMSY_PROJECTS_URL)
+ response.raise_for_status()
+ data = response.json()
+
+ return ApacheProjects.model_validate(data)
diff --git a/atr/blueprints/secret/secret.py b/atr/blueprints/secret/secret.py
index 5cb0e2c..c17f599 100644
--- a/atr/blueprints/secret/secret.py
+++ b/atr/blueprints/secret/secret.py
@@ -26,6 +26,7 @@ from werkzeug.wrappers.response import Response
from asfquart.base import ASFQuartException
from asfquart.session import read as session_read
+from atr.apache import get_apache_project_data
from atr.db import get_session
from atr.db.models import (
PMC,
@@ -99,44 +100,38 @@ async def secret_pmcs_update() -> str | Response:
if request.method == "POST":
# Fetch committee-info.json from Whimsy
- async with httpx.AsyncClient() as client:
- try:
- response = await client.get(_WHIMSY_COMMITTEE_URL)
- response.raise_for_status()
- data = response.json()
- except (httpx.RequestError, json.JSONDecodeError) as e:
- await flash(f"Failed to fetch committee data: {e!s}", "error")
- return redirect(url_for("secret_blueprint.secret_pmcs_update"))
-
- committees = data.get("committees", {})
+ try:
+ apache_projects = await get_apache_project_data()
+ except (httpx.RequestError, json.JSONDecodeError) as e:
+ await flash(f"Failed to fetch committee data: {e!s}", "error")
+ return redirect(url_for("secret_blueprint.secret_pmcs_update"))
+
updated_count = 0
try:
async with get_session() as db_session:
async with db_session.begin():
- for committee_id, info in committees.items():
+ for project in apache_projects.projects:
+ name = project.name
# Skip non-PMC committees
- if not info.get("pmc", False):
+ if not project.pmc:
continue
# Get or create PMC
- statement = select(PMC).where(PMC.project_name ==
committee_id)
+ statement = select(PMC).where(PMC.project_name == name)
pmc = (await
db_session.execute(statement)).scalar_one_or_none()
if not pmc:
- pmc = PMC(project_name=committee_id)
+ pmc = PMC(project_name=name)
db_session.add(pmc)
# Update PMC data
- roster = info.get("roster", {})
- # TODO: Here we say that roster == pmc_members ==
committers
- # We ought to do this more accurately instead
- pmc.pmc_members = list(roster.keys())
- pmc.committers = list(roster.keys())
+ pmc.pmc_members = project.owners
+ pmc.committers = project.members
# Mark chairs as release managers
# TODO: Who else is a release manager? How do we know?
- chairs = [m["id"] for m in info.get("chairs", [])]
- pmc.release_managers = chairs
+ # lets assume for now that all owners are also
release managers
+ pmc.release_managers = project.owners
updated_count += 1
diff --git a/atr/util.py b/atr/util.py
index fd40943..1e44012 100644
--- a/atr/util.py
+++ b/atr/util.py
@@ -16,9 +16,13 @@
# under the License.
import hashlib
+from dataclasses import dataclass
from functools import cache
from pathlib import Path
+from typing import Annotated, Any
+from pydantic import GetCoreSchemaHandler, TypeAdapter, create_model
+from pydantic_core import CoreSchema, core_schema
from quart import current_app
@@ -50,3 +54,51 @@ def compute_sha512(file_path: Path) -> str:
for chunk in iter(lambda: f.read(4096), b""):
sha512.update(chunk)
return sha512.hexdigest()
+
+
+def _get_dict_to_list_inner_type_adapter(source_type: Any, key: str) ->
TypeAdapter[dict[Any, Any]]:
+ root_adapter = TypeAdapter(source_type)
+ schema = root_adapter.core_schema
+
+ assert schema["type"] == "list"
+ assert (item_schema := schema["items_schema"])
+ assert item_schema["type"] == "model"
+ assert (cls := item_schema["cls"]) # noqa: RUF018
+
+ fields = cls.model_fields
+
+ assert (key_field := fields.get(key)) # noqa: RUF018
+ assert (other_fields := {k: v for k, v in fields.items() if k != key}) #
noqa: RUF018
+
+ model_name = f"{cls.__name__}Inner"
+ inner_model = create_model(model_name, **{k: (Any, v) for k, v in
other_fields.items()}) # type: ignore
+ return TypeAdapter(dict[Annotated[Any, key_field], inner_model]) # type:
ignore
+
+
+def _get_dict_to_list_validator(inner_adapter: TypeAdapter[dict[Any, Any]],
key: str) -> Any:
+ def validator(val: Any) -> Any:
+ if isinstance(val, dict):
+ validated = inner_adapter.validate_python(val)
+ return [{key: k, **{f: getattr(v, f) for f in v.model_fields}} for
k, v in validated.items()]
+
+ return val
+
+ return validator
+
+
+# from
https://github.com/pydantic/pydantic/discussions/8755#discussioncomment-8417979
+@dataclass
+class DictToList:
+ key: str
+
+ def __get_pydantic_core_schema__(
+ self,
+ source_type: Any,
+ handler: GetCoreSchemaHandler,
+ ) -> CoreSchema:
+ adapter = _get_dict_to_list_inner_type_adapter(source_type, self.key)
+
+ return core_schema.no_info_before_validator_function(
+ _get_dict_to_list_validator(adapter, self.key),
+ handler(source_type),
+ )
diff --git a/pyproject.toml b/pyproject.toml
index 6948f1b..b347a80 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -72,7 +72,14 @@ asfquart = { path = "./asfquart", editable = true }
[tool.pyright]
include = ["atr"]
-exclude = ["**/node_modules", "**/__pycache__", ".venv*", "asfquart", "tests"]
+exclude = [
+ "**/node_modules",
+ "**/__pycache__",
+ ".venv*",
+ "asfquart",
+ "tests",
+ "atr/util.py"
+]
ignore = []
defineConstant = { DEBUG = true }
stubPath = "typestubs"
@@ -100,6 +107,9 @@ lint.ignore = []
line-length = 120
exclude = ["asfquart"]
+[tool.ruff.lint.per-file-ignores]
+"atr/apache.py" = ["N815"]
+
[tool.mypy]
python_version = "3.13"
exclude = ["asfquart", "tests"]
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/test_apache.py b/tests/test_apache.py
new file mode 100644
index 0000000..8dd38ea
--- /dev/null
+++ b/tests/test_apache.py
@@ -0,0 +1,34 @@
+import json
+
+from atr.apache import ApacheProjects
+
+
+def test_model():
+ json_data = """
+{
+ "lastTimestamp": "20250219115218Z",
+ "project_count": 1,
+ "projects": {
+ "tooling": {
+ "createTimestamp": "20170713020428Z",
+ "modifyTimestamp": "20240725001829Z",
+ "member_count": 3,
+ "owner_count": 3,
+ "members": [
+ "wave",
+ "sbp",
+ "tn"
+ ],
+ "owners": [
+ "wave",
+ "sbp",
+ "tn"
+ ]
+ }
+ }
+}"""
+ projects = ApacheProjects.model_validate(json.loads(json_data))
+
+ assert projects is not None
+ assert projects.project_count == 1
+ assert projects.projects[0].name == "tooling"
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@tooling.apache.org
For additional commands, e-mail: dev-h...@tooling.apache.org
