This is an automated email from the ASF dual-hosted git repository. tn pushed a commit to branch add-blueprints in repository https://gitbox.apache.org/repos/asf/tooling-atr-experiments.git
commit c5f51d458e6b6acb3d8dc0b603c7cb1798184e01 Author: Thomas Neidhart <t...@apache.org> AuthorDate: Mon Feb 17 10:12:52 2025 +0100 add admin blueprint --- atr/blueprints/__init__.py | 15 ++++ atr/blueprints/admin/__init__.py | 24 +++++ atr/blueprints/admin/routes.py | 142 ++++++++++++++++++++++++++++++ atr/routes.py | 183 +++++---------------------------------- atr/server.py | 15 ++-- atr/templates/data-browser.html | 157 +++++++++++++++++---------------- atr/templates/page-404.html | 15 ++++ atr/templates/page-500.html | 15 ++++ atr/templates/pages.html | 11 ++- pyproject.toml | 9 +- 10 files changed, 334 insertions(+), 252 deletions(-) diff --git a/atr/blueprints/__init__.py b/atr/blueprints/__init__.py new file mode 100644 index 0000000..7b9f3f0 --- /dev/null +++ b/atr/blueprints/__init__.py @@ -0,0 +1,15 @@ +from importlib import import_module +from importlib.util import find_spec + +from asfquart.base import QuartApp + +_BLUEPRINT_MODULES = ["admin"] + + +def register_blueprints(app: QuartApp) -> None: + for module_name in _BLUEPRINT_MODULES: + routes_fqn = f"atr.blueprints.{module_name}.routes" + spec = find_spec(routes_fqn) + if spec is not None: + module = import_module(routes_fqn) + app.register_blueprint(module.blueprint) diff --git a/atr/blueprints/admin/__init__.py b/atr/blueprints/admin/__init__.py new file mode 100644 index 0000000..392442f --- /dev/null +++ b/atr/blueprints/admin/__init__.py @@ -0,0 +1,24 @@ +from quart import Blueprint + +from asfquart.auth import Requirements as R +from asfquart.auth import require +from asfquart.base import ASFQuartException +from asfquart.session import read as session_read + +blueprint = Blueprint("admin_blueprint", __name__, url_prefix="/admin") + +ALLOWED_USERS = {"cwells", "fluxo", "gmcdonald", "humbedooh", "sbp", "tn", "wave"} + + +@blueprint.before_request +async def before_request_func(): + @require(R.committer) + async def check_logged_in(): + session = await session_read() + if session is None: + raise ASFQuartException("Not authenticated", errorcode=401) + + if session.uid not in ALLOWED_USERS: + raise ASFQuartException("You are not authorized to access the admin interface", errorcode=403) + + await check_logged_in() diff --git a/atr/blueprints/admin/routes.py b/atr/blueprints/admin/routes.py new file mode 100644 index 0000000..bbd3b33 --- /dev/null +++ b/atr/blueprints/admin/routes.py @@ -0,0 +1,142 @@ +import json + +import httpx +from quart import current_app, render_template, request +from sqlmodel import select + +from asfquart.base import ASFQuartException +from atr.models import ( + PMC, + DistributionChannel, + Package, + PMCKeyLink, + ProductLine, + PublicSigningKey, + Release, + VotePolicy, +) + +from . import blueprint + + +@blueprint.route("/database") +@blueprint.route("/database/<model>") +async def root_admin_database(model: str = "PMC") -> str: + """Browse all records in the database.""" + + # Map of model names to their classes + models = { + "PMC": PMC, + "Release": Release, + "Package": Package, + "VotePolicy": VotePolicy, + "ProductLine": ProductLine, + "DistributionChannel": DistributionChannel, + "PublicSigningKey": PublicSigningKey, + "PMCKeyLink": PMCKeyLink, + } + + if model not in models: + raise ASFQuartException("model type not found", errorcode=404) + + async_session = current_app.config["async_session"] + async with async_session() as db_session: + # Get all records for the selected model + statement = select(models[model]) + records = (await db_session.execute(statement)).scalars().all() + + # Convert records to dictionaries for JSON serialization + records_dict = [] + for record in records: + if hasattr(record, "dict"): + record_dict = record.dict() + else: + # Fallback for models without dict() method + record_dict = { + "id": getattr(record, "id", None), + "storage_key": getattr(record, "storage_key", None), + } + for key in record.__dict__: + if not key.startswith("_"): + record_dict[key] = getattr(record, key) + records_dict.append(record_dict) + + return await render_template("data-browser.html", models=list(models.keys()), model=model, records=records_dict) + + +@blueprint.route("/update-pmcs", methods=["GET", "POST"]) +async def root_admin_update_pmcs() -> str: + """Update PMCs from remote, authoritative committee-info.json.""" + + if request.method == "POST": + # Fetch committee-info.json from Whimsy + WHIMSY_URL = "https://whimsy.apache.org/public/committee-info.json" + async with httpx.AsyncClient() as client: + try: + response = await client.get(WHIMSY_URL) + response.raise_for_status() + data = response.json() + except (httpx.RequestError, json.JSONDecodeError) as e: + raise ASFQuartException(f"Failed to fetch committee data: {str(e)}", errorcode=500) + + committees = data.get("committees", {}) + updated_count = 0 + + async_session = current_app.config["async_session"] + async with async_session() as db_session: + async with db_session.begin(): + for committee_id, info in committees.items(): + # Skip non-PMC committees + if not info.get("pmc", False): + continue + + # Get or create PMC + statement = select(PMC).where(PMC.project_name == committee_id) + pmc = (await db_session.execute(statement)).scalar_one_or_none() + if not pmc: + pmc = PMC(project_name=committee_id) + db_session.add(pmc) + + # Update PMC data + roster = info.get("roster", {}) + # TODO: Here we say that roster == pmc_members == committers + # We ought to do this more accurately instead + pmc.pmc_members = list(roster.keys()) + pmc.committers = list(roster.keys()) + + # Mark chairs as release managers + # TODO: Who else is a release manager? How do we know? + chairs = [m["id"] for m in info.get("chairs", [])] + pmc.release_managers = chairs + + updated_count += 1 + + # Add special entry for Tooling PMC + # Not clear why, but it's not in the Whimsy data + statement = select(PMC).where(PMC.project_name == "tooling") + tooling_pmc = (await db_session.execute(statement)).scalar_one_or_none() + if not tooling_pmc: + tooling_pmc = PMC(project_name="tooling") + db_session.add(tooling_pmc) + updated_count += 1 + + # Update Tooling PMC data + # Could put this in the "if not tooling_pmc" block, perhaps + tooling_pmc.pmc_members = ["wave", "tn", "sbp"] + tooling_pmc.committers = ["wave", "tn", "sbp"] + tooling_pmc.release_managers = ["wave"] + + return f"Successfully updated {updated_count} PMCs from Whimsy" + + # For GET requests, show the update form + return await render_template("update-pmcs.html") + + +@blueprint.get("/database/debug") +async def root_database_debug() -> str: + """Debug information about the database.""" + async_session = current_app.config["async_session"] + async with async_session() as db_session: + statement = select(PMC) + pmcs = (await db_session.execute(statement)).scalars().all() + return f"Database using {current_app.config['DATA_MODELS_FILE']} has {len(pmcs)} PMCs" diff --git a/atr/routes.py b/atr/routes.py index c5690fc..5b872fd 100644 --- a/atr/routes.py +++ b/atr/routes.py @@ -20,44 +20,40 @@ import asyncio import datetime import hashlib -import json import pprint import secrets import shutil import tempfile - from contextlib import asynccontextmanager from io import BufferedReader from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple, cast +from typing import Any, cast import aiofiles import aiofiles.os import gnupg -import httpx - -from asfquart import APP -from asfquart.auth import Requirements as R, require -from asfquart.base import ASFQuartException -from asfquart.session import read as session_read, ClientSession -from quart import current_app, render_template, request, Request -from sqlmodel import select +from quart import Request, current_app, render_template, request from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import selectinload from sqlalchemy.orm.attributes import InstrumentedAttribute +from sqlmodel import select from werkzeug.datastructures import FileStorage +from asfquart import APP +from asfquart.auth import Requirements as R +from asfquart.auth import require +from asfquart.base import ASFQuartException +from asfquart.session import ClientSession +from asfquart.session import read as session_read + from .models import ( - DistributionChannel, PMC, - PMCKeyLink, Package, - ProductLine, + PMCKeyLink, PublicSigningKey, Release, ReleasePhase, ReleaseStage, - VotePolicy, ) if APP is ...: @@ -198,145 +194,6 @@ async def root_add_release_candidate() -> str: ) -@APP.route("/admin/database") -@APP.route("/admin/database/<model>") -@require(R.committer) -async def root_admin_database(model: str = "PMC") -> str: - "Browse all records in the database." - session = await session_read() - if session is None: - raise ASFQuartException("Not authenticated", errorcode=401) - - if session.uid not in ALLOWED_USERS: - raise ASFQuartException("You are not authorized to browse data", errorcode=403) - - # Map of model names to their classes - models = { - "PMC": PMC, - "Release": Release, - "Package": Package, - "VotePolicy": VotePolicy, - "ProductLine": ProductLine, - "DistributionChannel": DistributionChannel, - "PublicSigningKey": PublicSigningKey, - "PMCKeyLink": PMCKeyLink, - } - - if model not in models: - # Default to PMC if invalid model specified - model = "PMC" - - async_session = current_app.config["async_session"] - async with async_session() as db_session: - # Get all records for the selected model - statement = select(models[model]) - records = (await db_session.execute(statement)).scalars().all() - - # Convert records to dictionaries for JSON serialization - records_dict = [] - for record in records: - if hasattr(record, "dict"): - record_dict = record.dict() - else: - # Fallback for models without dict() method - record_dict = { - "id": getattr(record, "id", None), - "storage_key": getattr(record, "storage_key", None), - } - for key in record.__dict__: - if not key.startswith("_"): - record_dict[key] = getattr(record, key) - records_dict.append(record_dict) - - return await render_template("data-browser.html", models=list(models.keys()), model=model, records=records_dict) - - -@APP.route("/admin/update-pmcs", methods=["GET", "POST"]) -@require(R.committer) -async def root_admin_update_pmcs() -> str: - "Update PMCs from remote, authoritative committee-info.json." - # Check authentication - session = await session_read() - if session is None: - raise ASFQuartException("Not authenticated", errorcode=401) - - if session.uid not in ALLOWED_USERS: - raise ASFQuartException("You are not authorized to update PMCs", errorcode=403) - - if request.method == "POST": - # Fetch committee-info.json from Whimsy - WHIMSY_URL = "https://whimsy.apache.org/public/committee-info.json" - async with httpx.AsyncClient() as client: - try: - response = await client.get(WHIMSY_URL) - response.raise_for_status() - data = response.json() - except (httpx.RequestError, json.JSONDecodeError) as e: - raise ASFQuartException(f"Failed to fetch committee data: {str(e)}", errorcode=500) - - committees = data.get("committees", {}) - updated_count = 0 - - async_session = current_app.config["async_session"] - async with async_session() as db_session: - async with db_session.begin(): - for committee_id, info in committees.items(): - # Skip non-PMC committees - if not info.get("pmc", False): - continue - - # Get or create PMC - statement = select(PMC).where(PMC.project_name == committee_id) - pmc = (await db_session.execute(statement)).scalar_one_or_none() - if not pmc: - pmc = PMC(project_name=committee_id) - db_session.add(pmc) - - # Update PMC data - roster = info.get("roster", {}) - # TODO: Here we say that roster == pmc_members == committers - # We ought to do this more accurately instead - pmc.pmc_members = list(roster.keys()) - pmc.committers = list(roster.keys()) - - # Mark chairs as release managers - # TODO: Who else is a release manager? How do we know? - chairs = [m["id"] for m in info.get("chairs", [])] - pmc.release_managers = chairs - - updated_count += 1 - - # Add special entry for Tooling PMC - # Not clear why, but it's not in the Whimsy data - statement = select(PMC).where(PMC.project_name == "tooling") - tooling_pmc = (await db_session.execute(statement)).scalar_one_or_none() - if not tooling_pmc: - tooling_pmc = PMC(project_name="tooling") - db_session.add(tooling_pmc) - updated_count += 1 - - # Update Tooling PMC data - # Could put this in the "if not tooling_pmc" block, perhaps - tooling_pmc.pmc_members = ["wave", "tn", "sbp"] - tooling_pmc.committers = ["wave", "tn", "sbp"] - tooling_pmc.release_managers = ["wave"] - - return f"Successfully updated {updated_count} PMCs from Whimsy" - - # For GET requests, show the update form - return await render_template("update-pmcs.html") - - -@APP.get("/database/debug") -async def root_database_debug() -> str: - """Debug information about the database.""" - async_session = current_app.config["async_session"] - async with async_session() as db_session: - statement = select(PMC) - pmcs = (await db_session.execute(statement)).scalars().all() - return f"Database using {current_app.config['DATA_MODELS_FILE']} has {len(pmcs)} PMCs" - - @APP.route("/release/signatures/verify/<release_key>") @require(R.committer) async def root_release_signatures_verify(release_key: str) -> str: @@ -348,10 +205,10 @@ async def root_release_signatures_verify(release_key: str) -> str: async_session = current_app.config["async_session"] async with async_session() as db_session: # Get the release and its packages, and PMC with its keys - release_packages = selectinload(cast(InstrumentedAttribute[List[Package]], Release.packages)) + release_packages = selectinload(cast(InstrumentedAttribute[list[Package]], Release.packages)) release_pmc = selectinload(cast(InstrumentedAttribute[PMC], Release.pmc)) pmc_keys_loader = selectinload(cast(InstrumentedAttribute[PMC], Release.pmc)).selectinload( - cast(InstrumentedAttribute[List[PublicSigningKey]], PMC.public_signing_keys) + cast(InstrumentedAttribute[list[PublicSigningKey]], PMC.public_signing_keys) ) # For now, for debugging, we'll just get all keys in the database @@ -465,7 +322,7 @@ async def root_pmc_directory() -> str: @APP.route("/pmc/list") -async def root_pmc_list() -> List[dict]: +async def root_pmc_list() -> list[dict]: "List all PMCs in the database." async_session = current_app.config["async_session"] async with async_session() as db_session: @@ -565,7 +422,7 @@ async def root_user_uploads() -> str: # TODO: We don't actually record who uploaded the release candidate # We should probably add that information! release_pmc = selectinload(cast(InstrumentedAttribute[PMC], Release.pmc)) - release_packages = selectinload(cast(InstrumentedAttribute[List[Package]], Release.packages)) + release_packages = selectinload(cast(InstrumentedAttribute[list[Package]], Release.packages)) statement = ( select(Release) .options(release_pmc, release_packages) @@ -623,7 +480,7 @@ async def save_file_by_hash(base_dir: Path, file: FileStorage) -> str: raise e -async def user_keys_add(session: ClientSession, public_key: str) -> Tuple[str, Optional[dict]]: +async def user_keys_add(session: ClientSession, public_key: str) -> tuple[str, dict | None]: if not public_key: return ("Public key is required", None) @@ -661,7 +518,7 @@ async def user_keys_add(session: ClientSession, public_key: str) -> Tuple[str, O async def user_keys_add_session( session: ClientSession, public_key: str, key: dict, db_session: AsyncSession -) -> Tuple[str, Optional[dict]]: +) -> tuple[str, dict | None]: # Check if key already exists statement = select(PublicSigningKey).where(PublicSigningKey.user_id == session.uid) @@ -709,7 +566,7 @@ async def user_keys_add_session( ) -async def verify_gpg_signature(artifact_path: Path, signature_path: Path, public_keys: List[str]) -> Dict[str, Any]: +async def verify_gpg_signature(artifact_path: Path, signature_path: Path, public_keys: list[str]) -> dict[str, Any]: """ Verify a GPG signature for a release artifact. Returns a dictionary with verification results and debug information. @@ -727,8 +584,8 @@ async def verify_gpg_signature(artifact_path: Path, signature_path: Path, public async def verify_gpg_signature_file( - sig_file: BufferedReader, artifact_path: Path, public_keys: List[str] -) -> Dict[str, Any]: + sig_file: BufferedReader, artifact_path: Path, public_keys: list[str] +) -> dict[str, Any]: # Run the blocking GPG verification in a thread async with ephemeral_gpg_home() as gpg_home: gpg = gnupg.GPG(gnupghome=gpg_home) diff --git a/atr/server.py b/atr/server.py index 13988a9..7fabc17 100644 --- a/atr/server.py +++ b/atr/server.py @@ -19,14 +19,16 @@ import os -import asfquart -import asfquart.generics -from asfquart.base import QuartApp -from sqlmodel import SQLModel -from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker from alembic import command from alembic.config import Config +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine from sqlalchemy.sql import text +from sqlmodel import SQLModel + +import asfquart +import asfquart.generics +from asfquart.base import QuartApp +from atr.blueprints import register_blueprints from .models import __file__ as data_models_file @@ -110,12 +112,15 @@ def create_app() -> QuartApp: await conn.run_sync(SQLModel.metadata.create_all) app.config["engine"] = engine + # TODO: apply this only for debug + app.config["TEMPLATES_AUTO_RELOAD"] = True @app.after_serving async def shutdown() -> None: app.background_tasks.clear() register_routes() + register_blueprints(app) return app diff --git a/atr/templates/data-browser.html b/atr/templates/data-browser.html index 3281b48..f0e78b5 100644 --- a/atr/templates/data-browser.html +++ b/atr/templates/data-browser.html @@ -1,89 +1,88 @@ -<!DOCTYPE html> -<html lang="en"> - <head> - <meta charset="utf-8" /> - <meta name="viewport" content="width=device-width,initial-scale=1.0" /> - <meta name="description" content="Browse all records in the database." /> - <title>ATR | Data Browser</title> - <link rel="stylesheet" href="{{ url_for('static', filename='root.css') }}" /> - <style> - .model-nav { - margin: 1rem 0; - padding: 0.5rem; - background: #f5f5f5; - border-radius: 4px; - } +{% extends "layouts/base.html" %} - .model-nav a { - margin-right: 1rem; - padding: 0.25rem 0.5rem; - text-decoration: none; - color: #333; - } +{% block title %} ATR | Data Browser {% endblock %} +{% block description %}Browse all records in the database.{% endblock %} - .model-nav a.active { - background: #333; - color: white; - border-radius: 2px; - } +{% block stylesheets %} + {{ super() }} + <style> + .model-nav { + margin: 1rem 0; + padding: 0.5rem; + background: #f5f5f5; + border-radius: 4px; + } - .record { - border: 1px solid #ddd; - padding: 1rem; - margin-bottom: 1rem; - border-radius: 4px; - } + .model-nav a { + margin-right: 1rem; + padding: 0.25rem 0.5rem; + text-decoration: none; + color: #333; + } - .record pre { - background: #f5f5f5; - padding: 0.5rem; - border-radius: 2px; - overflow-x: auto; - } + .model-nav a.active { + background: #333; + color: white; + border-radius: 2px; + } - .record-header { - display: flex; - justify-content: space-between; - align-items: center; - margin-bottom: 0.5rem; - } + .record { + border: 1px solid #ddd; + padding: 1rem; + margin-bottom: 1rem; + border-radius: 4px; + } - .record-meta { - color: #666; - font-size: 0.9em; - } + .record pre { + background: #f5f5f5; + padding: 0.5rem; + border-radius: 2px; + overflow-x: auto; + } - .no-records { - color: #666; - font-style: italic; - } - </style> - </head> - <body> - <h1>Data Browser</h1> - <p class="intro">Browse all records in the database.</p> + .record-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 0.5rem; + } - <div class="model-nav"> - {% for model_name in models %} - <a href="{{ url_for('root_admin_database', model=model_name) }}" - {% if model == model_name %}class="active"{% endif %}>{{ model_name }}</a> - {% endfor %} - </div> + .record-meta { + color: #666; + font-size: 0.9em; + } - {% if records %} - <div class="records"> - {% for record in records %} - <div class="record"> - <div class="record-header"> - <h3>{{ record.get('id', record.get('storage_key', 'Unknown ID') ) }}</h3> - <span class="record-meta">{{ model }}</span> - </div> - <pre>{{ record | tojson(indent=2) }}</pre> - </div> - {% endfor %} + .no-records { + color: #666; + font-style: italic; + } + </style> +{% endblock stylesheets %} + +{% block content %} +<h1>Data Browser</h1> +<p class="intro">Browse all records in the database.</p> + +<div class="model-nav"> + {% for model_name in models %} + <a href="{{ url_for('admin_blueprint.root_admin_database', model=model_name) }}" + {% if model == model_name %}class="active"{% endif %}>{{ model_name }}</a> + {% endfor %} +</div> + +{% if records %} + <div class="records"> + {% for record in records %} + <div class="record"> + <div class="record-header"> + <h3>{{ record.get('id', record.get('storage_key', 'Unknown ID') ) }}</h3> + <span class="record-meta">{{ model }}</span> + </div> + <pre>{{ record | tojson(indent=2) }}</pre> </div> - {% else %} - <p class="no-records">No records found for {{ model }}.</p> - {% endif %} - </body> -</html> + {% endfor %} + </div> +{% else %} + <p class="no-records">No records found for {{ model }}.</p> +{% endif %} +{% endblock content %} diff --git a/atr/templates/page-404.html b/atr/templates/page-404.html new file mode 100644 index 0000000..00c749f --- /dev/null +++ b/atr/templates/page-404.html @@ -0,0 +1,15 @@ +{% extends "layouts/base.html" %} + +{% block title %} Error 404 {% endblock %} + +{% block content %} +<div class="login-box"> + <div class="card"> + <div class="card-body login-card-body"> + <h5 class="login-box-msg"> + Error 404 - Page not found + </h5> + </div> + </div> +</div> +{% endblock content %} diff --git a/atr/templates/page-500.html b/atr/templates/page-500.html new file mode 100644 index 0000000..39ac5b8 --- /dev/null +++ b/atr/templates/page-500.html @@ -0,0 +1,15 @@ +{% extends "layouts/base.html" %} + +{% block title %} Error 500 {% endblock %} + +{% block content %} +<div class="login-box"> + <div class="card"> + <div class="card-body login-card-body"> + <h5 class="login-box-msg"> + Error 500 - Server Error + </h5> + </div> + </div> +</div> +{% endblock content %} diff --git a/atr/templates/pages.html b/atr/templates/pages.html index f7ced7f..f25aed5 100644 --- a/atr/templates/pages.html +++ b/atr/templates/pages.html @@ -201,7 +201,7 @@ <div class="endpoint"> <h3> - <a href="{{ url_for('root_admin_database') }}">/admin/database</a> + <a href="{{ url_for('admin_blueprint.root_admin_database') }}">/admin/database</a> </h3> <div class="endpoint-description">Browse all records in the database.</div> <div class="endpoint-meta"> @@ -214,7 +214,7 @@ <div class="endpoint"> <h3> - <a href="{{ url_for('root_admin_update_pmcs') }}">/admin/update-pmcs</a> + <a href="{{ url_for('admin_blueprint.root_admin_update_pmcs') }}">/admin/update-pmcs</a> </h3> <div class="endpoint-description">Update PMCs from remote, authoritative committee-info.json.</div> <div class="endpoint-meta"> @@ -241,11 +241,14 @@ <div class="endpoint"> <h3> - <a href="{{ url_for('root_database_debug') }}">/database/debug</a> + <a href="{{ url_for('admin_blueprint.root_database_debug') }}">/admin/database/debug</a> </h3> <div class="endpoint-description">Debug information about the database.</div> <div class="endpoint-meta"> - Access: <span class="access-requirement public">Public</span> + Access: <span class="access-requirement public">Committer</span> + <span class="access-requirement admin">Admin</span> + <br /> + Additional requirement: Must be in ALLOWED_USERS list </div> </div> </div> diff --git a/pyproject.toml b/pyproject.toml index 5b17146..720d0cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,7 +74,14 @@ executionEnvironments = [ ] [tool.ruff] -lint.select = ["E", "W", "F", "C90"] +lint.select = [ + "I", # isort + "E", + "W", + "F", + "C90", + "UP" # pyupgrade +] lint.ignore = [] line-length = 120 exclude = ["asfquart"] --------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@tooling.apache.org For additional commands, e-mail: dev-h...@tooling.apache.org