This is an automated email from the ASF dual-hosted git repository. sbp pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/tooling-atr-experiments.git
The following commit(s) were added to refs/heads/main by this push: new 1a11573 Add a PMC updater and release candidate processing code 1a11573 is described below commit 1a11573c714aa93c274fc1af4342af5104131ae1 Author: Sean B. Palmer <s...@miscoranda.com> AuthorDate: Thu Feb 13 15:46:58 2025 +0200 Add a PMC updater and release candidate processing code --- atr/models.py | 10 +- atr/routes.py | 191 +++++++++++++++++++++++++++++-- atr/server.py | 5 + atr/static/root.css | 4 +- atr/templates/add-release-candidate.html | 17 ++- atr/templates/update-pmcs.html | 30 +++++ poetry.lock | 49 +++++++- pyproject.toml | 2 + uv.lock | 30 +++++ 9 files changed, 318 insertions(+), 20 deletions(-) diff --git a/atr/models.py b/atr/models.py index 8552402..61c9dd4 100644 --- a/atr/models.py +++ b/atr/models.py @@ -120,11 +120,16 @@ class DistributionChannel(SQLModel, table=True): product_line: Optional[ProductLine] = Relationship(back_populates="distribution_channels") -class Package(BaseModel): +class Package(SQLModel, table=True): + id: Optional[int] = Field(default=None, primary_key=True) file: str signature: str checksum: str + # Many-to-one: A package belongs to one release + release_key: Optional[str] = Field(default=None, foreign_key="release.storage_key") + release: Optional["Release"] = Relationship(back_populates="packages") + class VoteEntry(BaseModel): result: bool @@ -172,7 +177,8 @@ class Release(SQLModel, table=True): package_managers: List[str] = Field(default_factory=list, sa_column=Column(JSON)) version: str - packages: List[Package] = Field(default_factory=list, sa_column=Column(JSON)) + # One-to-many: A release can have multiple packages + packages: List[Package] = Relationship(back_populates="release") sboms: List[str] = Field(default_factory=list, sa_column=Column(JSON)) # Many-to-one: A release can have one vote policy, a vote policy can be used by multiple releases diff --git a/atr/routes.py b/atr/routes.py index 680376f..eb01b55 100644 --- a/atr/routes.py +++ b/atr/routes.py @@ -17,7 +17,10 @@ "routes.py" -from typing import List +import hashlib +import json +from pathlib import Path +from typing import List, Tuple from asfquart import APP from asfquart.auth import Requirements as R, require @@ -26,13 +29,49 @@ from asfquart.session import read as session_read from quart import current_app, render_template, request from sqlmodel import Session, select from sqlalchemy.exc import IntegrityError +import httpx -from .models import PMC +from .models import PMC, Release, ReleaseStage, ReleasePhase, Package if APP is ...: raise ValueError("APP is not set") +def compute_sha3_256(file_data: bytes) -> str: + "Compute SHA3-256 hash of file data." + return hashlib.sha3_256(file_data).hexdigest() + + +def compute_sha512(file_path: Path) -> str: + "Compute SHA-512 hash of a file." + sha512 = hashlib.sha512() + with open(file_path, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + sha512.update(chunk) + return sha512.hexdigest() + + +async def save_file_by_hash(file, base_dir: Path) -> Tuple[Path, str]: + """ + Save a file using its SHA3-256 hash as the filename. + Returns the path where the file was saved and its hash. + """ + # FileStorage.read() returns bytes directly, no need to await + data = file.read() + file_hash = compute_sha3_256(data) + + # Create path with hash as filename + path = base_dir / file_hash + path.parent.mkdir(parents=True, exist_ok=True) + + # Only write if file doesn't exist + # If it does exist, it'll be the same content anyway + if not path.exists(): + path.write_bytes(data) + + return path, file_hash + + @APP.route("/add-release-candidate", methods=["GET", "POST"]) @require(R.committer) async def add_release_candidate() -> str: @@ -43,10 +82,7 @@ async def add_release_candidate() -> str: # For POST requests, handle the file upload if request.method == "POST": - # We'll implement the actual file handling later - # For now just return a message about what we would do form = await request.form - files = await request.files project_name = form.get("project_name") if not project_name: @@ -58,12 +94,67 @@ async def add_release_candidate() -> str: f"You must be a PMC member of {project_name} to submit a release candidate", errorcode=403 ) - release_file = files.get("release_file") - if not release_file: - raise ASFQuartException("Release file is required", errorcode=400) + # Get all uploaded files + files = await request.files + + # Get the release artifact and signature files + artifact_file = files.get("release_artifact") + signature_file = files.get("release_signature") + + if not artifact_file: + raise ASFQuartException("Release artifact file is required", errorcode=400) + if not signature_file: + raise ASFQuartException("Detached GPG signature file is required", errorcode=400) + if not signature_file.filename.endswith(".asc"): + # TODO: Could also check that it's artifact name + ".asc" + # And at least warn if it's not + raise ASFQuartException("Signature file must have .asc extension", errorcode=400) + + # Save files using their hashes as filenames + storage_dir = Path(current_app.config["RELEASE_STORAGE_DIR"]) / project_name + artifact_path, artifact_hash = await save_file_by_hash(artifact_file, storage_dir) + # TODO: Do we need to do anything with the signature hash? + # These should be identical, but path might be absolute? + # TODO: Need to check, ideally. Could have a data browser + signature_path, _ = await save_file_by_hash(signature_file, storage_dir) + + # Compute SHA-512 checksum of the artifact for the package record + # We're using SHA-3-256 for the filename, so we need to use SHA-3-512 for the checksum + checksum_512 = compute_sha512(artifact_path) + + # Store in database + with Session(current_app.config["engine"]) as db_session: + # Get PMC + statement = select(PMC).where(PMC.project_name == project_name) + pmc = db_session.exec(statement).first() + if not pmc: + raise ASFQuartException("PMC not found", errorcode=404) + + # Create release record using artifact hash as storage key + # At some point this presumably won't work, because we can have many artifacts + # But meanwhile it's fine + # TODO: Extract version from filename or add to form + release = Release( + storage_key=artifact_hash, + stage=ReleaseStage.CANDIDATE, + phase=ReleasePhase.RELEASE_CANDIDATE, + pmc_id=pmc.id, + version="", + ) + db_session.add(release) + + # Create package record + package = Package( + file=str(artifact_path.relative_to(current_app.config["RELEASE_STORAGE_DIR"])), + signature=str(signature_path.relative_to(current_app.config["RELEASE_STORAGE_DIR"])), + checksum=checksum_512, + release_key=release.storage_key, + ) + db_session.add(package) + + db_session.commit() - # TODO: Implement actual file handling - return f"Would process release candidate for {project_name} from file {release_file.filename}" + return f"Successfully uploaded release candidate for {project_name}" # For GET requests, show the form return await render_template( @@ -74,6 +165,86 @@ async def add_release_candidate() -> str: ) +@APP.route("/admin/update-pmcs", methods=["GET", "POST"]) +async def admin_update_pmcs() -> str: + "Update PMCs from remote, authoritative committee-info.json." + # Check authentication + session = await session_read() + if session is None: + raise ASFQuartException("Not authenticated", errorcode=401) + + # List of users allowed to update PMCs + ALLOWED_USERS = {"cwells", "fluxo", "gmcdonald", "humbedooh", "sbp", "tn", "wave"} + + if session.uid not in ALLOWED_USERS: + raise ASFQuartException("You are not authorized to update PMCs", errorcode=403) + + if request.method == "POST": + # TODO: We should probably lift this branch + # Or have the "GET" in a branch, and then we can happy path this POST branch + # Fetch committee-info.json from Whimsy + WHIMSY_URL = "https://whimsy.apache.org/public/committee-info.json" + async with httpx.AsyncClient() as client: + try: + response = await client.get(WHIMSY_URL) + response.raise_for_status() + data = response.json() + except (httpx.RequestError, json.JSONDecodeError) as e: + raise ASFQuartException(f"Failed to fetch committee data: {str(e)}", errorcode=500) + + committees = data.get("committees", {}) + updated_count = 0 + + with Session(current_app.config["engine"]) as db_session: + for committee_id, info in committees.items(): + # Skip non-PMC committees + if not info.get("pmc", False): + continue + + # Get or create PMC + statement = select(PMC).where(PMC.project_name == committee_id) + pmc = db_session.exec(statement).first() + if not pmc: + pmc = PMC(project_name=committee_id) + db_session.add(pmc) + + # Update PMC data + roster = info.get("roster", {}) + # All roster members are PMC members + pmc.pmc_members = list(roster.keys()) + # All PMC members are also committers + pmc.committers = list(roster.keys()) + + # Mark chairs as release managers + # TODO: Who else is a release manager? How do we know? + chairs = [m["id"] for m in info.get("chairs", [])] + pmc.release_managers = chairs + + updated_count += 1 + + # Add special entry for Tooling PMC + # Not clear why, but it's not in the Whimsy data + statement = select(PMC).where(PMC.project_name == "tooling") + tooling_pmc = db_session.exec(statement).first() + if not tooling_pmc: + tooling_pmc = PMC(project_name="tooling") + db_session.add(tooling_pmc) + updated_count += 1 + + # Update Tooling PMC data + # Could put this in the "if not tooling_pmc" block, perhaps + tooling_pmc.pmc_members = ["wave", "tn", "sbp"] + tooling_pmc.committers = ["wave", "tn", "sbp"] + tooling_pmc.release_managers = ["wave"] + + db_session.commit() + + return f"Successfully updated {updated_count} PMCs from Whimsy" + + # For GET requests, show the update form + return await render_template("update-pmcs.html") + + @APP.route("/pmc/create/<project_name>") async def pmc_create_arg(project_name: str) -> dict: "Create a new PMC with some sample data." diff --git a/atr/server.py b/atr/server.py index 79d98e4..7282064 100644 --- a/atr/server.py +++ b/atr/server.py @@ -53,6 +53,11 @@ def create_app() -> QuartApp: os.chdir(state_dir) print(f"Working directory changed to: {os.getcwd()}") + # Set up release storage directory + release_storage = os.path.join(state_dir, "releases") + os.makedirs(release_storage, exist_ok=True) + app.config["RELEASE_STORAGE_DIR"] = release_storage + sqlite_url = "sqlite:///./atr.db" engine = create_engine( sqlite_url, diff --git a/atr/static/root.css b/atr/static/root.css index 1dbc347..8da932b 100644 --- a/atr/static/root.css +++ b/atr/static/root.css @@ -5,7 +5,7 @@ body { } h1 { - color: #303284; + color: #036; } .pmc-list { @@ -32,7 +32,7 @@ h1 { .pmc-name { font-weight: bold; font-size: 1.2em; - color: #303284; + color: #036; margin-bottom: 0.5em; } diff --git a/atr/templates/add-release-candidate.html b/atr/templates/add-release-candidate.html index 871fd15..e58e0a6 100644 --- a/atr/templates/add-release-candidate.html +++ b/atr/templates/add-release-candidate.html @@ -76,12 +76,19 @@ </div> <div class="form-group"> - <label for="release_file">Release Candidate Archive:</label> - <input type="file" id="release_file" name="release_file" required + <label for="release_artifact">Release Candidate Archive:</label> + <input type="file" id="release_artifact" name="release_artifact" required accept="application/gzip,application/x-gzip,application/x-tar,application/zip,application/java-archive,.tar.gz,.tgz,.zip,.jar" - aria-describedby="file-help"> - <span id="file-help">Upload the release candidate archive (tar.gz, zip, or jar)</span> - <!-- TODO: What file names should we support here? Just any file name?--> + aria-describedby="artifact-help"> + <span id="artifact-help">Upload the release candidate archive (tar.gz, zip, or jar)</span> + </div> + + <div class="form-group"> + <label for="release_signature">Detached GPG Signature:</label> + <input type="file" id="release_signature" name="release_signature" required + accept="application/pgp-signature,.asc" + aria-describedby="signature-help"> + <span id="signature-help">Upload the detached GPG signature (.asc) file for the release candidate</span> </div> <button type="submit" {% if not pmc_memberships %}disabled{% endif %}> diff --git a/atr/templates/update-pmcs.html b/atr/templates/update-pmcs.html new file mode 100644 index 0000000..fd01175 --- /dev/null +++ b/atr/templates/update-pmcs.html @@ -0,0 +1,30 @@ +<!doctype html> +<html lang="en"> +<head> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width,initial-scale=1.0"> + <title>ATR | Update PMCs</title> + <link rel="stylesheet" href="{{ url_for('static', filename='root.css') }}"> + <style> + .form-group { + margin-bottom: 1rem; + } + button { + margin-top: 1rem; + padding: 0.5rem 1rem; + } + </style> +</head> +<body> + <h1>Update PMCs</h1> + <p class="intro">This page allows you to update the PMC information in the database from committee-info.json.</p> + + <div class="warning"> + <p><strong>Note:</strong> This operation will update all PMC information, including member lists and release manager assignments.</p> + </div> + + <form method="post"> + <button type="submit">Update PMCs</button> + </form> +</body> +</html> diff --git a/poetry.lock b/poetry.lock index eff2582..3503f30 100644 --- a/poetry.lock +++ b/poetry.lock @@ -958,6 +958,53 @@ files = [ {file = "hpack-4.1.0.tar.gz", hash = "sha256:ec5eca154f7056aa06f196a557655c5b009b382873ac8d1e66e79e87535f1dca"}, ] +[[package]] +name = "httpcore" +version = "1.0.7" +description = "A minimal low-level HTTP client." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "httpcore-1.0.7-py3-none-any.whl", hash = "sha256:a3fff8f43dc260d5bd363d9f9cf1830fa3a458b332856f34282de498ed420edd"}, + {file = "httpcore-1.0.7.tar.gz", hash = "sha256:8551cb62a169ec7162ac7be8d4817d561f60e08eaa485234898414bb5a8a0b4c"}, +] + +[package.dependencies] +certifi = "*" +h11 = ">=0.13,<0.15" + +[package.extras] +asyncio = ["anyio (>=4.0,<5.0)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +trio = ["trio (>=0.22.0,<1.0)"] + +[[package]] +name = "httpx" +version = "0.28.1" +description = "The next generation HTTP client." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, + {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, +] + +[package.dependencies] +anyio = "*" +certifi = "*" +httpcore = "==1.*" +idna = "*" + +[package.extras] +brotli = ["brotli", "brotlicffi"] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +zstd = ["zstandard (>=0.18.0)"] + [[package]] name = "hypercorn" version = "0.17.3" @@ -2293,4 +2340,4 @@ propcache = ">=0.2.0" [metadata] lock-version = "2.1" python-versions = "~=3.13" -content-hash = "0f091d55acfd7f3269b025aa80eadad13ff6b281f8c72c7cdd229d207f0b609d" +content-hash = "6be50693b57621af6fa35e34f54c89113075e48ee3c13ec418e92586fcc1c141" diff --git a/pyproject.toml b/pyproject.toml index 82b1887..0323bd2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ dependencies = [ "alembic~=1.14", "asfquart", "cryptography~=44.0", + "httpx~=0.27", "hypercorn~=0.17", "sqlmodel~=0.0", ] @@ -39,6 +40,7 @@ mypy = "^1.15.0" alembic = "~=1.14" asfquart = { path = "./asfquart", develop = true } cryptography = "~=44.0" +httpx = "~=0.27" hypercorn = "~=0.17" sqlmodel = "~=0.0" diff --git a/uv.lock b/uv.lock index 597cff8..ee6dade 100644 --- a/uv.lock +++ b/uv.lock @@ -451,6 +451,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/c6/80c95b1b2b94682a72cbdbfb85b81ae2daffa4291fbfa1b1464502ede10d/hpack-4.1.0-py3-none-any.whl", hash = "sha256:157ac792668d995c657d93111f46b4535ed114f0c9c8d672271bbec7eae1b496", size = 34357 }, ] +[[package]] +name = "httpcore" +version = "1.0.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6a/41/d7d0a89eb493922c37d343b607bc1b5da7f5be7e383740b4753ad8943e90/httpcore-1.0.7.tar.gz", hash = "sha256:8551cb62a169ec7162ac7be8d4817d561f60e08eaa485234898414bb5a8a0b4c", size = 85196 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/f5/72347bc88306acb359581ac4d52f23c0ef445b57157adedb9aee0cd689d2/httpcore-1.0.7-py3-none-any.whl", hash = "sha256:a3fff8f43dc260d5bd363d9f9cf1830fa3a458b332856f34282de498ed420edd", size = 78551 }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517 }, +] + [[package]] name = "hypercorn" version = "0.17.3" @@ -943,6 +971,7 @@ dependencies = [ { name = "alembic" }, { name = "asfquart" }, { name = "cryptography" }, + { name = "httpx" }, { name = "hypercorn" }, { name = "sqlmodel" }, ] @@ -960,6 +989,7 @@ requires-dist = [ { name = "alembic", specifier = "~=1.14" }, { name = "asfquart", editable = "asfquart" }, { name = "cryptography", specifier = "~=44.0" }, + { name = "httpx", specifier = "~=0.27" }, { name = "hypercorn", specifier = "~=0.17" }, { name = "sqlmodel", specifier = "~=0.0" }, ] --------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@tooling.apache.org For additional commands, e-mail: dev-h...@tooling.apache.org