This is an automated email from the ASF dual-hosted git repository.

sbp pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-atr-experiments.git


The following commit(s) were added to refs/heads/main by this push:
     new 1a11573  Add a PMC updater and release candidate processing code
1a11573 is described below

commit 1a11573c714aa93c274fc1af4342af5104131ae1
Author: Sean B. Palmer <s...@miscoranda.com>
AuthorDate: Thu Feb 13 15:46:58 2025 +0200

    Add a PMC updater and release candidate processing code
---
 atr/models.py                            |  10 +-
 atr/routes.py                            | 191 +++++++++++++++++++++++++++++--
 atr/server.py                            |   5 +
 atr/static/root.css                      |   4 +-
 atr/templates/add-release-candidate.html |  17 ++-
 atr/templates/update-pmcs.html           |  30 +++++
 poetry.lock                              |  49 +++++++-
 pyproject.toml                           |   2 +
 uv.lock                                  |  30 +++++
 9 files changed, 318 insertions(+), 20 deletions(-)

diff --git a/atr/models.py b/atr/models.py
index 8552402..61c9dd4 100644
--- a/atr/models.py
+++ b/atr/models.py
@@ -120,11 +120,16 @@ class DistributionChannel(SQLModel, table=True):
     product_line: Optional[ProductLine] = 
Relationship(back_populates="distribution_channels")
 
 
-class Package(BaseModel):
+class Package(SQLModel, table=True):
+    id: Optional[int] = Field(default=None, primary_key=True)
     file: str
     signature: str
     checksum: str
 
+    # Many-to-one: A package belongs to one release
+    release_key: Optional[str] = Field(default=None, 
foreign_key="release.storage_key")
+    release: Optional["Release"] = Relationship(back_populates="packages")
+
 
 class VoteEntry(BaseModel):
     result: bool
@@ -172,7 +177,8 @@ class Release(SQLModel, table=True):
 
     package_managers: List[str] = Field(default_factory=list, 
sa_column=Column(JSON))
     version: str
-    packages: List[Package] = Field(default_factory=list, 
sa_column=Column(JSON))
+    # One-to-many: A release can have multiple packages
+    packages: List[Package] = Relationship(back_populates="release")
     sboms: List[str] = Field(default_factory=list, sa_column=Column(JSON))
 
     # Many-to-one: A release can have one vote policy, a vote policy can be 
used by multiple releases
diff --git a/atr/routes.py b/atr/routes.py
index 680376f..eb01b55 100644
--- a/atr/routes.py
+++ b/atr/routes.py
@@ -17,7 +17,10 @@
 
 "routes.py"
 
-from typing import List
+import hashlib
+import json
+from pathlib import Path
+from typing import List, Tuple
 
 from asfquart import APP
 from asfquart.auth import Requirements as R, require
@@ -26,13 +29,49 @@ from asfquart.session import read as session_read
 from quart import current_app, render_template, request
 from sqlmodel import Session, select
 from sqlalchemy.exc import IntegrityError
+import httpx
 
-from .models import PMC
+from .models import PMC, Release, ReleaseStage, ReleasePhase, Package
 
 if APP is ...:
     raise ValueError("APP is not set")
 
 
+def compute_sha3_256(file_data: bytes) -> str:
+    "Compute SHA3-256 hash of file data."
+    return hashlib.sha3_256(file_data).hexdigest()
+
+
+def compute_sha512(file_path: Path) -> str:
+    "Compute SHA-512 hash of a file."
+    sha512 = hashlib.sha512()
+    with open(file_path, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            sha512.update(chunk)
+    return sha512.hexdigest()
+
+
+async def save_file_by_hash(file, base_dir: Path) -> Tuple[Path, str]:
+    """
+    Save a file using its SHA3-256 hash as the filename.
+    Returns the path where the file was saved and its hash.
+    """
+    # FileStorage.read() returns bytes directly, no need to await
+    data = file.read()
+    file_hash = compute_sha3_256(data)
+
+    # Create path with hash as filename
+    path = base_dir / file_hash
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+    # Only write if file doesn't exist
+    # If it does exist, it'll be the same content anyway
+    if not path.exists():
+        path.write_bytes(data)
+
+    return path, file_hash
+
+
 @APP.route("/add-release-candidate", methods=["GET", "POST"])
 @require(R.committer)
 async def add_release_candidate() -> str:
@@ -43,10 +82,7 @@ async def add_release_candidate() -> str:
 
     # For POST requests, handle the file upload
     if request.method == "POST":
-        # We'll implement the actual file handling later
-        # For now just return a message about what we would do
         form = await request.form
-        files = await request.files
 
         project_name = form.get("project_name")
         if not project_name:
@@ -58,12 +94,67 @@ async def add_release_candidate() -> str:
                 f"You must be a PMC member of {project_name} to submit a 
release candidate", errorcode=403
             )
 
-        release_file = files.get("release_file")
-        if not release_file:
-            raise ASFQuartException("Release file is required", errorcode=400)
+        # Get all uploaded files
+        files = await request.files
+
+        # Get the release artifact and signature files
+        artifact_file = files.get("release_artifact")
+        signature_file = files.get("release_signature")
+
+        if not artifact_file:
+            raise ASFQuartException("Release artifact file is required", 
errorcode=400)
+        if not signature_file:
+            raise ASFQuartException("Detached GPG signature file is required", 
errorcode=400)
+        if not signature_file.filename.endswith(".asc"):
+            # TODO: Could also check that it's artifact name + ".asc"
+            # And at least warn if it's not
+            raise ASFQuartException("Signature file must have .asc extension", 
errorcode=400)
+
+        # Save files using their hashes as filenames
+        storage_dir = Path(current_app.config["RELEASE_STORAGE_DIR"]) / 
project_name
+        artifact_path, artifact_hash = await save_file_by_hash(artifact_file, 
storage_dir)
+        # TODO: Do we need to do anything with the signature hash?
+        # These should be identical, but path might be absolute?
+        # TODO: Need to check, ideally. Could have a data browser
+        signature_path, _ = await save_file_by_hash(signature_file, 
storage_dir)
+
+        # Compute SHA-512 checksum of the artifact for the package record
+        # We're using SHA-3-256 for the filename, so we need to use SHA-3-512 
for the checksum
+        checksum_512 = compute_sha512(artifact_path)
+
+        # Store in database
+        with Session(current_app.config["engine"]) as db_session:
+            # Get PMC
+            statement = select(PMC).where(PMC.project_name == project_name)
+            pmc = db_session.exec(statement).first()
+            if not pmc:
+                raise ASFQuartException("PMC not found", errorcode=404)
+
+            # Create release record using artifact hash as storage key
+            # At some point this presumably won't work, because we can have 
many artifacts
+            # But meanwhile it's fine
+            # TODO: Extract version from filename or add to form
+            release = Release(
+                storage_key=artifact_hash,
+                stage=ReleaseStage.CANDIDATE,
+                phase=ReleasePhase.RELEASE_CANDIDATE,
+                pmc_id=pmc.id,
+                version="",
+            )
+            db_session.add(release)
+
+            # Create package record
+            package = Package(
+                
file=str(artifact_path.relative_to(current_app.config["RELEASE_STORAGE_DIR"])),
+                
signature=str(signature_path.relative_to(current_app.config["RELEASE_STORAGE_DIR"])),
+                checksum=checksum_512,
+                release_key=release.storage_key,
+            )
+            db_session.add(package)
+
+            db_session.commit()
 
-        # TODO: Implement actual file handling
-        return f"Would process release candidate for {project_name} from file 
{release_file.filename}"
+            return f"Successfully uploaded release candidate for 
{project_name}"
 
     # For GET requests, show the form
     return await render_template(
@@ -74,6 +165,86 @@ async def add_release_candidate() -> str:
     )
 
 
+@APP.route("/admin/update-pmcs", methods=["GET", "POST"])
+async def admin_update_pmcs() -> str:
+    "Update PMCs from remote, authoritative committee-info.json."
+    # Check authentication
+    session = await session_read()
+    if session is None:
+        raise ASFQuartException("Not authenticated", errorcode=401)
+
+    # List of users allowed to update PMCs
+    ALLOWED_USERS = {"cwells", "fluxo", "gmcdonald", "humbedooh", "sbp", "tn", 
"wave"}
+
+    if session.uid not in ALLOWED_USERS:
+        raise ASFQuartException("You are not authorized to update PMCs", 
errorcode=403)
+
+    if request.method == "POST":
+        # TODO: We should probably lift this branch
+        # Or have the "GET" in a branch, and then we can happy path this POST 
branch
+        # Fetch committee-info.json from Whimsy
+        WHIMSY_URL = "https://whimsy.apache.org/public/committee-info.json";
+        async with httpx.AsyncClient() as client:
+            try:
+                response = await client.get(WHIMSY_URL)
+                response.raise_for_status()
+                data = response.json()
+            except (httpx.RequestError, json.JSONDecodeError) as e:
+                raise ASFQuartException(f"Failed to fetch committee data: 
{str(e)}", errorcode=500)
+
+        committees = data.get("committees", {})
+        updated_count = 0
+
+        with Session(current_app.config["engine"]) as db_session:
+            for committee_id, info in committees.items():
+                # Skip non-PMC committees
+                if not info.get("pmc", False):
+                    continue
+
+                # Get or create PMC
+                statement = select(PMC).where(PMC.project_name == committee_id)
+                pmc = db_session.exec(statement).first()
+                if not pmc:
+                    pmc = PMC(project_name=committee_id)
+                    db_session.add(pmc)
+
+                # Update PMC data
+                roster = info.get("roster", {})
+                # All roster members are PMC members
+                pmc.pmc_members = list(roster.keys())
+                # All PMC members are also committers
+                pmc.committers = list(roster.keys())
+
+                # Mark chairs as release managers
+                # TODO: Who else is a release manager? How do we know?
+                chairs = [m["id"] for m in info.get("chairs", [])]
+                pmc.release_managers = chairs
+
+                updated_count += 1
+
+            # Add special entry for Tooling PMC
+            # Not clear why, but it's not in the Whimsy data
+            statement = select(PMC).where(PMC.project_name == "tooling")
+            tooling_pmc = db_session.exec(statement).first()
+            if not tooling_pmc:
+                tooling_pmc = PMC(project_name="tooling")
+                db_session.add(tooling_pmc)
+                updated_count += 1
+
+            # Update Tooling PMC data
+            # Could put this in the "if not tooling_pmc" block, perhaps
+            tooling_pmc.pmc_members = ["wave", "tn", "sbp"]
+            tooling_pmc.committers = ["wave", "tn", "sbp"]
+            tooling_pmc.release_managers = ["wave"]
+
+            db_session.commit()
+
+        return f"Successfully updated {updated_count} PMCs from Whimsy"
+
+    # For GET requests, show the update form
+    return await render_template("update-pmcs.html")
+
+
 @APP.route("/pmc/create/<project_name>")
 async def pmc_create_arg(project_name: str) -> dict:
     "Create a new PMC with some sample data."
diff --git a/atr/server.py b/atr/server.py
index 79d98e4..7282064 100644
--- a/atr/server.py
+++ b/atr/server.py
@@ -53,6 +53,11 @@ def create_app() -> QuartApp:
         os.chdir(state_dir)
         print(f"Working directory changed to: {os.getcwd()}")
 
+        # Set up release storage directory
+        release_storage = os.path.join(state_dir, "releases")
+        os.makedirs(release_storage, exist_ok=True)
+        app.config["RELEASE_STORAGE_DIR"] = release_storage
+
         sqlite_url = "sqlite:///./atr.db"
         engine = create_engine(
             sqlite_url,
diff --git a/atr/static/root.css b/atr/static/root.css
index 1dbc347..8da932b 100644
--- a/atr/static/root.css
+++ b/atr/static/root.css
@@ -5,7 +5,7 @@ body {
 }
 
 h1 {
-    color: #303284;
+    color: #036;
 }
 
 .pmc-list {
@@ -32,7 +32,7 @@ h1 {
 .pmc-name {
     font-weight: bold;
     font-size: 1.2em;
-    color: #303284;
+    color: #036;
     margin-bottom: 0.5em;
 }
 
diff --git a/atr/templates/add-release-candidate.html 
b/atr/templates/add-release-candidate.html
index 871fd15..e58e0a6 100644
--- a/atr/templates/add-release-candidate.html
+++ b/atr/templates/add-release-candidate.html
@@ -76,12 +76,19 @@
     </div>
 
     <div class="form-group">
-      <label for="release_file">Release Candidate Archive:</label>
-      <input type="file" id="release_file" name="release_file" required
+      <label for="release_artifact">Release Candidate Archive:</label>
+      <input type="file" id="release_artifact" name="release_artifact" required
              
accept="application/gzip,application/x-gzip,application/x-tar,application/zip,application/java-archive,.tar.gz,.tgz,.zip,.jar"
-             aria-describedby="file-help">
-      <span id="file-help">Upload the release candidate archive (tar.gz, zip, 
or jar)</span>
-      <!-- TODO: What file names should we support here? Just any file name?-->
+             aria-describedby="artifact-help">
+      <span id="artifact-help">Upload the release candidate archive (tar.gz, 
zip, or jar)</span>
+    </div>
+
+    <div class="form-group">
+      <label for="release_signature">Detached GPG Signature:</label>
+      <input type="file" id="release_signature" name="release_signature" 
required
+             accept="application/pgp-signature,.asc"
+             aria-describedby="signature-help">
+      <span id="signature-help">Upload the detached GPG signature (.asc) file 
for the release candidate</span>
     </div>
 
     <button type="submit" {% if not pmc_memberships %}disabled{% endif %}>
diff --git a/atr/templates/update-pmcs.html b/atr/templates/update-pmcs.html
new file mode 100644
index 0000000..fd01175
--- /dev/null
+++ b/atr/templates/update-pmcs.html
@@ -0,0 +1,30 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width,initial-scale=1.0">
+  <title>ATR | Update PMCs</title>
+  <link rel="stylesheet" href="{{ url_for('static', filename='root.css') }}">
+  <style>
+    .form-group {
+      margin-bottom: 1rem;
+    }
+    button {
+      margin-top: 1rem;
+      padding: 0.5rem 1rem;
+    }
+  </style>
+</head>
+<body>
+  <h1>Update PMCs</h1>
+  <p class="intro">This page allows you to update the PMC information in the 
database from committee-info.json.</p>
+
+  <div class="warning">
+    <p><strong>Note:</strong> This operation will update all PMC information, 
including member lists and release manager assignments.</p>
+  </div>
+
+  <form method="post">
+    <button type="submit">Update PMCs</button>
+  </form>
+</body>
+</html>
diff --git a/poetry.lock b/poetry.lock
index eff2582..3503f30 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -958,6 +958,53 @@ files = [
     {file = "hpack-4.1.0.tar.gz", hash = 
"sha256:ec5eca154f7056aa06f196a557655c5b009b382873ac8d1e66e79e87535f1dca"},
 ]
 
+[[package]]
+name = "httpcore"
+version = "1.0.7"
+description = "A minimal low-level HTTP client."
+optional = false
+python-versions = ">=3.8"
+groups = ["main"]
+files = [
+    {file = "httpcore-1.0.7-py3-none-any.whl", hash = 
"sha256:a3fff8f43dc260d5bd363d9f9cf1830fa3a458b332856f34282de498ed420edd"},
+    {file = "httpcore-1.0.7.tar.gz", hash = 
"sha256:8551cb62a169ec7162ac7be8d4817d561f60e08eaa485234898414bb5a8a0b4c"},
+]
+
+[package.dependencies]
+certifi = "*"
+h11 = ">=0.13,<0.15"
+
+[package.extras]
+asyncio = ["anyio (>=4.0,<5.0)"]
+http2 = ["h2 (>=3,<5)"]
+socks = ["socksio (==1.*)"]
+trio = ["trio (>=0.22.0,<1.0)"]
+
+[[package]]
+name = "httpx"
+version = "0.28.1"
+description = "The next generation HTTP client."
+optional = false
+python-versions = ">=3.8"
+groups = ["main"]
+files = [
+    {file = "httpx-0.28.1-py3-none-any.whl", hash = 
"sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"},
+    {file = "httpx-0.28.1.tar.gz", hash = 
"sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"},
+]
+
+[package.dependencies]
+anyio = "*"
+certifi = "*"
+httpcore = "==1.*"
+idna = "*"
+
+[package.extras]
+brotli = ["brotli", "brotlicffi"]
+cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"]
+http2 = ["h2 (>=3,<5)"]
+socks = ["socksio (==1.*)"]
+zstd = ["zstandard (>=0.18.0)"]
+
 [[package]]
 name = "hypercorn"
 version = "0.17.3"
@@ -2293,4 +2340,4 @@ propcache = ">=0.2.0"
 [metadata]
 lock-version = "2.1"
 python-versions = "~=3.13"
-content-hash = 
"0f091d55acfd7f3269b025aa80eadad13ff6b281f8c72c7cdd229d207f0b609d"
+content-hash = 
"6be50693b57621af6fa35e34f54c89113075e48ee3c13ec418e92586fcc1c141"
diff --git a/pyproject.toml b/pyproject.toml
index 82b1887..0323bd2 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -12,6 +12,7 @@ dependencies = [
   "alembic~=1.14",
   "asfquart",
   "cryptography~=44.0",
+  "httpx~=0.27",
   "hypercorn~=0.17",
   "sqlmodel~=0.0",
 ]
@@ -39,6 +40,7 @@ mypy = "^1.15.0"
 alembic = "~=1.14"
 asfquart = { path = "./asfquart", develop = true }
 cryptography = "~=44.0"
+httpx = "~=0.27"
 hypercorn = "~=0.17"
 sqlmodel = "~=0.0"
 
diff --git a/uv.lock b/uv.lock
index 597cff8..ee6dade 100644
--- a/uv.lock
+++ b/uv.lock
@@ -451,6 +451,34 @@ wheels = [
     { url = 
"https://files.pythonhosted.org/packages/07/c6/80c95b1b2b94682a72cbdbfb85b81ae2daffa4291fbfa1b1464502ede10d/hpack-4.1.0-py3-none-any.whl";,
 hash = 
"sha256:157ac792668d995c657d93111f46b4535ed114f0c9c8d672271bbec7eae1b496", size 
= 34357 },
 ]
 
+[[package]]
+name = "httpcore"
+version = "1.0.7"
+source = { registry = "https://pypi.org/simple"; }
+dependencies = [
+    { name = "certifi" },
+    { name = "h11" },
+]
+sdist = { url = 
"https://files.pythonhosted.org/packages/6a/41/d7d0a89eb493922c37d343b607bc1b5da7f5be7e383740b4753ad8943e90/httpcore-1.0.7.tar.gz";,
 hash = 
"sha256:8551cb62a169ec7162ac7be8d4817d561f60e08eaa485234898414bb5a8a0b4c", size 
= 85196 }
+wheels = [
+    { url = 
"https://files.pythonhosted.org/packages/87/f5/72347bc88306acb359581ac4d52f23c0ef445b57157adedb9aee0cd689d2/httpcore-1.0.7-py3-none-any.whl";,
 hash = 
"sha256:a3fff8f43dc260d5bd363d9f9cf1830fa3a458b332856f34282de498ed420edd", size 
= 78551 },
+]
+
+[[package]]
+name = "httpx"
+version = "0.28.1"
+source = { registry = "https://pypi.org/simple"; }
+dependencies = [
+    { name = "anyio" },
+    { name = "certifi" },
+    { name = "httpcore" },
+    { name = "idna" },
+]
+sdist = { url = 
"https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz";,
 hash = 
"sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size 
= 141406 }
+wheels = [
+    { url = 
"https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl";,
 hash = 
"sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size 
= 73517 },
+]
+
 [[package]]
 name = "hypercorn"
 version = "0.17.3"
@@ -943,6 +971,7 @@ dependencies = [
     { name = "alembic" },
     { name = "asfquart" },
     { name = "cryptography" },
+    { name = "httpx" },
     { name = "hypercorn" },
     { name = "sqlmodel" },
 ]
@@ -960,6 +989,7 @@ requires-dist = [
     { name = "alembic", specifier = "~=1.14" },
     { name = "asfquart", editable = "asfquart" },
     { name = "cryptography", specifier = "~=44.0" },
+    { name = "httpx", specifier = "~=0.27" },
     { name = "hypercorn", specifier = "~=0.17" },
     { name = "sqlmodel", specifier = "~=0.0" },
 ]


---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@tooling.apache.org
For additional commands, e-mail: dev-h...@tooling.apache.org

Reply via email to