This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-release.git
The following commit(s) were added to refs/heads/main by this push:
new 7c3eee1 Automatically run checks on candidate drafts after upload
7c3eee1 is described below
commit 7c3eee10df442b297eda1df676e3f5a429a84072
Author: Sean B. Palmer <[email protected]>
AuthorDate: Tue Mar 25 15:26:41 2025 +0200
Automatically run checks on candidate drafts after upload
Removes package routes and templates. Tasks are associated with
releases and file paths rather than with packages. The SSH server
automatically creates releases when needed, and starts checks on all
files in the candidate draft even after a partial upload.
---
atr/db/__init__.py | 38 +--
atr/db/models.py | 20 +-
atr/routes/candidate.py | 1 -
atr/routes/files.py | 27 +-
atr/routes/package.py | 254 -------------------
atr/server.py | 2 -
atr/ssh.py | 49 ++--
atr/tasks/__init__.py | 56 +++--
atr/tasks/rsync.py | 25 +-
.../{package-check.html => files-check.html} | 0
atr/templates/files-list.html | 9 +
atr/templates/package-add.html | 278 ---------------------
12 files changed, 147 insertions(+), 612 deletions(-)
diff --git a/atr/db/__init__.py b/atr/db/__init__.py
index d27f6bb..3a220d3 100644
--- a/atr/db/__init__.py
+++ b/atr/db/__init__.py
@@ -167,12 +167,11 @@ class Session(sqlalchemy.ext.asyncio.AsyncSession):
artifact_type: Opt[str] = NotSet,
filename: Opt[str] = NotSet,
sha512: Opt[str] = NotSet,
- signature_sha3: Opt[str] = NotSet,
- uploaded: Opt[bool] = NotSet,
+ signature_sha3: Opt[str | None] = NotSet,
+ uploaded: Opt[datetime] = NotSet,
bytes_size: Opt[int] = NotSet,
release_name: Opt[str] = NotSet,
_release: bool = False,
- _tasks: bool = False,
_release_project: bool = False,
_release_committee: bool = False,
) -> Query[models.Package]:
@@ -194,16 +193,16 @@ class Session(sqlalchemy.ext.asyncio.AsyncSession):
query = query.where(models.Package.bytes_size == bytes_size)
if is_defined(release_name):
query = query.where(models.Package.release_name == release_name)
+
if _release:
query = query.options(select_in_load(models.Package.release))
- if _tasks:
- query = query.options(select_in_load(models.Package.tasks))
if _release_project:
- query = query.options(select_in_load(models.Package.release,
models.Release.project))
+ query =
query.options(select_in_load_nested(models.Package.release,
models.Release.project))
if _release_committee:
query = query.options(
select_in_load_nested(models.Package.release,
models.Release.project, models.Project.committee)
)
+
return Query(self, query)
def project(
@@ -300,7 +299,7 @@ class Session(sqlalchemy.ext.asyncio.AsyncSession):
_packages: bool = False,
_vote_policy: bool = False,
_committee: bool = False,
- _packages_tasks: bool = False,
+ _tasks: bool = False,
) -> Query[models.Release]:
query = sqlmodel.select(models.Release)
@@ -333,8 +332,8 @@ class Session(sqlalchemy.ext.asyncio.AsyncSession):
query = query.options(select_in_load(models.Release.vote_policy))
if _committee:
query =
query.options(select_in_load_nested(models.Release.project,
models.Project.committee))
- if _packages_tasks:
- query =
query.options(select_in_load_nested(models.Release.packages,
models.Package.tasks))
+ if _tasks:
+ query = query.options(select_in_load(models.Release.tasks))
return Query(self, query)
@@ -367,9 +366,10 @@ class Session(sqlalchemy.ext.asyncio.AsyncSession):
completed: Opt[datetime | None] = NotSet,
result: Opt[Any | None] = NotSet,
error: Opt[str | None] = NotSet,
- package_sha3: Opt[str | None] = NotSet,
- _package: bool = False,
- _package_release: bool = False,
+ release_name: Opt[str | None] = NotSet,
+ path: Opt[str | None] = NotSet,
+ modified: Opt[int | None] = NotSet,
+ _release: bool = False,
) -> Query[models.Task]:
query = sqlmodel.select(models.Task)
@@ -393,13 +393,15 @@ class Session(sqlalchemy.ext.asyncio.AsyncSession):
query = query.where(models.Task.result == result)
if is_defined(error):
query = query.where(models.Task.error == error)
- if is_defined(package_sha3):
- query = query.where(models.Task.package_sha3 == package_sha3)
+ if is_defined(release_name):
+ query = query.where(models.Task.release_name == release_name)
+ if is_defined(path):
+ query = query.where(models.Task.path == path)
+ if is_defined(modified):
+ query = query.where(models.Task.modified == modified)
- if _package:
- query = query.options(select_in_load(models.Task.package))
- if _package_release:
- query = query.options(select_in_load_nested(models.Task.package,
models.Package.release))
+ if _release:
+ query = query.options(select_in_load(models.Task.release))
return Query(self, query)
diff --git a/atr/db/models.py b/atr/db/models.py
index 3a41d3c..f2d19fe 100644
--- a/atr/db/models.py
+++ b/atr/db/models.py
@@ -22,7 +22,7 @@
import datetime
import enum
-from typing import Any
+from typing import Any, Optional
import pydantic
import sqlalchemy
@@ -210,11 +210,6 @@ class Package(sqlmodel.SQLModel, table=True):
release_name: str = sqlmodel.Field(foreign_key="release.name")
release: "Release" = sqlmodel.Relationship(back_populates="packages")
- # One-to-many: A package can have multiple tasks
- tasks: list["Task"] = sqlmodel.Relationship(
- back_populates="package", sa_relationship_kwargs={"cascade": "all,
delete-orphan"}
- )
-
class VoteEntry(pydantic.BaseModel):
result: bool
@@ -287,10 +282,10 @@ class Task(sqlmodel.SQLModel, table=True):
completed: datetime.datetime | None = None
result: Any | None = sqlmodel.Field(default=None,
sa_column=sqlalchemy.Column(sqlalchemy.JSON))
error: str | None = None
-
- # Package relationship
- package_sha3: str | None = sqlmodel.Field(default=None,
foreign_key="package.artifact_sha3")
- package: Package | None = sqlmodel.Relationship(back_populates="tasks")
+ release_name: str | None = sqlmodel.Field(default=None,
foreign_key="release.name")
+ release: Optional["Release"] =
sqlmodel.Relationship(back_populates="tasks")
+ path: str | None = sqlmodel.Field(default=None)
+ modified: int | None = sqlmodel.Field(default=None)
# Create an index on status and added for efficient task claiming
__table_args__ = (
@@ -347,6 +342,11 @@ class Release(sqlmodel.SQLModel, table=True):
votes: list[VoteEntry] = sqlmodel.Field(default_factory=list,
sa_column=sqlalchemy.Column(sqlalchemy.JSON))
+ # One-to-many: A release can have multiple tasks
+ tasks: list["Task"] = sqlmodel.Relationship(
+ back_populates="release", sa_relationship_kwargs={"cascade": "all,
delete-orphan"}
+ )
+
# The combination of project_id and version must be unique
# Technically we want (project.name, version) to be unique
# But project.name is already unique, so project_id works as a proxy
thereof
diff --git a/atr/routes/candidate.py b/atr/routes/candidate.py
index 9d1cd4f..63e810d 100644
--- a/atr/routes/candidate.py
+++ b/atr/routes/candidate.py
@@ -185,7 +185,6 @@ async def root_candidate_review() -> str:
releases = await data.release(
stage=models.ReleaseStage.CANDIDATE,
_committee=True,
- _packages_tasks=True,
).all()
# Filter to only show releases for PMCs or PPMCs where the user is a
member or committer
diff --git a/atr/routes/files.py b/atr/routes/files.py
index 59eee8e..54307f1 100644
--- a/atr/routes/files.py
+++ b/atr/routes/files.py
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-"""package.py"""
+"""files.py"""
from __future__ import annotations
@@ -25,7 +25,7 @@ import pathlib
import re
from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypeVar
-import aiofiles
+import aiofiles.os
import asfquart.auth as auth
import asfquart.base as base
import asfquart.session as session
@@ -35,7 +35,6 @@ import werkzeug.wrappers.response as response
import wtforms
import atr.analysis as analysis
-import atr.config as config
import atr.db as db
import atr.db.models as models
import atr.routes as routes
@@ -47,7 +46,8 @@ if TYPE_CHECKING:
R = TypeVar("R", covariant=True)
-_CONFIG: Final = config.get()
+# _CONFIG: Final = config.get()
+_LOGGER: Final = logging.getLogger(__name__)
# This is the type of functions to which we apply @committer_get
@@ -353,6 +353,8 @@ async def root_files_list(session: CommitterSession,
project_name: str, version_
path_metadata = set()
path_warnings = {}
path_errors = {}
+ path_modified = {}
+ path_tasks: dict[pathlib.Path, dict[str, Any]] = {}
for path in paths:
# Get template and substitutions
elements = {
@@ -382,6 +384,21 @@ async def root_files_list(session: CommitterSession,
project_name: str, version_
# Get warnings and errors
path_warnings[path], path_errors[path] =
_path_warnings_errors(paths_set, path, ext_artifact, ext_metadata)
+ # Get modified time
+ full_path = str(util.get_candidate_draft_dir() / project_name /
version_name / path)
+ path_modified[path] = int(await aiofiles.os.path.getmtime(full_path))
+
+ # Get tasks
+ tasks = await data.task(
+ status=models.TaskStatus.COMPLETED,
+ release_name=f"{project_name}-{version_name}",
+ path=str(path),
+ modified=path_modified[path],
+ ).all()
+ path_tasks[path] = {}
+ for task in tasks:
+ path_tasks[path][task.task_type] = task.result
+
return await quart.render_template(
"files-list.html",
asf_id=session.uid,
@@ -396,4 +413,6 @@ async def root_files_list(session: CommitterSession,
project_name: str, version_
metadata=path_metadata,
warnings=path_warnings,
errors=path_errors,
+ modified=path_modified,
+ tasks=path_tasks,
)
diff --git a/atr/routes/package.py b/atr/routes/package.py
deleted file mode 100644
index a70d339..0000000
--- a/atr/routes/package.py
+++ /dev/null
@@ -1,254 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-"""package.py"""
-
-import asyncio
-import hashlib
-import logging
-import logging.handlers
-import os.path
-import pathlib
-import secrets
-from collections.abc import Sequence
-
-import aiofiles
-import aiofiles.os
-import asfquart.auth as auth
-import asfquart.base as base
-import asfquart.session as session
-import quart
-import werkzeug.datastructures as datastructures
-import werkzeug.wrappers.response as response
-
-import atr.db as db
-import atr.db.models as models
-import atr.routes as routes
-import atr.tasks as tasks
-
-
-async def file_hash_save(base_dir: pathlib.Path, file:
datastructures.FileStorage) -> tuple[str, int]:
- """
- Save a file using its SHA3-256 hash as the filename.
- Returns the hash and size in bytes of the saved file.
- """
- sha3 = hashlib.sha3_256()
- total_bytes = 0
-
- # Create temporary file to stream to while computing hash
- temp_path = base_dir / f"temp-{secrets.token_hex(8)}"
- try:
- stream = file.stream
-
- async with aiofiles.open(temp_path, "wb") as f:
- while True:
- chunk = await asyncio.to_thread(stream.read, 8192)
- if not chunk:
- break
- sha3.update(chunk)
- total_bytes += len(chunk)
- await f.write(chunk)
-
- file_hash = sha3.hexdigest()
- final_path = base_dir / file_hash
-
- # Only move to final location if it doesn't exist
- # This can race, but it's hash based so it's okay
- if not await aiofiles.os.path.exists(final_path):
- await aiofiles.os.rename(temp_path, final_path)
- else:
- # If file already exists, just remove the temp file
- await aiofiles.os.remove(temp_path)
-
- return file_hash, total_bytes
- except Exception as e:
- if await aiofiles.os.path.exists(temp_path):
- await aiofiles.os.remove(temp_path)
- raise e
-
-
-# Package functions
-
-
-async def package_data_get(data: db.Session, artifact_sha3: str, release_name:
str, session_uid: str) -> models.Package:
- """Validate package deletion request and return the package if valid."""
- # Get the package and its associated release
- package = await data.package(artifact_sha3=artifact_sha3,
_release_committee=True).demand(
- routes.FlashError("Package not found")
- )
-
- if package.release_name != release_name:
- raise routes.FlashError("Invalid release key")
-
- # Check permissions
- if package.release and package.release.committee:
- if (session_uid not in package.release.committee.committee_members)
and (
- session_uid not in package.release.committee.committers
- ):
- raise routes.FlashError("You don't have permission to access this
package")
-
- return package
-
-
-# Release functions
-
-
[email protected]_route("/package/check", methods=["GET", "POST"])
[email protected](auth.Requirements.committer)
-async def root_package_check() -> str | response.Response:
- """Show or create package verification tasks."""
- web_session = await session.read()
- if (web_session is None) or (web_session.uid is None):
- raise base.ASFQuartException("Not authenticated", errorcode=401)
-
- # Get parameters from either form data (POST) or query args (GET)
- if quart.request.method == "POST":
- form = await routes.get_form(quart.request)
- artifact_sha3 = form.get("artifact_sha3")
- release_name = form.get("release_name")
- else:
- artifact_sha3 = quart.request.args.get("artifact_sha3")
- release_name = quart.request.args.get("release_name")
-
- if not artifact_sha3 or not release_name:
- await quart.flash("Missing required parameters", "error")
- return quart.redirect(quart.url_for("root_candidate_review"))
-
- async with db.session() as data:
- async with data.begin():
- # Get the package and verify permissions
- try:
- package = await package_data_get(data, artifact_sha3,
release_name, web_session.uid)
- except routes.FlashError as e:
- logging.exception("FlashError:")
- await quart.flash(str(e), "error")
- return quart.redirect(quart.url_for("root_candidate_review"))
-
- if quart.request.method == "POST":
- # Check if package already has active tasks
- tasks, has_active_tasks = await task_package_status_get(data,
artifact_sha3)
- if has_active_tasks:
- await quart.flash("Package verification is already in
progress", "warning")
- return
quart.redirect(quart.url_for("root_candidate_review"))
-
- try:
- await task_verification_create(data, package)
- except routes.FlashError as e:
- logging.exception("FlashError:")
- await quart.flash(str(e), "error")
- return
quart.redirect(quart.url_for("root_candidate_review"))
-
- await quart.flash(f"Added verification tasks for package
{package.filename}", "success")
- return quart.redirect(
- quart.url_for("root_package_check",
artifact_sha3=artifact_sha3, release_name=release_name)
- )
- else:
- # Get all tasks for this package for GET request
- tasks, _ = await task_package_status_get(data, artifact_sha3)
- all_tasks_completed = bool(tasks) and all(
- task.status == models.TaskStatus.COMPLETED or task.status
== models.TaskStatus.FAILED
- for task in tasks
- )
- return await quart.render_template(
- "package-check.html",
- package=package,
- release=package.release,
- tasks=tasks,
- all_tasks_completed=all_tasks_completed,
- format_file_size=routes.format_file_size,
- )
-
-
[email protected]_route("/package/check/restart", methods=["POST"])
[email protected](auth.Requirements.committer)
-async def root_package_check_restart() -> response.Response:
- """Restart package verification tasks."""
- web_session = await session.read()
- if (web_session is None) or (web_session.uid is None):
- raise base.ASFQuartException("Not authenticated", errorcode=401)
-
- form = await routes.get_form(quart.request)
- artifact_sha3 = form.get("artifact_sha3")
- release_name = form.get("release_name")
-
- if not artifact_sha3 or not release_name:
- await quart.flash("Missing required parameters", "error")
- return quart.redirect(quart.url_for("root_candidate_review"))
-
- async with db.session() as data:
- async with data.begin():
- # Get the package and verify permissions
- try:
- package = await package_data_get(data, artifact_sha3,
release_name, web_session.uid)
- except routes.FlashError as e:
- logging.exception("FlashError:")
- await quart.flash(str(e), "error")
- return quart.redirect(quart.url_for("root_candidate_review"))
-
- # Check if package has any active tasks
- tasks, has_active_tasks = await task_package_status_get(data,
artifact_sha3)
- if has_active_tasks:
- await quart.flash("Cannot restart checks while tasks are still
in progress", "error")
- return quart.redirect(
- quart.url_for("root_package_check",
artifact_sha3=artifact_sha3, release_name=release_name)
- )
-
- # Delete existing tasks
- for task in tasks:
- await data.delete(task)
-
- try:
- await task_verification_create(data, package)
- except routes.FlashError as e:
- logging.exception("FlashError:")
- await quart.flash(str(e), "error")
- return quart.redirect(quart.url_for("root_candidate_review"))
-
- await quart.flash("Package checks restarted successfully",
"success")
- return quart.redirect(
- quart.url_for("root_package_check",
artifact_sha3=artifact_sha3, release_name=release_name)
- )
-
-
-async def task_package_status_get(data: db.Session, artifact_sha3: str) ->
tuple[Sequence[models.Task], bool]:
- """
- Get all tasks for a package and determine if any are still in progress.
- Returns tuple[Sequence[Task], bool]: List of tasks and whether any are
still in progress
- TODO: Could instead give active count and total count
- """
- tasks = await data.task(package_sha3=artifact_sha3).all()
- has_active_tasks = any(task.status in [models.TaskStatus.QUEUED,
models.TaskStatus.ACTIVE] for task in tasks)
- return tasks, has_active_tasks
-
-
-async def task_verification_create(data: db.Session, package: models.Package)
-> None:
- """Create verification tasks for a package."""
- # NOTE: A database session must be open when calling this function
- if not package.release or not package.release.committee:
- raise routes.FlashError("Could not determine committee for package")
-
- if package.signature_sha3 is None:
- raise routes.FlashError("Package has no signature")
-
- artifact_path = os.path.join("releases", package.artifact_sha3)
- signature_path = os.path.join("releases", package.signature_sha3)
- committee_name = package.release.committee.name
- artifact_tasks = await tasks.artifact_checks(
- artifact_path, signature_path=signature_path,
committee_name=committee_name
- )
- for task in artifact_tasks:
- data.add(task)
diff --git a/atr/server.py b/atr/server.py
index 5c011d6..6604c8d 100644
--- a/atr/server.py
+++ b/atr/server.py
@@ -64,7 +64,6 @@ def register_routes(app: base.QuartApp) -> tuple[str, ...]:
import atr.routes.download as download
import atr.routes.files as files
import atr.routes.keys as keys
- import atr.routes.package as package
import atr.routes.projects as projects
import atr.routes.release as release
import atr.routes.root as root
@@ -102,7 +101,6 @@ def register_routes(app: base.QuartApp) -> tuple[str, ...]:
download.__name__,
files.__name__,
keys.__name__,
- package.__name__,
projects.__name__,
release.__name__,
root.__name__,
diff --git a/atr/ssh.py b/atr/ssh.py
index 33585ff..9950f40 100644
--- a/atr/ssh.py
+++ b/atr/ssh.py
@@ -19,6 +19,7 @@
import asyncio
import asyncio.subprocess
+import datetime
import logging
import os
import string
@@ -31,6 +32,7 @@ import asyncssh
import atr.config as config
import atr.db as db
import atr.db.models as models
+import atr.tasks.rsync as rsync
import atr.user as user
import atr.util as util
@@ -284,25 +286,44 @@ async def _handle_client(process:
asyncssh.SSHServerProcess) -> None:
exit_status = await proc.wait()
# Start a task to process the new files
+ release_name = f"{project_name}-{release_version}"
async with db.session() as data:
- async with data.begin():
- from atr.tasks.rsync import Analyse
-
- data.add(
- models.Task(
- status=models.TaskStatus.QUEUED,
- task_type="rsync_analyse",
- task_args=Analyse(
- asf_uid=asf_uid,
- project_name=project_name,
- release_version=release_version,
- ).model_dump(),
- )
+ release = await data.release(name=release_name,
_committee=True).get()
+ # Create the release if it does not already exist
+ if release is None:
+ project = await data.project(name=project_name,
_committee=True).demand(
+ RuntimeError("Project not found")
)
+ release = models.Release(
+ name=release_name,
+ project_id=project.id,
+ project=project,
+ version=release_version,
+ stage=models.ReleaseStage.CANDIDATE,
+ phase=models.ReleasePhase.RELEASE_CANDIDATE,
+ created=datetime.datetime.now(),
+ )
+ data.add(release)
+ await data.commit()
+
+ # Add a task to analyse the new files
+ data.add(
+ models.Task(
+ status=models.TaskStatus.QUEUED,
+ task_type="rsync_analyse",
+ task_args=rsync.Analyse(
+ asf_uid=asf_uid,
+ project_name=project_name,
+ release_version=release_version,
+ ).model_dump(),
+ )
+ )
+ await data.commit()
+
# Exit the SSH process with the same status as the rsync process
process.exit(exit_status)
except Exception as e:
- _LOGGER.error(f"Error executing command: {e}")
+ _LOGGER.exception(f"Error executing command {process.command}")
process.stderr.write(f"Error: {e!s}\n")
process.exit(1)
diff --git a/atr/tasks/__init__.py b/atr/tasks/__init__.py
index 985ef2f..bd52b1d 100644
--- a/atr/tasks/__init__.py
+++ b/atr/tasks/__init__.py
@@ -24,15 +24,11 @@ import atr.tasks.archive as archive
import atr.util as util
-async def artifact_checks(
- path: str, signature_path: str | None = None, committee_name: str | None =
None
-) -> list[models.Task]:
+async def tar_gz_checks(release: models.Release, path: str, signature_path:
str | None = None) -> list[models.Task]:
# TODO: We should probably use an enum for task_type
- if path.startswith("releases/"):
- artifact_sha3 = path.split("/")[1]
- else:
- artifact_sha3 = await util.file_sha3(path)
+ full_path = str(util.get_candidate_draft_dir() / release.project.name /
release.version / path)
filename = os.path.basename(path)
+ modified = int(await aiofiles.os.path.getmtime(full_path))
if signature_path is None:
signature_path = path + ".asc"
if not (await aiofiles.os.path.exists(signature_path)):
@@ -42,52 +38,66 @@ async def artifact_checks(
models.Task(
status=models.TaskStatus.QUEUED,
task_type="verify_archive_integrity",
- task_args=archive.CheckIntegrity(path=path).model_dump(),
- package_sha3=artifact_sha3,
+ task_args=archive.CheckIntegrity(path=full_path).model_dump(),
+ release_name=release.name,
+ path=path,
+ modified=modified,
),
models.Task(
status=models.TaskStatus.QUEUED,
task_type="verify_archive_structure",
- task_args=[path, filename],
- package_sha3=artifact_sha3,
+ task_args=[full_path, filename],
+ release_name=release.name,
+ path=path,
+ modified=modified,
),
models.Task(
status=models.TaskStatus.QUEUED,
task_type="verify_license_files",
- task_args=[path],
- package_sha3=artifact_sha3,
+ task_args=[full_path],
+ release_name=release.name,
+ path=path,
+ modified=modified,
),
models.Task(
status=models.TaskStatus.QUEUED,
task_type="verify_license_headers",
- task_args=[path],
- package_sha3=artifact_sha3,
+ task_args=[full_path],
+ release_name=release.name,
+ path=path,
+ modified=modified,
),
models.Task(
status=models.TaskStatus.QUEUED,
task_type="verify_rat_license",
- task_args=[path],
- package_sha3=artifact_sha3,
+ task_args=[full_path],
+ release_name=release.name,
+ path=path,
+ modified=modified,
),
models.Task(
status=models.TaskStatus.QUEUED,
task_type="generate_cyclonedx_sbom",
- task_args=[path],
- package_sha3=artifact_sha3,
+ task_args=[full_path],
+ release_name=release.name,
+ path=path,
+ modified=modified,
),
]
- if signature_path and committee_name:
+ if signature_path and release.committee:
tasks.append(
models.Task(
status=models.TaskStatus.QUEUED,
task_type="verify_signature",
task_args=[
- committee_name,
- path,
+ release.committee.name,
+ full_path,
signature_path,
],
- package_sha3=artifact_sha3,
+ release_name=release.name,
+ path=path,
+ modified=modified,
),
)
diff --git a/atr/tasks/rsync.py b/atr/tasks/rsync.py
index 8ee50fa..80e18a9 100644
--- a/atr/tasks/rsync.py
+++ b/atr/tasks/rsync.py
@@ -20,12 +20,13 @@ from typing import Any, Final
import pydantic
-import atr.config as config
+import atr.db as db
import atr.db.models as models
+import atr.tasks as tasks
import atr.tasks.task as task
import atr.util as util
-_CONFIG: Final = config.get()
+# _CONFIG: Final = config.get()
_LOGGER: Final = logging.getLogger(__name__)
@@ -55,9 +56,17 @@ async def _analyse_core(asf_uid: str, project_name: str,
release_version: str) -
"""Analyse an rsync upload."""
base_path = util.get_candidate_draft_dir() / project_name / release_version
paths = await util.paths_recursive(base_path)
- # for path in paths:
- # # Add new tasks for each path
- # # We could use the SHA3 in input and output
- # # Or, less securely, we could use path and mtime instead
- # ...
- return {"paths": paths}
+ release_name = f"{project_name}-{release_version}"
+ async with db.session() as data:
+ release = await data.release(name=release_name,
_committee=True).demand(RuntimeError("Release not found"))
+ for path in paths:
+ # Add new tasks for each path
+ # We could use the SHA3 in input and output
+ # Or, less securely, we could use path and mtime instead
+ if not path.name.endswith(".tar.gz"):
+ continue
+ _LOGGER.info(f"Analyse {release_name} {path} {path!s}")
+ for task in await tasks.tar_gz_checks(release, str(path)):
+ data.add(task)
+ await data.commit()
+ return {"paths": [str(path) for path in paths]}
diff --git a/atr/templates/package-check.html b/atr/templates/files-check.html
similarity index 100%
rename from atr/templates/package-check.html
rename to atr/templates/files-check.html
diff --git a/atr/templates/files-list.html b/atr/templates/files-list.html
index bba4cbe..beede3f 100644
--- a/atr/templates/files-list.html
+++ b/atr/templates/files-list.html
@@ -59,6 +59,7 @@
<th>Path</th>
<th>Warnings</th>
<th>Errors</th>
+ <th>Tasks</th>
</tr>
</thead>
<tbody>
@@ -79,6 +80,14 @@
<td>
{% for error in errors[path] %}<div class="alert
alert-danger p-0 px-2 mt-2 mb-0">{{ error }}</div>{% endfor %}
</td>
+ <td>
+ {% if path in tasks %}
+ {% for task_type, task_result in tasks[path].items() %}
+ <pre>{{ task_type }}</pre>
+ <pre>{{ task_result }}</pre>
+ {% endfor %}
+ {% endif %}
+ </td>
</tr>
{% endfor %}
</tbody>
diff --git a/atr/templates/package-add.html b/atr/templates/package-add.html
deleted file mode 100644
index e164465..0000000
--- a/atr/templates/package-add.html
+++ /dev/null
@@ -1,278 +0,0 @@
-{% extends "layouts/base.html" %}
-
-{% block title %}
- Add a package ~ ATR
-{% endblock title %}
-
-{% block description %}
- Add a package to an existing release.
-{% endblock description %}
-
-{% block content %}
- <h1>Add package</h1>
- <p class="intro">
- Welcome, <strong>{{ asf_id }}</strong>! Use this form to add package
artifacts
- to an existing release candidate.
- </p>
-
- <form method="post" enctype="multipart/form-data" class="striking">
- <input type="hidden" name="form_type" value="single" />
- <table class="table">
- <tbody>
- <tr>
- <th class="text-end pe-3 align-top">
- <label for="release_name">Release:</label>
- </th>
- <td class="align-top">
- <select id="release_name"
- name="release_name"
- class="form-select mb-2"
- required>
- <option value="">Select a release...</option>
- {% for release in releases %}
- <option value="{{ release.name }}"
- {% if release.name == selected_release %}selected{%
endif %}>
- {{ release.committee.display_name }} - {{
release.project.name if release.project else "unknown" }} - {{ release.version
}}
- </option>
- {% endfor %}
- </select>
- {% if not releases %}<p class="text-danger mt-1">No releases found
that you can add a package to.</p>{% endif %}
- </td>
- </tr>
-
- <tr>
- <th class="text-end pe-3 align-top">
- <label for="release_artifact">Release candidate artifact:</label>
- </th>
- <td class="align-top">
- <input type="file"
- id="release_artifact"
- name="release_artifact"
- class="form-control mb-2"
- required
-
accept="application/gzip,application/x-gzip,application/x-tar,application/zip,application/java-archive,.tar.gz,.tgz,.zip,.jar"
- aria-describedby="artifact-help" />
- <span id="artifact-help" class="form-text text-muted">Upload the
release candidate archive (tar.gz, zip, or jar)</span>
- </td>
- </tr>
-
- <tr>
- <th class="text-end pe-3 align-top">
- <label>Artifact type:</label>
- </th>
- <td class="align-top">
- <div class="mb-2">
- <div class="form-check my-2">
- <input type="radio"
- class="form-check-input"
- id="source"
- name="artifact_type"
- value="source"
- required
- checked />
- <label class="form-check-label" for="source">Source
archive</label>
- </div>
- <div class="form-check my-2">
- <input type="radio"
- class="form-check-input"
- id="binary"
- name="artifact_type"
- value="binary" />
- <label class="form-check-label" for="binary">Binary
archive</label>
- </div>
- <div class="form-check my-2">
- <input type="radio"
- class="form-check-input"
- id="reproducible"
- name="artifact_type"
- value="reproducible" />
- <label class="form-check-label"
for="reproducible">Reproducible binary archive</label>
- </div>
- </div>
- </td>
- </tr>
-
- <tr>
- <th class="text-end pe-3 align-top">
- <label for="release_checksum">SHA2-512 hash file:</label>
- </th>
- <td class="align-top">
- <input type="file"
- id="release_checksum"
- name="release_checksum"
- class="form-control mb-2"
- accept=".sha512,.sha,.txt"
- aria-describedby="checksum-help" />
- <span id="checksum-help" class="form-text text-muted">Optional:
Upload the SHA-512 checksum file for verification</span>
- </td>
- </tr>
-
- <tr>
- <th class="text-end pe-3 align-top">
- <label for="release_signature">Detached signature:</label>
- </th>
- <td class="align-top">
- <input type="file"
- id="release_signature"
- name="release_signature"
- class="form-control mb-2"
- accept="application/pgp-signature,.asc"
- aria-describedby="signature-help" />
- <span id="signature-help" class="form-text text-muted">Optional:
Upload the detached GPG signature (.asc) file</span>
- </td>
- </tr>
-
- <tr>
- <td></td>
- <td>
- <button type="submit"
- class="btn btn-primary mt-3"
- {% if not releases %}disabled{% endif %}>Add
package</button>
- </td>
- </tr>
- </tbody>
- </table>
- </form>
-
- <div class="my-5 position-relative">
- <hr class="border-2 opacity-25" />
- <div class="position-absolute top-50 start-50 translate-middle bg-white
px-4 fs-5">
- Or add multiple packages from a URL
- </div>
- </div>
-
- <form method="post" class="striking">
- <input type="hidden" name="form_type" value="bulk" />
- <table class="table">
- <tbody>
- <tr>
- <th class="text-end pe-3 align-top">
- <label for="bulk_release_name">Release:</label>
- </th>
- <td class="align-top">
- <select id="bulk_release_name"
- name="release_name"
- class="form-select mb-2"
- required>
- <option value="">Select a release...</option>
- {% for release in releases %}
- <option value="{{ release.name }}"
- {% if release.name == selected_release %}selected{%
endif %}>
- {{ release.committee.display_name }} - {{
release.project.name if release.project else "unknown" }} - {{ release.version
}}
- </option>
- {% endfor %}
- </select>
- {% if not releases %}<p class="text-danger mt-1">No releases found
that you can add packages to.</p>{% endif %}
- </td>
- </tr>
-
- <tr>
- <th class="text-end pe-3 align-top">
- <label for="bulk_url">URL:</label>
- </th>
- <td class="align-top">
- <input type="url"
- id="bulk_url"
- name="url"
- class="form-control mb-2"
- required
- placeholder="https://example.org/path/to/packages/";
- aria-describedby="url-help" />
- <span id="url-help" class="form-text text-muted">Enter the URL of
the directory containing release packages</span>
- </td>
- </tr>
-
- <tr>
- <th class="text-end pe-3 align-top">
- <label>File types:</label>
- </th>
- <td class="align-top">
- <div class="mb-2">
- <div class="form-check my-2">
- <input type="checkbox"
- class="form-check-input"
- id="type_targz"
- name="file_types"
- value=".tar.gz"
- checked />
- <label class="form-check-label" for="type_targz">.tar.gz
files</label>
- </div>
- <div class="form-check my-2">
- <input type="checkbox"
- class="form-check-input"
- id="type_tgz"
- name="file_types"
- value=".tgz"
- checked />
- <label class="form-check-label" for="type_tgz">.tgz
files</label>
- </div>
- <div class="form-check my-2">
- <input type="checkbox"
- class="form-check-input"
- id="type_zip"
- name="file_types"
- value=".zip" />
- <label class="form-check-label" for="type_zip">.zip
files</label>
- </div>
- <div class="form-check my-2">
- <input type="checkbox"
- class="form-check-input"
- id="type_jar"
- name="file_types"
- value=".jar" />
- <label class="form-check-label" for="type_jar">.jar
files</label>
- </div>
- </div>
- </td>
- </tr>
-
- <tr>
- <th class="text-end pe-3 align-top">
- <label for="bulk_max_depth">Maximum depth:</label>
- </th>
- <td class="align-top">
- <input type="number"
- id="bulk_max_depth"
- name="max_depth"
- class="form-control mb-2 w-25"
- value="1"
- min="1"
- max="10"
- required
- aria-describedby="depth-help" />
- <span id="depth-help" class="form-text text-muted">Maximum request
depth to search for packages (1-10)</span>
- </td>
- </tr>
-
- <tr>
- <th class="text-end pe-3 align-top">
- <label for="bulk_require_signatures">Require signatures:</label>
- </th>
- <td class="align-top">
- <div class="mb-2">
- <div class="form-check my-2">
- <input type="checkbox"
- class="form-check-input"
- id="bulk_require_signatures"
- name="require_signatures"
- checked />
- <label class="form-check-label" for="bulk_require_signatures">
- Only download packages that have corresponding .asc
signature files
- </label>
- </div>
- </div>
- </td>
- </tr>
-
- <tr>
- <td></td>
- <td>
- <button type="submit"
- class="btn btn-primary mt-3"
- {% if not releases %}disabled{% endif %}>Add packages from
URL</button>
- </td>
- </tr>
- </tbody>
- </table>
- </form>
-{% endblock content %}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
