This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-release.git
The following commit(s) were added to refs/heads/main by this push:
new 6b1dc4a Allow one task to specify multiple results
6b1dc4a is described below
commit 6b1dc4aee1367f880e5f09d783a0c01a9f10fc85
Author: Sean B. Palmer <[email protected]>
AuthorDate: Fri Mar 28 20:25:12 2025 +0200
Allow one task to specify multiple results
---
atr/blueprints/admin/admin.py | 1 +
atr/db/__init__.py | 37 +++++++++++++++++++
atr/db/models.py | 22 +++++++++++
atr/tasks/archive.py | 11 ++++++
atr/tasks/task.py | 85 ++++++++++++++++++++++++++++++++++++++++++-
atr/util.py | 13 +++++++
6 files changed, 168 insertions(+), 1 deletion(-)
diff --git a/atr/blueprints/admin/admin.py b/atr/blueprints/admin/admin.py
index f6c501e..267bb69 100644
--- a/atr/blueprints/admin/admin.py
+++ b/atr/blueprints/admin/admin.py
@@ -128,6 +128,7 @@ async def admin_data(model: str = "Committee") -> str:
# Map of model names to their classes
# TODO: Add distribution channel, key link, and any others
model_methods: dict[str, Callable[[], db.Query[Any]]] = {
+ "CheckResult": data.check_result,
"Committee": data.committee,
"Package": data.package,
"Project": data.project,
diff --git a/atr/db/__init__.py b/atr/db/__init__.py
index 55845fa..cda85c4 100644
--- a/atr/db/__init__.py
+++ b/atr/db/__init__.py
@@ -114,6 +114,43 @@ class Query(Generic[T]):
class Session(sqlalchemy.ext.asyncio.AsyncSession):
# TODO: Need to type all of these arguments correctly
+
+ def check_result(
+ self,
+ id: Opt[int] = NotSet,
+ release_name: Opt[str] = NotSet,
+ checker: Opt[str] = NotSet,
+ path: Opt[str | None] = NotSet,
+ created: Opt[datetime.datetime] = NotSet,
+ status: Opt[models.CheckResultStatus] = NotSet,
+ message: Opt[str] = NotSet,
+ data: Opt[Any] = NotSet,
+ _release: bool = False,
+ ) -> Query[models.CheckResult]:
+ query = sqlmodel.select(models.CheckResult)
+
+ if is_defined(id):
+ query = query.where(models.CheckResult.id == id)
+ if is_defined(release_name):
+ query = query.where(models.CheckResult.release_name ==
release_name)
+ if is_defined(checker):
+ query = query.where(models.CheckResult.checker == checker)
+ if is_defined(path):
+ query = query.where(models.CheckResult.path == path)
+ if is_defined(created):
+ query = query.where(models.CheckResult.created == created)
+ if is_defined(status):
+ query = query.where(models.CheckResult.status == status)
+ if is_defined(message):
+ query = query.where(models.CheckResult.message == message)
+ if is_defined(data):
+ query = query.where(models.CheckResult.data == data)
+
+ if _release:
+ query = query.options(select_in_load(models.CheckResult.release))
+
+ return Query(self, query)
+
def committee(
self,
id: Opt[int] = NotSet,
diff --git a/atr/db/models.py b/atr/db/models.py
index 689f6e2..e27b4a6 100644
--- a/atr/db/models.py
+++ b/atr/db/models.py
@@ -329,6 +329,9 @@ class Release(sqlmodel.SQLModel, table=True):
back_populates="release", sa_relationship_kwargs={"cascade": "all,
delete-orphan"}
)
+ # One-to-many: A release can have multiple check results
+ check_results: list["CheckResult"] =
sqlmodel.Relationship(back_populates="release")
+
# The combination of project_id and version must be unique
# Technically we want (project.name, version) to be unique
# But project.name is already unique, so project_id works as a proxy
thereof
@@ -353,3 +356,22 @@ class SSHKey(sqlmodel.SQLModel, table=True):
fingerprint: str = sqlmodel.Field(primary_key=True)
key: str
asf_uid: str
+
+
+class CheckResultStatus(str, enum.Enum):
+ EXCEPTION = "exception"
+ FAILURE = "failure"
+ SUCCESS = "success"
+ WARNING = "warning"
+
+
+class CheckResult(sqlmodel.SQLModel, table=True):
+ id: int = sqlmodel.Field(default=None, primary_key=True)
+ release_name: str = sqlmodel.Field(foreign_key="release.name")
+ release: Release = sqlmodel.Relationship(back_populates="check_results")
+ checker: str
+ path: str | None = None
+ created: datetime.datetime
+ status: CheckResultStatus
+ message: str
+ data: Any = sqlmodel.Field(sa_column=sqlalchemy.Column(sqlalchemy.JSON))
diff --git a/atr/tasks/archive.py b/atr/tasks/archive.py
index 493d367..6e70b67 100644
--- a/atr/tasks/archive.py
+++ b/atr/tasks/archive.py
@@ -25,6 +25,7 @@ import pydantic
import atr.db.models as models
import atr.tasks.task as task
+import atr.util as util
_LOGGER: Final = logging.getLogger(__name__)
@@ -42,6 +43,16 @@ async def check_integrity(args: dict[str, Any]) ->
tuple[models.TaskStatus, str
# Then we can have a single task wrapper for all tasks
# TODO: We should use task.TaskError as standard, and maybe typeguard each
function
data = CheckIntegrity(**args)
+ # TODO: Check arguments should have release_name and path as standard
+ # Followed by any necessary additional arguments
+ release_name, rel_path = util.abs_path_to_release_and_rel_path(data.path)
+ check = await task.Check.create(checker=check_integrity,
release_name=release_name, path=rel_path)
+ try:
+ size = await asyncio.to_thread(_check_integrity_core, data.path,
data.chunk_size)
+ await check.success("Able to read all entries of the archive using
tarfile", {"size": size})
+ except Exception as e:
+ await check.failure("Unable to read all entries of the archive using
tarfile", {"error": str(e)})
+ return task.FAILED, str(e), ()
task_results = task.results_as_tuple(await
asyncio.to_thread(_check_integrity_core, data.path, data.chunk_size))
_LOGGER.info(f"Verified {data.path} and computed size {task_results[0]}")
return task.COMPLETED, None, task_results
diff --git a/atr/tasks/task.py b/atr/tasks/task.py
index 78dbb74..833e48a 100644
--- a/atr/tasks/task.py
+++ b/atr/tasks/task.py
@@ -15,10 +15,19 @@
# specific language governing permissions and limitations
# under the License.
-from typing import Any, Final
+from __future__ import annotations
+import datetime
+from typing import TYPE_CHECKING, Any, Final
+
+import sqlmodel
+
+import atr.db as db
import atr.db.models as models
+if TYPE_CHECKING:
+ from collections.abc import Callable
+
QUEUED: Final = models.TaskStatus.QUEUED
ACTIVE: Final = models.TaskStatus.ACTIVE
COMPLETED: Final = models.TaskStatus.COMPLETED
@@ -33,6 +42,80 @@ class Error(Exception):
self.result = tuple(result)
+class Check:
+ def __init__(
+ self, checker: Callable[..., Any], release_name: str, path: str | None
= None, afresh: bool = True
+ ) -> None:
+ self.checker = checker.__module__ + "." + checker.__name__
+ self.release_name = release_name
+ self.path = path
+ self.afresh = afresh
+ self._constructed = False
+
+ @classmethod
+ async def create(
+ cls, checker: Callable[..., Any], release_name: str, path: str | None
= None, afresh: bool = True
+ ) -> Check:
+ check = cls(checker, release_name, path, afresh)
+ if afresh is True:
+ # Clear outer path whether it's specified or not
+ await check._clear(path)
+ check._constructed = True
+ return check
+
+ async def _add(
+ self, status: models.CheckResultStatus, message: str, data: Any, path:
str | None = None
+ ) -> models.CheckResult:
+ if self._constructed is False:
+ raise RuntimeError("Cannot add check result to a check that has
not been constructed")
+ if path is not None:
+ if self.path is not None:
+ raise ValueError("Cannot specify path twice")
+ if self.afresh is True:
+ # Clear inner path only if it's specified
+ await self._clear(path)
+
+ result = models.CheckResult(
+ release_name=self.release_name,
+ checker=self.checker,
+ path=path or self.path,
+ created=datetime.datetime.now(),
+ status=status,
+ message=message,
+ data=data,
+ )
+
+ # It would be more efficient to keep a session open
+ # But, we prefer in this case to maintain a simpler interface
+ # If performance is unacceptable, we can revisit this design
+ async with db.session() as session:
+ session.add(result)
+ await session.commit()
+ return result
+
+ async def _clear(self, path: str | None = None) -> None:
+ async with db.session() as data:
+ stmt = sqlmodel.delete(models.CheckResult).where(
+
db.validate_instrumented_attribute(models.CheckResult.release_name) ==
self.release_name,
+ db.validate_instrumented_attribute(models.CheckResult.checker)
== self.checker,
+ db.validate_instrumented_attribute(models.CheckResult.path) ==
path,
+ )
+ await data.execute(stmt)
+ await data.commit()
+
+ async def exception(self, message: str, data: Any, path: str | None =
None) -> models.CheckResult:
+ return await self._add(models.CheckResultStatus.EXCEPTION, message,
data, path=path)
+
+ async def failure(self, message: str, data: Any, path: str | None = None)
-> models.CheckResult:
+ return await self._add(models.CheckResultStatus.FAILURE, message,
data, path=path)
+
+ async def success(self, message: str, data: Any, path: str | None = None)
-> models.CheckResult:
+ return await self._add(models.CheckResultStatus.SUCCESS, message,
data, path=path)
+
+ async def warning(self, message: str, data: Any, path: str | None = None)
-> models.CheckResult:
+ return await self._add(models.CheckResultStatus.WARNING, message,
data, path=path)
+
+
def results_as_tuple(item: Any) -> tuple[Any, ...]:
"""Ensure that returned results are structured as a tuple."""
if not isinstance(item, tuple):
diff --git a/atr/util.py b/atr/util.py
index 539b1b0..260df0c 100644
--- a/atr/util.py
+++ b/atr/util.py
@@ -92,6 +92,19 @@ class QuartFormTyped(quart_wtf.QuartForm):
return form
+def abs_path_to_release_and_rel_path(abs_path: str) -> tuple[str, str]:
+ """Return the release name and relative path for a given path."""
+ conf = config.get()
+ phase_dir = pathlib.Path(conf.PHASE_STORAGE_DIR)
+ phase_sub_dir = pathlib.Path(abs_path).relative_to(phase_dir)
+ # Skip the first component, which is the phase name
+ # The next two components are the project name and version name
+ project_name = phase_sub_dir.parts[1]
+ version_name = phase_sub_dir.parts[2]
+ release_name = f"{project_name}-{version_name}"
+ return release_name, str(pathlib.Path(*phase_sub_dir.parts[3:]))
+
+
def as_url(func: Callable, **kwargs: Any) -> str:
"""Return the URL for a function."""
return quart.url_for(func.__annotations__["endpoint"], **kwargs)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
