This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-release.git
The following commit(s) were added to refs/heads/main by this push:
new bdbf4d3 Remove external check caching to prepare for internal caching
bdbf4d3 is described below
commit bdbf4d3553d3a7c33b7c63a4d50e53dcf733d45f
Author: Sean B. Palmer <[email protected]>
AuthorDate: Mon Apr 7 18:51:04 2025 +0100
Remove external check caching to prepare for internal caching
---
atr/db/__init__.py | 38 +++++++++++++++++---------------------
atr/db/models.py | 2 --
atr/tasks/__init__.py | 35 +----------------------------------
3 files changed, 18 insertions(+), 57 deletions(-)
diff --git a/atr/db/__init__.py b/atr/db/__init__.py
index 9e81ee9..4a02c77 100644
--- a/atr/db/__init__.py
+++ b/atr/db/__init__.py
@@ -428,10 +428,6 @@ class Session(sqlalchemy.ext.asyncio.AsyncSession):
query = query.where(models.Task.error == error)
if is_defined(release_name):
query = query.where(models.Task.release_name == release_name)
- if is_defined(path):
- query = query.where(models.Task.path == path)
- if is_defined(modified):
- query = query.where(models.Task.modified == modified)
if _release:
query = query.options(select_in_load(models.Task.release))
@@ -550,23 +546,23 @@ async def create_async_engine(app_config:
type[config.AppConfig]) -> sqlalchemy.
return engine
-async def recent_tasks(data: Session, release_name: str, file_path: str,
modified: int) -> dict[str, models.Task]:
- """Get the most recent task for each task type for a specific file."""
- tasks = await data.task(
- release_name=release_name,
- path=str(file_path),
- modified=modified,
- ).all()
-
- # Group by task_type and keep the most recent one
- # We use the highest id to determine the most recent task
- recent_tasks: dict[str, models.Task] = {}
- for task in tasks:
- # If we haven't seen this task type before or if this task is newer
- if (task.task_type.value not in recent_tasks) or (task.id >
recent_tasks[task.task_type.value].id):
- recent_tasks[task.task_type.value] = task
-
- return recent_tasks
+# async def recent_tasks(data: Session, release_name: str, file_path: str,
modified: int) -> dict[str, models.Task]:
+# """Get the most recent task for each task type for a specific file."""
+# tasks = await data.task(
+# release_name=release_name,
+# path=str(file_path),
+# modified=modified,
+# ).all()
+#
+# # Group by task_type and keep the most recent one
+# # We use the highest id to determine the most recent task
+# recent_tasks: dict[str, models.Task] = {}
+# for task in tasks:
+# # If we haven't seen this task type before or if this task is newer
+# if (task.task_type.value not in recent_tasks) or (task.id >
recent_tasks[task.task_type.value].id):
+# recent_tasks[task.task_type.value] = task
+#
+# return recent_tasks
def select_in_load(*entities: Any) -> orm.strategy_options._AbstractLoad:
diff --git a/atr/db/models.py b/atr/db/models.py
index fb6f79b..0dc3da3 100644
--- a/atr/db/models.py
+++ b/atr/db/models.py
@@ -327,8 +327,6 @@ class Task(sqlmodel.SQLModel, table=True):
error: str | None = None
release_name: str | None = sqlmodel.Field(default=None,
foreign_key="release.name")
release: Optional["Release"] =
sqlmodel.Relationship(back_populates="tasks")
- path: str | None = sqlmodel.Field(default=None)
- modified: int | None = sqlmodel.Field(default=None)
# Create an index on status and added for efficient task claiming
__table_args__ = (
diff --git a/atr/tasks/__init__.py b/atr/tasks/__init__.py
index b5a6fd0..6ac2436 100644
--- a/atr/tasks/__init__.py
+++ b/atr/tasks/__init__.py
@@ -19,8 +19,6 @@ import contextlib
from collections.abc import Awaitable, Callable, Coroutine
from typing import Any, Final
-import aiofiles.os
-
import atr.db as db
import atr.db.models as models
import atr.tasks.checks as checks
@@ -42,7 +40,6 @@ async def asc_checks(release: models.Release, signature_path:
str) -> list[model
draft_dir = util.get_release_candidate_draft_dir() / release.project.name
/ release.version
full_signature_path = str(draft_dir / signature_path)
- modified = int(await aiofiles.os.path.getmtime(full_signature_path))
if release.committee:
tasks.append(
@@ -55,8 +52,6 @@ async def asc_checks(release: models.Release, signature_path:
str) -> list[model
abs_signature_path=full_signature_path,
).model_dump(),
release_name=release.name,
- path=signature_path,
- modified=modified,
),
)
@@ -74,15 +69,10 @@ async def draft_checks(project_name: str, release_version:
str, caller_data: db.
RuntimeError("Release not found")
)
for path in paths_recursive:
- full_path = base_path / path
- modified = int(await aiofiles.os.path.getmtime(full_path))
- cached_tasks = await db.recent_tasks(data, release.name,
str(path), modified)
-
for task_type, task_function in TASK_FUNCTIONS.items():
if path.name.endswith(task_type):
for task in await task_function(release, str(path)):
- if task.task_type not in cached_tasks:
- data.add(task)
+ data.add(task)
path_check_task = models.Task(
status=models.TaskStatus.QUEUED,
@@ -141,7 +131,6 @@ async def sha_checks(release: models.Release, hash_file:
str) -> list[models.Tas
full_hash_file_path = str(
util.get_release_candidate_draft_dir() / release.project.name /
release.version / hash_file
)
- modified = int(await aiofiles.os.path.getmtime(full_hash_file_path))
algorithm = "sha512"
if hash_file.endswith(".sha512"):
original_file = full_hash_file_path.removesuffix(".sha512")
@@ -162,8 +151,6 @@ async def sha_checks(release: models.Release, hash_file:
str) -> list[models.Tas
algorithm=algorithm,
).model_dump(),
release_name=release.name,
- path=hash_file,
- modified=modified,
),
)
@@ -174,7 +161,6 @@ async def tar_gz_checks(release: models.Release, path: str)
-> list[models.Task]
# TODO: We should probably use an enum for task_type
full_path = str(util.get_release_candidate_draft_dir() /
release.project.name / release.version / path)
# filename = os.path.basename(path)
- modified = int(await aiofiles.os.path.getmtime(full_path))
tasks = [
models.Task(
@@ -182,40 +168,30 @@ async def tar_gz_checks(release: models.Release, path:
str) -> list[models.Task]
task_type=models.TaskType.LICENSE_FILES,
task_args=license.Files(release_name=release.name,
abs_path=full_path).model_dump(),
release_name=release.name,
- path=path,
- modified=modified,
),
models.Task(
status=models.TaskStatus.QUEUED,
task_type=models.TaskType.LICENSE_HEADERS,
task_args=license.Headers(release_name=release.name,
abs_path=full_path).model_dump(),
release_name=release.name,
- path=path,
- modified=modified,
),
models.Task(
status=models.TaskStatus.QUEUED,
task_type=models.TaskType.RAT_CHECK,
task_args=rat.Check(release_name=release.name,
abs_path=full_path).model_dump(),
release_name=release.name,
- path=path,
- modified=modified,
),
models.Task(
status=models.TaskStatus.QUEUED,
task_type=models.TaskType.TARGZ_INTEGRITY,
task_args=targz.Integrity(release_name=release.name,
abs_path=full_path).model_dump(),
release_name=release.name,
- path=path,
- modified=modified,
),
models.Task(
status=models.TaskStatus.QUEUED,
task_type=models.TaskType.TARGZ_STRUCTURE,
task_args=checks.ReleaseAndAbsPath(release_name=release.name,
abs_path=full_path).model_dump(),
release_name=release.name,
- path=path,
- modified=modified,
),
]
@@ -225,7 +201,6 @@ async def tar_gz_checks(release: models.Release, path: str)
-> list[models.Task]
async def zip_checks(release: models.Release, path: str) -> list[models.Task]:
"""Create check tasks for a .zip file."""
full_path = str(util.get_release_candidate_draft_dir() /
release.project.name / release.version / path)
- modified = int(await aiofiles.os.path.getmtime(full_path))
tasks = [
models.Task(
@@ -233,32 +208,24 @@ async def zip_checks(release: models.Release, path: str)
-> list[models.Task]:
task_type=models.TaskType.ZIPFORMAT_INTEGRITY,
task_args=checks.ReleaseAndAbsPath(release_name=release.name,
abs_path=full_path).model_dump(),
release_name=release.name,
- path=path,
- modified=modified,
),
models.Task(
status=models.TaskStatus.QUEUED,
task_type=models.TaskType.ZIPFORMAT_LICENSE_FILES,
task_args=checks.ReleaseAndAbsPath(release_name=release.name,
abs_path=full_path).model_dump(),
release_name=release.name,
- path=path,
- modified=modified,
),
models.Task(
status=models.TaskStatus.QUEUED,
task_type=models.TaskType.ZIPFORMAT_LICENSE_HEADERS,
task_args=checks.ReleaseAndAbsPath(release_name=release.name,
abs_path=full_path).model_dump(),
release_name=release.name,
- path=path,
- modified=modified,
),
models.Task(
status=models.TaskStatus.QUEUED,
task_type=models.TaskType.ZIPFORMAT_STRUCTURE,
task_args=checks.ReleaseAndAbsPath(release_name=release.name,
abs_path=full_path).model_dump(),
release_name=release.name,
- path=path,
- modified=modified,
),
]
return tasks
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
