This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch sbp
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
The following commit(s) were added to refs/heads/sbp by this push:
new ece6e9d4 Fix more function ordering by improving the order fixing
script
ece6e9d4 is described below
commit ece6e9d425c625588e4bc009d4ec33fd0510f6b2
Author: Sean B. Palmer <[email protected]>
AuthorDate: Mon Feb 23 20:33:45 2026 +0000
Fix more function ordering by improving the order fixing script
---
atr/admin/__init__.py | 10 ++--
atr/archives.py | 74 +++++++++++------------
atr/attestable.py | 92 ++++++++++++++--------------
atr/datasources/apache.py | 22 +++----
atr/db/__init__.py | 19 ------
atr/db/interaction.py | 42 ++++++-------
atr/hashes.py | 16 ++---
atr/log.py | 40 +++----------
atr/models/tabulate.py | 58 ++++++++----------
atr/post/draft.py | 54 ++++++++---------
atr/principal.py | 28 ++++-----
atr/sbom/osv.py | 36 +++++------
atr/sbom/tool.py | 42 ++++++-------
atr/sbom/utilities.py | 54 ++++++++---------
atr/storage/__init__.py | 67 ++++++++++-----------
atr/storage/writers/checks.py | 14 ++---
atr/svn/__init__.py | 80 ++++++++++++-------------
atr/tasks/__init__.py | 118 ++++++++++++++++++------------------
atr/tasks/checks/__init__.py | 12 ++--
atr/tasks/gha.py | 135 +++++++++++++++++-------------------------
atr/tasks/sbom.py | 36 +++++------
atr/template.py | 6 +-
atr/validate.py | 74 +++++++++++------------
scripts/fix_order.py | 35 ++++++-----
scripts/fix_order.sh | 3 +-
25 files changed, 547 insertions(+), 620 deletions(-)
diff --git a/atr/admin/__init__.py b/atr/admin/__init__.py
index 21f95172..27dd52d2 100644
--- a/atr/admin/__init__.py
+++ b/atr/admin/__init__.py
@@ -577,11 +577,6 @@ async def logs(session: web.Committer) ->
web.QuartResponse:
return web.TextResponse("\n".join(recent_logs))
[email protected]("/raise-error")
-async def raise_error(session: web.Committer) -> web.QuartResponse:
- raise RuntimeError("Admin test route deliberately raised an unhandled
error")
-
-
@admin.get("/ongoing-tasks/<project_name>/<version_name>/<revision>")
async def ongoing_tasks_get(
session: web.Committer, project_name: str, version_name: str, revision: str
@@ -706,6 +701,11 @@ async def projects_update_post(session: web.Committer) ->
str | web.WerkzeugResp
}, 200
[email protected]("/raise-error")
+async def raise_error(session: web.Committer) -> web.QuartResponse:
+ raise RuntimeError("Admin test route deliberately raised an unhandled
error")
+
+
@admin.get("/revoke-user-tokens")
async def revoke_user_tokens_get(session: web.Committer) -> str:
"""Revoke all Personal Access Tokens for a specified user."""
diff --git a/atr/archives.py b/atr/archives.py
index 0ed69824..6b71ae34 100644
--- a/atr/archives.py
+++ b/atr/archives.py
@@ -91,6 +91,43 @@ def total_size(tgz_path: str, chunk_size: int = 4096) -> int:
return total_size
+def _safe_path(base_dir: str, *paths: str) -> str | None:
+ """Return an absolute path within the base_dir built from the given paths,
or None if it escapes."""
+ target = os.path.abspath(os.path.join(base_dir, *paths))
+ abs_base = os.path.abspath(base_dir)
+ if (target == abs_base) or target.startswith(abs_base + os.sep):
+ return target
+ return None
+
+
+def _size_tar(archive: tarzip.Archive, chunk_size: int) -> int:
+ total_size = 0
+ for member in archive:
+ if not isinstance(member, tarzip.TarMember):
+ continue
+ total_size += member.size
+ if member.isfile():
+ fileobj = archive.extractfile(member)
+ if fileobj is not None:
+ while fileobj.read(chunk_size):
+ pass
+ return total_size
+
+
+def _size_zip(archive: tarzip.Archive, chunk_size: int) -> int:
+ total_size = 0
+ for member in archive:
+ if not isinstance(member, tarzip.ZipMember):
+ continue
+ total_size += member.size
+ if member.isfile():
+ fileobj = archive.extractfile(member)
+ if fileobj is not None:
+ while fileobj.read(chunk_size):
+ pass
+ return total_size
+
+
def _tar_archive_extract_member( # noqa: C901
archive: tarzip.Archive,
member: tarzip.TarMember,
@@ -242,43 +279,6 @@ def _tar_archive_extract_safe_process_symlink(member:
tarzip.TarMember, extract_
log.warning(f"Failed to create symlink {target_path} -> {link_target}:
{e}")
-def _safe_path(base_dir: str, *paths: str) -> str | None:
- """Return an absolute path within the base_dir built from the given paths,
or None if it escapes."""
- target = os.path.abspath(os.path.join(base_dir, *paths))
- abs_base = os.path.abspath(base_dir)
- if (target == abs_base) or target.startswith(abs_base + os.sep):
- return target
- return None
-
-
-def _size_tar(archive: tarzip.Archive, chunk_size: int) -> int:
- total_size = 0
- for member in archive:
- if not isinstance(member, tarzip.TarMember):
- continue
- total_size += member.size
- if member.isfile():
- fileobj = archive.extractfile(member)
- if fileobj is not None:
- while fileobj.read(chunk_size):
- pass
- return total_size
-
-
-def _size_zip(archive: tarzip.Archive, chunk_size: int) -> int:
- total_size = 0
- for member in archive:
- if not isinstance(member, tarzip.ZipMember):
- continue
- total_size += member.size
- if member.isfile():
- fileobj = archive.extractfile(member)
- if fileobj is not None:
- while fileobj.read(chunk_size):
- pass
- return total_size
-
-
def _zip_archive_extract_member(
archive: tarzip.Archive,
member: tarzip.ZipMember,
diff --git a/atr/attestable.py b/atr/attestable.py
index 9f80690d..91a65470 100644
--- a/atr/attestable.py
+++ b/atr/attestable.py
@@ -33,6 +33,10 @@ if TYPE_CHECKING:
import pathlib
+def attestable_checks_path(project_name: str, version_name: str,
revision_number: str) -> pathlib.Path:
+ return util.get_attestable_dir() / project_name / version_name /
f"{revision_number}.checks.json"
+
+
def attestable_path(project_name: str, version_name: str, revision_number:
str) -> pathlib.Path:
return util.get_attestable_dir() / project_name / version_name /
f"{revision_number}.json"
@@ -45,10 +49,6 @@ def github_tp_payload_path(project_name: str, version_name:
str, revision_number
return util.get_attestable_dir() / project_name / version_name /
f"{revision_number}.github-tp.json"
-def attestable_checks_path(project_name: str, version_name: str,
revision_number: str) -> pathlib.Path:
- return util.get_attestable_dir() / project_name / version_name /
f"{revision_number}.checks.json"
-
-
async def github_tp_payload_write(
project_name: str, version_name: str, revision_number: str,
github_payload: dict[str, Any]
) -> None:
@@ -73,26 +73,6 @@ async def load(
return None
-async def load_paths(
- project_name: str,
- version_name: str,
- revision_number: str,
-) -> dict[str, str] | None:
- file_path = attestable_paths_path(project_name, version_name,
revision_number)
- if await aiofiles.os.path.isfile(file_path):
- try:
- async with aiofiles.open(file_path, encoding="utf-8") as f:
- data = json.loads(await f.read())
- return models.AttestablePathsV1.model_validate(data).paths
- except (json.JSONDecodeError, pydantic.ValidationError) as e:
- # log.warning(f"Could not parse {file_path}, trying combined file:
{e}")
- log.warning(f"Could not parse {file_path}: {e}")
- # combined = await load(project_name, version_name, revision_number)
- # if combined is not None:
- # return combined.paths
- return None
-
-
async def load_checks(
project_name: str,
version_name: str,
@@ -115,6 +95,26 @@ async def load_checks(
return {}
+async def load_paths(
+ project_name: str,
+ version_name: str,
+ revision_number: str,
+) -> dict[str, str] | None:
+ file_path = attestable_paths_path(project_name, version_name,
revision_number)
+ if await aiofiles.os.path.isfile(file_path):
+ try:
+ async with aiofiles.open(file_path, encoding="utf-8") as f:
+ data = json.loads(await f.read())
+ return models.AttestablePathsV1.model_validate(data).paths
+ except (json.JSONDecodeError, pydantic.ValidationError) as e:
+ # log.warning(f"Could not parse {file_path}, trying combined file:
{e}")
+ log.warning(f"Could not parse {file_path}: {e}")
+ # combined = await load(project_name, version_name, revision_number)
+ # if combined is not None:
+ # return combined.paths
+ return None
+
+
def migrate_to_paths_files() -> int:
attestable_dir = util.get_attestable_dir()
if not attestable_dir.is_dir():
@@ -161,28 +161,6 @@ async def paths_to_hashes_and_sizes(directory:
pathlib.Path) -> tuple[dict[str,
return path_to_hash, path_to_size
-async def write_files_data(
- project_name: str,
- version_name: str,
- revision_number: str,
- release_policy: dict[str, Any] | None,
- uploader_uid: str,
- previous: models.AttestableV1 | None,
- path_to_hash: dict[str, str],
- path_to_size: dict[str, int],
-) -> None:
- result = _generate_files_data(path_to_hash, path_to_size, revision_number,
release_policy, uploader_uid, previous)
- file_path = attestable_path(project_name, version_name, revision_number)
- await util.atomic_write_file(file_path, result.model_dump_json(indent=2))
- paths_result = models.AttestablePathsV1(paths=result.paths)
- paths_file_path = attestable_paths_path(project_name, version_name,
revision_number)
- await util.atomic_write_file(paths_file_path,
paths_result.model_dump_json(indent=2))
- checks_file_path = attestable_checks_path(project_name, version_name,
revision_number)
- if not checks_file_path.exists():
- async with aiofiles.open(checks_file_path, "w", encoding="utf-8") as f:
- await
f.write(models.AttestableChecksV2().model_dump_json(indent=2))
-
-
async def write_checks_data(
project_name: str,
version_name: str,
@@ -207,6 +185,28 @@ async def write_checks_data(
await util.atomic_modify_file(attestable_checks_path(project_name,
version_name, revision_number), modify)
+async def write_files_data(
+ project_name: str,
+ version_name: str,
+ revision_number: str,
+ release_policy: dict[str, Any] | None,
+ uploader_uid: str,
+ previous: models.AttestableV1 | None,
+ path_to_hash: dict[str, str],
+ path_to_size: dict[str, int],
+) -> None:
+ result = _generate_files_data(path_to_hash, path_to_size, revision_number,
release_policy, uploader_uid, previous)
+ file_path = attestable_path(project_name, version_name, revision_number)
+ await util.atomic_write_file(file_path, result.model_dump_json(indent=2))
+ paths_result = models.AttestablePathsV1(paths=result.paths)
+ paths_file_path = attestable_paths_path(project_name, version_name,
revision_number)
+ await util.atomic_write_file(paths_file_path,
paths_result.model_dump_json(indent=2))
+ checks_file_path = attestable_checks_path(project_name, version_name,
revision_number)
+ if not checks_file_path.exists():
+ async with aiofiles.open(checks_file_path, "w", encoding="utf-8") as f:
+ await
f.write(models.AttestableChecksV2().model_dump_json(indent=2))
+
+
def _compute_hashes_with_attribution(
current_hash_to_paths: dict[str, set[str]],
path_to_size: dict[str, int],
diff --git a/atr/datasources/apache.py b/atr/datasources/apache.py
index f1dfcd37..0db6c0e8 100644
--- a/atr/datasources/apache.py
+++ b/atr/datasources/apache.py
@@ -321,17 +321,6 @@ async def update_metadata() -> tuple[int, int]:
return added_count, updated_count
-def _project_status(pmc: sql.Committee, project_name: str, project_status:
ProjectStatus) -> sql.ProjectStatus:
- if pmc.name == "attic":
- # This must come first, because attic is also a standing committee
- return sql.ProjectStatus.RETIRED
- elif ("_dormant_" in project_name) or
project_status.name.endswith("(Dormant)"):
- return sql.ProjectStatus.DORMANT
- elif util.committee_is_standing(pmc.name):
- return sql.ProjectStatus.STANDING
- return sql.ProjectStatus.ACTIVE
-
-
async def _process_undiscovered(data: db.Session) -> tuple[int, int]:
added_count = 0
updated_count = 0
@@ -358,6 +347,17 @@ async def _process_undiscovered(data: db.Session) ->
tuple[int, int]:
return added_count, updated_count
+def _project_status(pmc: sql.Committee, project_name: str, project_status:
ProjectStatus) -> sql.ProjectStatus:
+ if pmc.name == "attic":
+ # This must come first, because attic is also a standing committee
+ return sql.ProjectStatus.RETIRED
+ elif ("_dormant_" in project_name) or
project_status.name.endswith("(Dormant)"):
+ return sql.ProjectStatus.DORMANT
+ elif util.committee_is_standing(pmc.name):
+ return sql.ProjectStatus.STANDING
+ return sql.ProjectStatus.ACTIVE
+
+
async def _update_committees(
data: db.Session, ldap_projects: LDAPProjectsData, committees_by_name:
Mapping[str, Committee]
) -> tuple[int, int]:
diff --git a/atr/db/__init__.py b/atr/db/__init__.py
index 76df7d4e..97b1f967 100644
--- a/atr/db/__init__.py
+++ b/atr/db/__init__.py
@@ -951,25 +951,6 @@ def log_queries() -> Iterator[None]:
global_log_query = original_global_log_query_state
-# async def recent_tasks(data: Session, release_name: str, file_path: str,
modified: int) -> dict[str, models.Task]:
-# """Get the most recent task for each task type for a specific file."""
-# tasks = await data.task(
-# release_name=release_name,
-# path=str(file_path),
-# modified=modified,
-# ).all()
-#
-# # Group by task_type and keep the most recent one
-# # We use the highest id to determine the most recent task
-# recent_tasks: dict[str, models.Task] = {}
-# for task in tasks:
-# # If we haven't seen this task type before or if this task is newer
-# if (task.task_type.value not in recent_tasks) or (task.id >
recent_tasks[task.task_type.value].id):
-# recent_tasks[task.task_type.value] = task
-#
-# return recent_tasks
-
-
def select_in_load(*entities: Any) -> orm.strategy_options._AbstractLoad:
"""Eagerly load the given entities from the query."""
validated_entities = []
diff --git a/atr/db/interaction.py b/atr/db/interaction.py
index bb3ff3ac..6e7e17a5 100644
--- a/atr/db/interaction.py
+++ b/atr/db/interaction.py
@@ -192,6 +192,27 @@ async def checks_for(
return list(check_results)
+async def count_checks_for_revision_by_status(
+ status: sql.CheckResultStatus, release: sql.Release, revision_number: str,
caller_data: db.Session | None = None
+):
+ file_path_checks = await attestable.load_checks(release.project_name,
release.version, revision_number)
+ check_hashes = [h for inner in file_path_checks.values() for h in
inner.values()]
+ if len(check_hashes) == 0:
+ return 0
+ async with db.ensure_session(caller_data) as data:
+ via = sql.validate_instrumented_attribute
+ query = (
+ sqlmodel.select(sqlalchemy.func.count())
+ .select_from(sql.CheckResult)
+ .where(
+ via(sql.CheckResult.inputs_hash).in_(check_hashes),
+ sql.CheckResult.status == status,
+ )
+ )
+ result = await data.execute(query)
+ return result.scalar_one()
+
+
@contextlib.asynccontextmanager
async def ephemeral_gpg_home() -> AsyncGenerator[str]:
"""Create a temporary directory for an isolated GPG home, and clean it up
on exit."""
@@ -550,27 +571,6 @@ async def wait_for_task(
return False
-async def count_checks_for_revision_by_status(
- status: sql.CheckResultStatus, release: sql.Release, revision_number: str,
caller_data: db.Session | None = None
-):
- file_path_checks = await attestable.load_checks(release.project_name,
release.version, revision_number)
- check_hashes = [h for inner in file_path_checks.values() for h in
inner.values()]
- if len(check_hashes) == 0:
- return 0
- async with db.ensure_session(caller_data) as data:
- via = sql.validate_instrumented_attribute
- query = (
- sqlmodel.select(sqlalchemy.func.count())
- .select_from(sql.CheckResult)
- .where(
- via(sql.CheckResult.inputs_hash).in_(check_hashes),
- sql.CheckResult.status == status,
- )
- )
- result = await data.execute(query)
- return result.scalar_one()
-
-
async def _trusted_project(repository: str, workflow_ref: str, phase:
TrustedProjectPhase) -> sql.Project:
# Debugging
log.info(f"GitHub OIDC JWT payload: {repository} {workflow_ref}")
diff --git a/atr/hashes.py b/atr/hashes.py
index 2970e086..de29a760 100644
--- a/atr/hashes.py
+++ b/atr/hashes.py
@@ -25,6 +25,14 @@ import blake3
_HASH_CHUNK_SIZE: Final[int] = 4 * 1024 * 1024
+def compute_dict_hash(to_hash: dict[Any, Any]) -> str:
+ hasher = blake3.blake3()
+ for k in sorted(to_hash.keys()):
+ hasher.update(str(k).encode("utf-8"))
+ hasher.update(str(to_hash[k]).encode("utf-8"))
+ return f"blake3:{hasher.hexdigest()}"
+
+
async def compute_file_hash(path: str | pathlib.Path) -> str:
path = pathlib.Path(path)
hasher = blake3.blake3()
@@ -32,11 +40,3 @@ async def compute_file_hash(path: str | pathlib.Path) -> str:
while chunk := await f.read(_HASH_CHUNK_SIZE):
hasher.update(chunk)
return f"blake3:{hasher.hexdigest()}"
-
-
-def compute_dict_hash(to_hash: dict[Any, Any]) -> str:
- hasher = blake3.blake3()
- for k in sorted(to_hash.keys()):
- hasher.update(str(k).encode("utf-8"))
- hasher.update(str(to_hash[k]).encode("utf-8"))
- return f"blake3:{hasher.hexdigest()}"
diff --git a/atr/log.py b/atr/log.py
index eb81d986..480f5159 100644
--- a/atr/log.py
+++ b/atr/log.py
@@ -94,6 +94,15 @@ def clear_context():
structlog.contextvars.clear_contextvars()
+def create_debug_handler() -> logging.Handler:
+ global _global_recent_logs
+ _global_recent_logs = collections.deque(maxlen=100)
+ handler = BufferingHandler()
+ handler.setFormatter(logging.Formatter("%(asctime)s %(name)s
%(levelname)s: %(message)s"))
+ handler.setLevel(logging.DEBUG)
+ return handler
+
+
def critical(msg: str, **kwargs) -> None:
_event(logging.CRITICAL, msg, **kwargs)
@@ -126,15 +135,6 @@ def info(msg: str, **kwargs) -> None:
_event(logging.INFO, msg, **kwargs)
-def create_debug_handler() -> logging.Handler:
- global _global_recent_logs
- _global_recent_logs = collections.deque(maxlen=100)
- handler = BufferingHandler()
- handler.setFormatter(logging.Formatter("%(asctime)s %(name)s
%(levelname)s: %(message)s"))
- handler.setLevel(logging.DEBUG)
- return handler
-
-
def interface_name(depth: int = 1) -> str:
return caller_name(depth=depth)
@@ -158,28 +158,6 @@ def python_repr(object_name: str) -> str:
return f"<{object_name}>"
-# def secret(msg: str, data: bytes) -> None:
-# import base64
-
-# import nacl.encoding as encoding
-# import nacl.public as public
-
-# import atr.config as config
-
-# conf = config.get()
-# public_key_b64 = conf.LOG_PUBLIC_KEY
-# if public_key_b64 is None:
-# raise ValueError("LOG_PUBLIC_KEY is not set")
-
-# recipient_pk = public.PublicKey(
-# public_key_b64.encode("ascii"),
-# encoder=encoding.Base64Encoder,
-# )
-# ciphertext = public.SealedBox(recipient_pk).encrypt(data)
-# encoded_ciphertext = base64.b64encode(ciphertext).decode("ascii")
-# _event(logging.INFO, f"{msg} {encoded_ciphertext}")
-
-
def warning(msg: str, **kwargs) -> None:
_event(logging.WARNING, msg, **kwargs)
diff --git a/atr/models/tabulate.py b/atr/models/tabulate.py
index fdd0ff17..da6fac30 100644
--- a/atr/models/tabulate.py
+++ b/atr/models/tabulate.py
@@ -16,7 +16,6 @@
# under the License.
import enum
-from typing import Any, Literal
import pydantic
@@ -37,19 +36,15 @@ class VoteStatus(enum.Enum):
UNKNOWN = "Unknown"
-def example(value: Any) -> dict[Literal["json_schema_extra"], dict[str, Any]]:
- return {"json_schema_extra": {"example": value}}
-
-
class VoteEmail(schema.Strict):
- asf_uid_or_email: str = schema.Field(..., **example("user"))
- from_email: str = schema.Field(..., **example("[email protected]"))
- status: VoteStatus = schema.Field(..., **example(VoteStatus.BINDING))
- asf_eid: str = schema.Field(...,
**example("[email protected]"))
- iso_datetime: str = schema.Field(..., **example("2025-05-01T12:00:00Z"))
- vote: Vote = schema.Field(..., **example(Vote.YES))
- quotation: str = schema.Field(..., **example("+1 (Binding)"))
- updated: bool = schema.Field(..., **example(True))
+ asf_uid_or_email: str = schema.example("user")
+ from_email: str = schema.example("[email protected]")
+ status: VoteStatus = schema.example(VoteStatus.BINDING)
+ asf_eid: str =
schema.example("[email protected]")
+ iso_datetime: str = schema.example("2025-05-01T12:00:00Z")
+ vote: Vote = schema.example(Vote.YES)
+ quotation: str = schema.example("+1 (Binding)")
+ updated: bool = schema.example(True)
@pydantic.field_validator("status", mode="before")
@classmethod
@@ -63,24 +58,21 @@ class VoteEmail(schema.Strict):
class VoteDetails(schema.Strict):
- start_unixtime: int | None = schema.Field(..., **example(1714435200))
- votes: dict[str, VoteEmail] = schema.Field(
- ...,
- **example(
- {
- "user": VoteEmail(
- asf_uid_or_email="user",
- from_email="[email protected]",
- status=VoteStatus.BINDING,
- asf_eid="[email protected]",
- iso_datetime="2025-05-01T12:00:00Z",
- vote=Vote.YES,
- quotation="+1 (Binding)",
- updated=True,
- )
- }
- ),
+ start_unixtime: int | None = schema.example(1714435200)
+ votes: dict[str, VoteEmail] = schema.example(
+ {
+ "user": VoteEmail(
+ asf_uid_or_email="user",
+ from_email="[email protected]",
+ status=VoteStatus.BINDING,
+ asf_eid="[email protected]",
+ iso_datetime="2025-05-01T12:00:00Z",
+ vote=Vote.YES,
+ quotation="+1 (Binding)",
+ updated=True,
+ )
+ }
)
- summary: dict[str, int] = schema.Field(..., **example({"user": 1}))
- passed: bool = schema.Field(..., **example(True))
- outcome: str = schema.Field(..., **example("The vote passed."))
+ summary: dict[str, int] = schema.example({"user": 1})
+ passed: bool = schema.example(True)
+ outcome: str = schema.example("The vote passed.")
diff --git a/atr/post/draft.py b/atr/post/draft.py
index 9b57b708..c35a3234 100644
--- a/atr/post/draft.py
+++ b/atr/post/draft.py
@@ -38,6 +38,33 @@ if TYPE_CHECKING:
import pathlib
[email protected]("/draft/reset/<project_name>/<version_name>")
[email protected]()
+async def cache_reset(session: web.Committer, project_name: str, version_name:
str) -> web.WerkzeugResponse:
+ """Start a new draft revision and switch this release to global caching"""
+ await session.check_access(project_name)
+ if not session.is_admin:
+ raise base.ASFQuartException("Admin access required", errorcode=403)
+
+ description = "Empty revision to restart all checks without cache for the
whole release candidate draft"
+ async with storage.write(session) as write:
+ wacp = await write.as_project_committee_participant(project_name)
+ await wacp.revision.create_revision(
+ project_name,
+ version_name,
+ session.uid,
+ description=description,
+ reset_to_global_cache=True,
+ )
+
+ return await session.redirect(
+ get.compose.selected,
+ project_name=project_name,
+ version_name=version_name,
+ success="Release set back to global caching",
+ )
+
+
@post.committer("/compose/<project_name>/<version_name>")
@post.empty()
async def delete(session: web.Committer, project_name: str, version_name: str)
-> web.WerkzeugResponse:
@@ -145,33 +172,6 @@ async def recheck(session: web.Committer, project_name:
str, version_name: str)
)
[email protected]("/draft/reset/<project_name>/<version_name>")
[email protected]()
-async def cache_reset(session: web.Committer, project_name: str, version_name:
str) -> web.WerkzeugResponse:
- """Start a new draft revision and switch this release to global caching"""
- await session.check_access(project_name)
- if not session.is_admin:
- raise base.ASFQuartException("Admin access required", errorcode=403)
-
- description = "Empty revision to restart all checks without cache for the
whole release candidate draft"
- async with storage.write(session) as write:
- wacp = await write.as_project_committee_participant(project_name)
- await wacp.revision.create_revision(
- project_name,
- version_name,
- session.uid,
- description=description,
- reset_to_global_cache=True,
- )
-
- return await session.redirect(
- get.compose.selected,
- project_name=project_name,
- version_name=version_name,
- success="Release set back to global caching",
- )
-
-
@post.committer("/draft/sbomgen/<project_name>/<version_name>/<path:file_path>")
@post.empty()
async def sbomgen(session: web.Committer, project_name: str, version_name:
str, file_path: str) -> web.WerkzeugResponse:
diff --git a/atr/principal.py b/atr/principal.py
index 30b3027a..2e602b33 100644
--- a/atr/principal.py
+++ b/atr/principal.py
@@ -63,20 +63,6 @@ ArgumentNone = ArgumentNoneType()
type UID = web.Committer | str | None | ArgumentNoneType
-def attr_to_list(attr):
- """Converts a list of bytestring attribute values to a unique list of
strings."""
- return list(set([value for value in attr or []]))
-
-
-def get_ldap_bind_dn_and_password() -> tuple[str, str]:
- conf = config.get()
- bind_dn = conf.LDAP_BIND_DN
- bind_password = conf.LDAP_BIND_PASSWORD
- if (not bind_dn) or (not bind_password):
- raise CommitterError("LDAP bind DN or password not set")
- return bind_dn, bind_password
-
-
class Committer:
"""Verifies and loads a committer's credentials via LDAP."""
@@ -418,6 +404,20 @@ class Authorisation(AsyncObject):
return self.__authoriser.member_of(self.__asf_uid)
+def attr_to_list(attr):
+ """Converts a list of bytestring attribute values to a unique list of
strings."""
+ return list(set([value for value in attr or []]))
+
+
+def get_ldap_bind_dn_and_password() -> tuple[str, str]:
+ conf = config.get()
+ bind_dn = conf.LDAP_BIND_DN
+ bind_password = conf.LDAP_BIND_PASSWORD
+ if (not bind_dn) or (not bind_password):
+ raise CommitterError("LDAP bind DN or password not set")
+ return bind_dn, bind_password
+
+
def _augment_test_membership(
committees: frozenset[str],
projects: frozenset[str],
diff --git a/atr/sbom/osv.py b/atr/sbom/osv.py
index 028e7574..227c7816 100644
--- a/atr/sbom/osv.py
+++ b/atr/sbom/osv.py
@@ -100,14 +100,6 @@ async def scan_bundle(bundle: models.bundle.Bundle) ->
tuple[list[models.osv.Com
return result, ignored
-def vulns_from_bundle(bundle: models.bundle.Bundle) ->
list[models.osv.CdxVulnerabilityDetail]:
- vulns = get_pointer(bundle.doc, "/vulnerabilities")
- if vulns is None:
- return []
- print(vulns)
- return [models.osv.CdxVulnerabilityDetail.model_validate(v) for v in vulns]
-
-
async def vuln_patch(
doc: yyjson.Document,
components: list[models.osv.ComponentVulnerabilities],
@@ -122,16 +114,12 @@ async def vuln_patch(
return patch_ops
-def _assemble_vulnerabilities(doc: yyjson.Document, patch_ops:
models.patch.Patch) -> None:
- if get_pointer(doc, "/vulnerabilities") is not None:
- patch_ops.append(models.patch.RemoveOp(op="remove",
path="/vulnerabilities"))
- patch_ops.append(
- models.patch.AddOp(
- op="add",
- path="/vulnerabilities",
- value=[],
- )
- )
+def vulns_from_bundle(bundle: models.bundle.Bundle) ->
list[models.osv.CdxVulnerabilityDetail]:
+ vulns = get_pointer(bundle.doc, "/vulnerabilities")
+ if vulns is None:
+ return []
+ print(vulns)
+ return [models.osv.CdxVulnerabilityDetail.model_validate(v) for v in vulns]
def _assemble_component_vulnerability(
@@ -165,6 +153,18 @@ def _assemble_component_vulnerability(
)
+def _assemble_vulnerabilities(doc: yyjson.Document, patch_ops:
models.patch.Patch) -> None:
+ if get_pointer(doc, "/vulnerabilities") is not None:
+ patch_ops.append(models.patch.RemoveOp(op="remove",
path="/vulnerabilities"))
+ patch_ops.append(
+ models.patch.AddOp(
+ op="add",
+ path="/vulnerabilities",
+ value=[],
+ )
+ )
+
+
def _component_purl_with_version(component: models.bom.Component) -> str |
None:
if component.purl is None:
return None
diff --git a/atr/sbom/tool.py b/atr/sbom/tool.py
index b7f84ae1..08e98802 100644
--- a/atr/sbom/tool.py
+++ b/atr/sbom/tool.py
@@ -38,6 +38,27 @@ _KNOWN_TOOLS: Final[dict[str, models.tool.Tool]] = {
}
+def outdated_version_core(
+ isotime: str, version: str, version_as_of: Callable[[str], str | None]
+) -> semver.VersionInfo | None:
+ expected_version = version_as_of(isotime)
+ if expected_version is None:
+ return None
+ if version == expected_version:
+ return None
+ expected_version_comparable = version_parse(expected_version)
+ version_comparable = version_parse(version)
+ if (expected_version_comparable is None) or (version_comparable is None):
+ # Couldn't parse the version
+ return None
+ # If the version used is less than the version available
+ if version_comparable < expected_version_comparable:
+ # Then note the version available
+ return expected_version_comparable
+ # Otherwise, the user is using the latest version
+ return None
+
+
def plugin_outdated_version(bom_value: models.bom.Bom) ->
list[models.tool.Outdated] | None:
if bom_value.metadata is None:
return [models.tool.OutdatedMissingMetadata()]
@@ -76,27 +97,6 @@ def plugin_outdated_version(bom_value: models.bom.Bom) ->
list[models.tool.Outda
return errors
-def outdated_version_core(
- isotime: str, version: str, version_as_of: Callable[[str], str | None]
-) -> semver.VersionInfo | None:
- expected_version = version_as_of(isotime)
- if expected_version is None:
- return None
- if version == expected_version:
- return None
- expected_version_comparable = version_parse(expected_version)
- version_comparable = version_parse(version)
- if (expected_version_comparable is None) or (version_comparable is None):
- # Couldn't parse the version
- return None
- # If the version used is less than the version available
- if version_comparable < expected_version_comparable:
- # Then note the version available
- return expected_version_comparable
- # Otherwise, the user is using the latest version
- return None
-
-
def version_parse(version_str: str) -> semver.VersionInfo | None:
try:
return semver.VersionInfo.parse(version_str.lstrip("v"))
diff --git a/atr/sbom/utilities.py b/atr/sbom/utilities.py
index 027ebc0c..5d4e94a7 100644
--- a/atr/sbom/utilities.py
+++ b/atr/sbom/utilities.py
@@ -33,18 +33,18 @@ import atr.util as util
from . import constants, models
+if True:
-def get_atr_version():
- try:
- from atr import metadata
+ def get_atr_version():
+ try:
+ from atr import metadata
- return metadata.version
- except ImportError:
- return "cli"
+ return metadata.version
+ except ImportError:
+ return "cli"
_ATR_VERSION = get_atr_version()
-
_SCORING_METHODS_OSV = {"CVSS_V2": "CVSSv2", "CVSS_V3": "CVSSv3", "CVSS_V4":
"CVSSv4"}
_SCORING_METHODS_CDX = {"CVSSv2": "CVSS_V2", "CVSSv3": "CVSS_V3", "CVSSv4":
"CVSS_V4", "other": "Other"}
_CDX_SEVERITIES = ["critical", "high", "medium", "low", "info", "none",
"unknown"]
@@ -89,6 +89,18 @@ async def bundle_to_vuln_patch(
return patch_ops
+def cdx_severity_to_osv(severity: list[dict[str, str | float]]) -> tuple[str |
None, list[dict[str, str]]]:
+ severities = [
+ {
+ "score": str(s.get("score", str(s.get("vector", "")))),
+ "type": _SCORING_METHODS_CDX.get(str(s.get("method", "other"))),
+ }
+ for s in severity
+ ]
+ textual = severity[0].get("severity")
+ return str(textual), severities
+
+
def get_pointer(doc: yyjson.Document, path: str) -> Any | None:
try:
return doc.get_pointer(path)
@@ -109,16 +121,6 @@ def get_props_from_bundle(bundle_value:
models.bundle.Bundle) -> tuple[int, list
return version, [p for p in properties if "asf:atr:" in p.get("name", "")]
-def patch_to_data(patch_ops: models.patch.Patch) -> list[dict[str, Any]]:
- return [op.model_dump(by_alias=True, exclude_none=True) for op in
patch_ops]
-
-
-def path_to_bundle(path: pathlib.Path) -> models.bundle.Bundle:
- text = path.read_text(encoding="utf-8")
- bom = models.bom.Bom.model_validate_json(text)
- return models.bundle.Bundle(doc=yyjson.Document(text), bom=bom, path=path,
text=text)
-
-
def osv_severity_to_cdx(severity: list[dict[str, Any]] | None, textual: str)
-> list[dict[str, str | float]] | None:
if severity is not None:
return [
@@ -132,16 +134,14 @@ def osv_severity_to_cdx(severity: list[dict[str, Any]] |
None, textual: str) ->
return None
-def cdx_severity_to_osv(severity: list[dict[str, str | float]]) -> tuple[str |
None, list[dict[str, str]]]:
- severities = [
- {
- "score": str(s.get("score", str(s.get("vector", "")))),
- "type": _SCORING_METHODS_CDX.get(str(s.get("method", "other"))),
- }
- for s in severity
- ]
- textual = severity[0].get("severity")
- return str(textual), severities
+def patch_to_data(patch_ops: models.patch.Patch) -> list[dict[str, Any]]:
+ return [op.model_dump(by_alias=True, exclude_none=True) for op in
patch_ops]
+
+
+def path_to_bundle(path: pathlib.Path) -> models.bundle.Bundle:
+ text = path.read_text(encoding="utf-8")
+ bom = models.bom.Bom.model_validate_json(text)
+ return models.bundle.Bundle(doc=yyjson.Document(text), bom=bom, path=path,
text=text)
def _extract_cdx_score(type: str, score_str: str) -> dict[str, str | float]:
diff --git a/atr/storage/__init__.py b/atr/storage/__init__.py
index 7c758a94..15314b26 100644
--- a/atr/storage/__init__.py
+++ b/atr/storage/__init__.py
@@ -41,27 +41,6 @@ import atr.user as user
## Access credentials
-# Do not rename this interface
-# It is named to reserve the atr.storage.audit logger name
-def audit(**kwargs: basic.JSON) -> None:
- now =
datetime.datetime.now(datetime.UTC).isoformat(timespec="milliseconds")
- now = now.replace("+00:00", "Z")
- action = log.caller_name(depth=2)
- request_user_id = log.get_context("user_id")
- admin_user_id = log.get_context("admin_id")
- kwargs = {"datetime": now, "action": action, **kwargs}
- if request_user_id:
- kwargs["request_user_id"] = request_user_id
- if admin_user_id and (request_user_id != admin_user_id):
- kwargs["admin_user_id"] = admin_user_id
- msg = json.dumps(kwargs, allow_nan=False)
- # The atr.log logger should give the same name
- # But to be extra sure, we set it manually
- logger = logging.getLogger("atr.storage.audit")
- # TODO: Convert to async
- logger.info(msg)
-
-
class AccessAs:
def append_to_audit_log(self, **kwargs: basic.JSON) -> None:
audit(**kwargs)
@@ -404,7 +383,25 @@ class Write:
return committees
-# Context managers
+# Do not rename this interface
+# It is named to reserve the atr.storage.audit logger name
+def audit(**kwargs: basic.JSON) -> None:
+ now =
datetime.datetime.now(datetime.UTC).isoformat(timespec="milliseconds")
+ now = now.replace("+00:00", "Z")
+ action = log.caller_name(depth=2)
+ request_user_id = log.get_context("user_id")
+ admin_user_id = log.get_context("admin_id")
+ kwargs = {"datetime": now, "action": action, **kwargs}
+ if request_user_id:
+ kwargs["request_user_id"] = request_user_id
+ if admin_user_id and (request_user_id != admin_user_id):
+ kwargs["admin_user_id"] = admin_user_id
+ msg = json.dumps(kwargs, allow_nan=False)
+ # The atr.log logger should give the same name
+ # But to be extra sure, we set it manually
+ logger = logging.getLogger("atr.storage.audit")
+ # TODO: Convert to async
+ logger.info(msg)
@contextlib.asynccontextmanager
@@ -418,6 +415,19 @@ async def read(asf_uid: principal.UID =
principal.ArgumentNone) -> AsyncGenerato
yield Read(authorisation, data)
[email protected]
+async def read_and_write(asf_uid: principal.UID = principal.ArgumentNone) ->
AsyncGenerator[tuple[Read, Write]]:
+ if asf_uid is principal.ArgumentNone:
+ authorisation = await principal.Authorisation()
+ else:
+ authorisation = await principal.Authorisation(asf_uid)
+ async with db.session() as data:
+ # TODO: Replace data with a DatabaseWriter instance
+ r = Read(authorisation, data)
+ w = Write(authorisation, data)
+ yield r, w
+
+
@contextlib.asynccontextmanager
async def read_as_foundation_committer(
asf_uid: principal.UID = principal.ArgumentNone,
@@ -434,19 +444,6 @@ async def read_as_general_public(
yield r.as_general_public()
[email protected]
-async def read_and_write(asf_uid: principal.UID = principal.ArgumentNone) ->
AsyncGenerator[tuple[Read, Write]]:
- if asf_uid is principal.ArgumentNone:
- authorisation = await principal.Authorisation()
- else:
- authorisation = await principal.Authorisation(asf_uid)
- async with db.session() as data:
- # TODO: Replace data with a DatabaseWriter instance
- r = Read(authorisation, data)
- w = Write(authorisation, data)
- yield r, w
-
-
@contextlib.asynccontextmanager
async def write(asf_uid: principal.UID = principal.ArgumentNone) ->
AsyncGenerator[Write]:
if asf_uid is principal.ArgumentNone:
diff --git a/atr/storage/writers/checks.py b/atr/storage/writers/checks.py
index 452559a4..5b2dc0a6 100644
--- a/atr/storage/writers/checks.py
+++ b/atr/storage/writers/checks.py
@@ -28,13 +28,6 @@ import atr.models.validation as validation
import atr.storage as storage
-def _validate_ignore_patterns(*patterns: str | None) -> None:
- for pattern in patterns:
- if pattern is None:
- continue
- validation.validate_ignore_pattern(pattern)
-
-
class GeneralPublic:
def __init__(
self,
@@ -191,3 +184,10 @@ class CommitteeMember(CommitteeParticipant):
raise storage.AccessError(f"Project {project_name} not found")
if project.committee_name != self.__committee_name:
raise storage.AccessError(f"Project {project_name} is not in
committee {self.__committee_name}")
+
+
+def _validate_ignore_patterns(*patterns: str | None) -> None:
+ for pattern in patterns:
+ if pattern is None:
+ continue
+ validation.validate_ignore_pattern(pattern)
diff --git a/atr/svn/__init__.py b/atr/svn/__init__.py
index b887a793..a41a6bc0 100644
--- a/atr/svn/__init__.py
+++ b/atr/svn/__init__.py
@@ -90,46 +90,6 @@ class SvnLog(pydantic_xml.BaseXmlModel, tag="log"):
entries: list[SvnLogEntry] = pydantic_xml.element(tag="logentry")
-async def _run_svnmucc_command(*args: str) -> str:
- return await run_command("svnmucc", *args)
-
-
-async def _run_svn_command(sub_cmd: str, path: str, *args: str) -> str:
- # Do not log this command, as it may contain a password or secret token
- return await run_command("svn", *[sub_cmd, *args, path])
-
-
-async def _run_svn_info(path_or_url: str) -> str:
- log.debug(f"fetching svn info for '{path_or_url}'")
- return await _run_svn_command("info", path_or_url)
-
-
-async def update(path: pathlib.Path) -> str:
- log.debug(f"running svn update for '{path}'")
- return await _run_svn_command("update", str(path), "--parents")
-
-
-async def get_log(path: pathlib.Path) -> SvnLog:
- log.debug(f"running svn log for '{path}'")
- svn_token = config.get().SVN_TOKEN
- if svn_token is None:
- raise ValueError("SVN_TOKEN must be set")
- # TODO: Or omit username entirely?
- log_output = await _run_svn_command("log", str(path), "--xml",
"--username", _ASF_TOOL, "--password", svn_token)
- return SvnLog.from_xml(log_output)
-
-
-async def get_diff(path: pathlib.Path, revision: int) -> str:
- log.debug(f"running svn diff for '{path}': r{revision}")
- svn_token = config.get().SVN_TOKEN
- if svn_token is None:
- raise ValueError("SVN_TOKEN must be set")
- # TODO: Or omit username entirely?
- return await _run_svn_command(
- "diff", str(path), "-c", str(revision), "--username", _ASF_TOOL,
"--password", svn_token
- )
-
-
async def commit(path: pathlib.Path, url: str, username: str, revision: str,
message: str) -> str:
log.debug(f"running svn commit for user '{username}' to '{url}'")
# The username here is the ASF UID of the committer
@@ -154,6 +114,27 @@ async def commit(path: pathlib.Path, url: str, username:
str, revision: str, mes
)
+async def get_diff(path: pathlib.Path, revision: int) -> str:
+ log.debug(f"running svn diff for '{path}': r{revision}")
+ svn_token = config.get().SVN_TOKEN
+ if svn_token is None:
+ raise ValueError("SVN_TOKEN must be set")
+ # TODO: Or omit username entirely?
+ return await _run_svn_command(
+ "diff", str(path), "-c", str(revision), "--username", _ASF_TOOL,
"--password", svn_token
+ )
+
+
+async def get_log(path: pathlib.Path) -> SvnLog:
+ log.debug(f"running svn log for '{path}'")
+ svn_token = config.get().SVN_TOKEN
+ if svn_token is None:
+ raise ValueError("SVN_TOKEN must be set")
+ # TODO: Or omit username entirely?
+ log_output = await _run_svn_command("log", str(path), "--xml",
"--username", _ASF_TOOL, "--password", svn_token)
+ return SvnLog.from_xml(log_output)
+
+
async def run_command(cmd: str, *args: str) -> str:
"""Run a svn command asynchronously.
@@ -177,3 +158,22 @@ async def run_command(cmd: str, *args: str) -> str:
else:
output = stdout.decode().strip()
return output
+
+
+async def update(path: pathlib.Path) -> str:
+ log.debug(f"running svn update for '{path}'")
+ return await _run_svn_command("update", str(path), "--parents")
+
+
+async def _run_svn_command(sub_cmd: str, path: str, *args: str) -> str:
+ # Do not log this command, as it may contain a password or secret token
+ return await run_command("svn", *[sub_cmd, *args, path])
+
+
+async def _run_svn_info(path_or_url: str) -> str:
+ log.debug(f"fetching svn info for '{path_or_url}'")
+ return await _run_svn_command("info", path_or_url)
+
+
+async def _run_svnmucc_command(*args: str) -> str:
+ return await run_command("svnmucc", *args)
diff --git a/atr/tasks/__init__.py b/atr/tasks/__init__.py
index 3acb79df..282db9cf 100644
--- a/atr/tasks/__init__.py
+++ b/atr/tasks/__init__.py
@@ -197,50 +197,6 @@ async def draft_checks(
return len(relative_paths)
-async def _draft_file_checks(
- asf_uid: str,
- caller_data: db.Session | None,
- data: db.Session,
- path: pathlib.Path,
- previous_version: sql.Release | None,
- project_name: str,
- release: sql.Release,
- release_version: str,
- revision_number: str,
-):
- path_str = str(path)
- task_function: Callable[[str, sql.Release, str, str, db.Session],
Awaitable[list[sql.Task | None]]] | None = None
- for suffix, func in TASK_FUNCTIONS.items():
- if path.name.endswith(suffix):
- task_function = func
- break
- if task_function:
- for task in await task_function(asf_uid, release, revision_number,
path_str, data):
- if task:
- task.revision_number = revision_number
- await _add_task(data, task)
- # TODO: Should we check .json files for their content?
- # Ideally we would not have to do that
- if path.name.endswith(".cdx.json"):
- cdx_task = await queued(
- asf_uid,
- sql.TaskType.SBOM_TOOL_SCORE,
- release,
- revision_number,
- path_str,
- extra_args={
- "project_name": project_name,
- "version_name": release_version,
- "revision_number": revision_number,
- "previous_release_version": previous_version.version if
previous_version else None,
- "file_path": path_str,
- "asf_uid": asf_uid,
- },
- )
- if cdx_task:
- await _add_task(data, cdx_task)
-
-
async def keys_import_file(
asf_uid: str, project_name: str, version_name: str, revision_number: str,
caller_data: db.Session | None = None
) -> None:
@@ -324,21 +280,6 @@ async def queued(
)
-async def _add_task(data: db.Session, task: sql.Task) -> None:
- try:
- async with data.begin_nested():
- data.add(task)
- await data.flush()
- except sqlalchemy.exc.IntegrityError as e:
- if (
- isinstance(e.orig, sqlite3.IntegrityError)
- and (e.orig.sqlite_errorcode == sqlite3.SQLITE_CONSTRAINT_UNIQUE)
- and ("task.inputs_hash" in str(e.orig))
- ):
- return
- raise
-
-
def resolve(task_type: sql.TaskType) -> Callable[...,
Awaitable[results.Results | None]]: # noqa: C901
match task_type:
case sql.TaskType.COMPARE_SOURCE_TREES:
@@ -608,6 +549,65 @@ async def zip_checks(
return await asyncio.gather(*tasks)
+async def _add_task(data: db.Session, task: sql.Task) -> None:
+ try:
+ async with data.begin_nested():
+ data.add(task)
+ await data.flush()
+ except sqlalchemy.exc.IntegrityError as e:
+ if (
+ isinstance(e.orig, sqlite3.IntegrityError)
+ and (e.orig.sqlite_errorcode == sqlite3.SQLITE_CONSTRAINT_UNIQUE)
+ and ("task.inputs_hash" in str(e.orig))
+ ):
+ return
+ raise
+
+
+async def _draft_file_checks(
+ asf_uid: str,
+ caller_data: db.Session | None,
+ data: db.Session,
+ path: pathlib.Path,
+ previous_version: sql.Release | None,
+ project_name: str,
+ release: sql.Release,
+ release_version: str,
+ revision_number: str,
+):
+ path_str = str(path)
+ task_function: Callable[[str, sql.Release, str, str, db.Session],
Awaitable[list[sql.Task | None]]] | None = None
+ for suffix, func in TASK_FUNCTIONS.items():
+ if path.name.endswith(suffix):
+ task_function = func
+ break
+ if task_function:
+ for task in await task_function(asf_uid, release, revision_number,
path_str, data):
+ if task:
+ task.revision_number = revision_number
+ await _add_task(data, task)
+ # TODO: Should we check .json files for their content?
+ # Ideally we would not have to do that
+ if path.name.endswith(".cdx.json"):
+ cdx_task = await queued(
+ asf_uid,
+ sql.TaskType.SBOM_TOOL_SCORE,
+ release,
+ revision_number,
+ path_str,
+ extra_args={
+ "project_name": project_name,
+ "version_name": release_version,
+ "revision_number": revision_number,
+ "previous_release_version": previous_version.version if
previous_version else None,
+ "file_path": path_str,
+ "asf_uid": asf_uid,
+ },
+ )
+ if cdx_task:
+ await _add_task(data, cdx_task)
+
+
TASK_FUNCTIONS: Final[dict[str, Callable[..., Coroutine[Any, Any,
list[sql.Task | None]]]]] = {
".asc": asc_checks,
".sha256": sha_checks,
diff --git a/atr/tasks/checks/__init__.py b/atr/tasks/checks/__init__.py
index a0335aa6..4bdcf40b 100644
--- a/atr/tasks/checks/__init__.py
+++ b/atr/tasks/checks/__init__.py
@@ -393,8 +393,10 @@ async def _resolve_all_files(release: sql.Release,
rel_path: str | None = None)
return list(sorted(relative_paths_set))
-async def _resolve_is_podling(release: sql.Release, rel_path: str | None =
None) -> bool:
- return (release.committee is not None) and release.committee.is_podling
+async def _resolve_committee_name(release: sql.Release, rel_path: str | None =
None) -> str:
+ if release.committee is None:
+ raise ValueError("Release has no committee")
+ return release.committee.name
async def _resolve_github_tp_sha(release: sql.Release, rel_path: str | None =
None) -> str:
@@ -421,10 +423,8 @@ async def _resolve_github_tp_sha(release: sql.Release,
rel_path: str | None = No
return ""
-async def _resolve_committee_name(release: sql.Release, rel_path: str | None =
None) -> str:
- if release.committee is None:
- raise ValueError("Release has no committee")
- return release.committee.name
+async def _resolve_is_podling(release: sql.Release, rel_path: str | None =
None) -> bool:
+ return (release.committee is not None) and release.committee.is_podling
async def _resolve_unsuffixed_file_hash(release: sql.Release, rel_path: str |
None = None) -> str:
diff --git a/atr/tasks/gha.py b/atr/tasks/gha.py
index 0830dfd8..6be367b3 100644
--- a/atr/tasks/gha.py
+++ b/atr/tasks/gha.py
@@ -66,61 +66,6 @@ class WorkflowStatusCheck(schema.Strict):
asf_uid: str = schema.description("ASF UID of the user triggering the
workflow")
[email protected]_model(DistributionWorkflow)
-async def trigger_workflow(args: DistributionWorkflow, *, task_id: int | None
= None) -> results.Results | None:
- unique_id = f"atr-dist-{args.name}-{uuid.uuid4()}"
- try:
- sql_platform = sql.DistributionPlatform[args.platform]
- except KeyError:
- _fail(f"Invalid platform: {args.platform}")
- workflow = f"distribute-{sql_platform.value.gh_slug}{'-stg' if
args.staging else ''}.yml"
- payload = {
- "ref": "main",
- "inputs": {
- "atr-id": unique_id,
- "asf-uid": args.asf_uid,
- "project": args.project_name,
- "phase": args.phase,
- "version": args.version_name,
- "distribution-owner-namespace": args.namespace,
- "distribution-package": args.package,
- "distribution-version": args.version,
- **args.arguments,
- },
- }
- headers = {"Accept": "application/vnd.github+json", "Authorization":
f"Bearer {config.get().GITHUB_TOKEN}"}
- log.info(
- f"Triggering Github workflow apache/tooling-actions/{workflow} with
args: {
- json.dumps(args.arguments, indent=2)
- }"
- )
- async with util.create_secure_session() as session:
- try:
- async with session.post(
-
f"{_BASE_URL}/apache/tooling-actions/actions/workflows/{workflow}/dispatches",
- headers=headers,
- json=payload,
- ) as response:
- response.raise_for_status()
- except aiohttp.ClientResponseError as e:
- _fail(f"Failed to trigger GitHub workflow: {e.message}
({e.status})")
-
- run, run_id = await _find_triggered_run(session, headers, unique_id)
-
- if run.get("status") in _FAILED_STATUSES:
- _fail(f"Github workflow apache/tooling-actions/{workflow} run
{run_id} failed with error")
- async with storage.write_as_committee_member(args.committee_name,
args.asf_uid) as w:
- try:
- await w.workflowstatus.add_workflow_status(
- workflow, run_id, args.project_name, task_id,
status=run.get("status")
- )
- except storage.AccessError as e:
- _fail(f"Failed to record distribution: {e}")
- return results.DistributionWorkflow(
- kind="distribution_workflow", name=args.name, run_id=run_id,
url=run.get("html_url", "")
- )
-
-
@checks.with_model(WorkflowStatusCheck)
async def status_check(args: WorkflowStatusCheck) ->
DistributionWorkflowStatus:
"""Check remote workflow statuses."""
@@ -193,6 +138,61 @@ async def status_check(args: WorkflowStatusCheck) ->
DistributionWorkflowStatus:
_fail(f"Unexpected error during workflow status update: {e!s}")
[email protected]_model(DistributionWorkflow)
+async def trigger_workflow(args: DistributionWorkflow, *, task_id: int | None
= None) -> results.Results | None:
+ unique_id = f"atr-dist-{args.name}-{uuid.uuid4()}"
+ try:
+ sql_platform = sql.DistributionPlatform[args.platform]
+ except KeyError:
+ _fail(f"Invalid platform: {args.platform}")
+ workflow = f"distribute-{sql_platform.value.gh_slug}{'-stg' if
args.staging else ''}.yml"
+ payload = {
+ "ref": "main",
+ "inputs": {
+ "atr-id": unique_id,
+ "asf-uid": args.asf_uid,
+ "project": args.project_name,
+ "phase": args.phase,
+ "version": args.version_name,
+ "distribution-owner-namespace": args.namespace,
+ "distribution-package": args.package,
+ "distribution-version": args.version,
+ **args.arguments,
+ },
+ }
+ headers = {"Accept": "application/vnd.github+json", "Authorization":
f"Bearer {config.get().GITHUB_TOKEN}"}
+ log.info(
+ f"Triggering Github workflow apache/tooling-actions/{workflow} with
args: {
+ json.dumps(args.arguments, indent=2)
+ }"
+ )
+ async with util.create_secure_session() as session:
+ try:
+ async with session.post(
+
f"{_BASE_URL}/apache/tooling-actions/actions/workflows/{workflow}/dispatches",
+ headers=headers,
+ json=payload,
+ ) as response:
+ response.raise_for_status()
+ except aiohttp.ClientResponseError as e:
+ _fail(f"Failed to trigger GitHub workflow: {e.message}
({e.status})")
+
+ run, run_id = await _find_triggered_run(session, headers, unique_id)
+
+ if run.get("status") in _FAILED_STATUSES:
+ _fail(f"Github workflow apache/tooling-actions/{workflow} run
{run_id} failed with error")
+ async with storage.write_as_committee_member(args.committee_name,
args.asf_uid) as w:
+ try:
+ await w.workflowstatus.add_workflow_status(
+ workflow, run_id, args.project_name, task_id,
status=run.get("status")
+ )
+ except storage.AccessError as e:
+ _fail(f"Failed to record distribution: {e}")
+ return results.DistributionWorkflow(
+ kind="distribution_workflow", name=args.name, run_id=run_id,
url=run.get("html_url", "")
+ )
+
+
def _fail(message: str) -> NoReturn:
log.error(message)
raise RuntimeError(message)
@@ -222,31 +222,6 @@ async def _find_triggered_run(
return run, run_id
-#
-# async def _record_distribution(
-# committee_name: str,
-# release: str,
-# platform: sql.DistributionPlatform,
-# namespace: str,
-# package: str,
-# version: str,
-# staging: bool,
-# ):
-# log.info("Creating distribution record")
-# dd = distribution.Data(
-# platform=platform,
-# owner_namespace=namespace,
-# package=package,
-# version=version,
-# details=False,
-# )
-# async with
storage.write_as_committee_member(committee_name=committee_name) as w:
-# try:
-# _dist, _added, _metadata = await
w.distributions.record_from_data(release=release, staging=staging, dd=dd)
-# except storage.AccessError as e:
-# _fail(f"Failed to record distribution: {e}")
-
-
async def _request_and_retry(
session: aiohttp.client.ClientSession,
url: str,
diff --git a/atr/tasks/sbom.py b/atr/tasks/sbom.py
index 9b242d0a..7c60d88d 100644
--- a/atr/tasks/sbom.py
+++ b/atr/tasks/sbom.py
@@ -293,6 +293,24 @@ async def score_tool(args: ScoreArgs) -> results.Results |
None:
)
+def _extracted_dir(temp_dir: str) -> str | None:
+ # Loop through all the dirs in temp_dir
+ extract_dir = None
+ log.info(f"Checking directories in {temp_dir}: {os.listdir(temp_dir)}")
+ for dir_name in os.listdir(temp_dir):
+ if dir_name.startswith("."):
+ continue
+ dir_path = os.path.join(temp_dir, dir_name)
+ if os.path.isdir(dir_path):
+ if extract_dir is None:
+ extract_dir = dir_path
+ else:
+ return temp_dir
+ if extract_dir is None:
+ extract_dir = temp_dir
+ return extract_dir
+
+
async def _generate_cyclonedx_core(artifact_path: str, output_path: str) ->
dict[str, Any]:
"""Core logic to generate CycloneDX SBOM on failure."""
log.info(f"Generating CycloneDX SBOM for {artifact_path} -> {output_path}")
@@ -393,21 +411,3 @@ async def _generate_cyclonedx_core(artifact_path: str,
output_path: str) -> dict
except FileNotFoundError:
log.error("syft command not found. Is it installed and in PATH?")
raise SBOMGenerationError("syft command not found")
-
-
-def _extracted_dir(temp_dir: str) -> str | None:
- # Loop through all the dirs in temp_dir
- extract_dir = None
- log.info(f"Checking directories in {temp_dir}: {os.listdir(temp_dir)}")
- for dir_name in os.listdir(temp_dir):
- if dir_name.startswith("."):
- continue
- dir_path = os.path.join(temp_dir, dir_name)
- if os.path.isdir(dir_path):
- if extract_dir is None:
- extract_dir = dir_path
- else:
- return temp_dir
- if extract_dir is None:
- extract_dir = temp_dir
- return extract_dir
diff --git a/atr/template.py b/atr/template.py
index df194bf8..d71d5707 100644
--- a/atr/template.py
+++ b/atr/template.py
@@ -57,9 +57,6 @@ async def render_sync(
return await _render_in_thread(template, context_vars, app_instance)
-render = render_sync
-
-
async def _render_in_thread(template: jinja2.Template, context: dict, app:
app.Quart) -> str:
if template.environment.is_async is False:
raise RuntimeError("Template environment is not async")
@@ -77,3 +74,6 @@ async def _render_in_thread(template: jinja2.Template,
context: dict, app: app.Q
context=context,
)
return rendered_template
+
+
+render = render_sync
diff --git a/atr/validate.py b/atr/validate.py
index 575f2e5c..760dfc56 100644
--- a/atr/validate.py
+++ b/atr/validate.py
@@ -50,31 +50,50 @@ type ReleaseAnnotatedDivergences = Callable[[sql.Release],
AnnotatedDivergences]
T = TypeVar("T")
+if True:
-def committee(c: sql.Committee) -> AnnotatedDivergences:
- """Check that a committee is valid."""
+ def committee_components(
+ *components: str,
+ ) -> Callable[[CommitteeDivergences], CommitteeAnnotatedDivergences]:
+ """Wrap a Committee divergence generator to yield annotated
divergences."""
- yield from committee_child_committees(c)
- yield from committee_full_name(c)
+ def wrap(original: CommitteeDivergences) ->
CommitteeAnnotatedDivergences:
+ def replacement(c: sql.Committee) -> AnnotatedDivergences:
+ yield from divergences_with_annotations(
+ components,
+ original.__name__,
+ c.name,
+ original(c),
+ )
+ return replacement
-def committee_components(
- *components: str,
-) -> Callable[[CommitteeDivergences], CommitteeAnnotatedDivergences]:
- """Wrap a Committee divergence generator to yield annotated divergences."""
+ return wrap
- def wrap(original: CommitteeDivergences) -> CommitteeAnnotatedDivergences:
- def replacement(c: sql.Committee) -> AnnotatedDivergences:
- yield from divergences_with_annotations(
- components,
- original.__name__,
- c.name,
- original(c),
- )
+ def project_components(
+ *components: str,
+ ) -> Callable[[ProjectDivergences], ProjectAnnotatedDivergences]:
+ """Wrap a Project divergence generator to yield annotated
divergences."""
- return replacement
+ def wrap(original: ProjectDivergences) -> ProjectAnnotatedDivergences:
+ def replacement(p: sql.Project) -> AnnotatedDivergences:
+ yield from divergences_with_annotations(
+ components,
+ original.__name__,
+ p.name,
+ original(p),
+ )
- return wrap
+ return replacement
+
+ return wrap
+
+
+def committee(c: sql.Committee) -> AnnotatedDivergences:
+ """Check that a committee is valid."""
+
+ yield from committee_child_committees(c)
+ yield from committee_full_name(c)
@committee_components("Committee.child_committees")
@@ -176,25 +195,6 @@ def project(p: sql.Project) -> AnnotatedDivergences:
yield from project_release_policy(p)
-def project_components(
- *components: str,
-) -> Callable[[ProjectDivergences], ProjectAnnotatedDivergences]:
- """Wrap a Project divergence generator to yield annotated divergences."""
-
- def wrap(original: ProjectDivergences) -> ProjectAnnotatedDivergences:
- def replacement(p: sql.Project) -> AnnotatedDivergences:
- yield from divergences_with_annotations(
- components,
- original.__name__,
- p.name,
- original(p),
- )
-
- return replacement
-
- return wrap
-
-
@project_components("Project.category")
def project_category(p: sql.Project) -> Divergences:
"""Check that the category string uses 'label, label' syntax without
colons."""
diff --git a/scripts/fix_order.py b/scripts/fix_order.py
index 9774f35b..9c2c33c6 100755
--- a/scripts/fix_order.py
+++ b/scripts/fix_order.py
@@ -30,11 +30,13 @@ def main() -> None:
blocks = _parse_blocks(path.read_text(encoding="utf-8"))
nonfunc, func = [], []
seen_func = False
- main_guard: list[str] | None = None
- if blocks and _is_main_guard(blocks[-1][1]):
- main_guard = blocks[-1][1]
- blocks = blocks[:-1]
+ trailing: list[tuple[int, list[str]]] = []
+ while blocks and not _is_func(blocks[-1][1]):
+ trailing.insert(0, blocks.pop())
+ if not blocks:
+ blocks = trailing
+ trailing = []
for lineno, lines in blocks:
count = _count_defs(lines)
@@ -49,6 +51,14 @@ def main() -> None:
nonfunc.append((lineno, lines))
func.sort(key=_sort_key)
+ print(_assemble(nonfunc, func, trailing), end="")
+
+
+def _assemble(
+ nonfunc: list[tuple[int, list[str]]],
+ func: list[tuple[int, list[str]]],
+ trailing: list[tuple[int, list[str]]],
+) -> str:
nonfunc_text = "".join("".join(b) for _, b in nonfunc)
func_parts = [_normalise(b) for _, b in func]
func_text = "\n\n".join(func_parts)
@@ -56,12 +66,14 @@ def main() -> None:
output = nonfunc_text.rstrip("\n") + "\n\n\n" + func_text
else:
output = nonfunc_text + func_text
- if main_guard:
+ if trailing:
+ trailing_parts = [_normalise(b) for _, b in trailing]
+ trailing_text = "\n\n".join(trailing_parts)
if output:
- output = output.rstrip("\n") + "\n\n\n" + _normalise(main_guard)
+ output = output.rstrip("\n") + "\n\n\n" + trailing_text
else:
- output = _normalise(main_guard)
- print(output, end="")
+ output = trailing_text
+ return output
def _count_defs(block: list[str]) -> int:
@@ -79,13 +91,6 @@ def _is_func(block: list[str]) -> bool:
return False
-def _is_main_guard(block: list[str]) -> bool:
- for line in block:
- if line.strip():
- return line.startswith("if __name__")
- return False
-
-
def _normalise(block: list[str]) -> str:
while block and (not block[-1].strip()):
block = block[:-1]
diff --git a/scripts/fix_order.sh b/scripts/fix_order.sh
index b2bf2711..679faedc 100755
--- a/scripts/fix_order.sh
+++ b/scripts/fix_order.sh
@@ -12,8 +12,7 @@ tmp="$$.tmp.py"
backup="$HOME/.fix_order.backup.py"
script_dir="$(dirname "$0")"
-# TODO: Use uv here?
-python3 "$script_dir/fix_order.py" "$file" > "$tmp"
+uv run --frozen python3 "$script_dir/fix_order.py" "$file" > "$tmp"
status=$?
if [ $status -ne 0 ]
then
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
