This is an automated email from the ASF dual-hosted git repository. arm pushed a commit to branch arm in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
commit 21b35c790e2e7fa8e44158fc235c1069e19bcdb6 Author: Alastair McFarlane <[email protected]> AuthorDate: Thu Mar 12 09:58:53 2026 +0000 #765: use safe values for distribution params --- atr/blueprints/common.py | 8 ++++++- atr/models/api.py | 18 +++++++-------- atr/models/distribution.py | 8 +++---- atr/models/sql.py | 17 +++++++++++++- atr/shared/distribution.py | 43 ++++++++++++++++++++---------------- atr/storage/writers/distributions.py | 33 +++++++++++++-------------- atr/tasks/distribution.py | 9 +------- 7 files changed, 78 insertions(+), 58 deletions(-) diff --git a/atr/blueprints/common.py b/atr/blueprints/common.py index b9c2d219..0fd2073b 100644 --- a/atr/blueprints/common.py +++ b/atr/blueprints/common.py @@ -38,7 +38,13 @@ QUART_CONVERTERS: dict[Any, str] = { unsafe.Path: "path", } -VALIDATED_TYPES: set[Any] = {safe.ProjectName, safe.RevisionNumber, safe.VersionName, unsafe.UnsafeStr} +VALIDATED_TYPES: set[Any] = { + safe.Alphanumeric, + safe.ProjectName, + safe.RevisionNumber, + safe.VersionName, + unsafe.UnsafeStr, +} async def authenticate() -> web.Committer: diff --git a/atr/models/api.py b/atr/models/api.py index 342773b2..514a7f6d 100644 --- a/atr/models/api.py +++ b/atr/models/api.py @@ -103,9 +103,9 @@ class DistributionRecordArgs(schema.Strict): project: safe.ProjectName = schema.example("example") version: safe.VersionName = schema.example("0.0.1") platform: sql.DistributionPlatform = schema.example(sql.DistributionPlatform.ARTIFACT_HUB) - distribution_owner_namespace: str | None = schema.default_example(None, "example") - distribution_package: str = schema.example("example") - distribution_version: str = schema.example("0.0.1") + distribution_owner_namespace: safe.Alphanumeric | None = schema.default_example(None, "example") + distribution_package: safe.Alphanumeric = schema.example("example") + distribution_version: safe.VersionName = schema.example("0.0.1") staging: bool = schema.example(False) details: bool = schema.example(False) @@ -131,9 +131,9 @@ class DistributionRecordFromWorkflowArgs(schema.Strict): project: safe.ProjectName = schema.example("example") version: safe.VersionName = schema.example("0.0.1") platform: sql.DistributionPlatform = schema.example(sql.DistributionPlatform.ARTIFACT_HUB) - distribution_owner_namespace: str | None = schema.default_example(None, "example") - distribution_package: str = schema.example("example") - distribution_version: str = schema.example("0.0.1") + distribution_owner_namespace: safe.Alphanumeric | None = schema.default_example(None, "example") + distribution_package: safe.Alphanumeric = schema.example("example") + distribution_version: safe.VersionName = schema.example("0.0.1") phase: str = schema.Field(strict=False, default="compose", json_schema_extra={"examples": ["compose", "finish"]}) staging: bool = schema.example(False) details: bool = schema.example(False) @@ -327,9 +327,9 @@ class PublisherDistributionRecordArgs(schema.Strict): jwt: str = schema.example("eyJhbGciOiJIUzI1[...]mMjLiuyu5CSpyHI=") version: safe.VersionName = schema.example("0.0.1") platform: sql.DistributionPlatform = schema.example(sql.DistributionPlatform.ARTIFACT_HUB) - distribution_owner_namespace: str | None = schema.default_example(None, "example") - distribution_package: str = schema.example("example") - distribution_version: str = schema.example("0.0.1") + distribution_owner_namespace: safe.Alphanumeric | None = schema.default_example(None, "example") + distribution_package: safe.Alphanumeric = schema.example("example") + distribution_version: safe.VersionName = schema.example("0.0.1") staging: bool = schema.example(False) details: bool = schema.example(False) diff --git a/atr/models/distribution.py b/atr/models/distribution.py index 3dfae1b3..7e5f3149 100644 --- a/atr/models/distribution.py +++ b/atr/models/distribution.py @@ -19,7 +19,7 @@ import datetime import pydantic -from . import basic, schema, sql +from . import basic, safe, schema, sql class ArtifactHubAvailableVersion(schema.Subset): @@ -93,9 +93,9 @@ class PyPIResponse(schema.Subset): # Including all of the enum properties class Data(schema.Subset): platform: sql.DistributionPlatform - owner_namespace: str | None = None - package: str - version: str + owner_namespace: safe.Alphanumeric | None = None + package: safe.Alphanumeric + version: safe.VersionName details: bool @pydantic.field_validator("owner_namespace", mode="before") diff --git a/atr/models/sql.py b/atr/models/sql.py index ac70f282..29eac4ac 100644 --- a/atr/models/sql.py +++ b/atr/models/sql.py @@ -24,7 +24,7 @@ import dataclasses import datetime import enum -from typing import Any, Final, Literal, Optional, TypeVar, overload +from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypeVar, overload import pydantic import sqlalchemy @@ -36,6 +36,9 @@ import sqlmodel from . import results, safe, schema +if TYPE_CHECKING: + from . import distribution + T = TypeVar("T") sqlmodel.SQLModel.metadata = sqlalchemy.MetaData( @@ -1066,6 +1069,18 @@ class Distribution(sqlmodel.SQLModel, table=True): # So we do not store it in the database # api_response: Any = sqlmodel.Field(sa_column=sqlalchemy.Column(sqlalchemy.JSON)) + def distribution_data(self, details: bool = False) -> "distribution.Data": + """Get a distribution data object""" + from . import distribution + + return distribution.Data( + platform=self.platform, + owner_namespace=safe.Alphanumeric(self.owner_namespace), + package=safe.Alphanumeric(self.package), + version=safe.VersionName(self.version), + details=details, + ) + @property def identifier(self) -> str: def normal(text: str) -> str: diff --git a/atr/shared/distribution.py b/atr/shared/distribution.py index b565f6ba..922e02db 100644 --- a/atr/shared/distribution.py +++ b/atr/shared/distribution.py @@ -126,13 +126,13 @@ class DistributionAutomateForm(form.Form): platform: form.Enum[DistributionPlatform] = form.label( "Platform", widget=form.Widget.SELECT, enum_filter_include=[DistributionPlatform.MAVEN.value] ) - owner_namespace: str = form.label( + owner_namespace: safe.Alphanumeric = form.label( "Owner or Namespace", "Who owns or names the package (Maven groupId, npm @scope, Docker namespace, " "GitHub owner, ArtifactHub repo). Leave blank if not used.", ) - package: str = form.label("Package") - version: str = form.label("Version") + package: safe.Alphanumeric = form.label("Package") + version: safe.VersionName = form.label("Version") details: form.Bool = form.label( "Include details", "Include the details of the distribution in the response", @@ -159,13 +159,13 @@ class DistributionAutomateForm(form.Form): class DistributionRecordForm(form.Form): platform: form.Enum[DistributionPlatform] = form.label("Platform", widget=form.Widget.SELECT) - owner_namespace: str = form.label( + owner_namespace: safe.Alphanumeric = form.label( "Owner or Namespace", "Who owns or names the package (Maven groupId, npm @scope, Docker namespace, " "GitHub owner, ArtifactHub repo). Leave blank if not used.", ) - package: str = form.label("Package") - version: str = form.label("Version") + package: safe.Alphanumeric = form.label("Package") + version: safe.VersionName = form.label("Version") details: form.Bool = form.label( "Include details", "Include the details of the distribution in the response", @@ -193,8 +193,9 @@ class DistributionRecordForm(form.Form): def distribution_upload_date( # noqa: C901 platform: sql.DistributionPlatform, data: basic.JSON, - version: str, + version_name: safe.VersionName, ) -> datetime.datetime | None: + version = str(version_name) match platform: case sql.DistributionPlatform.ARTIFACT_HUB: if not (versions := distribution.ArtifactHubResponse.model_validate(data).available_versions): @@ -236,7 +237,7 @@ def distribution_upload_date( # noqa: C901 def distribution_web_url( # noqa: C901 platform: sql.DistributionPlatform, data: basic.JSON, - version: str, + version: safe.VersionName, ) -> str | None: match platform: case sql.DistributionPlatform.ARTIFACT_HUB: @@ -247,7 +248,7 @@ def distribution_web_url( # noqa: C901 if repo_name and pkg_name: if ver: return f"https://artifacthub.io/packages/helm/{repo_name}/{pkg_name}/{ver}"; - return f"https://artifacthub.io/packages/helm/{repo_name}/{pkg_name}/{version}"; + return f"https://artifacthub.io/packages/helm/{repo_name}/{pkg_name}/{version!s}"; if ah.home_url: return ah.home_url for link in ah.links: @@ -266,7 +267,7 @@ def distribution_web_url( # noqa: C901 case sql.DistributionPlatform.NPM: nr = distribution.NpmResponse.model_validate(data) # return nr.homepage - return f"https://www.npmjs.com/package/{nr.name}/v/{version}"; + return f"https://www.npmjs.com/package/{nr.name}/v/{version!s}"; case sql.DistributionPlatform.NPM_SCOPED: nr = distribution.NpmResponse.model_validate(data) # TODO: This is not correct @@ -278,17 +279,18 @@ def distribution_web_url( # noqa: C901 def get_api_url(dd: distribution.Data, staging: bool | None = None): + namespace = str(dd.owner_namespace) if dd.owner_namespace else None template_url = _template_url(dd, staging) - package = urllib.parse.quote(dd.package) - version = urllib.parse.quote(dd.version) + package = urllib.parse.quote(str(dd.package)) + version = urllib.parse.quote(str(dd.version)) api_url = template_url.format( - owner_namespace=dd.owner_namespace, + owner_namespace=namespace, package=package, version=version, ) if dd.platform == sql.DistributionPlatform.MAVEN: # We do this here because the CDNs break the namespace up into a / delimited URL - owner = (dd.owner_namespace or "").replace(".", "/") + owner = (namespace or "").replace(".", "/") api_url = template_url.format( owner_namespace=owner, package=package, @@ -298,11 +300,12 @@ def get_api_url(dd: distribution.Data, staging: bool | None = None): def html_submitted_values_table(block: htm.Block, dd: distribution.Data) -> None: + namespace = str(dd.owner_namespace) if dd.owner_namespace else "-" tbody = htm.tbody[ html_tr("Platform", dd.platform.name), - html_tr("Owner or Namespace", dd.owner_namespace or "-"), - html_tr("Package", dd.package), - html_tr("Version", dd.version), + html_tr("Owner or Namespace", namespace), + html_tr("Package", str(dd.package)), + html_tr("Version", str(dd.version)), ] block.table(".table.table-striped.table-bordered")[tbody] @@ -316,8 +319,9 @@ def html_tr_a(label: str, value: str | None) -> htm.Element: async def json_from_distribution_platform( - api_url: str, platform: sql.DistributionPlatform, version: str + api_url: str, platform: sql.DistributionPlatform, version_name: safe.VersionName ) -> outcome.Outcome[basic.JSON]: + version = str(version_name) try: async with util.create_secure_session() as session: async with session.get(api_url) as response: @@ -334,11 +338,12 @@ async def json_from_distribution_platform( return outcome.Result(result) -async def json_from_maven_xml(api_url: str, version: str) -> outcome.Outcome[basic.JSON]: +async def json_from_maven_xml(api_url: str, version_name: safe.VersionName) -> outcome.Outcome[basic.JSON]: import datetime import defusedxml.ElementTree as ElementTree + version = str(version_name) try: async with util.create_secure_session() as session: async with session.get(api_url) as response: diff --git a/atr/storage/writers/distributions.py b/atr/storage/writers/distributions.py index 120241b4..f9707c83 100644 --- a/atr/storage/writers/distributions.py +++ b/atr/storage/writers/distributions.py @@ -97,22 +97,22 @@ class CommitteeMember(CommitteeParticipant): release_name: models.safe.ReleaseName, platform: models.sql.DistributionPlatform, committee_name: str, - owner_namespace: str | None, + owner_namespace: models.safe.Alphanumeric | None, project_name: models.safe.ProjectName, version_name: models.safe.VersionName, phase: str, revision_number: str | None, - package: str, - version: str, + package: models.safe.Alphanumeric, + version: models.safe.VersionName, staging: bool, ) -> models.sql.Task: dist_task = models.sql.Task( task_type=models.sql.TaskType.DISTRIBUTION_WORKFLOW, task_args=gha.DistributionWorkflow( name=str(release_name), - namespace=owner_namespace or "", - package=package, - version=version, + namespace=str(owner_namespace) if owner_namespace else "", + package=str(package), + version=str(version), project_name=str(project_name), version_name=str(version_name), phase=phase, @@ -138,24 +138,25 @@ class CommitteeMember(CommitteeParticipant): self, release_name: models.safe.ReleaseName, platform: models.sql.DistributionPlatform, - owner_namespace: str | None, - package: str, - version: str, + owner_namespace: models.safe.Alphanumeric | None, + package: models.safe.Alphanumeric, + version: models.safe.VersionName, staging: bool, pending: bool, upload_date: datetime.datetime | None, api_url: str | None = None, web_url: str | None = None, ) -> tuple[models.sql.Distribution, bool]: + namespace = str(owner_namespace) if owner_namespace else "" existing = await self.__data.distribution( - str(release_name), platform, owner_namespace or "", package, version + str(release_name), platform, namespace, str(package), str(version) ).get() dist = models.sql.Distribution( platform=platform, release_name=str(release_name), - owner_namespace=owner_namespace or "", - package=package, - version=version, + owner_namespace=namespace, + package=str(package), + version=str(version), staging=staging, pending=pending, retries=0, @@ -173,9 +174,9 @@ class CommitteeMember(CommitteeParticipant): upgraded = await self.__upgrade_staging_to_final( release_name, platform, - owner_namespace, - package, - version, + namespace, + str(package), + str(version), pending, upload_date, api_url, diff --git a/atr/tasks/distribution.py b/atr/tasks/distribution.py index 0c962e9c..b6a0caf1 100644 --- a/atr/tasks/distribution.py +++ b/atr/tasks/distribution.py @@ -20,7 +20,6 @@ import pydantic import atr.db as db import atr.log as log -import atr.models as models import atr.models.results as results import atr.models.schema as schema import atr.shared.distribution as distribution @@ -46,13 +45,7 @@ async def status_check(args: DistributionStatusCheckArgs, *, task_id: int | None dists = await data.distribution(pending=True, _with_release=True, _with_release_project=True).all() for dist in dists: name = f"{dist.platform} {dist.owner_namespace} {dist.package} {dist.version}" - dd = models.distribution.Data( - platform=dist.platform, - owner_namespace=dist.owner_namespace, - package=dist.package, - version=dist.version, - details=False, - ) + dd = dist.distribution_data() if not dist.created_by: log.warning(f"Distribution {name} has no creator, skipping") continue --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
