This is an automated email from the ASF dual-hosted git repository.
sbp pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-trusted-release.git
The following commit(s) were added to refs/heads/main by this push:
new f2f9e75 Add project validation
f2f9e75 is described below
commit f2f9e75978ef3c3a96b9fd0810f70f74a7806861
Author: Sean B. Palmer <[email protected]>
AuthorDate: Wed Jun 25 16:30:36 2025 +0100
Add project validation
---
atr/blueprints/admin/admin.py | 6 +-
atr/validate.py | 139 +++++++++++++++++++++++++++++++++++++++++-
scripts/integrity_check.py | 11 ++--
3 files changed, 146 insertions(+), 10 deletions(-)
diff --git a/atr/blueprints/admin/admin.py b/atr/blueprints/admin/admin.py
index d2b04ef..3322f3a 100644
--- a/atr/blueprints/admin/admin.py
+++ b/atr/blueprints/admin/admin.py
@@ -638,13 +638,11 @@ async def admin_validate() -> str:
"""Run validators and display any divergences."""
async with db.session() as data:
- releases = await data.release().order_by(models.Release.name).all()
-
- results = list(validate.releases(releases))
+ divergences = [d async for d in validate.everything(data)]
return await template.render(
"validation.html",
- divergences=results,
+ divergences=divergences,
)
diff --git a/atr/validate.py b/atr/validate.py
index 02b708d..a134c41 100644
--- a/atr/validate.py
+++ b/atr/validate.py
@@ -15,11 +15,13 @@
# specific language governing permissions and limitations
# under the License.
+import asyncio
import datetime
import pathlib
-from collections.abc import Callable, Generator, Iterable, Sequence
+from collections.abc import AsyncGenerator, Callable, Generator, Iterable,
Sequence
from typing import NamedTuple, TypeVar
+import atr.db as db
import atr.db.models as models
import atr.util as util
@@ -38,6 +40,9 @@ class AnnotatedDivergence(NamedTuple):
Divergences = Generator[Divergence]
AnnotatedDivergences = Generator[AnnotatedDivergence]
+AsyncAnnotatedDivergences = AsyncGenerator[AnnotatedDivergence]
+ProjectDivergences = Callable[[models.Project], Divergences]
+ProjectAnnotatedDivergences = Callable[[models.Project], AnnotatedDivergences]
ReleaseDivergences = Callable[[models.Release], Divergences]
ReleaseAnnotatedDivergences = Callable[[models.Release], AnnotatedDivergences]
@@ -67,6 +72,138 @@ def divergences_with_annotations(
yield AnnotatedDivergence(list(components), validator, source, d)
+async def everything(data: db.Session) -> AsyncAnnotatedDivergences:
+ """Yield divergences for all projects and releases in the DB."""
+ projects_sorted = await
data.project(_distribution_channels=True).order_by(models.Project.name).all()
+ releases_sorted = await data.release().order_by(models.Release.name).all()
+
+ for p in await asyncio.to_thread(projects, projects_sorted):
+ yield p
+
+ for r in await asyncio.to_thread(releases, releases_sorted):
+ yield r
+
+
+def project(p: models.Project) -> AnnotatedDivergences:
+ """Check that a project is valid."""
+
+ yield from project_category(p)
+ yield from project_committee(p)
+ yield from project_created(p)
+ yield from project_distribution_channels(p)
+ yield from project_full_name(p)
+ yield from project_programming_languages(p)
+ yield from project_release_policy(p)
+
+
+def project_components(
+ *components: str,
+) -> Callable[[ProjectDivergences], ProjectAnnotatedDivergences]:
+ """Wrap a Project divergence generator to yield annotated divergences."""
+
+ def wrap(original: ProjectDivergences) -> ProjectAnnotatedDivergences:
+ def replacement(p: models.Project) -> AnnotatedDivergences:
+ yield from divergences_with_annotations(
+ components,
+ original.__name__,
+ p.name,
+ original(p),
+ )
+
+ return replacement
+
+ return wrap
+
+
+@project_components("Project.category")
+def project_category(p: models.Project) -> Divergences:
+ """Check that the category string uses 'label, label' syntax without
colons."""
+
+ def okay(cat: str | None) -> bool:
+ if not cat:
+ return True
+ tokens = [t.strip() for t in cat.split(",")]
+ if any((not t) or (":" in t) for t in tokens):
+ return False
+ return True
+
+ expected = "comma separated labels without colon"
+ yield from divergences_predicate(okay, expected, p.category)
+
+
+@project_components("Project.committee_name")
+def project_committee(p: models.Project) -> Divergences:
+ """Check that the project is linked to a committee."""
+
+ def okay(cn: str | None) -> bool:
+ return cn is not None
+
+ expected = "committee_name to be set"
+ yield from divergences_predicate(okay, expected, p.committee_name)
+
+
+@project_components("Project.created")
+def project_created(p: models.Project) -> Divergences:
+ """Check that the project created timestamp is in the past."""
+ now = datetime.datetime.now(datetime.UTC)
+
+ def predicate(dt: datetime.datetime) -> bool:
+ return dt < now
+
+ expected = "value to be in the past"
+ yield from divergences_predicate(predicate, expected, p.created)
+
+
+@project_components("Project.distribution_channels")
+def project_distribution_channels(p: models.Project) -> Divergences:
+ """Check that distribution_channels is empty."""
+ expected: list[object] = []
+ actual = p.distribution_channels
+ yield from divergences(expected, actual)
+
+
+@project_components("Project.full_name")
+def project_full_name(p: models.Project) -> Divergences:
+ """Check that the project full_name is present and starts with 'Apache
'."""
+
+ def okay(fn: str | None) -> bool:
+ return (fn is not None) and fn.startswith("Apache ")
+
+ expected = "full_name to be set and start with 'Apache '"
+ yield from divergences_predicate(okay, expected, p.full_name)
+
+
+@project_components("Project.programming_languages")
+def project_programming_languages(p: models.Project) -> Divergences:
+ """Check that programming_languages uses 'label, label' syntax without
colons."""
+
+ def okay(pl: str | None) -> bool:
+ if not pl:
+ return True
+ tokens = [t.strip() for t in pl.split(",")]
+ if any((not t) or (":" in t) for t in tokens):
+ return False
+ return True
+
+ expected = "comma separated labels without colon"
+ yield from divergences_predicate(okay, expected, p.programming_languages)
+
+
+@project_components("Project.release_policy")
+def project_release_policy(p: models.Project) -> Divergences:
+ """Ensure that release_policy is None."""
+
+ expected = None
+ actual = p.release_policy_id
+ yield from divergences(expected, actual)
+
+
+def projects(ps: Iterable[models.Project]) -> AnnotatedDivergences:
+ """Validate multiple projects."""
+ for p in ps:
+ yield from project(p)
+
+
def release(r: models.Release) -> AnnotatedDivergences:
"""Check that a release is valid."""
yield from release_created(r)
diff --git a/scripts/integrity_check.py b/scripts/integrity_check.py
index 2090dd2..436121e 100644
--- a/scripts/integrity_check.py
+++ b/scripts/integrity_check.py
@@ -14,12 +14,13 @@ import atr.validate as validate
async def amain() -> None:
await db.init_database_for_worker()
async with db.session() as data:
- releases = await data.release().all()
- divergences = 0
- for divergence in validate.releases(releases):
+ divergences = [d async for d in validate.everything(data)]
+ for divergence in divergences:
print(divergence)
- divergences += 1
- print(len(releases), "releases,", divergences, "errors")
+ print(len(divergences), "errors")
+
+ if divergences:
+ sys.exit(1)
def main() -> None:
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
