This is an automated email from the ASF dual-hosted git repository. sbp pushed a commit to branch sbp in repository https://gitbox.apache.org/repos/asf/tooling-trusted-releases.git
commit a77ac42090e07c28c2caf568e88117db86b09469 Author: Sean B. Palmer <[email protected]> AuthorDate: Wed Mar 4 15:17:09 2026 +0000 Migrate all revision creators to use quarantine --- atr/get/test.py | 4 +- atr/post/draft.py | 21 +++++-- atr/post/revisions.py | 8 ++- atr/storage/writers/keys.py | 2 +- atr/storage/writers/release.py | 14 +++-- atr/storage/writers/revision.py | 114 ------------------------------------- atr/storage/writers/vote.py | 4 +- atr/tasks/sbom.py | 4 +- tests/unit/test_create_revision.py | 8 ++- 9 files changed, 41 insertions(+), 138 deletions(-) diff --git a/atr/get/test.py b/atr/get/test.py index 4022add6..99385ce2 100644 --- a/atr/get/test.py +++ b/atr/get/test.py @@ -111,7 +111,7 @@ async def test_merge( async with aiofiles.open(path_prior / "from_prior.txt", "w") as f: await f.write("prior content") - await wacp_p.revision.create_revision( + await wacp_p.revision.create_revision_with_quarantine( str(project_name), str(version_name), session.uid, @@ -119,7 +119,7 @@ async def test_merge( modify=modify_prior, ) - await wacp_n.revision.create_revision( + await wacp_n.revision.create_revision_with_quarantine( str(project_name), str(version_name), session.uid, diff --git a/atr/post/draft.py b/atr/post/draft.py index af36550b..533b5242 100644 --- a/atr/post/draft.py +++ b/atr/post/draft.py @@ -58,7 +58,7 @@ async def cache_reset( description = "Empty revision to restart all checks without cache for the whole release candidate draft" async with storage.write(session) as write: wacp = await write.as_project_committee_participant(str(project_name)) - await wacp.revision.create_revision( + result = await wacp.revision.create_revision_with_quarantine( str(project_name), str(version_name), session.uid, @@ -66,11 +66,14 @@ async def cache_reset( reset_to_global_cache=True, ) + success = "Release set back to global caching" + if isinstance(result, sql.Quarantined): + success += ". Archive validation in progress." return await session.redirect( get.compose.selected, project_name=str(project_name), version_name=str(version_name), - success="Release set back to global caching", + success=success, ) @@ -203,7 +206,7 @@ async def recheck( description = "Empty revision to restart all checks without cache for the whole release candidate draft" async with storage.write(session) as write: wacp = await write.as_project_committee_participant(str(project_name)) - await wacp.revision.create_revision( + result = await wacp.revision.create_revision_with_quarantine( str(project_name), str(version_name), session.uid, @@ -211,11 +214,14 @@ async def recheck( set_local_cache=True, ) + success = "All checks restarted with release-local cache" + if isinstance(result, sql.Quarantined): + success += ". Archive validation in progress." return await session.redirect( get.compose.selected, project_name=str(project_name), version_name=str(version_name), - success="All checks restarted with release-local cache", + success=success, ) @@ -285,7 +291,7 @@ async def sbomgen( if not success: raise web.FlashError("Internal error: SBOM generation timed out") - await wacp.revision.create_revision( + result = await wacp.revision.create_revision_with_quarantine( str(project_name), str(version_name), session.uid, description=description, modify=modify ) @@ -296,9 +302,12 @@ async def sbomgen( get.compose.selected, project_name=str(project_name), version_name=str(version_name) ) + success = f"SBOM generated for {rel_path.name}" + if isinstance(result, sql.Quarantined): + success += ". Archive validation in progress." return await session.redirect( get.compose.selected, - success=f"SBOM generated for {rel_path.name}", + success=success, project_name=str(project_name), version_name=str(version_name), ) diff --git a/atr/post/revisions.py b/atr/post/revisions.py index 1c994030..72f6497d 100644 --- a/atr/post/revisions.py +++ b/atr/post/revisions.py @@ -70,12 +70,16 @@ async def _set_revision( description = f"Copy of revision {selected_revision_number} through web interface" async with storage.write(session) as write: wacp = await write.as_project_committee_participant(project_name) - new_revision = await wacp.revision.create_revision( + result = await wacp.revision.create_revision_with_quarantine( project_name, version_name, session.uid, description=description, clone_from=selected_revision_number ) + if isinstance(result, sql.Quarantined): + success = f"Revision copy from {selected_revision_number} received. Archive validation in progress." + else: + success = f"Copied revision {selected_revision_number} to new latest revision, {result.number}" return await session.redirect( get.revisions.selected, - success=f"Copied revision {selected_revision_number} to new latest revision, {new_revision.number}", + success=success, project_name=project_name, version_name=version_name, ) diff --git a/atr/storage/writers/keys.py b/atr/storage/writers/keys.py index 4b3cff77..ca1e4f58 100644 --- a/atr/storage/writers/keys.py +++ b/atr/storage/writers/keys.py @@ -490,7 +490,7 @@ class CommitteeParticipant(FoundationCommitter): path_in_new_revision = path / "KEYS" await aiofiles.os.remove(path_in_new_revision) - await self.__write_as.revision.create_revision( + await self.__write_as.revision.create_revision_with_quarantine( project_name, version_name, self.__asf_uid, description=description, modify=modify ) return outcomes diff --git a/atr/storage/writers/release.py b/atr/storage/writers/release.py index 79735012..963fe9ab 100644 --- a/atr/storage/writers/release.py +++ b/atr/storage/writers/release.py @@ -172,7 +172,7 @@ class CommitteeParticipant(FoundationCommitter): await aiofiles.os.rmdir(path_to_remove) try: - await self.__write_as.revision.create_revision( + await self.__write_as.revision.create_revision_with_quarantine( project_name, version_name, self.__asf_uid, description=description, modify=modify ) except types.FailedError as e: @@ -212,7 +212,7 @@ class CommitteeParticipant(FoundationCommitter): # Delete the file await aiofiles.os.remove(path_in_new_revision) - await self.__write_as.revision.create_revision( + await self.__write_as.revision.create_revision_with_quarantine( project_name, version, self.__asf_uid, description=description, modify=modify ) return metadata_files_deleted @@ -251,7 +251,7 @@ class CommitteeParticipant(FoundationCommitter): async with aiofiles.open(hash_path_in_new_revision, "w") as f: await f.write(f"{hash_value} {rel_path.name}\n") - await self.__write_as.revision.create_revision( + await self.__write_as.revision.create_revision_with_quarantine( project_name, version_name, self.__asf_uid, description=description, modify=modify ) @@ -297,7 +297,7 @@ class CommitteeParticipant(FoundationCommitter): ) try: - await self.__write_as.revision.create_revision( + await self.__write_as.revision.create_revision_with_quarantine( project_name, version_name, self.__asf_uid, description=description, modify=modify ) except types.FailedError as e: @@ -377,7 +377,7 @@ class CommitteeParticipant(FoundationCommitter): renamed_count = await self.__remove_rc_tags_revision(path, error_messages) try: - await self.__write_as.revision.create_revision( + await self.__write_as.revision.create_revision_with_quarantine( project_name, version_name, self.__asf_uid, description=description, modify=modify ) except types.FailedError as e: @@ -444,7 +444,9 @@ class CommitteeParticipant(FoundationCommitter): await self.__data.refresh(release) description = "Creation of empty release candidate draft through web interface" - await self.__write_as.revision.create_revision(project_name, version, self.__asf_uid, description=description) + await self.__write_as.revision.create_revision_with_quarantine( + project_name, version, self.__asf_uid, description=description + ) self.__write_as.append_to_audit_log( asf_uid=self.__asf_uid, project_name=project_name, diff --git a/atr/storage/writers/revision.py b/atr/storage/writers/revision.py index 909a9bd2..e3bcde32 100644 --- a/atr/storage/writers/revision.py +++ b/atr/storage/writers/revision.py @@ -323,120 +323,6 @@ class CommitteeParticipant(FoundationCommitter): self.__asf_uid = asf_uid self.__committee_name = committee_name - async def create_revision( # noqa: C901 - self, - project_name: str, - version_name: str, - asf_uid: str, - description: str | None = None, - set_local_cache: bool = False, - reset_to_global_cache: bool = False, - modify: Callable[[pathlib.Path, sql.Revision | None], Awaitable[None]] | None = None, - clone_from: str | None = None, - ) -> sql.Revision: - """Create a new revision.""" - # Get the release - release_name = sql.release_name(project_name, version_name) - async with db.session() as data: - release = await data.release(name=release_name, _release_policy=True, _project_release_policy=True).demand( - RuntimeError("Release does not exist for new revision creation") - ) - if clone_from is not None: - old_revision = await data.revision(release_name=release_name, number=clone_from).demand( - RuntimeError(f"Revision {clone_from} does not exist") - ) - else: - old_revision = await interaction.latest_revision(release) - if set_local_cache: - release.check_cache_key = str(uuid.uuid4()) - if reset_to_global_cache: - release.check_cache_key = None - - if clone_from is not None: - old_release_dir = paths.release_directory_base(release) / clone_from - else: - old_release_dir = paths.release_directory(release) - merge_enabled = clone_from is None - - # Create a temporary directory - # We ensure, below, that it's removed on any exception - # Use the tmp subdirectory of state, to ensure that it is on the same filesystem - prefix_token = secrets.token_hex(16) - temp_dir: str = await asyncio.to_thread(tempfile.mkdtemp, prefix=prefix_token + "-", dir=paths.get_tmp_dir()) - temp_dir_path = pathlib.Path(temp_dir) - - try: - # The directory was created by mkdtemp, but it's empty - if old_revision is not None: - # If this is not the first revision, hard link the previous revision - await util.create_hard_link_clone(old_release_dir, temp_dir_path, do_not_create_dest_dir=True) - # The directory is either empty or its files are hard linked to the previous revision - if modify is not None: - await modify(temp_dir_path, old_revision) - except types.FailedError: - await aioshutil.rmtree(temp_dir) - raise - except Exception: - await aioshutil.rmtree(temp_dir) - raise - - validation_errors = await asyncio.to_thread(detection.validate_directory, temp_dir_path) - if validation_errors: - await aioshutil.rmtree(temp_dir) - raise types.FailedError("File validation failed:\n" + "\n".join(validation_errors)) - - # Ensure that the permissions of every directory are 755 - try: - await asyncio.to_thread(util.chmod_directories, temp_dir_path) - except Exception: - await aioshutil.rmtree(temp_dir) - raise - - # Make files read only to prevent them from being modified through hard links - try: - await asyncio.to_thread(util.chmod_files, temp_dir_path, 0o444) - except Exception: - await aioshutil.rmtree(temp_dir) - raise - - try: - path_to_hash, path_to_size = await attestable.paths_to_hashes_and_sizes(temp_dir_path) - parent_revision_number = old_revision.number if old_revision else None - previous_attestable = None - if parent_revision_number is not None: - previous_attestable = await attestable.load(project_name, version_name, parent_revision_number) - base_inodes: dict[str, int] = {} - base_hashes: dict[str, str] = {} - if merge_enabled and (old_revision is not None): - base_dir = old_release_dir - base_inodes = await asyncio.to_thread(util.paths_to_inodes, base_dir) - base_hashes = dict(previous_attestable.paths) if (previous_attestable is not None) else {} - n_inodes = await asyncio.to_thread(util.paths_to_inodes, temp_dir_path) - except Exception: - await aioshutil.rmtree(temp_dir) - raise - - async with SafeSession(temp_dir) as data: - return await finalise_revision( - data, - asf_uid=asf_uid, - base_hashes=base_hashes, - base_inodes=base_inodes, - description=description, - merge_enabled=merge_enabled, - n_inodes=n_inodes, - old_revision=old_revision, - path_to_hash=path_to_hash, - path_to_size=path_to_size, - previous_attestable=previous_attestable, - project_name=project_name, - release=release, - release_name=release_name, - temp_dir=temp_dir, - temp_dir_path=temp_dir_path, - version_name=version_name, - ) - async def create_revision_with_quarantine( # noqa: C901 self, project_name: str, diff --git a/atr/storage/writers/vote.py b/atr/storage/writers/vote.py index 1f276071..f221bc1e 100644 --- a/atr/storage/writers/vote.py +++ b/atr/storage/writers/vote.py @@ -294,7 +294,7 @@ class CommitteeMember(CommitteeParticipant): success_message = "Vote marked as passed" description = "Create a preview revision from the last candidate draft" - await self.__write_as.revision.create_revision( + await self.__write_as.revision.create_revision_with_quarantine( project_name, release.version, self.__asf_uid, description=description ) else: @@ -381,7 +381,7 @@ class CommitteeMember(CommitteeParticipant): success_message = "Vote marked as passed" description = "Create a preview revision from the last candidate draft" - await self.__write_as.revision.create_revision( + await self.__write_as.revision.create_revision_with_quarantine( project_name, release.version, self.__asf_uid, description=description ) if (voting_round == 2) and (release.podling_thread_id is not None): diff --git a/atr/tasks/sbom.py b/atr/tasks/sbom.py index 5a6539f0..7c5ac7e2 100644 --- a/atr/tasks/sbom.py +++ b/atr/tasks/sbom.py @@ -111,7 +111,7 @@ async def augment(args: FileArgs) -> results.Results | None: async with aiofiles.open(new_full_path, "w", encoding="utf-8") as f: await f.write(merged.dumps()) - await wacp.revision.create_revision( + await wacp.revision.create_revision_with_quarantine( args.project_name, args.version_name, args.asf_uid or "unknown", description=description, modify=modify ) @@ -180,7 +180,7 @@ async def osv_scan(args: FileArgs) -> results.Results | None: async with aiofiles.open(new_full_path, "w", encoding="utf-8") as f: await f.write(merged.dumps()) - await wacp.revision.create_revision( + await wacp.revision.create_revision_with_quarantine( args.project_name, args.version_name, args.asf_uid or "unknown", description=description, modify=modify ) diff --git a/tests/unit/test_create_revision.py b/tests/unit/test_create_revision.py index 13f84a66..9b1ec01f 100644 --- a/tests/unit/test_create_revision.py +++ b/tests/unit/test_create_revision.py @@ -128,6 +128,7 @@ async def test_clone_from_older_revision_skips_merge_without_intervening_change( ), mock.patch.object(revision.attestable, "write_files_data", new_callable=mock.AsyncMock), mock.patch.object(revision.db, "session", return_value=mock_session), + mock.patch.object(revision.detection, "detect_archives_requiring_quarantine", return_value=[]), mock.patch.object(revision.detection, "validate_directory", return_value=[]), mock.patch.object( revision.interaction, "latest_revision", new_callable=mock.AsyncMock, return_value=latest_revision @@ -146,7 +147,7 @@ async def test_clone_from_older_revision_skips_merge_without_intervening_change( mock.patch.object(revision.paths, "release_directory", return_value=tmp_path / "releases" / "00006"), mock.patch.object(revision.paths, "release_directory_base", return_value=tmp_path / "releases"), ): - await participant.create_revision("proj", "1.0", "test", clone_from="00002") + await participant.create_revision_with_quarantine("proj", "1.0", "test", clone_from="00002") if merge_mock.called: raise AssertionError( @@ -213,6 +214,7 @@ async def test_intervening_revision_triggers_merge_and_uses_latest_parent(tmp_pa ), mock.patch.object(revision.attestable, "write_files_data", new_callable=mock.AsyncMock), mock.patch.object(revision.db, "session", return_value=mock_session), + mock.patch.object(revision.detection, "detect_archives_requiring_quarantine", return_value=[]), mock.patch.object(revision.detection, "validate_directory", return_value=[]), mock.patch.object( revision.interaction, @@ -232,7 +234,7 @@ async def test_intervening_revision_triggers_merge_and_uses_latest_parent(tmp_pa mock.patch.object(revision.paths, "release_directory", return_value=tmp_path / "releases" / "00007"), mock.patch.object(revision.paths, "release_directory_base", return_value=tmp_path / "releases"), ): - created_revision = await participant.create_revision("proj", "1.0", "test") + created_revision = await participant.create_revision_with_quarantine("proj", "1.0", "test") assert isinstance(created_revision, FakeRevision) assert merge_mock.await_count == 1 @@ -262,7 +264,7 @@ async def test_modify_failed_error_propagates_and_cleans_up(tmp_path: pathlib.Pa mock.patch.object(revision.paths, "get_tmp_dir", return_value=tmp_path), ): with pytest.raises(types.FailedError, match="Intentional error"): - await participant.create_revision("proj", "1.0", "test", modify=modify) + await participant.create_revision_with_quarantine("proj", "1.0", "test", modify=modify) assert isinstance(received_args["path"], pathlib.Path) assert received_args["old_rev"] is None --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
