As a means for binhost clients to select source repo revisions which are consistent with binhosts, inject REPO_REVISIONS from a package into the index header, using a history of synced revisions to guarantee forward progress. This queries the relevant repos to check if any new revisions have appeared in the absence of a proper sync operation.
Bug: https://bugs.gentoo.org/924772 Signed-off-by: Zac Medico <zmed...@gentoo.org> --- lib/portage/dbapi/bintree.py | 66 ++++++++++++++++++++- lib/portage/tests/sync/test_sync_local.py | 71 +++++++++++++++++++---- 2 files changed, 123 insertions(+), 14 deletions(-) diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 7bc1f60f6d..fbf60e74eb 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -48,6 +48,7 @@ from portage.exception import ( from portage.localization import _ from portage.output import colorize from portage.package.ebuild.profile_iuse import iter_iuse_vars +from portage.sync.revision_history import get_repo_revision_history from portage.util import ensure_dirs from portage.util.file_copy import copyfile from portage.util.futures import asyncio @@ -62,6 +63,7 @@ from portage import _unicode_encode import codecs import errno import io +import json import re import stat import subprocess @@ -134,13 +136,19 @@ class bindbapi(fakedbapi): "USE", "_mtime_", } + # Keys required only when initially adding a package. + self._init_aux_keys = { + "REPO_REVISIONS", + } self._aux_cache = {} self._aux_cache_slot_dict_cache = None @property def _aux_cache_slot_dict(self): if self._aux_cache_slot_dict_cache is None: - self._aux_cache_slot_dict_cache = slot_dict_class(self._aux_cache_keys) + self._aux_cache_slot_dict_cache = slot_dict_class( + chain(self._aux_cache_keys, self._init_aux_keys) + ) return self._aux_cache_slot_dict_cache def __getstate__(self): @@ -1791,6 +1799,10 @@ class binarytree: pkgindex = self._new_pkgindex() d = self._inject_file(pkgindex, cpv, full_path) + repo_revisions = d.get("REPO_REVISIONS") + if repo_revisions: + repo_revisions = json.loads(repo_revisions) + self._inject_repo_revisions(pkgindex.header, repo_revisions) self._update_pkgindex_header(pkgindex.header) self._pkgindex_write(pkgindex) @@ -1872,7 +1884,7 @@ class binarytree: @return: package metadata """ if keys is None: - keys = self.dbapi._aux_cache_keys + keys = chain(self.dbapi._aux_cache_keys, self.dbapi._init_aux_keys) metadata = self.dbapi._aux_cache_slot_dict() else: metadata = {} @@ -1916,6 +1928,56 @@ class binarytree: return metadata + def _inject_repo_revisions(self, header, repo_revisions): + """ + Inject REPO_REVISIONS from a package into the index header, + using a history of synced revisions to guarantee forward + progress. This queries the relevant repos to check if any + new revisions have appeared in the absence of a proper sync + operation. + + This does not expose REPO_REVISIONS that do not appear in + the sync history, since such revisions suggest that the + package was not built locally, and in this case its + REPO_REVISIONS are not intended to be exposed. + """ + synced_repo_revisions = get_repo_revision_history( + self.settings["EROOT"], + [self.settings.repositories[repo_name] for repo_name in repo_revisions], + ) + header_repo_revisions = ( + json.loads(header["REPO_REVISIONS"]) if header.get("REPO_REVISIONS") else {} + ) + for repo_name, repo_revision in repo_revisions.items(): + rev_list = synced_repo_revisions.get(repo_name, []) + header_rev = header_repo_revisions.get(repo_name) + if not rev_list or header_rev in (repo_revision, rev_list[0]): + continue + try: + header_rev_index = ( + None if header_rev is None else rev_list.index(header_rev) + ) + except ValueError: + header_rev_index = None + try: + repo_revision_index = rev_list.index(repo_revision) + except ValueError: + repo_revision_index = None + if repo_revision_index is not None and ( + header_rev_index is None or repo_revision_index < header_rev_index + ): + # There is forward progress when repo_revision is more recent + # than header_rev or header_rev was not found in the history. + # Do not expose repo_revision here if it does not appear in + # the history, since this suggests that the package was not + # built locally and in this case its REPO_REVISIONS are not + # intended to be exposed here. + header_repo_revisions[repo_name] = repo_revision + if header_repo_revisions: + header["REPO_REVISIONS"] = json.dumps( + header_repo_revisions, ensure_ascii=False, sort_keys=True + ) + def _inject_file(self, pkgindex, cpv, filename): """ Add a package to internal data structures, and add an diff --git a/lib/portage/tests/sync/test_sync_local.py b/lib/portage/tests/sync/test_sync_local.py index 91649398de..7e6158ee45 100644 --- a/lib/portage/tests/sync/test_sync_local.py +++ b/lib/portage/tests/sync/test_sync_local.py @@ -380,6 +380,45 @@ class SyncLocalTestCase(TestCase): (homedir, lambda: self.assertTrue(bool(get_revision_history()))), ) + def assert_latest_rev_in_packages_index(positive): + """ + If we build a binary package then its REPO_REVISIONS should + propagate into $PKGDIR/Packages as long as it results in + forward progress according to the repo revision history. + """ + revision_history = get_revision_history() + prefix = "REPO_REVISIONS:" + header_repo_revisions = None + try: + with open( + os.path.join(settings["PKGDIR"], "Packages"), encoding="utf8" + ) as f: + for line in f: + if line.startswith(prefix): + header_repo_revisions = line[len(prefix) :].strip() + break + except FileNotFoundError: + pass + + if positive: + self.assertFalse(header_repo_revisions is None) + + if header_repo_revisions is None: + header_repo_revisions = {} + else: + header_repo_revisions = json.loads(header_repo_revisions) + + (self.assertEqual if positive else self.assertNotEqual)( + revision_history.get(repo.name, [False])[0], + header_repo_revisions.get(repo.name, None), + ) + + pkgindex_revisions_cmds = ( + (homedir, lambda: assert_latest_rev_in_packages_index(False)), + (homedir, cmds["emerge"] + ("-B", "dev-libs/A")), + (homedir, lambda: assert_latest_rev_in_packages_index(True)), + ) + def hg_init_global_config(): with open(os.path.join(homedir, ".hgrc"), "w") as f: f.write(f"[ui]\nusername = {committer_name} <{committer_email}>\n") @@ -447,18 +486,25 @@ class SyncLocalTestCase(TestCase): pythonpath = ":" + pythonpath pythonpath = PORTAGE_PYM_PATH + pythonpath - env = { - "PORTAGE_OVERRIDE_EPREFIX": eprefix, - "DISTDIR": distdir, - "GENTOO_COMMITTER_NAME": committer_name, - "GENTOO_COMMITTER_EMAIL": committer_email, - "HOME": homedir, - "PATH": settings["PATH"], - "PORTAGE_GRPNAME": os.environ["PORTAGE_GRPNAME"], - "PORTAGE_USERNAME": os.environ["PORTAGE_USERNAME"], - "PYTHONDONTWRITEBYTECODE": os.environ.get("PYTHONDONTWRITEBYTECODE", ""), - "PYTHONPATH": pythonpath, - } + env = settings.environ() + env.update( + { + "PORTAGE_OVERRIDE_EPREFIX": eprefix, + "DISTDIR": distdir, + "GENTOO_COMMITTER_NAME": committer_name, + "GENTOO_COMMITTER_EMAIL": committer_email, + "HOME": homedir, + "PORTAGE_INST_GID": str(os.getgid()), + "PORTAGE_INST_UID": str(os.getuid()), + "PORTAGE_GRPNAME": os.environ["PORTAGE_GRPNAME"], + "PORTAGE_USERNAME": os.environ["PORTAGE_USERNAME"], + "PYTHONDONTWRITEBYTECODE": os.environ.get( + "PYTHONDONTWRITEBYTECODE", "" + ), + "PYTHONPATH": pythonpath, + } + ) + repos_set_conf("rsync") if os.environ.get("SANDBOX_ON") == "1": @@ -518,6 +564,7 @@ class SyncLocalTestCase(TestCase): + upstream_git_commit + sync_cmds + repo_revisions_cmds + + pkgindex_revisions_cmds + mercurial_tests ): if hasattr(cmd, "__call__"): -- 2.41.0
