commit: 445f10f4214c673f8fe0a9cc518c12767be4f159 Author: Sheng Yu <syu.os <AT> protonmail <DOT> com> AuthorDate: Wed Nov 2 18:46:03 2022 +0000 Commit: Sam James <sam <AT> gentoo <DOT> org> CommitDate: Wed Nov 2 22:58:15 2022 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=445f10f4
Use binpkg extensions and header to get format This deprecates the BINPKG_FORMAT in the metadata and Package index. Also fixed a bug that obsolete packages that have been moved may be attempted to be read during package searching. Closes: https://bugs.gentoo.org/877357 Closes: https://bugs.gentoo.org/877419 Signed-off-by: Sheng Yu <syu.os <AT> protonmail.com> Closes: https://github.com/gentoo/portage/pull/928 Signed-off-by: Sam James <sam <AT> gentoo.org> lib/_emerge/BinpkgExtractorAsync.py | 4 +- lib/_emerge/BinpkgFetcher.py | 32 ++------ lib/_emerge/Package.py | 1 - lib/_emerge/actions.py | 15 +++- lib/_emerge/depgraph.py | 15 +++- lib/_emerge/search.py | 6 +- lib/portage/binpkg.py | 23 ++++-- lib/portage/dbapi/bintree.py | 142 +++++++++++++++--------------------- lib/portage/gpkg.py | 4 + lib/portage/versions.py | 16 ---- lib/portage/xpak.py | 18 +++++ 11 files changed, 141 insertions(+), 135 deletions(-) diff --git a/lib/_emerge/BinpkgExtractorAsync.py b/lib/_emerge/BinpkgExtractorAsync.py index 65b383998..ca7ccf73a 100644 --- a/lib/_emerge/BinpkgExtractorAsync.py +++ b/lib/_emerge/BinpkgExtractorAsync.py @@ -34,7 +34,9 @@ class BinpkgExtractorAsync(SpawnProcess): if binpkg_format == "xpak": self._xpak_start() else: - raise InvalidBinaryPackageFormat(self.pkg_path) + raise InvalidBinaryPackageFormat( + f"{self.pkg_path} is not a valid xpak binary package" + ) def _xpak_start(self): tar_options = "" diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py index b7021e276..9018d0ee2 100644 --- a/lib/_emerge/BinpkgFetcher.py +++ b/lib/_emerge/BinpkgFetcher.py @@ -11,9 +11,8 @@ import stat import sys import portage from portage import os -from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS -from portage.const import SUPPORTED_XPAK_EXTENSIONS, SUPPORTED_GPKG_EXTENSIONS -from portage.exception import FileNotFound, InvalidBinaryPackageFormat +from portage.binpkg import get_binpkg_format +from portage.exception import FileNotFound from portage.util._async.AsyncTaskFuture import AsyncTaskFuture from portage.util._pty import _create_pty_or_pipe @@ -28,18 +27,9 @@ class BinpkgFetcher(CompositeTask): pkg = self.pkg bintree = pkg.root_config.trees["bintree"] instance_key = bintree.dbapi._instance_key(pkg.cpv) - binpkg_format = bintree._remotepkgs[instance_key].get("BINPKG_FORMAT", None) - - if binpkg_format is None: - binpkg_path = bintree._remotepkgs[instance_key].get("PATH") - if binpkg_path.endswith(SUPPORTED_XPAK_EXTENSIONS): - binpkg_format = "xpak" - elif binpkg_path.endswith(SUPPORTED_GPKG_EXTENSIONS): - binpkg_format = "gpkg" - else: - raise InvalidBinaryPackageFormat( - f"Unsupported binary package format from '{binpkg_path}'" - ) + + binpkg_path = bintree._remotepkgs[instance_key].get("PATH") + binpkg_format = get_binpkg_format(binpkg_path) self.pkg_allocated_path = pkg.root_config.trees["bintree"].getname( pkg.cpv, allocate_new=True, remote_binpkg_format=binpkg_format @@ -128,17 +118,11 @@ class _BinpkgFetcherProcess(SpawnProcess): resumecommand = None if bintree._remote_has_index: remote_metadata = bintree._remotepkgs[bintree.dbapi._instance_key(pkg.cpv)] - binpkg_format = remote_metadata.get( - "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0] - ) - if binpkg_format not in SUPPORTED_GENTOO_BINPKG_FORMATS: - raise InvalidBinaryPackageFormat(binpkg_format) rel_uri = remote_metadata.get("PATH") if not rel_uri: - if binpkg_format == "xpak": - rel_uri = pkg.cpv + ".tbz2" - elif binpkg_format == "gpkg": - rel_uri = pkg.cpv + ".gpkg.tar" + # Assume that the remote index is out of date. No path should + # never happen in new portage versions. + rel_uri = pkg.cpv + ".tbz2" remote_base_uri = remote_metadata["BASE_URI"] uri = remote_base_uri.rstrip("/") + "/" + rel_uri.lstrip("/") fetchcommand = remote_metadata.get("FETCHCOMMAND") diff --git a/lib/_emerge/Package.py b/lib/_emerge/Package.py index a19904113..c50349e27 100644 --- a/lib/_emerge/Package.py +++ b/lib/_emerge/Package.py @@ -63,7 +63,6 @@ class Package(Task): metadata_keys = [ "BDEPEND", - "BINPKG_FORMAT", "BUILD_ID", "BUILD_TIME", "CHOST", diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py index b45a6361a..073e3ae7e 100644 --- a/lib/_emerge/actions.py +++ b/lib/_emerge/actions.py @@ -41,7 +41,13 @@ from portage.dbapi._expand_new_virt import expand_new_virt from portage.dbapi.IndexedPortdb import IndexedPortdb from portage.dbapi.IndexedVardb import IndexedVardb from portage.dep import Atom, _repo_separator, _slot_separator -from portage.exception import InvalidAtom, InvalidData, ParseError, GPGException +from portage.exception import ( + InvalidAtom, + InvalidData, + ParseError, + GPGException, + InvalidBinaryPackageFormat, +) from portage.output import ( colorize, create_color_func, @@ -2320,9 +2326,12 @@ def action_info(settings, trees, myopts, myfiles): elif pkg_type == "binary": binpkg_file = bindb.bintree.getname(pkg.cpv) ebuild_file_name = pkg.cpv.split("/")[1] + ".ebuild" - binpkg_format = pkg.cpv._metadata.get("BINPKG_FORMAT", None) - if not binpkg_format: + try: binpkg_format = get_binpkg_format(binpkg_file) + except InvalidBinaryPackageFormat as e: + out.ewarn(e) + continue + if binpkg_format == "xpak": ebuild_file_contents = portage.xpak.tbz2(binpkg_file).getfile( ebuild_file_name diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py index 79fa1f3df..c976f8205 100644 --- a/lib/_emerge/depgraph.py +++ b/lib/_emerge/depgraph.py @@ -39,6 +39,7 @@ from portage.dep._slot_operator import ignore_built_slot_operator_deps, strip_sl from portage.eapi import eapi_has_strong_blocks, eapi_has_required_use, _get_eapi_attrs from portage.exception import ( InvalidAtom, + InvalidBinaryPackageFormat, InvalidData, InvalidDependString, PackageNotFound, @@ -4580,7 +4581,19 @@ class depgraph: noiselevel=-1, ) return 0, myfavorites - binpkg_format = get_binpkg_format(x) + + try: + binpkg_format = get_binpkg_format(x) + except InvalidBinaryPackageFormat as e: + writemsg( + colorize( + "BAD", + "\n{e}\n" % x, + ), + noiselevel=-1, + ) + continue + if binpkg_format == "xpak": mytbz2 = portage.xpak.tbz2(x) mykey = None diff --git a/lib/_emerge/search.py b/lib/_emerge/search.py index 8550b0487..4989ed787 100644 --- a/lib/_emerge/search.py +++ b/lib/_emerge/search.py @@ -8,6 +8,7 @@ from portage import os from portage.dbapi.porttree import _parse_uri_map from portage.dbapi.IndexedPortdb import IndexedPortdb from portage.dbapi.IndexedVardb import IndexedVardb +from portage.exception import InvalidBinaryPackageFormat from portage.localization import localized_size from portage.output import bold, darkgreen, green, red from portage.util import writemsg_stdout @@ -490,7 +491,10 @@ class search: if db is not vardb and db.cpv_exists(mycpv): available = True if not myebuild and hasattr(db, "bintree"): - myebuild = db.bintree.getname(mycpv) + try: + myebuild = db.bintree.getname(mycpv) + except InvalidBinaryPackageFormat: + break try: mysum[0] = os.stat(myebuild).st_size except OSError: diff --git a/lib/portage/binpkg.py b/lib/portage/binpkg.py index f3f149b14..001a2e277 100644 --- a/lib/portage/binpkg.py +++ b/lib/portage/binpkg.py @@ -2,12 +2,14 @@ # Distributed under the terms of the GNU General Public License v2 import tarfile +from portage import os from portage.const import SUPPORTED_XPAK_EXTENSIONS, SUPPORTED_GPKG_EXTENSIONS +from portage.exception import InvalidBinaryPackageFormat from portage.output import colorize from portage.util import writemsg -def get_binpkg_format(binpkg_path): +def get_binpkg_format(binpkg_path, check_file=False, remote=False): if binpkg_path.endswith(SUPPORTED_XPAK_EXTENSIONS): file_ext_format = "xpak" elif binpkg_path.endswith(SUPPORTED_GPKG_EXTENSIONS): @@ -15,6 +17,17 @@ def get_binpkg_format(binpkg_path): else: file_ext_format = None + if remote: + if file_ext_format is not None: + return file_ext_format + else: + raise InvalidBinaryPackageFormat( + f"Unsupported binary package format from '{binpkg_path}'" + ) + + if file_ext_format is not None and not check_file: + return file_ext_format + try: with open(binpkg_path, "rb") as binpkg_file: header = binpkg_file.read(100) @@ -32,9 +45,7 @@ def get_binpkg_format(binpkg_path): if file_format is None: try: with tarfile.open(binpkg_path) as gpkg_tar: - if "gpkg-1" in [ - f.split("/", maxsplit=1)[-1] for f in gpkg_tar.getnames() - ]: + if "gpkg-1" in (os.path.basename(f) for f in gpkg_tar.getnames()): file_format = "gpkg" except tarfile.TarError: pass @@ -43,7 +54,9 @@ def get_binpkg_format(binpkg_path): file_format = None if file_format is None: - return None + raise InvalidBinaryPackageFormat( + f"Unsupported binary package format from '{binpkg_path}'" + ) if (file_ext_format is not None) and (file_ext_format != file_format): writemsg( diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index c8bb2c88e..771abedd5 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -85,7 +85,6 @@ class bindbapi(fakedbapi): list(fakedbapi._known_keys) + ["CHOST", "repository", "USE"] ) _pkg_str_aux_keys = fakedbapi._pkg_str_aux_keys + ( - "BINPKG_FORMAT", "BUILD_ID", "BUILD_TIME", "_mtime_", @@ -108,7 +107,6 @@ class bindbapi(fakedbapi): self._aux_cache_keys = set( [ "BDEPEND", - "BINPKG_FORMAT", "BUILD_ID", "BUILD_TIME", "CHOST", @@ -193,7 +191,8 @@ class bindbapi(fakedbapi): st = os.lstat(binpkg_path) except OSError: raise KeyError(mycpv) - binpkg_format = self.cpvdict[instance_key]["BINPKG_FORMAT"] + + binpkg_format = get_binpkg_format(binpkg_path) if binpkg_format == "xpak": metadata_bytes = portage.xpak.tbz2(binpkg_path).get_data() decode_metadata_name = False @@ -202,10 +201,6 @@ class bindbapi(fakedbapi): self.settings, mycpv, binpkg_path ).get_metadata() decode_metadata_name = True - else: - raise InvalidBinaryPackageFormat( - "Unknown binary package format %s" % binpkg_path - ) def getitem(k): if k == "_mtime_": @@ -266,7 +261,7 @@ class bindbapi(fakedbapi): if not os.path.exists(binpkg_path): raise KeyError(cpv) - binpkg_format = cpv.binpkg_format + binpkg_format = get_binpkg_format(binpkg_path) if binpkg_format == "xpak": mytbz2 = portage.xpak.tbz2(binpkg_path) mydata = mytbz2.get_data() @@ -325,7 +320,7 @@ class bindbapi(fakedbapi): await add_pkg._db.unpack_metadata(pkg, dest_dir, loop=loop) else: binpkg_file = self.bintree.getname(cpv) - binpkg_format = cpv.binpkg_format + binpkg_format = get_binpkg_format(binpkg_file) if binpkg_format == "xpak": await loop.run_in_executor( ForkExecutor(loop=loop), @@ -362,7 +357,7 @@ class bindbapi(fakedbapi): pkg_path = self.bintree.getname(cpv) if pkg_path is not None: - binpkg_format = cpv.binpkg_format + binpkg_format = get_binpkg_format(pkg_path) if binpkg_format == "xpak": extractor = BinpkgExtractorAsync( background=settings.get("PORTAGE_BACKGROUND") == "1", @@ -502,7 +497,6 @@ class binarytree: self._pkgindex_aux_keys = [ "BASE_URI", "BDEPEND", - "BINPKG_FORMAT", "BUILD_ID", "BUILD_TIME", "CHOST", @@ -546,7 +540,6 @@ class binarytree: "ACCEPT_LICENSE", "ACCEPT_PROPERTIES", "ACCEPT_RESTRICT", - "BINPKG_FORMAT", "CBUILD", "CONFIG_PROTECT", "CONFIG_PROTECT_MASK", @@ -582,13 +575,10 @@ class binarytree: "SLOT": "0", "USE": "", } - self._pkgindex_inherited_keys = ["BINPKG_FORMAT", "CHOST", "repository"] + self._pkgindex_inherited_keys = ["CHOST", "repository"] # Populate the header with appropriate defaults. self._pkgindex_default_header_data = { - "BINPKG_FORMAT": self.settings.get( - "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0] - ), "CHOST": self.settings.get("CHOST", ""), "repository": "", } @@ -679,7 +669,7 @@ class binarytree: continue moves += 1 - binpkg_format = mycpv.binpkg_format + binpkg_format = get_binpkg_format(binpkg_path) if binpkg_format == "xpak": mytbz2 = portage.xpak.tbz2(binpkg_path) mydata = mytbz2.get_data() @@ -924,24 +914,19 @@ class binarytree: metadata[_instance_key(cpv)] = d path = d.get("PATH") if not path: - binpkg_format = d["BINPKG_FORMAT"] - if binpkg_format == "xpak": - if gpkg_only: - if not gpkg_only_warned: - writemsg( - colorize( - "WARN", - "Local XPAK packages are ignored due to 'binpkg-request-signature'.\n", - ), - noiselevel=-1, - ) - gpkg_only_warned = True - continue - path = cpv + ".tbz2" - elif binpkg_format == "gpkg": - path = cpv + ".gpkg.tar" - else: + if gpkg_only: + if not gpkg_only_warned: + writemsg( + colorize( + "WARN", + "Local XPAK packages are ignored due to 'binpkg-request-signature'.\n", + ), + noiselevel=-1, + ) + gpkg_only_warned = True continue + else: + path = cpv + ".tbz2" if reindex: basename = os.path.basename(path) @@ -1040,15 +1025,11 @@ class binarytree: self.invalids.append(myfile[:-5]) continue - binpkg_format = None - if match: - binpkg_format = match.get("BINPKG_FORMAT", None) - - if not binpkg_format: - if myfile.endswith(SUPPORTED_XPAK_EXTENSIONS): - binpkg_format = "xpak" - elif myfile.endswith(SUPPORTED_GPKG_EXTENSIONS): - binpkg_format = "gpkg" + try: + binpkg_format = get_binpkg_format(myfile) + except InvalidBinaryPackageFormat: + self.invalids.append(myfile[:-5]) + continue if gpkg_only: if binpkg_format != "gpkg": @@ -1550,7 +1531,19 @@ class binarytree: continue if gpkg_only: - binpkg_format = d.get("BINPKG_FORMAT", "xpak") + try: + binpkg_format = get_binpkg_format( + d.get("PATH"), remote=True + ) + except InvalidBinaryPackageFormat: + writemsg( + colorize( + "WARN", + f"{e}\n", + ), + noiselevel=-1, + ) + continue if binpkg_format != "gpkg": if not gpkg_only_warned: writemsg( @@ -1637,7 +1630,15 @@ class binarytree: noiselevel=-1, ) return - binpkg_format = metadata["BINPKG_FORMAT"] + + try: + binpkg_format = get_binpkg_format(full_path) + except InvalidBinaryPackageFormat as e: + writemsg( + f"!!! Invalid binary package: '{full_path}'\n", + noiselevel=-1, + ) + return invalid_depend = False try: @@ -1765,8 +1766,6 @@ class binarytree: v = _unicode_decode(v) metadata[k] = " ".join(v.split()) - metadata["BINPKG_FORMAT"] = binpkg_format - return metadata def _inject_file(self, pkgindex, cpv, filename): @@ -1848,10 +1847,6 @@ class binarytree: """ pkg_path = self.getname(cpv) - try: - binpkg_format = cpv.binpkg_format - except AttributeError: - raise KeyError("{} metadata not found!".format(cpv)) d = dict(cpv._metadata.items()) d.update(perform_multiple_checksums(pkg_path, hashes=self._pkgindex_hashes)) @@ -1860,18 +1855,10 @@ class binarytree: st = os.lstat(pkg_path) d["_mtime_"] = str(st[stat.ST_MTIME]) d["SIZE"] = str(st.st_size) - d["BINPKG_FORMAT"] = binpkg_format rel_path = pkg_path[len(self.pkgdir) + 1 :] - # record location if it's non-default - if binpkg_format == "xpak": - if rel_path != cpv + ".tbz2": - d["PATH"] = rel_path - elif binpkg_format == "gpkg": - if rel_path != cpv + ".gpkg.tar": - d["PATH"] = rel_path - else: - raise InvalidBinaryPackageFormat(binpkg_format) + # Always record location + d["PATH"] = rel_path return d @@ -2079,25 +2066,14 @@ class binarytree: return (None, None) if filename is None: - try: - binpkg_format = cpv.binpkg_format - except AttributeError: - # In order to force the caller to clarify its intent, do not - # use default BINPKG_FORMAT unless allocate_new is True. - # The caller can set cpv.binpkg_format in advance if something - # other than the default is desired here. - if allocate_new: - binpkg_format = self.settings.get( - "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0] - ) - else: - binpkg_format = None + binpkg_format = self.settings.get( + "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0] + ) if not binpkg_format: - # Raise an error if the desired binpkg_format is not clear. - # The caller should either set allocate_new to True or else - # ensure that cpv.binpkg_format is set to a particular format. - raise InvalidBinaryPackageFormat(binpkg_format) + raise InvalidBinaryPackageFormat( + "Unable to determine the binpkg format." + ) elif binpkg_format == "xpak": if self._multi_instance: pf = catsplit(cpv)[1] @@ -2117,7 +2093,7 @@ class binarytree: else: filename = os.path.join(self.pkgdir, cpv + ".gpkg.tar") else: - raise InvalidBinaryPackageFormat(binpkg_format) + raise InvalidBinaryPackageFormat(f"{binpkg_format}") return (filename, build_id) @@ -2143,8 +2119,8 @@ class binarytree: def _allocate_filename(self, cpv, remote_binpkg_format=None): if remote_binpkg_format is None: try: - binpkg_format = cpv.binpkg_format - except AttributeError: + binpkg_format = get_binpkg_format(cpv._metadata["PATH"]) + except (AttributeError, KeyError): binpkg_format = self.settings.get( "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0] ) @@ -2173,8 +2149,8 @@ class binarytree: if remote_binpkg_format is None: try: - binpkg_format = cpv.binpkg_format - except AttributeError: + binpkg_format = get_binpkg_format(cpv._metadata["PATH"]) + except (AttributeError, KeyError): binpkg_format = self.settings.get( "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0] ) diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py index 42e1c60fd..5a8c16186 100644 --- a/lib/portage/gpkg.py +++ b/lib/portage/gpkg.py @@ -21,6 +21,7 @@ from portage import normalize_path from portage import _encodings from portage import _unicode_decode from portage import _unicode_encode +from portage.binpkg import get_binpkg_format from portage.exception import ( FileNotFound, InvalidBinaryPackageFormat, @@ -1560,6 +1561,7 @@ class gpkg: with open(self.gpkg_file, "rb") as container: container_tar_format = self._get_tar_format(container) if container_tar_format is None: + get_binpkg_format(self.gpkg_file, check_file=True) raise InvalidBinaryPackageFormat( f"Cannot identify tar format: {self.gpkg_file}" ) @@ -1569,12 +1571,14 @@ class gpkg: try: container_files = container.getnames() except tarfile.ReadError: + get_binpkg_format(self.gpkg_file, check_file=True) raise InvalidBinaryPackageFormat( f"Cannot read tar file: {self.gpkg_file}" ) # Check if gpkg version file exists in any place if self.gpkg_version not in (os.path.basename(f) for f in container_files): + get_binpkg_format(self.gpkg_file, check_file=True) raise InvalidBinaryPackageFormat(f"Invalid gpkg file: {self.gpkg_file}") # Check how many layers are in the container diff --git a/lib/portage/versions.py b/lib/portage/versions.py index 3d8694a0e..e1f798bdf 100644 --- a/lib/portage/versions.py +++ b/lib/portage/versions.py @@ -491,22 +491,6 @@ class _pkg_str(str): self.__dict__["_stable"] = stable return stable - @property - def binpkg_format(self): - """ - Returns the BINPKG_FORMAT metadata. A return value of None means - that the format is unset. If there is no metadata available or the - BINPKG_FORMAT key is missing from the metadata, then raise - AttributeError. - - @rtype: str or None - @return: a non-empty BINPKG_FORMAT string, or None - """ - try: - return self._metadata["BINPKG_FORMAT"] or None - except (AttributeError, KeyError): - raise AttributeError("binpkg_format") - def pkgsplit(mypkg, silent=1, eapi=None): """ diff --git a/lib/portage/xpak.py b/lib/portage/xpak.py index b20429b6f..b586c0be8 100644 --- a/lib/portage/xpak.py +++ b/lib/portage/xpak.py @@ -43,6 +43,8 @@ from portage import normalize_path from portage import _encodings from portage import _unicode_decode from portage import _unicode_encode +from portage.binpkg import get_binpkg_format +from portage.exception import InvalidBinaryPackageFormat from portage.util.file_copy import copyfile @@ -435,14 +437,26 @@ class tbz2: self.infosize = 0 self.xpaksize = 0 if trailer[-4:] != b"STOP": + try: + get_binpkg_format(self.file, check_file=True) + except InvalidBinaryPackageFormat: + pass return 0 if trailer[0:8] != b"XPAKSTOP": + try: + get_binpkg_format(self.file, check_file=True) + except InvalidBinaryPackageFormat: + pass return 0 self.infosize = decodeint(trailer[8:12]) self.xpaksize = self.infosize + 8 a.seek(-(self.xpaksize), 2) header = a.read(16) if header[0:8] != b"XPAKPACK": + try: + get_binpkg_format(self.file, check_file=True) + except InvalidBinaryPackageFormat: + pass return 0 self.indexsize = decodeint(header[8:12]) self.datasize = decodeint(header[12:16]) @@ -453,6 +467,10 @@ class tbz2: except SystemExit: raise except: + try: + get_binpkg_format(self.file, check_file=True) + except InvalidBinaryPackageFormat: + pass return 0 finally: if a is not None:
