commit: fb1d0a22f65747b750143080536a4129e8654f97 Author: Zac Medico <zmedico <AT> gentoo <DOT> org> AuthorDate: Fri Dec 29 00:29:40 2023 +0000 Commit: Zac Medico <zmedico <AT> gentoo <DOT> org> CommitDate: Sat Dec 30 21:46:04 2023 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=fb1d0a22
dbapi: KeyError tolerance during package moves Raise a new CorruptionKeyError exception type instead of a plain KeyError when os.stat fails. Treat this type of exception as a warning during package moves. Also fix this error that the test case triggered in the binarytree.remove method: NameError: name 'binpkg_path' is not defined Bug: https://bugs.gentoo.org/920828 Signed-off-by: Zac Medico <zmedico <AT> gentoo.org> lib/portage/dbapi/__init__.py | 14 ++++-- lib/portage/dbapi/bintree.py | 28 +++++++++--- lib/portage/dbapi/vartree.py | 9 ++-- lib/portage/exception.py | 6 ++- lib/portage/tests/update/test_update_dbentry.py | 60 +++++++++++++++++++++++++ 5 files changed, 103 insertions(+), 14 deletions(-) diff --git a/lib/portage/dbapi/__init__.py b/lib/portage/dbapi/__init__.py index 09163e94d8..6f95b93a21 100644 --- a/lib/portage/dbapi/__init__.py +++ b/lib/portage/dbapi/__init__.py @@ -25,7 +25,11 @@ from portage.const import MERGING_IDENTIFIER from portage import os from portage import auxdbkeys from portage.eapi import _get_eapi_attrs -from portage.exception import InvalidBinaryPackageFormat, InvalidData +from portage.exception import ( + CorruptionKeyError, + InvalidBinaryPackageFormat, + InvalidData, +) from portage.localization import _ from _emerge.Package import Package @@ -424,7 +428,7 @@ class dbapi: if metadata_updates: try: aux_update(cpv, metadata_updates) - except InvalidBinaryPackageFormat as e: + except (InvalidBinaryPackageFormat, CorruptionKeyError) as e: warnings.warn(e) if onUpdate: onUpdate(maxval, i + 1) @@ -470,5 +474,9 @@ class dbapi: ): newslot = f"{newslot}/{mycpv.sub_slot}" mydata = {"SLOT": newslot + "\n"} - self.aux_update(mycpv, mydata) + try: + self.aux_update(mycpv, mydata) + except CorruptionKeyError as e: + warnings.warn(e) + continue return moves diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index a20c8dfe26..a139e37659 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -37,6 +37,7 @@ from portage.dbapi.virtual import fakedbapi from portage.dep import Atom, use_reduce, paren_enclose from portage.exception import ( AlarmSignal, + CorruptionKeyError, InvalidPackageName, InvalidBinaryPackageFormat, ParseError, @@ -210,9 +211,9 @@ class bindbapi(fakedbapi): raise KeyError(mycpv) binpkg_path = os.path.join(self.bintree.pkgdir, binpkg_path) try: - st = os.lstat(binpkg_path) - except OSError: - raise KeyError(mycpv) + st = os.stat(binpkg_path) + except OSError as oe: + raise CorruptionKeyError(mycpv) from oe binpkg_format = get_binpkg_format(binpkg_path) if binpkg_format == "xpak": @@ -283,8 +284,10 @@ class bindbapi(fakedbapi): cpv_str += f"-{build_id}" binpkg_path = self.bintree.getname(cpv) - if not os.path.exists(binpkg_path): - raise KeyError(cpv) + try: + os.stat(binpkg_path) + except OSError as oe: + raise CorruptionKeyError(cpv) from oe binpkg_format = get_binpkg_format(binpkg_path) if binpkg_format == "xpak": @@ -694,7 +697,18 @@ class binarytree: continue binpkg_path = self.getname(mycpv) - if os.path.exists(binpkg_path) and not os.access(binpkg_path, os.W_OK): + try: + os.stat(binpkg_path) + except FileNotFoundError: + writemsg(_("!!! File not found: %s\n") % binpkg_path, noiselevel=-1) + continue + except OSError as oe: + writemsg( + _("!!! File os error (path %s): %s\n") % (binpkg_path, oe), + noiselevel=-1, + ) + continue + if not os.access(binpkg_path, os.W_OK): writemsg( _("!!! Cannot update readonly binary: %s\n") % mycpv, noiselevel=-1 ) @@ -1829,7 +1843,7 @@ class binarytree: writemsg( colorize( "WARN", - f"Failed to remove package: {binpkg_path} {str(err)}", + f"Failed to remove package: {pkg_path} {str(err)}", ) ) finally: diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index 11767a9f91..5d39ca1965 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -1,4 +1,4 @@ -# Copyright 1998-2021 Gentoo Authors +# Copyright 1998-2023 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 __all__ = ["vardbapi", "vartree", "dblink"] + ["write_contents", "tar_contents"] @@ -60,6 +60,7 @@ from portage.const import ( from portage.dbapi import dbapi from portage.exception import ( CommandNotFound, + CorruptionKeyError, InvalidData, InvalidLocation, InvalidPackageName, @@ -990,8 +991,10 @@ class vardbapi(dbapi): def aux_update(self, cpv, values): mylink = self._dblink(cpv) - if not mylink.exists(): - raise KeyError(cpv) + try: + os.stat(mylink.dbdir) + except OSError as oe: + raise CorruptionKeyError(cpv) from oe self._bump_mtime(cpv) self._clear_pkg_cache(mylink) for k, v in values.items(): diff --git a/lib/portage/exception.py b/lib/portage/exception.py index 153a9f9a55..7b48aa919e 100644 --- a/lib/portage/exception.py +++ b/lib/portage/exception.py @@ -1,4 +1,4 @@ -# Copyright 1998-2020 Gentoo Authors +# Copyright 1998-2023 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import signal @@ -30,6 +30,10 @@ class CorruptionError(PortageException): """Corruption indication""" +class CorruptionKeyError(CorruptionError, PortageKeyError): + """KeyError raised when corruption is detected (cause should be accesssible as __cause__)""" + + class InvalidDependString(PortageException): """An invalid depend string has been encountered""" diff --git a/lib/portage/tests/update/test_update_dbentry.py b/lib/portage/tests/update/test_update_dbentry.py index a3c9a37e8c..3b3f0caae6 100644 --- a/lib/portage/tests/update/test_update_dbentry.py +++ b/lib/portage/tests/update/test_update_dbentry.py @@ -1,6 +1,7 @@ # Copyright 2012-2023 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 +import shutil import sys import re import textwrap @@ -9,6 +10,7 @@ import portage from portage import os from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS from portage.dep import Atom +from portage.exception import CorruptionKeyError from portage.tests import TestCase from portage.tests.resolver.ResolverPlayground import ResolverPlayground from portage.update import update_dbentry @@ -186,6 +188,11 @@ class UpdateDbentryTestCase(TestCase): "EAPI": "4", "SLOT": "2", }, + "dev-libs/B-2::test_repo": { + "SLOT": "2", + "RDEPEND": "dev-libs/M dev-libs/N dev-libs/P", + "EAPI": "4", + }, "dev-libs/B-1::test_repo": { "RDEPEND": "dev-libs/M dev-libs/N dev-libs/P", "EAPI": "4", @@ -215,6 +222,11 @@ class UpdateDbentryTestCase(TestCase): "RDEPEND": "dev-libs/M dev-libs/N dev-libs/P", "EAPI": "4", }, + "dev-libs/B-2::test_repo": { + "SLOT": "2", + "RDEPEND": "dev-libs/M dev-libs/N dev-libs/P", + "EAPI": "4", + }, } world = ["dev-libs/M", "dev-libs/N"] @@ -269,6 +281,34 @@ class UpdateDbentryTestCase(TestCase): ) ) + # Delete some things in order to trigger CorruptionKeyError during package moves. + corruption_atom = Atom("dev-libs/B:2") + # Demonstrate initial state. + self.assertEqual(bindb.match(corruption_atom), ["dev-libs/B-2"]) + for cpv in bindb.match(corruption_atom): + os.unlink(bindb.bintree.getname(cpv)) + self.assertRaises( + CorruptionKeyError, + bindb.aux_update, + cpv, + {"RDEPEND": "dev-libs/M-moved"}, + ) + # Demonstrate corrupt state. + self.assertEqual(bindb.match(corruption_atom), ["dev-libs/B-2"]) + + # Demonstrate initial state. + self.assertEqual(vardb.match(corruption_atom), ["dev-libs/B-2"]) + for cpv in vardb.match(corruption_atom): + shutil.rmtree(vardb.getpath(cpv)) + self.assertRaises( + CorruptionKeyError, + vardb.aux_update, + cpv, + {"RDEPEND": "dev-libs/M-moved"}, + ) + # Demonstrate correct state because vardbapi checks the disk. + self.assertEqual(vardb.match(corruption_atom), []) + global_noiselimit = portage.util.noiselimit portage.util.noiselimit = -2 try: @@ -304,6 +344,26 @@ class UpdateDbentryTestCase(TestCase): self.assertTrue("dev-libs/M" in rdepend) self.assertTrue("dev-libs/M-moved" not in rdepend) + # Demonstrate that match still returns stale results + # due to intentional corruption. + self.assertEqual(bindb.match(corruption_atom), ["dev-libs/B-2"]) + + # Update bintree state so aux_get will properly raise KeyError. + for cpv in bindb.match(corruption_atom): + # Demonstrate that aux_get returns stale results. + self.assertEqual( + ["dev-libs/M dev-libs/N dev-libs/P"], + bindb.aux_get(cpv, ["RDEPEND"]), + ) + bindb.bintree.remove(cpv) + self.assertEqual(bindb.match(corruption_atom), []) + self.assertRaises( + KeyError, bindb.aux_get, "dev-libs/B-2", ["RDEPEND"] + ) + self.assertRaises( + KeyError, vardb.aux_get, "dev-libs/B-2", ["RDEPEND"] + ) + selected_set.load() self.assertTrue("dev-libs/M" not in selected_set) self.assertTrue("dev-libs/M-moved" in selected_set)
