Users may not want some symlinks to get clobbered, so protect them with CONFIG_PROTECT. Changes were required in the dblink.mergeme method and the new_protect_filename function.
The changes to dblink.mergeme do 3 things: * Move the bulk of config protection logic from dblink.mergeme to a new dblink._protect method. The new method only returns 3 variables, which makes it easier to understand how config protection interacts with the dblink.mergeme code that uses those variables. This is important, since dblink.mergeme has so many variables. * Initialize more variables at the beginning of dblink.mergeme, since those variables are used by the dblink._protect method. * Use the variables returned from dblink._protect to trigger appropriate behavior later in dblink.mergeme. The new_protect_filename changes are required since this function compares the new file to old ._cfg* files that may already exist, in order to avoid creating duplicate ._cfg* files. In these comparisons, it needs to handle symlinks differently from regular files. The unit tests demonstrate operation in many different scenarios, including: * regular file replaces regular file * regular file replaces symlink * regular file replaces directory * symlink replaces symlink * symlink replaces regular file * symlink replaces directory * directory replaces regular file * directory replaces symlink X-Gentoo-Bug: 485598 X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=485598 --- This updated patch only adds to the commit message in order to provide some information that may be helpful to reviewers of the patch. There are no changes to the code. pym/portage/dbapi/vartree.py | 255 ++++++++++++--------- pym/portage/tests/emerge/test_config_protect.py | 292 ++++++++++++++++++++++++ pym/portage/util/__init__.py | 35 ++- 3 files changed, 463 insertions(+), 119 deletions(-) create mode 100644 pym/portage/tests/emerge/test_config_protect.py diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index e21135a..219ca16 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -4461,21 +4461,17 @@ class dblink(object): # stat file once, test using S_* macros many times (faster that way) mystat = os.lstat(mysrc) mymode = mystat[stat.ST_MODE] - # handy variables; mydest is the target object on the live filesystems; - # mysrc is the source object in the temporary install dir - try: - mydstat = os.lstat(mydest) - mydmode = mydstat.st_mode - except OSError as e: - if e.errno != errno.ENOENT: - raise - del e - #dest file doesn't exist - mydstat = None - mydmode = None + mymd5 = None + myto = None - if stat.S_ISLNK(mymode): - # we are merging a symbolic link + if sys.hexversion >= 0x3030000: + mymtime = mystat.st_mtime_ns + else: + mymtime = mystat[stat.ST_MTIME] + + if stat.S_ISREG(mymode): + mymd5 = perform_md5(mysrc, calc_prelink=calc_prelink) + elif stat.S_ISLNK(mymode): # The file name of mysrc and the actual file that it points to # will have earlier been forcefully converted to the 'merge' # encoding if necessary, but the content of the symbolic link @@ -4495,6 +4491,69 @@ class dblink(object): os.unlink(mysrc) os.symlink(myto, mysrc) + mymd5 = portage.checksum._new_md5( + _unicode_encode(myto)).hexdigest() + + protected = False + if stat.S_ISLNK(mymode) or stat.S_ISREG(mymode): + protected = self.isprotected(mydest) + + if stat.S_ISREG(mymode) and \ + mystat.st_size == 0 and \ + os.path.basename(mydest).startswith(".keep"): + protected = False + + destmd5 = None + mydest_link = None + # handy variables; mydest is the target object on the live filesystems; + # mysrc is the source object in the temporary install dir + try: + mydstat = os.lstat(mydest) + mydmode = mydstat.st_mode + if protected: + if stat.S_ISLNK(mydmode): + # Read symlink target as bytes, in case the + # target path has a bad encoding. + mydest_link = _os.readlink( + _unicode_encode(mydest, + encoding=_encodings['merge'], + errors='strict')) + mydest_link = _unicode_decode(mydest_link, + encoding=_encodings['merge'], + errors='replace') + + # For protection of symlinks, the md5 + # of the link target path string is used + # for cfgfiledict (symlinks are + # protected since bug #485598). + destmd5 = portage.checksum._new_md5( + _unicode_encode(mydest_link)).hexdigest() + + elif stat.S_ISREG(mydmode): + destmd5 = perform_md5(mydest, + calc_prelink=calc_prelink) + except (FileNotFound, OSError) as e: + if isinstance(e, OSError) and e.errno != errno.ENOENT: + raise + #dest file doesn't exist + mydstat = None + mydmode = None + mydest_link = None + destmd5 = None + + moveme = True + if protected: + mydest, protected, moveme = self._protect(cfgfiledict, + protect_if_modified, mymd5, myto, mydest, + myrealdest, mydmode, destmd5, mydest_link) + + zing = "!!!" + if not moveme: + # confmem rejected this update + zing = "---" + + if stat.S_ISLNK(mymode): + # we are merging a symbolic link # Pass in the symlink target in order to bypass the # os.readlink() call inside abssymlink(), since that # call is unsafe if the merge encoding is not ascii @@ -4510,9 +4569,8 @@ class dblink(object): # myrealto contains the path of the real file to which this symlink points. # we can simply test for existence of this file to see if the target has been merged yet myrealto = normalize_path(os.path.join(destroot, myabsto)) - if mydmode!=None: - #destination exists - if stat.S_ISDIR(mydmode): + if mydmode is not None and stat.S_ISDIR(mydmode): + if not protected: # we can't merge a symlink over a directory newdest = self._new_backup_path(mydest) msg = [] @@ -4525,22 +4583,6 @@ class dblink(object): self._eerror("preinst", msg) mydest = newdest - elif not stat.S_ISLNK(mydmode): - if os.path.exists(mysrc) and stat.S_ISDIR(os.stat(mysrc)[stat.ST_MODE]): - # Kill file blocking installation of symlink to dir #71787 - pass - elif self.isprotected(mydest): - # Use md5 of the target in ${D} if it exists... - try: - newmd5 = perform_md5(join(srcroot, myabsto)) - except FileNotFound: - # Maybe the target is merged already. - try: - newmd5 = perform_md5(myrealto) - except FileNotFound: - newmd5 = None - mydest = new_protect_filename(mydest, newmd5=newmd5) - # if secondhand is None it means we're operating in "force" mode and should not create a second hand. if (secondhand != None) and (not os.path.exists(myrealto)): # either the target directory doesn't exist yet or the target file doesn't exist -- or @@ -4549,9 +4591,11 @@ class dblink(object): secondhand.append(mysrc[len(srcroot):]) continue # unlinking no longer necessary; "movefile" will overwrite symlinks atomically and correctly - mymtime = movefile(mysrc, mydest, newmtime=thismtime, - sstat=mystat, mysettings=self.settings, - encoding=_encodings['merge']) + if moveme: + zing = ">>>" + mymtime = movefile(mysrc, mydest, newmtime=thismtime, + sstat=mystat, mysettings=self.settings, + encoding=_encodings['merge']) try: self._merged_path(mydest, os.lstat(mydest)) @@ -4567,7 +4611,7 @@ class dblink(object): [_("QA Notice: Symbolic link /%s points to /%s which does not exist.") % (relative_path, myabsto)]) - showMessage(">>> %s -> %s\n" % (mydest, myto)) + showMessage("%s %s -> %s\n" % (zing, mydest, myto)) if sys.hexversion >= 0x3030000: outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime // 1000000000)+"\n") else: @@ -4589,7 +4633,8 @@ class dblink(object): if dflags != 0: bsd_chflags.lchflags(mydest, 0) - if not os.access(mydest, os.W_OK): + if not stat.S_ISLNK(mydmode) and \ + not os.access(mydest, os.W_OK): pkgstuff = pkgsplit(self.pkg) writemsg(_("\n!!! Cannot write to '%s'.\n") % mydest, noiselevel=-1) writemsg(_("!!! Please check permissions and directories for broken symlinks.\n")) @@ -4678,14 +4723,8 @@ class dblink(object): elif stat.S_ISREG(mymode): # we are merging a regular file - mymd5 = perform_md5(mysrc, calc_prelink=calc_prelink) - # calculate config file protection stuff - mydestdir = os.path.dirname(mydest) - moveme = 1 - zing = "!!!" - mymtime = None - protected = self.isprotected(mydest) - if mydmode is not None and stat.S_ISDIR(mydmode): + if not protected and \ + mydmode is not None and stat.S_ISDIR(mydmode): # install of destination is blocked by an existing directory with the same name newdest = self._new_backup_path(mydest) msg = [] @@ -4698,73 +4737,6 @@ class dblink(object): self._eerror("preinst", msg) mydest = newdest - elif mydmode is None or stat.S_ISREG(mydmode) or \ - (stat.S_ISLNK(mydmode) and os.path.exists(mydest) - and stat.S_ISREG(os.stat(mydest)[stat.ST_MODE])): - # install of destination is blocked by an existing regular file, - # or by a symlink to an existing regular file; - # now, config file management may come into play. - # we only need to tweak mydest if cfg file management is in play. - destmd5 = None - if protected and mydmode is not None: - destmd5 = perform_md5(mydest, calc_prelink=calc_prelink) - if protect_if_modified: - contents_key = \ - self._installed_instance._match_contents(myrealdest) - if contents_key: - inst_info = self._installed_instance.getcontents()[contents_key] - if inst_info[0] == "obj" and inst_info[2] == destmd5: - protected = False - - if protected: - # we have a protection path; enable config file management. - cfgprot = 0 - cfgprot_force = False - if mydmode is None: - if self._installed_instance is not None and \ - self._installed_instance._match_contents( - myrealdest) is not False: - # If the file doesn't exist, then it may - # have been deleted or renamed by the - # admin. Therefore, force the file to be - # merged with a ._cfg name, so that the - # admin will be prompted for this update - # (see bug #523684). - cfgprot_force = True - moveme = True - cfgprot = True - elif mymd5 == destmd5: - #file already in place; simply update mtimes of destination - moveme = 1 - else: - if mymd5 == cfgfiledict.get(myrealdest, [None])[0]: - """ An identical update has previously been - merged. Skip it unless the user has chosen - --noconfmem.""" - moveme = cfgfiledict["IGNORE"] - cfgprot = cfgfiledict["IGNORE"] - if not moveme: - zing = "---" - if sys.hexversion >= 0x3030000: - mymtime = mystat.st_mtime_ns - else: - mymtime = mystat[stat.ST_MTIME] - else: - moveme = 1 - cfgprot = 1 - if moveme: - # Merging a new file, so update confmem. - cfgfiledict[myrealdest] = [mymd5] - elif destmd5 == cfgfiledict.get(myrealdest, [None])[0]: - """A previously remembered update has been - accepted, so it is removed from confmem.""" - del cfgfiledict[myrealdest] - - if cfgprot: - mydest = new_protect_filename(mydest, - newmd5=mymd5, - force=cfgprot_force) - # whether config protection or not, we merge the new file the # same way. Unless moveme=0 (blocking directory) if moveme: @@ -4820,6 +4792,63 @@ class dblink(object): outfile.write("dev %s\n" % myrealdest) showMessage(zing + " " + mydest + "\n") + def _protect(self, cfgfiledict, protect_if_modified, mymd5, myto, + mydest, myrealdest, mydmode, destmd5, mydest_link): + + moveme = True + protected = True + force = False + k = False + if self._installed_instance is not None: + k = self._installed_instance._match_contents(myrealdest) + if k is not False: + if mydmode is None: + # If the file doesn't exist, then it may + # have been deleted or renamed by the + # admin. Therefore, force the file to be + # merged with a ._cfg name, so that the + # admin will be prompted for this update + # (see bug #523684). + force = True + + elif protect_if_modified: + data = self._installed_instance.getcontents()[k] + if data[0] == "obj" and data[2] == destmd5: + protected = False + elif data[0] == "sym" and data[2] == mydest_link: + protected = False + + if protected and mydmode is not None: + # we have a protection path; enable config file management. + if mymd5 != destmd5 and \ + mymd5 == cfgfiledict.get(myrealdest, [None])[0]: + # An identical update has previously been + # merged. Skip it unless the user has chosen + # --noconfmem. + moveme = protected = bool(cfgfiledict["IGNORE"]) + + if protected and \ + (mydest_link is not None or myto is not None) and \ + mydest_link != myto: + # If either one is a symlink, and they are not + # identical symlinks, then force config protection. + force = True + + if moveme: + # Merging a new file, so update confmem. + cfgfiledict[myrealdest] = [mymd5] + elif destmd5 == cfgfiledict.get(myrealdest, [None])[0]: + # A previously remembered update has been + # accepted, so it is removed from confmem. + del cfgfiledict[myrealdest] + + if protected and moveme: + mydest = new_protect_filename(mydest, + newmd5=(mydest_link or mymd5), + force=force) + + return mydest, protected, moveme + def _merged_path(self, path, lstatobj, exists=True): previous_path = self._device_path_map.get(lstatobj.st_dev) if previous_path is None or previous_path is False or \ diff --git a/pym/portage/tests/emerge/test_config_protect.py b/pym/portage/tests/emerge/test_config_protect.py new file mode 100644 index 0000000..5d7d8e9 --- /dev/null +++ b/pym/portage/tests/emerge/test_config_protect.py @@ -0,0 +1,292 @@ +# Copyright 2014 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import unicode_literals + +import io +from functools import partial +import shutil +import stat +import subprocess +import sys +import time + +import portage +from portage import os +from portage import _encodings, _unicode_decode +from portage.const import BASH_BINARY, PORTAGE_PYM_PATH +from portage.process import find_binary +from portage.tests import TestCase +from portage.tests.resolver.ResolverPlayground import ResolverPlayground +from portage.util import (ensure_dirs, find_updated_config_files, + shlex_split) + +class ConfigProtectTestCase(TestCase): + + def testConfigProtect(self): + """ + Demonstrates many different scenarios. For example: + + * regular file replaces regular file + * regular file replaces symlink + * regular file replaces directory + * symlink replaces symlink + * symlink replaces regular file + * symlink replaces directory + * directory replaces regular file + * directory replaces symlink + """ + + debug = False + + content_A_1 = """ +S="${WORKDIR}" + +src_install() { + insinto /etc/A + keepdir /etc/A/dir_a + keepdir /etc/A/symlink_replaces_dir + keepdir /etc/A/regular_replaces_dir + echo regular_a_1 > "${T}"/regular_a + doins "${T}"/regular_a + echo regular_b_1 > "${T}"/regular_b + doins "${T}"/regular_b + dosym regular_a /etc/A/regular_replaces_symlink + dosym regular_b /etc/A/symlink_replaces_symlink + echo regular_replaces_regular_1 > \ + "${T}"/regular_replaces_regular + doins "${T}"/regular_replaces_regular + echo symlink_replaces_regular > \ + "${T}"/symlink_replaces_regular + doins "${T}"/symlink_replaces_regular +} + +""" + + content_A_2 = """ +S="${WORKDIR}" + +src_install() { + insinto /etc/A + keepdir /etc/A/dir_a + dosym dir_a /etc/A/symlink_replaces_dir + echo regular_replaces_dir > "${T}"/regular_replaces_dir + doins "${T}"/regular_replaces_dir + echo regular_a_2 > "${T}"/regular_a + doins "${T}"/regular_a + echo regular_b_2 > "${T}"/regular_b + doins "${T}"/regular_b + echo regular_replaces_symlink > \ + "${T}"/regular_replaces_symlink + doins "${T}"/regular_replaces_symlink + dosym regular_b /etc/A/symlink_replaces_symlink + echo regular_replaces_regular_2 > \ + "${T}"/regular_replaces_regular + doins "${T}"/regular_replaces_regular + dosym regular_a /etc/A/symlink_replaces_regular +} + +""" + + ebuilds = { + "dev-libs/A-1": { + "EAPI" : "5", + "IUSE" : "+flag", + "KEYWORDS": "x86", + "LICENSE": "GPL-2", + "MISC_CONTENT": content_A_1, + }, + "dev-libs/A-2": { + "EAPI" : "5", + "IUSE" : "+flag", + "KEYWORDS": "x86", + "LICENSE": "GPL-2", + "MISC_CONTENT": content_A_2, + }, + } + + playground = ResolverPlayground( + ebuilds=ebuilds, debug=debug) + settings = playground.settings + eprefix = settings["EPREFIX"] + eroot = settings["EROOT"] + var_cache_edb = os.path.join(eprefix, "var", "cache", "edb") + + portage_python = portage._python_interpreter + dispatch_conf_cmd = (portage_python, "-b", "-Wd", + os.path.join(self.sbindir, "dispatch-conf")) + emerge_cmd = (portage_python, "-b", "-Wd", + os.path.join(self.bindir, "emerge")) + etc_update_cmd = (BASH_BINARY, + os.path.join(self.sbindir, "etc-update")) + etc_update_auto = etc_update_cmd + ("--automode", "-5",) + + config_protect = "/etc" + + def modify_files(dir_path): + for name in os.listdir(dir_path): + path = os.path.join(dir_path, name) + st = os.lstat(path) + if stat.S_ISREG(st.st_mode): + with io.open(path, mode='a', + encoding=_encodings["stdio"]) as f: + f.write("modified at %d\n" % time.time()) + elif stat.S_ISLNK(st.st_mode): + old_dest = os.readlink(path) + os.unlink(path) + os.symlink(old_dest + + " modified at %d" % time.time(), path) + + def updated_config_files(count): + self.assertEqual(count, + sum(len(x[1]) for x in find_updated_config_files(eroot, + shlex_split(config_protect)))) + + test_commands = ( + etc_update_cmd, + dispatch_conf_cmd, + emerge_cmd + ("-1", "=dev-libs/A-1"), + partial(updated_config_files, 0), + emerge_cmd + ("-1", "=dev-libs/A-2"), + partial(updated_config_files, 2), + etc_update_auto, + partial(updated_config_files, 0), + emerge_cmd + ("-1", "=dev-libs/A-2"), + partial(updated_config_files, 0), + # Test bug #523684, where a file renamed or removed by the + # admin forces replacement files to be merged with config + # protection. + partial(shutil.rmtree, + os.path.join(eprefix, "etc", "A")), + emerge_cmd + ("-1", "=dev-libs/A-2"), + partial(updated_config_files, 8), + etc_update_auto, + partial(updated_config_files, 0), + # Modify some config files, and verify that it triggers + # config protection. + partial(modify_files, + os.path.join(eroot, "etc", "A")), + emerge_cmd + ("-1", "=dev-libs/A-2"), + partial(updated_config_files, 6), + etc_update_auto, + partial(updated_config_files, 0), + # Modify some config files, downgrade to A-1, and verify + # that config protection works properly when the file + # types are changing. + partial(modify_files, + os.path.join(eroot, "etc", "A")), + emerge_cmd + ("-1", "--noconfmem", "=dev-libs/A-1"), + partial(updated_config_files, 6), + etc_update_auto, + partial(updated_config_files, 0), + ) + + distdir = playground.distdir + fake_bin = os.path.join(eprefix, "bin") + portage_tmpdir = os.path.join(eprefix, "var", "tmp", "portage") + + path = os.environ.get("PATH") + if path is not None and not path.strip(): + path = None + if path is None: + path = "" + else: + path = ":" + path + path = fake_bin + path + + pythonpath = os.environ.get("PYTHONPATH") + if pythonpath is not None and not pythonpath.strip(): + pythonpath = None + if pythonpath is not None and \ + pythonpath.split(":")[0] == PORTAGE_PYM_PATH: + pass + else: + if pythonpath is None: + pythonpath = "" + else: + pythonpath = ":" + pythonpath + pythonpath = PORTAGE_PYM_PATH + pythonpath + + env = { + "PORTAGE_OVERRIDE_EPREFIX" : eprefix, + "CLEAN_DELAY" : "0", + "CONFIG_PROTECT": config_protect, + "DISTDIR" : distdir, + "EMERGE_DEFAULT_OPTS": "-v", + "EMERGE_WARNING_DELAY" : "0", + "INFODIR" : "", + "INFOPATH" : "", + "PATH" : path, + "PORTAGE_INST_GID" : str(portage.data.portage_gid), + "PORTAGE_INST_UID" : str(portage.data.portage_uid), + "PORTAGE_PYTHON" : portage_python, + "PORTAGE_REPOSITORIES" : settings.repositories.config_string(), + "PORTAGE_TMPDIR" : portage_tmpdir, + "PYTHONPATH" : pythonpath, + "__PORTAGE_TEST_PATH_OVERRIDE" : fake_bin, + } + + if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ: + env["__PORTAGE_TEST_HARDLINK_LOCKS"] = \ + os.environ["__PORTAGE_TEST_HARDLINK_LOCKS"] + + dirs = [distdir, fake_bin, portage_tmpdir, + var_cache_edb] + etc_symlinks = ("dispatch-conf.conf", "etc-update.conf") + # Override things that may be unavailable, or may have portability + # issues when running tests in exotic environments. + # prepstrip - bug #447810 (bash read builtin EINTR problem) + true_symlinks = ["prepstrip", "scanelf"] + true_binary = find_binary("true") + self.assertEqual(true_binary is None, False, + "true command not found") + try: + for d in dirs: + ensure_dirs(d) + for x in true_symlinks: + os.symlink(true_binary, os.path.join(fake_bin, x)) + for x in etc_symlinks: + os.symlink(os.path.join(self.cnf_etc_path, x), + os.path.join(eprefix, "etc", x)) + with open(os.path.join(var_cache_edb, "counter"), 'wb') as f: + f.write(b"100") + + if debug: + # The subprocess inherits both stdout and stderr, for + # debugging purposes. + stdout = None + else: + # The subprocess inherits stderr so that any warnings + # triggered by python -Wd will be visible. + stdout = subprocess.PIPE + + for args in test_commands: + + if hasattr(args, '__call__'): + args() + continue + + if isinstance(args[0], dict): + local_env = env.copy() + local_env.update(args[0]) + args = args[1:] + else: + local_env = env + + proc = subprocess.Popen(args, + env=local_env, stdout=stdout) + + if debug: + proc.wait() + else: + output = proc.stdout.readlines() + proc.wait() + proc.stdout.close() + if proc.returncode != os.EX_OK: + for line in output: + sys.stderr.write(_unicode_decode(line)) + + self.assertEqual(os.EX_OK, proc.returncode, + "emerge failed with args %s" % (args,)) + finally: + playground.cleanup() diff --git a/pym/portage/util/__init__.py b/pym/portage/util/__init__.py index fe79942..ad3a351 100644 --- a/pym/portage/util/__init__.py +++ b/pym/portage/util/__init__.py @@ -1676,13 +1676,36 @@ def new_protect_filename(mydest, newmd5=None, force=False): old_pfile = normalize_path(os.path.join(real_dirname, last_pfile)) if last_pfile and newmd5: try: - last_pfile_md5 = portage.checksum._perform_md5_merge(old_pfile) - except FileNotFound: - # The file suddenly disappeared or it's a broken symlink. - pass + old_pfile_st = _os_merge.lstat(old_pfile) + except OSError as e: + if e.errno != errno.ENOENT: + raise else: - if last_pfile_md5 == newmd5: - return old_pfile + if stat.S_ISLNK(old_pfile_st.st_mode): + try: + # Read symlink target as bytes, in case the + # target path has a bad encoding. + pfile_link = _os.readlink(_unicode_encode(old_pfile, + encoding=_encodings['merge'], errors='strict')) + except OSError: + if e.errno != errno.ENOENT: + raise + else: + pfile_link = _unicode_decode( + encoding=_encodings['merge'], errors='replace') + if pfile_link == newmd5: + return old_pfile + else: + try: + last_pfile_md5 = \ + portage.checksum._perform_md5_merge(old_pfile) + except FileNotFound: + # The file suddenly disappeared or it's a + # broken symlink. + pass + else: + if last_pfile_md5 == newmd5: + return old_pfile return new_pfile def find_updated_config_files(target_root, config_protect): -- 2.0.4
