commit: 02417188225758b1822d176abd8902a92300a371 Author: Zac Medico <zmedico <AT> gentoo <DOT> org> AuthorDate: Sun Oct 26 09:52:22 2014 +0000 Commit: Zac Medico <zmedico <AT> gentoo <DOT> org> CommitDate: Mon Nov 3 04:17:09 2014 +0000 URL: http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=02417188
CONFIG_PROTECT: protect symlinks, bug #485598 Users may not want some symlinks to get clobbered, so protect them with CONFIG_PROTECT. Changes were required in the dblink.mergeme method and the new_protect_filename function. The changes to dblink.mergeme do 3 things: * Move the bulk of config protection logic from dblink.mergeme to a new dblink._protect method. The new method only returns 3 variables, which makes it easier to understand how config protection interacts with the dblink.mergeme code that uses those variables. This is important, since dblink.mergeme has so many variables. * Initialize more variables at the beginning of dblink.mergeme, since those variables are used by the dblink._protect method. * Use the variables returned from dblink._protect to trigger appropriate behavior later in dblink.mergeme. The new_protect_filename changes are required since this function compares the new file to old ._cfg* files that may already exist, in order to avoid creating duplicate ._cfg* files. In these comparisons, it needs to handle symlinks differently from regular files. The unit tests demonstrate operation in many different scenarios, including: * regular file replaces regular file * regular file replaces symlink * regular file replaces directory * symlink replaces symlink * symlink replaces regular file * symlink replaces directory * directory replaces regular file * directory replaces symlink X-Gentoo-Bug: 485598 X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=485598 --- pym/portage/dbapi/vartree.py | 255 ++++++++++++--------- pym/portage/tests/emerge/test_config_protect.py | 292 ++++++++++++++++++++++++ pym/portage/util/__init__.py | 35 ++- 3 files changed, 463 insertions(+), 119 deletions(-) diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index e21135a..8b06f4c 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -4461,21 +4461,17 @@ class dblink(object): # stat file once, test using S_* macros many times (faster that way) mystat = os.lstat(mysrc) mymode = mystat[stat.ST_MODE] - # handy variables; mydest is the target object on the live filesystems; - # mysrc is the source object in the temporary install dir - try: - mydstat = os.lstat(mydest) - mydmode = mydstat.st_mode - except OSError as e: - if e.errno != errno.ENOENT: - raise - del e - #dest file doesn't exist - mydstat = None - mydmode = None + mymd5 = None + myto = None - if stat.S_ISLNK(mymode): - # we are merging a symbolic link + if sys.hexversion >= 0x3030000: + mymtime = mystat.st_mtime_ns + else: + mymtime = mystat[stat.ST_MTIME] + + if stat.S_ISREG(mymode): + mymd5 = perform_md5(mysrc, calc_prelink=calc_prelink) + elif stat.S_ISLNK(mymode): # The file name of mysrc and the actual file that it points to # will have earlier been forcefully converted to the 'merge' # encoding if necessary, but the content of the symbolic link @@ -4495,6 +4491,69 @@ class dblink(object): os.unlink(mysrc) os.symlink(myto, mysrc) + mymd5 = portage.checksum._new_md5( + _unicode_encode(myto)).hexdigest() + + protected = False + if stat.S_ISLNK(mymode) or stat.S_ISREG(mymode): + protected = self.isprotected(mydest) + + if stat.S_ISREG(mymode) and \ + mystat.st_size == 0 and \ + os.path.basename(mydest).startswith(".keep"): + protected = False + + destmd5 = None + mydest_link = None + # handy variables; mydest is the target object on the live filesystems; + # mysrc is the source object in the temporary install dir + try: + mydstat = os.lstat(mydest) + mydmode = mydstat.st_mode + if protected: + if stat.S_ISLNK(mydmode): + # Read symlink target as bytes, in case the + # target path has a bad encoding. + mydest_link = _os.readlink( + _unicode_encode(mydest, + encoding=_encodings['merge'], + errors='strict')) + mydest_link = _unicode_decode(mydest_link, + encoding=_encodings['merge'], + errors='replace') + + # For protection of symlinks, the md5 + # of the link target path string is used + # for cfgfiledict (symlinks are + # protected since bug #485598). + destmd5 = portage.checksum._new_md5( + _unicode_encode(mydest_link)).hexdigest() + + elif stat.S_ISREG(mydmode): + destmd5 = perform_md5(mydest, + calc_prelink=calc_prelink) + except (FileNotFound, OSError) as e: + if isinstance(e, OSError) and e.errno != errno.ENOENT: + raise + #dest file doesn't exist + mydstat = None + mydmode = None + mydest_link = None + destmd5 = None + + moveme = True + if protected: + mydest, protected, moveme = self._protect(cfgfiledict, + protect_if_modified, mymd5, myto, mydest, + myrealdest, mydmode, destmd5, mydest_link) + + zing = "!!!" + if not moveme: + # confmem rejected this update + zing = "---" + + if stat.S_ISLNK(mymode): + # we are merging a symbolic link # Pass in the symlink target in order to bypass the # os.readlink() call inside abssymlink(), since that # call is unsafe if the merge encoding is not ascii @@ -4510,9 +4569,8 @@ class dblink(object): # myrealto contains the path of the real file to which this symlink points. # we can simply test for existence of this file to see if the target has been merged yet myrealto = normalize_path(os.path.join(destroot, myabsto)) - if mydmode!=None: - #destination exists - if stat.S_ISDIR(mydmode): + if mydmode is not None and stat.S_ISDIR(mydmode): + if not protected: # we can't merge a symlink over a directory newdest = self._new_backup_path(mydest) msg = [] @@ -4525,22 +4583,6 @@ class dblink(object): self._eerror("preinst", msg) mydest = newdest - elif not stat.S_ISLNK(mydmode): - if os.path.exists(mysrc) and stat.S_ISDIR(os.stat(mysrc)[stat.ST_MODE]): - # Kill file blocking installation of symlink to dir #71787 - pass - elif self.isprotected(mydest): - # Use md5 of the target in ${D} if it exists... - try: - newmd5 = perform_md5(join(srcroot, myabsto)) - except FileNotFound: - # Maybe the target is merged already. - try: - newmd5 = perform_md5(myrealto) - except FileNotFound: - newmd5 = None - mydest = new_protect_filename(mydest, newmd5=newmd5) - # if secondhand is None it means we're operating in "force" mode and should not create a second hand. if (secondhand != None) and (not os.path.exists(myrealto)): # either the target directory doesn't exist yet or the target file doesn't exist -- or @@ -4549,9 +4591,11 @@ class dblink(object): secondhand.append(mysrc[len(srcroot):]) continue # unlinking no longer necessary; "movefile" will overwrite symlinks atomically and correctly - mymtime = movefile(mysrc, mydest, newmtime=thismtime, - sstat=mystat, mysettings=self.settings, - encoding=_encodings['merge']) + if moveme: + zing = ">>>" + mymtime = movefile(mysrc, mydest, newmtime=thismtime, + sstat=mystat, mysettings=self.settings, + encoding=_encodings['merge']) try: self._merged_path(mydest, os.lstat(mydest)) @@ -4567,7 +4611,7 @@ class dblink(object): [_("QA Notice: Symbolic link /%s points to /%s which does not exist.") % (relative_path, myabsto)]) - showMessage(">>> %s -> %s\n" % (mydest, myto)) + showMessage("%s %s -> %s\n" % (zing, mydest, myto)) if sys.hexversion >= 0x3030000: outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime // 1000000000)+"\n") else: @@ -4589,7 +4633,8 @@ class dblink(object): if dflags != 0: bsd_chflags.lchflags(mydest, 0) - if not os.access(mydest, os.W_OK): + if not stat.S_ISLNK(mydmode) and \ + not os.access(mydest, os.W_OK): pkgstuff = pkgsplit(self.pkg) writemsg(_("\n!!! Cannot write to '%s'.\n") % mydest, noiselevel=-1) writemsg(_("!!! Please check permissions and directories for broken symlinks.\n")) @@ -4678,14 +4723,8 @@ class dblink(object): elif stat.S_ISREG(mymode): # we are merging a regular file - mymd5 = perform_md5(mysrc, calc_prelink=calc_prelink) - # calculate config file protection stuff - mydestdir = os.path.dirname(mydest) - moveme = 1 - zing = "!!!" - mymtime = None - protected = self.isprotected(mydest) - if mydmode is not None and stat.S_ISDIR(mydmode): + if not protected and \ + mydmode is not None and stat.S_ISDIR(mydmode): # install of destination is blocked by an existing directory with the same name newdest = self._new_backup_path(mydest) msg = [] @@ -4698,73 +4737,6 @@ class dblink(object): self._eerror("preinst", msg) mydest = newdest - elif mydmode is None or stat.S_ISREG(mydmode) or \ - (stat.S_ISLNK(mydmode) and os.path.exists(mydest) - and stat.S_ISREG(os.stat(mydest)[stat.ST_MODE])): - # install of destination is blocked by an existing regular file, - # or by a symlink to an existing regular file; - # now, config file management may come into play. - # we only need to tweak mydest if cfg file management is in play. - destmd5 = None - if protected and mydmode is not None: - destmd5 = perform_md5(mydest, calc_prelink=calc_prelink) - if protect_if_modified: - contents_key = \ - self._installed_instance._match_contents(myrealdest) - if contents_key: - inst_info = self._installed_instance.getcontents()[contents_key] - if inst_info[0] == "obj" and inst_info[2] == destmd5: - protected = False - - if protected: - # we have a protection path; enable config file management. - cfgprot = 0 - cfgprot_force = False - if mydmode is None: - if self._installed_instance is not None and \ - self._installed_instance._match_contents( - myrealdest) is not False: - # If the file doesn't exist, then it may - # have been deleted or renamed by the - # admin. Therefore, force the file to be - # merged with a ._cfg name, so that the - # admin will be prompted for this update - # (see bug #523684). - cfgprot_force = True - moveme = True - cfgprot = True - elif mymd5 == destmd5: - #file already in place; simply update mtimes of destination - moveme = 1 - else: - if mymd5 == cfgfiledict.get(myrealdest, [None])[0]: - """ An identical update has previously been - merged. Skip it unless the user has chosen - --noconfmem.""" - moveme = cfgfiledict["IGNORE"] - cfgprot = cfgfiledict["IGNORE"] - if not moveme: - zing = "---" - if sys.hexversion >= 0x3030000: - mymtime = mystat.st_mtime_ns - else: - mymtime = mystat[stat.ST_MTIME] - else: - moveme = 1 - cfgprot = 1 - if moveme: - # Merging a new file, so update confmem. - cfgfiledict[myrealdest] = [mymd5] - elif destmd5 == cfgfiledict.get(myrealdest, [None])[0]: - """A previously remembered update has been - accepted, so it is removed from confmem.""" - del cfgfiledict[myrealdest] - - if cfgprot: - mydest = new_protect_filename(mydest, - newmd5=mymd5, - force=cfgprot_force) - # whether config protection or not, we merge the new file the # same way. Unless moveme=0 (blocking directory) if moveme: @@ -4820,6 +4792,63 @@ class dblink(object): outfile.write("dev %s\n" % myrealdest) showMessage(zing + " " + mydest + "\n") + def _protect(self, cfgfiledict, protect_if_modified, src_md5, + src_link, dest, dest_real, dest_mode, dest_md5, dest_link): + + move_me = True + protected = True + force = False + k = False + if self._installed_instance is not None: + k = self._installed_instance._match_contents(dest_real) + if k is not False: + if dest_mode is None: + # If the file doesn't exist, then it may + # have been deleted or renamed by the + # admin. Therefore, force the file to be + # merged with a ._cfg name, so that the + # admin will be prompted for this update + # (see bug #523684). + force = True + + elif protect_if_modified: + data = self._installed_instance.getcontents()[k] + if data[0] == "obj" and data[2] == dest_md5: + protected = False + elif data[0] == "sym" and data[2] == dest_link: + protected = False + + if protected and dest_mode is not None: + # we have a protection path; enable config file management. + if src_md5 != dest_md5 and \ + src_md5 == cfgfiledict.get(dest_real, [None])[0]: + # An identical update has previously been + # merged. Skip it unless the user has chosen + # --noconfmem. + move_me = protected = bool(cfgfiledict["IGNORE"]) + + if protected and \ + (dest_link is not None or src_link is not None) and \ + dest_link != src_link: + # If either one is a symlink, and they are not + # identical symlinks, then force config protection. + force = True + + if move_me: + # Merging a new file, so update confmem. + cfgfiledict[dest_real] = [src_md5] + elif dest_md5 == cfgfiledict.get(dest_real, [None])[0]: + # A previously remembered update has been + # accepted, so it is removed from confmem. + del cfgfiledict[dest_real] + + if protected and move_me: + dest = new_protect_filename(dest, + newmd5=(dest_link or src_md5), + force=force) + + return dest, protected, move_me + def _merged_path(self, path, lstatobj, exists=True): previous_path = self._device_path_map.get(lstatobj.st_dev) if previous_path is None or previous_path is False or \ diff --git a/pym/portage/tests/emerge/test_config_protect.py b/pym/portage/tests/emerge/test_config_protect.py new file mode 100644 index 0000000..5d7d8e9 --- /dev/null +++ b/pym/portage/tests/emerge/test_config_protect.py @@ -0,0 +1,292 @@ +# Copyright 2014 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import unicode_literals + +import io +from functools import partial +import shutil +import stat +import subprocess +import sys +import time + +import portage +from portage import os +from portage import _encodings, _unicode_decode +from portage.const import BASH_BINARY, PORTAGE_PYM_PATH +from portage.process import find_binary +from portage.tests import TestCase +from portage.tests.resolver.ResolverPlayground import ResolverPlayground +from portage.util import (ensure_dirs, find_updated_config_files, + shlex_split) + +class ConfigProtectTestCase(TestCase): + + def testConfigProtect(self): + """ + Demonstrates many different scenarios. For example: + + * regular file replaces regular file + * regular file replaces symlink + * regular file replaces directory + * symlink replaces symlink + * symlink replaces regular file + * symlink replaces directory + * directory replaces regular file + * directory replaces symlink + """ + + debug = False + + content_A_1 = """ +S="${WORKDIR}" + +src_install() { + insinto /etc/A + keepdir /etc/A/dir_a + keepdir /etc/A/symlink_replaces_dir + keepdir /etc/A/regular_replaces_dir + echo regular_a_1 > "${T}"/regular_a + doins "${T}"/regular_a + echo regular_b_1 > "${T}"/regular_b + doins "${T}"/regular_b + dosym regular_a /etc/A/regular_replaces_symlink + dosym regular_b /etc/A/symlink_replaces_symlink + echo regular_replaces_regular_1 > \ + "${T}"/regular_replaces_regular + doins "${T}"/regular_replaces_regular + echo symlink_replaces_regular > \ + "${T}"/symlink_replaces_regular + doins "${T}"/symlink_replaces_regular +} + +""" + + content_A_2 = """ +S="${WORKDIR}" + +src_install() { + insinto /etc/A + keepdir /etc/A/dir_a + dosym dir_a /etc/A/symlink_replaces_dir + echo regular_replaces_dir > "${T}"/regular_replaces_dir + doins "${T}"/regular_replaces_dir + echo regular_a_2 > "${T}"/regular_a + doins "${T}"/regular_a + echo regular_b_2 > "${T}"/regular_b + doins "${T}"/regular_b + echo regular_replaces_symlink > \ + "${T}"/regular_replaces_symlink + doins "${T}"/regular_replaces_symlink + dosym regular_b /etc/A/symlink_replaces_symlink + echo regular_replaces_regular_2 > \ + "${T}"/regular_replaces_regular + doins "${T}"/regular_replaces_regular + dosym regular_a /etc/A/symlink_replaces_regular +} + +""" + + ebuilds = { + "dev-libs/A-1": { + "EAPI" : "5", + "IUSE" : "+flag", + "KEYWORDS": "x86", + "LICENSE": "GPL-2", + "MISC_CONTENT": content_A_1, + }, + "dev-libs/A-2": { + "EAPI" : "5", + "IUSE" : "+flag", + "KEYWORDS": "x86", + "LICENSE": "GPL-2", + "MISC_CONTENT": content_A_2, + }, + } + + playground = ResolverPlayground( + ebuilds=ebuilds, debug=debug) + settings = playground.settings + eprefix = settings["EPREFIX"] + eroot = settings["EROOT"] + var_cache_edb = os.path.join(eprefix, "var", "cache", "edb") + + portage_python = portage._python_interpreter + dispatch_conf_cmd = (portage_python, "-b", "-Wd", + os.path.join(self.sbindir, "dispatch-conf")) + emerge_cmd = (portage_python, "-b", "-Wd", + os.path.join(self.bindir, "emerge")) + etc_update_cmd = (BASH_BINARY, + os.path.join(self.sbindir, "etc-update")) + etc_update_auto = etc_update_cmd + ("--automode", "-5",) + + config_protect = "/etc" + + def modify_files(dir_path): + for name in os.listdir(dir_path): + path = os.path.join(dir_path, name) + st = os.lstat(path) + if stat.S_ISREG(st.st_mode): + with io.open(path, mode='a', + encoding=_encodings["stdio"]) as f: + f.write("modified at %d\n" % time.time()) + elif stat.S_ISLNK(st.st_mode): + old_dest = os.readlink(path) + os.unlink(path) + os.symlink(old_dest + + " modified at %d" % time.time(), path) + + def updated_config_files(count): + self.assertEqual(count, + sum(len(x[1]) for x in find_updated_config_files(eroot, + shlex_split(config_protect)))) + + test_commands = ( + etc_update_cmd, + dispatch_conf_cmd, + emerge_cmd + ("-1", "=dev-libs/A-1"), + partial(updated_config_files, 0), + emerge_cmd + ("-1", "=dev-libs/A-2"), + partial(updated_config_files, 2), + etc_update_auto, + partial(updated_config_files, 0), + emerge_cmd + ("-1", "=dev-libs/A-2"), + partial(updated_config_files, 0), + # Test bug #523684, where a file renamed or removed by the + # admin forces replacement files to be merged with config + # protection. + partial(shutil.rmtree, + os.path.join(eprefix, "etc", "A")), + emerge_cmd + ("-1", "=dev-libs/A-2"), + partial(updated_config_files, 8), + etc_update_auto, + partial(updated_config_files, 0), + # Modify some config files, and verify that it triggers + # config protection. + partial(modify_files, + os.path.join(eroot, "etc", "A")), + emerge_cmd + ("-1", "=dev-libs/A-2"), + partial(updated_config_files, 6), + etc_update_auto, + partial(updated_config_files, 0), + # Modify some config files, downgrade to A-1, and verify + # that config protection works properly when the file + # types are changing. + partial(modify_files, + os.path.join(eroot, "etc", "A")), + emerge_cmd + ("-1", "--noconfmem", "=dev-libs/A-1"), + partial(updated_config_files, 6), + etc_update_auto, + partial(updated_config_files, 0), + ) + + distdir = playground.distdir + fake_bin = os.path.join(eprefix, "bin") + portage_tmpdir = os.path.join(eprefix, "var", "tmp", "portage") + + path = os.environ.get("PATH") + if path is not None and not path.strip(): + path = None + if path is None: + path = "" + else: + path = ":" + path + path = fake_bin + path + + pythonpath = os.environ.get("PYTHONPATH") + if pythonpath is not None and not pythonpath.strip(): + pythonpath = None + if pythonpath is not None and \ + pythonpath.split(":")[0] == PORTAGE_PYM_PATH: + pass + else: + if pythonpath is None: + pythonpath = "" + else: + pythonpath = ":" + pythonpath + pythonpath = PORTAGE_PYM_PATH + pythonpath + + env = { + "PORTAGE_OVERRIDE_EPREFIX" : eprefix, + "CLEAN_DELAY" : "0", + "CONFIG_PROTECT": config_protect, + "DISTDIR" : distdir, + "EMERGE_DEFAULT_OPTS": "-v", + "EMERGE_WARNING_DELAY" : "0", + "INFODIR" : "", + "INFOPATH" : "", + "PATH" : path, + "PORTAGE_INST_GID" : str(portage.data.portage_gid), + "PORTAGE_INST_UID" : str(portage.data.portage_uid), + "PORTAGE_PYTHON" : portage_python, + "PORTAGE_REPOSITORIES" : settings.repositories.config_string(), + "PORTAGE_TMPDIR" : portage_tmpdir, + "PYTHONPATH" : pythonpath, + "__PORTAGE_TEST_PATH_OVERRIDE" : fake_bin, + } + + if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ: + env["__PORTAGE_TEST_HARDLINK_LOCKS"] = \ + os.environ["__PORTAGE_TEST_HARDLINK_LOCKS"] + + dirs = [distdir, fake_bin, portage_tmpdir, + var_cache_edb] + etc_symlinks = ("dispatch-conf.conf", "etc-update.conf") + # Override things that may be unavailable, or may have portability + # issues when running tests in exotic environments. + # prepstrip - bug #447810 (bash read builtin EINTR problem) + true_symlinks = ["prepstrip", "scanelf"] + true_binary = find_binary("true") + self.assertEqual(true_binary is None, False, + "true command not found") + try: + for d in dirs: + ensure_dirs(d) + for x in true_symlinks: + os.symlink(true_binary, os.path.join(fake_bin, x)) + for x in etc_symlinks: + os.symlink(os.path.join(self.cnf_etc_path, x), + os.path.join(eprefix, "etc", x)) + with open(os.path.join(var_cache_edb, "counter"), 'wb') as f: + f.write(b"100") + + if debug: + # The subprocess inherits both stdout and stderr, for + # debugging purposes. + stdout = None + else: + # The subprocess inherits stderr so that any warnings + # triggered by python -Wd will be visible. + stdout = subprocess.PIPE + + for args in test_commands: + + if hasattr(args, '__call__'): + args() + continue + + if isinstance(args[0], dict): + local_env = env.copy() + local_env.update(args[0]) + args = args[1:] + else: + local_env = env + + proc = subprocess.Popen(args, + env=local_env, stdout=stdout) + + if debug: + proc.wait() + else: + output = proc.stdout.readlines() + proc.wait() + proc.stdout.close() + if proc.returncode != os.EX_OK: + for line in output: + sys.stderr.write(_unicode_decode(line)) + + self.assertEqual(os.EX_OK, proc.returncode, + "emerge failed with args %s" % (args,)) + finally: + playground.cleanup() diff --git a/pym/portage/util/__init__.py b/pym/portage/util/__init__.py index fe79942..ad3a351 100644 --- a/pym/portage/util/__init__.py +++ b/pym/portage/util/__init__.py @@ -1676,13 +1676,36 @@ def new_protect_filename(mydest, newmd5=None, force=False): old_pfile = normalize_path(os.path.join(real_dirname, last_pfile)) if last_pfile and newmd5: try: - last_pfile_md5 = portage.checksum._perform_md5_merge(old_pfile) - except FileNotFound: - # The file suddenly disappeared or it's a broken symlink. - pass + old_pfile_st = _os_merge.lstat(old_pfile) + except OSError as e: + if e.errno != errno.ENOENT: + raise else: - if last_pfile_md5 == newmd5: - return old_pfile + if stat.S_ISLNK(old_pfile_st.st_mode): + try: + # Read symlink target as bytes, in case the + # target path has a bad encoding. + pfile_link = _os.readlink(_unicode_encode(old_pfile, + encoding=_encodings['merge'], errors='strict')) + except OSError: + if e.errno != errno.ENOENT: + raise + else: + pfile_link = _unicode_decode( + encoding=_encodings['merge'], errors='replace') + if pfile_link == newmd5: + return old_pfile + else: + try: + last_pfile_md5 = \ + portage.checksum._perform_md5_merge(old_pfile) + except FileNotFound: + # The file suddenly disappeared or it's a + # broken symlink. + pass + else: + if last_pfile_md5 == newmd5: + return old_pfile return new_pfile def find_updated_config_files(target_root, config_protect):
