Use the cached portage.getpid() function to avoid unnecessary syscalls, and update the cache after each call to os.fork() where the fork may invoke portage APIs.
Bug: https://bugs.gentoo.org/739540 Signed-off-by: Zac Medico <zmed...@gentoo.org> --- bin/quickpkg | 2 +- lib/_emerge/EbuildBinpkg.py | 4 +++- lib/_emerge/actions.py | 2 +- lib/portage/_emirrordist/FetchTask.py | 4 ++-- lib/portage/cache/metadata.py | 4 +++- lib/portage/elog/mod_mail_summary.py | 5 ++--- lib/portage/elog/mod_save_summary.py | 2 +- lib/portage/locks.py | 2 +- lib/portage/package/ebuild/doebuild.py | 2 +- lib/portage/process.py | 16 +++++++++++----- lib/portage/tests/locks/test_lock_nonblock.py | 1 + .../futures/asyncio/test_wakeup_fd_sigchld.py | 2 +- lib/portage/util/__init__.py | 4 ++-- lib/portage/util/_eventloop/EventLoop.py | 6 +++--- .../util/_eventloop/asyncio_event_loop.py | 4 ++-- lib/portage/util/_eventloop/global_event_loop.py | 7 +++---- lib/portage/util/locale.py | 1 + lib/portage/util/movefile.py | 2 +- lib/portage/util/socks5.py | 2 +- lib/portage/xpak.py | 2 +- 20 files changed, 42 insertions(+), 32 deletions(-) diff --git a/bin/quickpkg b/bin/quickpkg index be7d1d7af..a171b3bd5 100755 --- a/bin/quickpkg +++ b/bin/quickpkg @@ -111,7 +111,7 @@ def quickpkg_atom(options, infos, arg, eout): vardb.aux_update(cpv, update_metadata) xpdata = xpak.xpak(dblnk.dbdir) binpkg_tmpfile = os.path.join(bintree.pkgdir, - cpv + ".tbz2." + str(os.getpid())) + cpv + ".tbz2." + str(portage.getpid())) ensure_dirs(os.path.dirname(binpkg_tmpfile)) binpkg_compression = settings.get("BINPKG_COMPRESS", "bzip2") try: diff --git a/lib/_emerge/EbuildBinpkg.py b/lib/_emerge/EbuildBinpkg.py index 6e098eb8a..879b3a9aa 100644 --- a/lib/_emerge/EbuildBinpkg.py +++ b/lib/_emerge/EbuildBinpkg.py @@ -3,6 +3,8 @@ from _emerge.CompositeTask import CompositeTask from _emerge.EbuildPhase import EbuildPhase + +import portage from portage import os class EbuildBinpkg(CompositeTask): @@ -17,7 +19,7 @@ class EbuildBinpkg(CompositeTask): root_config = pkg.root_config bintree = root_config.trees["bintree"] binpkg_tmpfile = os.path.join(bintree.pkgdir, - pkg.cpv + ".tbz2." + str(os.getpid())) + pkg.cpv + ".tbz2." + str(portage.getpid())) bintree._ensure_dir(os.path.dirname(binpkg_tmpfile)) self._binpkg_tmpfile = binpkg_tmpfile diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py index 063f5d4a0..a4ecfe43d 100644 --- a/lib/_emerge/actions.py +++ b/lib/_emerge/actions.py @@ -2623,7 +2623,7 @@ def ionice(settings): if not ionice_cmd: return - variables = {"PID" : str(os.getpid())} + variables = {"PID" : str(portage.getpid())} cmd = [varexpand(x, mydict=variables) for x in ionice_cmd] try: diff --git a/lib/portage/_emirrordist/FetchTask.py b/lib/portage/_emirrordist/FetchTask.py index 457ca2ac6..41f96b962 100644 --- a/lib/portage/_emirrordist/FetchTask.py +++ b/lib/portage/_emirrordist/FetchTask.py @@ -415,7 +415,7 @@ class FetchTask(CompositeTask): self._fetch_tmp_dir_info = 'distfiles' distdir = self.config.options.distfiles - tmp_basename = self.distfile + '._emirrordist_fetch_.%s' % os.getpid() + tmp_basename = self.distfile + '._emirrordist_fetch_.%s' % portage.getpid() variables = { "DISTDIR": distdir, @@ -622,7 +622,7 @@ class FetchTask(CompositeTask): head, tail = os.path.split(dest) hardlink_tmp = os.path.join(head, ".%s._mirrordist_hardlink_.%s" % \ - (tail, os.getpid())) + (tail, portage.getpid())) try: try: diff --git a/lib/portage/cache/metadata.py b/lib/portage/cache/metadata.py index db81b8ba1..bd1b70fa0 100644 --- a/lib/portage/cache/metadata.py +++ b/lib/portage/cache/metadata.py @@ -6,6 +6,8 @@ import errno import re import stat from operator import attrgetter + +import portage from portage import os from portage import _encodings from portage import _unicode_encode @@ -122,7 +124,7 @@ class database(flat_hash.database): s = cpv.rfind("/") fp = os.path.join(self.location,cpv[:s], - ".update.%i.%s" % (os.getpid(), cpv[s+1:])) + ".update.%i.%s" % (portage.getpid(), cpv[s+1:])) try: myf = open(_unicode_encode(fp, encoding=_encodings['fs'], errors='strict'), 'wb') diff --git a/lib/portage/elog/mod_mail_summary.py b/lib/portage/elog/mod_mail_summary.py index ac260880e..789f55f4d 100644 --- a/lib/portage/elog/mod_mail_summary.py +++ b/lib/portage/elog/mod_mail_summary.py @@ -6,7 +6,6 @@ import portage from portage.exception import AlarmSignal, PortageException from portage.localization import _ from portage.util import writemsg -from portage import os from portage import _encodings from portage import _unicode_decode @@ -22,7 +21,7 @@ def process(mysettings, key, logentries, fulltext): time.strftime("%Y%m%d-%H%M%S %Z", time.localtime(time.time())), encoding=_encodings['content'], errors='replace') header = _(">>> Messages generated for package %(pkg)s by process %(pid)d on %(time)s:\n\n") % \ - {"pkg": key, "pid": os.getpid(), "time": time_str} + {"pkg": key, "pid": portage.getpid(), "time": time_str} config_root = mysettings["PORTAGE_CONFIGROOT"] # Copy needed variables from the config instance, @@ -66,7 +65,7 @@ def _finalize(mysettings, items): mysubject = mysubject.replace("${HOST}", socket.getfqdn()) mybody = _("elog messages for the following packages generated by " - "process %(pid)d on host %(host)s:\n") % {"pid": os.getpid(), "host": socket.getfqdn()} + "process %(pid)d on host %(host)s:\n") % {"pid": portage.getpid(), "host": socket.getfqdn()} for key in items: mybody += "- %s\n" % key diff --git a/lib/portage/elog/mod_save_summary.py b/lib/portage/elog/mod_save_summary.py index 946a1ad4c..6e24b608f 100644 --- a/lib/portage/elog/mod_save_summary.py +++ b/lib/portage/elog/mod_save_summary.py @@ -79,7 +79,7 @@ def process(mysettings, key, logentries, fulltext): encoding=_encodings['content'], errors='replace') elogfile.write(_(">>> Messages generated by process " "%(pid)d on %(time)s for package %(pkg)s:\n\n") % - {"pid": os.getpid(), "time": time_str, "pkg": key}) + {"pid": portage.getpid(), "time": time_str, "pkg": key}) elogfile.write(_unicode_decode(fulltext)) elogfile.write("\n") elogfile.close() diff --git a/lib/portage/locks.py b/lib/portage/locks.py index 701093024..1073343be 100644 --- a/lib/portage/locks.py +++ b/lib/portage/locks.py @@ -501,7 +501,7 @@ def unlockfile(mytuple): def hardlock_name(path): base, tail = os.path.split(path) return os.path.join(base, ".%s.hardlock-%s-%s" % - (tail, portage._decode_argv([os.uname()[1]])[0], os.getpid())) + (tail, portage._decode_argv([os.uname()[1]])[0], portage.getpid())) def hardlink_is_mine(link, lock): try: diff --git a/lib/portage/package/ebuild/doebuild.py b/lib/portage/package/ebuild/doebuild.py index 7bb942966..3b1991b28 100644 --- a/lib/portage/package/ebuild/doebuild.py +++ b/lib/portage/package/ebuild/doebuild.py @@ -1178,7 +1178,7 @@ def doebuild(myebuild, mydo, _unused=DeprecationWarning, settings=None, debug=0, bintree = portage.db[mysettings['EROOT']]['bintree'] mysettings["PORTAGE_BINPKG_TMPFILE"] = \ bintree.getname(mysettings.mycpv) + \ - ".%s" % (os.getpid(),) + ".%s" % (portage.getpid(),) bintree._ensure_dir(os.path.dirname( mysettings["PORTAGE_BINPKG_TMPFILE"])) else: diff --git a/lib/portage/process.py b/lib/portage/process.py index 8d4cf164e..48548bacc 100644 --- a/lib/portage/process.py +++ b/lib/portage/process.py @@ -79,11 +79,11 @@ if _fd_dir is not None: raise return range(max_fd_limit) -elif os.path.isdir("/proc/%s/fd" % os.getpid()): +elif os.path.isdir("/proc/%s/fd" % portage.getpid()): # In order for this function to work in forked subprocesses, # os.getpid() must be called from inside the function. def get_open_fds(): - return (int(fd) for fd in os.listdir("/proc/%s/fd" % os.getpid()) + return (int(fd) for fd in os.listdir("/proc/%s/fd" % portage.getpid()) if fd.isdigit()) else: @@ -363,12 +363,13 @@ def spawn(mycommand, env=None, opt_name=None, fd_pipes=None, returnpid=False, # fork, so that the result is cached in the main process. bool(groups) - parent_pid = os.getpid() + parent_pid = portage.getpid() pid = None try: pid = os.fork() if pid == 0: + portage._ForkWatcher.hook(portage._ForkWatcher) try: _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, cwd, pre_exec, close_fds, @@ -386,7 +387,9 @@ def spawn(mycommand, env=None, opt_name=None, fd_pipes=None, returnpid=False, sys.stderr.flush() finally: - if pid == 0 or (pid is None and os.getpid() != parent_pid): + # Don't used portage.getpid() here, due to a race with the above + # portage._ForkWatcher cache update. + if pid == 0 or (pid is None and _os.getpid() != parent_pid): # Call os._exit() from a finally block in order # to suppress any finally blocks from earlier # in the call stack (see bug #345289). This @@ -603,7 +606,7 @@ def _exec(binary, mycommand, opt_name, fd_pipes, # it is done before we start forking children if cgroup: with open(os.path.join(cgroup, 'cgroup.procs'), 'a') as f: - f.write('%d\n' % os.getpid()) + f.write('%d\n' % portage.getpid()) # Unshare (while still uid==0) if unshare_net or unshare_ipc or unshare_mount or unshare_pid: @@ -640,6 +643,9 @@ def _exec(binary, mycommand, opt_name, fd_pipes, if unshare_pid: main_child_pid = os.fork() if main_child_pid == 0: + # The portage.getpid() cache may need to be updated here, + # in case the pre_exec function invokes portage APIs. + portage._ForkWatcher.hook(portage._ForkWatcher) # pid namespace requires us to become init binary, myargs = portage._python_interpreter, [ portage._python_interpreter, diff --git a/lib/portage/tests/locks/test_lock_nonblock.py b/lib/portage/tests/locks/test_lock_nonblock.py index 02ba16ad9..3448b84f6 100644 --- a/lib/portage/tests/locks/test_lock_nonblock.py +++ b/lib/portage/tests/locks/test_lock_nonblock.py @@ -19,6 +19,7 @@ class LockNonblockTestCase(TestCase): lock1 = portage.locks.lockfile(path) pid = os.fork() if pid == 0: + portage._ForkWatcher.hook(portage._ForkWatcher) portage.locks._close_fds() # Disable close_fds since we don't exec # (see _setup_pipes docstring). diff --git a/lib/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py b/lib/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py index e5b104e0f..c37a6338b 100644 --- a/lib/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py +++ b/lib/portage/tests/util/futures/asyncio/test_wakeup_fd_sigchld.py @@ -39,7 +39,7 @@ proc = loop.run_until_complete(asyncio.create_subprocess_exec('sleep', '0', loop loop.run_until_complete(proc.wait()) for i in range(8192): - os.kill(os.getpid(), signal.SIGCHLD) + os.kill(portage.getpid(), signal.SIGCHLD) # Verify that the child watcher still works correctly # (this will hang if it doesn't). diff --git a/lib/portage/util/__init__.py b/lib/portage/util/__init__.py index c14c15fe8..0412b2b59 100644 --- a/lib/portage/util/__init__.py +++ b/lib/portage/util/__init__.py @@ -1266,7 +1266,7 @@ class atomic_ofstream(ObjectProxy): if follow_links: canonical_path = os.path.realpath(filename) object.__setattr__(self, '_real_name', canonical_path) - tmp_name = "%s.%i" % (canonical_path, os.getpid()) + tmp_name = "%s.%i" % (canonical_path, portage.getpid()) try: object.__setattr__(self, '_file', open_func(_unicode_encode(tmp_name, @@ -1281,7 +1281,7 @@ class atomic_ofstream(ObjectProxy): # new error if necessary. object.__setattr__(self, '_real_name', filename) - tmp_name = "%s.%i" % (filename, os.getpid()) + tmp_name = "%s.%i" % (filename, portage.getpid()) object.__setattr__(self, '_file', open_func(_unicode_encode(tmp_name, encoding=_encodings['fs'], errors='strict'), diff --git a/lib/portage/util/_eventloop/EventLoop.py b/lib/portage/util/_eventloop/EventLoop.py index 94e637853..ff2b73255 100644 --- a/lib/portage/util/_eventloop/EventLoop.py +++ b/lib/portage/util/_eventloop/EventLoop.py @@ -188,7 +188,7 @@ class EventLoop: self._sigchld_read = None self._sigchld_write = None self._sigchld_src_id = None - self._pid = os.getpid() + self._pid = portage.getpid() self._asyncio_wrapper = _PortageEventLoop(loop=self) self._asyncio_child_watcher = _PortageChildWatcher(self) @@ -431,7 +431,7 @@ class EventLoop: # If this signal handler was not installed by the # current process then the signal doesn't belong to # this EventLoop instance. - if os.getpid() == self._pid: + if portage.getpid() == self._pid: os.write(self._sigchld_write, b'\0') def _sigchld_io_cb(self, fd, events): @@ -1026,7 +1026,7 @@ class EventLoop: log_lines.append('{}: {}'.format(key, value)) logging.error('\n'.join(log_lines), exc_info=exc_info) - os.kill(os.getpid(), signal.SIGTERM) + os.kill(portage.getpid(), signal.SIGTERM) def call_exception_handler(self, context): """ diff --git a/lib/portage/util/_eventloop/asyncio_event_loop.py b/lib/portage/util/_eventloop/asyncio_event_loop.py index 605e7243b..836f1c30a 100644 --- a/lib/portage/util/_eventloop/asyncio_event_loop.py +++ b/lib/portage/util/_eventloop/asyncio_event_loop.py @@ -69,7 +69,7 @@ class AsyncioEventLoop(_AbstractEventLoop): # in order to ensure that emerge exits immediately (though # uncleanly). signal.signal(signal.SIGTERM, signal.SIG_DFL) - os.kill(os.getpid(), signal.SIGTERM) + os.kill(portage.getpid(), signal.SIGTERM) def _create_future(self): """ @@ -117,7 +117,7 @@ class AsyncioEventLoop(_AbstractEventLoop): self._wakeup_fd = -1 # Account for any signals that may have arrived between # set_wakeup_fd calls. - os.kill(os.getpid(), signal.SIGCHLD) + os.kill(portage.getpid(), signal.SIGCHLD) try: return self._loop.run_until_complete(future) finally: diff --git a/lib/portage/util/_eventloop/global_event_loop.py b/lib/portage/util/_eventloop/global_event_loop.py index 1db958d2e..21a1d1970 100644 --- a/lib/portage/util/_eventloop/global_event_loop.py +++ b/lib/portage/util/_eventloop/global_event_loop.py @@ -1,13 +1,12 @@ # Copyright 2012-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -import os - +import portage from .EventLoop import EventLoop from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop -_MAIN_PID = os.getpid() +_MAIN_PID = portage.getpid() _instances = {} @@ -17,7 +16,7 @@ def global_event_loop(): belongs exclusively to the current process. """ - pid = os.getpid() + pid = portage.getpid() instance = _instances.get(pid) if instance is not None: return instance diff --git a/lib/portage/util/locale.py b/lib/portage/util/locale.py index 99c8f7ae7..58c687139 100644 --- a/lib/portage/util/locale.py +++ b/lib/portage/util/locale.py @@ -102,6 +102,7 @@ def check_locale(silent=False, env=None): pid = os.fork() if pid == 0: + portage._ForkWatcher.hook(portage._ForkWatcher) try: if env is not None: try: diff --git a/lib/portage/util/movefile.py b/lib/portage/util/movefile.py index 4f8054f29..a251d369d 100644 --- a/lib/portage/util/movefile.py +++ b/lib/portage/util/movefile.py @@ -209,7 +209,7 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, if hardlink_candidates: head, tail = os.path.split(dest) hardlink_tmp = os.path.join(head, ".%s._portage_merge_.%s" % \ - (tail, os.getpid())) + (tail, portage.getpid())) try: os.unlink(hardlink_tmp) except OSError as e: diff --git a/lib/portage/util/socks5.py b/lib/portage/util/socks5.py index 9f22c1dbe..a76d1a741 100644 --- a/lib/portage/util/socks5.py +++ b/lib/portage/util/socks5.py @@ -42,7 +42,7 @@ class ProxyManager: portage.util.ensure_dirs(tmpdir, **ensure_dirs_kwargs) self.socket_path = os.path.join(tmpdir, - '.portage.%d.net.sock' % os.getpid()) + '.portage.%d.net.sock' % portage.getpid()) server_bin = os.path.join(settings['PORTAGE_BIN_PATH'], 'socks5-server.py') spawn_kwargs = {} # The portage_uid check solves EPERM failures in Travis CI. diff --git a/lib/portage/xpak.py b/lib/portage/xpak.py index 2a4bcda21..9063c4c56 100644 --- a/lib/portage/xpak.py +++ b/lib/portage/xpak.py @@ -325,7 +325,7 @@ class tbz2: self.scan() # Don't care about condition... We'll rewrite the data anyway. if break_hardlinks and self.filestat and self.filestat.st_nlink > 1: - tmp_fname = "%s.%d" % (self.file, os.getpid()) + tmp_fname = "%s.%d" % (self.file, portage.getpid()) copyfile(self.file, tmp_fname) try: portage.util.apply_stat_permissions(self.file, self.filestat) -- 2.25.3
