commit: 0896ede9663d1ffb10434ee163205e7d9a909667
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Feb 26 00:12:13 2024 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Tue Feb 27 02:52:19 2024 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=0896ede9
Scheduler: Support parallel-install with merge-wait
For system packages, always serialize install regardless of
parallel-install, in order to mitigate failures triggered
by fragile states as in bug 256616. For other packages,
continue to populate self._task_queues.merge, which will
serialize install unless parallel-install is enabled.
Fixes: 825db01b91a3 ("Add merge-wait FEATURES setting enabled by default")
Bug: https://bugs.gentoo.org/256616
Bug: https://bugs.gentoo.org/925213
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>
lib/_emerge/PackageMerge.py | 4 ++--
lib/_emerge/Scheduler.py | 44 +++++++++++++++++++++++++++++++++-----------
man/make.conf.5 | 9 ++++++---
3 files changed, 41 insertions(+), 16 deletions(-)
diff --git a/lib/_emerge/PackageMerge.py b/lib/_emerge/PackageMerge.py
index 82725c66a5..11d0ff2f37 100644
--- a/lib/_emerge/PackageMerge.py
+++ b/lib/_emerge/PackageMerge.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2014 Gentoo Foundation
+# Copyright 1999-2024 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from _emerge.CompositeTask import CompositeTask
@@ -7,7 +7,7 @@ from portage.output import colorize
class PackageMerge(CompositeTask):
- __slots__ = ("merge", "postinst_failure")
+ __slots__ = ("is_system_pkg", "merge", "postinst_failure")
def _should_show_status(self):
return (
diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py
index 9950792dc9..5c318f89b9 100644
--- a/lib/_emerge/Scheduler.py
+++ b/lib/_emerge/Scheduler.py
@@ -1519,17 +1519,20 @@ class Scheduler(PollScheduler):
self._deallocate_config(build.settings)
elif build.returncode == os.EX_OK:
self.curval += 1
- merge = PackageMerge(merge=build, scheduler=self._sched_iface)
+ merge = PackageMerge(
+ is_system_pkg=(build.pkg in self._deep_system_deps),
+ merge=build,
+ scheduler=self._sched_iface,
+ )
self._running_tasks[id(merge)] = merge
# By default, merge-wait only allows merge when no builds are
executing.
# As a special exception, dependencies on system packages are
frequently
# unspecified and will therefore force merge-wait.
- is_system_pkg = build.pkg in self._deep_system_deps
if not build.build_opts.buildpkgonly and (
- "merge-wait" in build.settings.features or is_system_pkg
+ "merge-wait" in build.settings.features or merge.is_system_pkg
):
self._merge_wait_queue.append(merge)
- if is_system_pkg:
+ if merge.is_system_pkg:
merge.addStartListener(self._system_merge_started)
else:
self._task_queues.merge.add(merge)
@@ -1804,13 +1807,32 @@ class Scheduler(PollScheduler):
and not self._jobs
and not self._task_queues.merge
):
- task = self._merge_wait_queue.popleft()
- task.scheduler = self._sched_iface
- self._merge_wait_scheduled.append(task)
- self._task_queues.merge.add(task)
- task.addExitListener(self._merge_wait_exit_handler)
- self._status_display.merges = len(self._task_queues.merge)
- state_change += 1
+ while self._merge_wait_queue:
+ # If we added non-system packages to the merge queue in a
+ # previous iteration of this loop, then for system
packages we
+ # need to come back later when the merge queue is empty.
+ # TODO: Maybe promote non-system packages to the front of
the
+ # queue and process them within the current loop, though
that
+ # causes merge order to differ from the order builds
finish.
+ if (
+ self._task_queues.merge
+ and self._merge_wait_queue[0].is_system_pkg
+ ):
+ break
+ task = self._merge_wait_queue.popleft()
+ task.scheduler = self._sched_iface
+ self._merge_wait_scheduled.append(task)
+ self._task_queues.merge.add(task)
+ task.addExitListener(self._merge_wait_exit_handler)
+ self._status_display.merges = len(self._task_queues.merge)
+ state_change += 1
+ # For system packages, always serialize install regardless
of
+ # parallel-install, in order to mitigate failures triggered
+ # by fragile states as in bug 256616. For other packages,
+ # continue to populate self._task_queues.merge, which will
+ # serialize install unless parallel-install is enabled.
+ if task.is_system_pkg:
+ break
if self._schedule_tasks_imp():
state_change += 1
diff --git a/man/make.conf.5 b/man/make.conf.5
index e13f6eec4f..21ae09e574 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -652,9 +652,12 @@ terminal to view parallel-fetch progress.
.TP
.B parallel\-install
Use finer\-grained locks when installing packages, allowing for greater
-parallelism. Note that \fIparallel\-install\fR currently has no effect
-unless \fImerge\-wait\fR is disabled. For additional parallelism,
-disable \fIebuild\-locks\fR.
+parallelism. For additional parallelism disable \fIebuild\-locks\fR.
+Also disable \fImerge\-wait\fR for additional parallelism if desired,
+but that increases the possibility of random build failures. When
+\fIparallel\-install\fR is used together with \fImerge\-wait\fR,
+parallel installation occurs in batches when there are no builds
+running.
.TP
.B pid\-sandbox
Isolate the process space for the ebuild processes. This makes it
