[gentoo-portage-dev] [PATCH] _slot_confict_backtrack: minimize conflict atoms (bug 743631)

[Zac Medico]

Prefer choices that minimize conflict atoms, so that choices
which satisfy all parents are preferred. This reduces the
minimum necessary backtrack tries from 21 to 7 for the unit
test related to bug 743115.

Bug: https://bugs.gentoo.org/743115
Bug: https://bugs.gentoo.org/743631
Signed-off-by: Zac Medico 
---
 lib/_emerge/depgraph.py | 6 ++
 .../tests/resolver/test_slot_operator_missed_update.py  | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 3f864aefc..40e7d1325 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -1797,6 +1797,12 @@ class depgraph:
if parent_atom not in parent_atoms)
backtrack_data.append((to_be_masked, conflict_atoms))
 
+   # Prefer choices that minimize conflict atoms. This is intended
+   # to take precedence over the earlier package version sort. The
+   # package version sort is still needed or else the
+   # testOverlapSlotConflict method of 
VirtualMinimizeChildrenTestCase
+   # will not succeed reliably with the default backtrack limit.
+   backtrack_data.sort(key=lambda item: len(item[1]))
to_be_masked = backtrack_data[-1][0]
 
self._dynamic_config._backtrack_infos.setdefault(
diff --git a/lib/portage/tests/resolver/test_slot_operator_missed_update.py 
b/lib/portage/tests/resolver/test_slot_operator_missed_update.py
index fce012f62..1ea701003 100644
--- a/lib/portage/tests/resolver/test_slot_operator_missed_update.py
+++ b/lib/portage/tests/resolver/test_slot_operator_missed_update.py
@@ -90,7 +90,7 @@ class BacktrackMissedUpdateTestCase(TestCase):
# Bug 743115: missed updates trigger excessive 
backtracking
ResolverPlaygroundTestCase(
[">=dev-python/pypy3-7.3.2_rc", "@world"],
-   options={"--update": True, "--deep": True, 
"--backtrack": 25},
+   options={"--update": True, "--deep": True, 
"--backtrack": 10},
success=True,
mergelist=[
"dev-python/pypy3-7.3.2_rc2_p37-r1",
-- 
2.25.3

[gentoo-portage-dev] Re: [PATCH] _slot_confict_backtrack: minimize conflict atoms (bug 743631)

[Zac Medico]

On 9/19/20 2:40 PM, Zac Medico wrote:
> diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
> index 3f864aefc..40e7d1325 100644
> --- a/lib/_emerge/depgraph.py
> +++ b/lib/_emerge/depgraph.py
> @@ -1797,6 +1797,12 @@ class depgraph:
>   if parent_atom not in parent_atoms)
>   backtrack_data.append((to_be_masked, conflict_atoms))
>  
> + # Prefer choices that minimize conflict atoms. This is intended
> + # to take precedence over the earlier package version sort. The
> + # package version sort is still needed or else the
> + # testOverlapSlotConflict method of 
> VirtualMinimizeChildrenTestCase
> + # will not succeed reliably with the default backtrack limit.

Updated this comment to indicate that the package version sort sort is actually
needed for deterministic results:

# Prefer choices that minimize conflict atoms. This is intended
# to take precedence over the earlier package version sort. The
# package version sort is still needed or else choices for the
# testOverlapSlotConflict method of 
VirtualMinimizeChildrenTestCase
# become non-deterministic.

-- 
Thanks,
Zac



signature.asc
Description: OpenPGP digital signature

[gentoo-portage-dev] [PATCH] _slot_confict_backtrack: group similar missed updates (bug 743115)

[Zac Medico]

When a slot conflict occurs due to a missed update, and some other
similar update(s) are available, add the similar update(s) to the
runtime package mask for the same backtracking choice. This reduces
minimum number of backtrack tries required to solve the test case
for bug 743115 from 7 to 4, where the difference of 3 corresponds
to the number of other similar setuptools updates available.

Bug: https://bugs.gentoo.org/743115
Signed-off-by: Zac Medico 
---
 lib/_emerge/depgraph.py   | 25 ---
 lib/_emerge/resolver/backtracking.py  |  7 +++---
 .../test_slot_operator_missed_update.py   |  2 +-
 3 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 7281d8692..2a840b2ca 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -1795,15 +1795,32 @@ class depgraph:

self._dynamic_config._parent_atoms.get(to_be_masked, set())
conflict_atoms = set(parent_atom for parent_atom in 
all_parents \
if parent_atom not in parent_atoms)
-   backtrack_data.append((to_be_masked, conflict_atoms))
+
+   similar_pkgs = []
+   if conflict_atoms:
+   # If the conflict has been triggered by a 
missed update, then
+   # we can avoid excessive backtracking if we 
detect similar missed
+   # updates and mask them as part of the same 
backtracking choice.
+   for similar_pkg in 
self._iter_similar_available(to_be_masked, slot_atom):
+   if similar_pkg in conflict_pkgs:
+   continue
+   similar_conflict_atoms = []
+   for parent_atom in conflict_atoms:
+   parent, atom = parent_atom
+   if not atom.match(similar_pkg):
+   
similar_conflict_atoms.append(parent_atom)
+   if similar_conflict_atoms:
+   
similar_pkgs.append((similar_pkg, set(similar_conflict_atoms)))
+   similar_pkgs.append((to_be_masked, conflict_atoms))
+   backtrack_data.append(tuple(similar_pkgs))
 
# Prefer choices that minimize conflict atoms. This is intended
# to take precedence over the earlier package version sort. The
# package version sort is still needed or else choices for the
# testOverlapSlotConflict method of 
VirtualMinimizeChildrenTestCase
# become non-deterministic.
-   backtrack_data.sort(key=lambda item: len(item[1]))
-   to_be_masked = backtrack_data[-1][0]
+   backtrack_data.sort(key=lambda similar_pkgs: max(len(item[1]) 
for item in similar_pkgs))
+   to_be_masked = [item[0] for item in backtrack_data[-1]]
 
self._dynamic_config._backtrack_infos.setdefault(
"slot conflict", []).append(backtrack_data)
@@ -1814,7 +1831,7 @@ class depgraph:
"",
"backtracking due to slot conflict:",
"   first package:  %s" % existing_node,
-   "  package to mask: %s" % to_be_masked,
+   "  package(s) to mask: %s" % str(to_be_masked),
"  slot: %s" % slot_atom,
"   parents: %s" % ", ".join(
"(%s, '%s')" % (ppkg, atom) for ppkg, 
atom in all_parents
diff --git a/lib/_emerge/resolver/backtracking.py 
b/lib/_emerge/resolver/backtracking.py
index bc3fb3206..ca94623ac 100644
--- a/lib/_emerge/resolver/backtracking.py
+++ b/lib/_emerge/resolver/backtracking.py
@@ -166,13 +166,14 @@ class Backtracker:
self._feedback_slot_conflict(conflicts_data[0])
 
def _feedback_slot_conflict(self, conflict_data):
-   for pkg, parent_atoms in conflict_data:
+   for similar_pkgs in conflict_data:
new_node = copy.deepcopy(self._current_node)
new_node.depth += 1
new_node.mask_steps += 1
new_node.terminal = False
-   new_node.parameter.runtime_pkg_mask.setdefault(
-   pkg, {})["slot conflict"] = parent_atoms
+   for pkg, parent_atoms in similar_pkgs:
+   new_node.parameter.runtime_pkg_mask.setdefault(
+   pkg, {})["slot co

[gentoo-portage-dev] [PATCH] emerge: enable parallel-fetch during pkg_pretend (bug 710432)

[Zac Medico]

Execute pkg_pretend phases in a coroutine while parallel-fetch
is running concurrently. When it's time to execute the pkg_pretend
phase for a remote binary package, use a Scheduler _get_prefetcher
method to get a running prefetcher if available, and otherwise
start a new fetcher.

Bug: https://bugs.gentoo.org/710432
Signed-off-by: Zac Medico 
---
 lib/_emerge/Scheduler.py | 94 +---
 1 file changed, 58 insertions(+), 36 deletions(-)

diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py
index a69421288..20884986f 100644
--- a/lib/_emerge/Scheduler.py
+++ b/lib/_emerge/Scheduler.py
@@ -25,6 +25,7 @@ from portage._sets import SETPREFIX
 from portage._sets.base import InternalPackageSet
 from portage.util import ensure_dirs, writemsg, writemsg_level
 from portage.util.futures import asyncio
+from portage.util.futures.compat_coroutine import coroutine, coroutine_return
 from portage.util.SlotObject import SlotObject
 from portage.util._async.SchedulerInterface import SchedulerInterface
 from portage.package.ebuild.digestcheck import digestcheck
@@ -766,7 +767,8 @@ class Scheduler(PollScheduler):
 
return prefetcher
 
-   def _run_pkg_pretend(self):
+   @coroutine
+   def _run_pkg_pretend(self, loop=None):
"""
Since pkg_pretend output may be important, this method sends all
output directly to stdout (regardless of options like --quiet or
@@ -774,7 +776,7 @@ class Scheduler(PollScheduler):
"""
 
failures = 0
-   sched_iface = self._sched_iface
+   sched_iface = loop = asyncio._wrap_loop(loop or 
self._sched_iface)
 
for x in self._mergelist:
if not isinstance(x, Package):
@@ -795,12 +797,18 @@ class Scheduler(PollScheduler):
root_config = x.root_config
settings = self.pkgsettings[root_config.root]
settings.setcpv(x)
+   if not x.built:
+   # Get required SRC_URI metadata (it's not 
cached in x.metadata
+   # because some packages have an extremely large 
SRC_URI value).
+   portdb = root_config.trees["porttree"].dbapi
+   settings.configdict["pkg"]["SRC_URI"], = (yield 
portdb.async_aux_get(
+   x.cpv, ["SRC_URI"], myrepo=x.repo, 
loop=loop))
 
# setcpv/package.env allows for per-package 
PORTAGE_TMPDIR so we
# have to validate it for each package
rval = _check_temp_dir(settings)
if rval != os.EX_OK:
-   return rval
+   coroutine_return(rval)
 
build_dir_path = os.path.join(
os.path.realpath(settings["PORTAGE_TMPDIR"]),
@@ -809,7 +817,7 @@ class Scheduler(PollScheduler):
settings["PORTAGE_BUILDDIR"] = build_dir_path
build_dir = EbuildBuildDir(scheduler=sched_iface,
settings=settings)
-   sched_iface.run_until_complete(build_dir.async_lock())
+   yield build_dir.async_lock()
current_task = None
 
try:
@@ -835,7 +843,7 @@ class Scheduler(PollScheduler):
phase='clean', 
scheduler=sched_iface, settings=settings)
current_task = clean_phase
clean_phase.start()
-   clean_phase.wait()
+   yield clean_phase.async_wait()
 
if x.built:
tree = "bintree"
@@ -845,10 +853,11 @@ class Scheduler(PollScheduler):
# Display fetch on stdout, so that it's 
always clear what
# is consuming time here.
if bintree.isremote(x.cpv):
-   fetcher = BinpkgFetcher(pkg=x,
-   scheduler=sched_iface)
-   fetcher.start()
-   if fetcher.wait() != os.EX_OK:
+   fetcher = 
self._get_prefetcher(x)
+   if fetcher is None:
+   fetcher = 
BinpkgFetcher(pkg=x, scheduler=loop)
+   fetcher.start()
+   if (yield fetcher.async_wait()) 
!= os.EX_