[gentoo-portage-dev] [PATCH] emerge: enable parallel-fetch during pkg_pretend (bug 710432)
Execute pkg_pretend phases in a coroutine while parallel-fetch is running concurrently. When it's time to execute the pkg_pretend phase for a remote binary package, use a Scheduler _get_prefetcher method to get a running prefetcher if available, and otherwise start a new fetcher. Bug: https://bugs.gentoo.org/710432 Signed-off-by: Zac Medico --- lib/_emerge/Scheduler.py | 94 +--- 1 file changed, 58 insertions(+), 36 deletions(-) diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py index a69421288..20884986f 100644 --- a/lib/_emerge/Scheduler.py +++ b/lib/_emerge/Scheduler.py @@ -25,6 +25,7 @@ from portage._sets import SETPREFIX from portage._sets.base import InternalPackageSet from portage.util import ensure_dirs, writemsg, writemsg_level from portage.util.futures import asyncio +from portage.util.futures.compat_coroutine import coroutine, coroutine_return from portage.util.SlotObject import SlotObject from portage.util._async.SchedulerInterface import SchedulerInterface from portage.package.ebuild.digestcheck import digestcheck @@ -766,7 +767,8 @@ class Scheduler(PollScheduler): return prefetcher - def _run_pkg_pretend(self): + @coroutine + def _run_pkg_pretend(self, loop=None): """ Since pkg_pretend output may be important, this method sends all output directly to stdout (regardless of options like --quiet or @@ -774,7 +776,7 @@ class Scheduler(PollScheduler): """ failures = 0 - sched_iface = self._sched_iface + sched_iface = loop = asyncio._wrap_loop(loop or self._sched_iface) for x in self._mergelist: if not isinstance(x, Package): @@ -795,12 +797,18 @@ class Scheduler(PollScheduler): root_config = x.root_config settings = self.pkgsettings[root_config.root] settings.setcpv(x) + if not x.built: + # Get required SRC_URI metadata (it's not cached in x.metadata + # because some packages have an extremely large SRC_URI value). + portdb = root_config.trees["porttree"].dbapi + settings.configdict["pkg"]["SRC_URI"], = (yield portdb.async_aux_get( + x.cpv, ["SRC_URI"], myrepo=x.repo, loop=loop)) # setcpv/package.env allows for per-package PORTAGE_TMPDIR so we # have to validate it for each package rval = _check_temp_dir(settings) if rval != os.EX_OK: - return rval + coroutine_return(rval) build_dir_path = os.path.join( os.path.realpath(settings["PORTAGE_TMPDIR"]), @@ -809,7 +817,7 @@ class Scheduler(PollScheduler): settings["PORTAGE_BUILDDIR"] = build_dir_path build_dir = EbuildBuildDir(scheduler=sched_iface, settings=settings) - sched_iface.run_until_complete(build_dir.async_lock()) + yield build_dir.async_lock() current_task = None try: @@ -835,7 +843,7 @@ class Scheduler(PollScheduler): phase='clean', scheduler=sched_iface, settings=settings) current_task = clean_phase clean_phase.start() - clean_phase.wait() + yield clean_phase.async_wait() if x.built: tree = "bintree" @@ -845,10 +853,11 @@ class Scheduler(PollScheduler): # Display fetch on stdout, so that it's always clear what # is consuming time here. if bintree.isremote(x.cpv): - fetcher = BinpkgFetcher(pkg=x, - scheduler=sched_iface) - fetcher.start() - if fetcher.wait() != os.EX_OK: + fetcher = self._get_prefetcher(x) + if fetcher is None: + fetcher = BinpkgFetcher(pkg=x, scheduler=loop) + fetcher.start() + if (yield fetcher.async_wait()) !=
[gentoo-dev] Last rites: sys-block/rts5229
# David Seifert (2020-09-20) # EAPI 4, last release in 2012, sandbox violations and # full of bugs. Mainlined since 3.14, Removal in 30 days. # Bug #679502, #701406, #701408, #742116. sys-block/rts5229 signature.asc Description: This is a digitally signed message part
[gentoo-portage-dev] [PATCH] _slot_confict_backtrack: group similar missed updates (bug 743115)
When a slot conflict occurs due to a missed update, and some other
similar update(s) are available, add the similar update(s) to the
runtime package mask for the same backtracking choice. This reduces
minimum number of backtrack tries required to solve the test case
for bug 743115 from 7 to 4, where the difference of 3 corresponds
to the number of other similar setuptools updates available.
Bug: https://bugs.gentoo.org/743115
Signed-off-by: Zac Medico
---
lib/_emerge/depgraph.py | 25 ---
lib/_emerge/resolver/backtracking.py | 7 +++---
.../test_slot_operator_missed_update.py | 2 +-
3 files changed, 26 insertions(+), 8 deletions(-)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 7281d8692..2a840b2ca 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -1795,15 +1795,32 @@ class depgraph:
self._dynamic_config._parent_atoms.get(to_be_masked, set())
conflict_atoms = set(parent_atom for parent_atom in
all_parents \
if parent_atom not in parent_atoms)
- backtrack_data.append((to_be_masked, conflict_atoms))
+
+ similar_pkgs = []
+ if conflict_atoms:
+ # If the conflict has been triggered by a
missed update, then
+ # we can avoid excessive backtracking if we
detect similar missed
+ # updates and mask them as part of the same
backtracking choice.
+ for similar_pkg in
self._iter_similar_available(to_be_masked, slot_atom):
+ if similar_pkg in conflict_pkgs:
+ continue
+ similar_conflict_atoms = []
+ for parent_atom in conflict_atoms:
+ parent, atom = parent_atom
+ if not atom.match(similar_pkg):
+
similar_conflict_atoms.append(parent_atom)
+ if similar_conflict_atoms:
+
similar_pkgs.append((similar_pkg, set(similar_conflict_atoms)))
+ similar_pkgs.append((to_be_masked, conflict_atoms))
+ backtrack_data.append(tuple(similar_pkgs))
# Prefer choices that minimize conflict atoms. This is intended
# to take precedence over the earlier package version sort. The
# package version sort is still needed or else choices for the
# testOverlapSlotConflict method of
VirtualMinimizeChildrenTestCase
# become non-deterministic.
- backtrack_data.sort(key=lambda item: len(item[1]))
- to_be_masked = backtrack_data[-1][0]
+ backtrack_data.sort(key=lambda similar_pkgs: max(len(item[1])
for item in similar_pkgs))
+ to_be_masked = [item[0] for item in backtrack_data[-1]]
self._dynamic_config._backtrack_infos.setdefault(
"slot conflict", []).append(backtrack_data)
@@ -1814,7 +1831,7 @@ class depgraph:
"",
"backtracking due to slot conflict:",
" first package: %s" % existing_node,
- " package to mask: %s" % to_be_masked,
+ " package(s) to mask: %s" % str(to_be_masked),
" slot: %s" % slot_atom,
" parents: %s" % ", ".join(
"(%s, '%s')" % (ppkg, atom) for ppkg,
atom in all_parents
diff --git a/lib/_emerge/resolver/backtracking.py
b/lib/_emerge/resolver/backtracking.py
index bc3fb3206..ca94623ac 100644
--- a/lib/_emerge/resolver/backtracking.py
+++ b/lib/_emerge/resolver/backtracking.py
@@ -166,13 +166,14 @@ class Backtracker:
self._feedback_slot_conflict(conflicts_data[0])
def _feedback_slot_conflict(self, conflict_data):
- for pkg, parent_atoms in conflict_data:
+ for similar_pkgs in conflict_data:
new_node = copy.deepcopy(self._current_node)
new_node.depth += 1
new_node.mask_steps += 1
new_node.terminal = False
- new_node.parameter.runtime_pkg_mask.setdefault(
- pkg, {})["slot conflict"] = parent_atoms
+ for pkg, parent_atoms in similar_pkgs:
+ new_node.parameter.runtime_pkg_mask.setdefault(
+ pkg, {})["slot
[gentoo-portage-dev] Re: [PATCH] _slot_confict_backtrack: minimize conflict atoms (bug 743631)
On 9/19/20 2:40 PM, Zac Medico wrote: > diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py > index 3f864aefc..40e7d1325 100644 > --- a/lib/_emerge/depgraph.py > +++ b/lib/_emerge/depgraph.py > @@ -1797,6 +1797,12 @@ class depgraph: > if parent_atom not in parent_atoms) > backtrack_data.append((to_be_masked, conflict_atoms)) > > + # Prefer choices that minimize conflict atoms. This is intended > + # to take precedence over the earlier package version sort. The > + # package version sort is still needed or else the > + # testOverlapSlotConflict method of > VirtualMinimizeChildrenTestCase > + # will not succeed reliably with the default backtrack limit. Updated this comment to indicate that the package version sort sort is actually needed for deterministic results: # Prefer choices that minimize conflict atoms. This is intended # to take precedence over the earlier package version sort. The # package version sort is still needed or else choices for the # testOverlapSlotConflict method of VirtualMinimizeChildrenTestCase # become non-deterministic. -- Thanks, Zac signature.asc Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH] _slot_confict_backtrack: minimize conflict atoms (bug 743631)
Prefer choices that minimize conflict atoms, so that choices
which satisfy all parents are preferred. This reduces the
minimum necessary backtrack tries from 21 to 7 for the unit
test related to bug 743115.
Bug: https://bugs.gentoo.org/743115
Bug: https://bugs.gentoo.org/743631
Signed-off-by: Zac Medico
---
lib/_emerge/depgraph.py | 6 ++
.../tests/resolver/test_slot_operator_missed_update.py | 2 +-
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 3f864aefc..40e7d1325 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -1797,6 +1797,12 @@ class depgraph:
if parent_atom not in parent_atoms)
backtrack_data.append((to_be_masked, conflict_atoms))
+ # Prefer choices that minimize conflict atoms. This is intended
+ # to take precedence over the earlier package version sort. The
+ # package version sort is still needed or else the
+ # testOverlapSlotConflict method of
VirtualMinimizeChildrenTestCase
+ # will not succeed reliably with the default backtrack limit.
+ backtrack_data.sort(key=lambda item: len(item[1]))
to_be_masked = backtrack_data[-1][0]
self._dynamic_config._backtrack_infos.setdefault(
diff --git a/lib/portage/tests/resolver/test_slot_operator_missed_update.py
b/lib/portage/tests/resolver/test_slot_operator_missed_update.py
index fce012f62..1ea701003 100644
--- a/lib/portage/tests/resolver/test_slot_operator_missed_update.py
+++ b/lib/portage/tests/resolver/test_slot_operator_missed_update.py
@@ -90,7 +90,7 @@ class BacktrackMissedUpdateTestCase(TestCase):
# Bug 743115: missed updates trigger excessive
backtracking
ResolverPlaygroundTestCase(
[">=dev-python/pypy3-7.3.2_rc", "@world"],
- options={"--update": True, "--deep": True,
"--backtrack": 25},
+ options={"--update": True, "--deep": True,
"--backtrack": 10},
success=True,
mergelist=[
"dev-python/pypy3-7.3.2_rc2_p37-r1",
--
2.25.3
Re: [gentoo-dev] How to stabilize packages with frequent release cycles?
On Wed, Sep 16, 2020 at 4:08 PM Jonas Stein wrote: > > Hi, > > > When the latest release remains 'latest ~arch' for less than 3 days, > > stabilizing it after 30 days makes little sense. After all, people with > > frequent upgrade cycle will test it for no more than that, and people > > with infrequent upgrade cycle may miss the version entirely. > > > Do you have any suggestions how we could improve this? > > At first we need a strict definition of "stable" and "testing", then we > can discuss how to stabilize. > Not sure it is a definition issue so much that the concept doesn't fit with these sorts of packages. Normally the idea of stable is that you're willing to trade speed for quality. The problem is that in these sorts of packages you're often getting neither. For example, you're not going to have a more-bug-free experience with youtube-dl if you run a two month old version, because the APIs are all changing and you're just losing the cat and mouse game. IMO these sorts of packages probably shouldn't have stable versions at all. Then users will accept ~arch, and both know what they're getting into, and also not get stuck with old versions that give them suboptimal results. Now, if somebody can come up with a better interface for that which is cleaner than having to stick foo/bar in accept_keywords that would be nice. But that almost suggests another class of keyword entirely. These packages aren't really "stable" - so much as stable not being an option. -- Rich
[gentoo-dev] Last rites: sci-mathematics/axiom
# David Seifert (2020-09-19) # EAPI 4, last release in 2008, upstream pretty much dead, # tons of bugs, broken since at least 2016, lots of weird # dead/alive/redead forks all over the internet. Use # sci-mathematics/fricas as spiritual successor fork. # Removal in 30 days. Bug #326575, #514762, #532498, # #574956, #581250, #586402, #587878, #740966. sci-mathematics/axiom signature.asc Description: This is a digitally signed message part
Re: [gentoo-dev] Last rites: next batch of py2 packages
On 2020-09-19 16:36, Azamat Hackimov wrote: > Hello. > > сб, 19 сент. 2020 г. в 15:51, Michał Górny : >> sys-firmware/nvidia-firmware >> > Created PR https://github.com/gentoo/gentoo/pull/17600 > Excellent! Thats an important firmware package, I would have started a PR also in a few minutes! 0xCE5F3E9B6DE05D12.asc Description: application/pgp-keys
Re: [gentoo-dev] Last rites: next batch of py2 packages
Hello. сб, 19 сент. 2020 г. в 15:51, Michał Górny : > sys-firmware/nvidia-firmware > Created PR https://github.com/gentoo/gentoo/pull/17600 -- >From Siberia with Love!
[gentoo-dev] Last rites: next batch of py2 packages
# Michał Górny (2020-09-19) # These packages (or package versions) still require Python 2.7. # They are either dead upstream, their Python 3 porting efforts are # not progressing or their maintainers are simply unresponsive. # Please do not remove any packages from this list unless you actually # port them to Python 3. # Removal in 30 days. Please find relevant bugs on tracker bug #694800. app-admin/github-backup-utils app-backup/genbackupdata app-i18n/pology app-pda/gtkpod app-text/pdf2djvu app-text/sgmltools-lite dev-util/anjuta dev-util/gyp app-i18n/mozc sci-libs/magma sys-firmware/nvidia-firmware -- Best regards, Michał Górny signature.asc Description: This is a digitally signed message part
Re: [gentoo-dev] Last rites: dev-java/oracle-{jdk,jre}-bin and revdeps
took a while, removed. I was able to save jabref-bin, works fine with openjdk:8 geogebra now availabie as geogebra-bin and works with openjdk8 and 11. sleuthkit was spared. rest is gone, but if someone wants to restore - patches welcome. On 4/18/20 9:10 PM, Georgy Yakovlev wrote: > # Georgy Yakovlev (2020-04-18) > # Unmaintained, vulnerable oracle java ebuilds, even fetching distfiles > # requires agreement to restrictive license > # Revdeps that still depend on oracle variants require javafx > # Please use icedtea or openjdk instead. > # Removal in 30 days. > # https://bugs.gentoo.org/681828 > dev-java/oracle-jdk-bin > dev-java/oracle-jre-bin > app-forensics/sleuthkit > app-text/jabref-bin > dev-java/netbeans-platform > dev-java/netbeans-harness > games-util/pogo-manager-bin > net-p2p/bisq > sci-mathematics/geogebra > > > > Oracle java has been maintainer-needed since august 2019, > no one stepped up. > Removal in 30 days. > > If someone wants to save the javafx revdeps, best way will be > packaging zulufx community [1], I can provide some guidance > on packaging it, should not be too hard. >
