[gentoo-portage-dev] [PATCH] emerge: use asyncio interfaces for spinner during owner lookup (bug 591760)

[Zac Medico]

X-Gentoo-bug: 591760
X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=591760
---
 pym/_emerge/depgraph.py  | 13 -
 pym/portage/dbapi/vartree.py | 22 +-
 2 files changed, 17 insertions(+), 18 deletions(-)

diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
index 7c9130b..04e724d 100644
--- a/pym/_emerge/depgraph.py
+++ b/pym/_emerge/depgraph.py
@@ -3668,17 +3668,20 @@ class depgraph(object):
def select_files(self, args):
# Use the global event loop for spinner progress
# indication during file owner lookups (bug #461412).
-   spinner_id = None
+   def spinner_cb():
+   self._frozen_config.spinner.update()
+   spinner_cb.handle = 
self._event_loop.call_soon(spinner_cb)
+
+   spinner_cb.handle = None
try:
spinner = self._frozen_config.spinner
if spinner is not None and \
spinner.update is not spinner.update_quiet:
-   spinner_id = self._event_loop.idle_add(
-   self._frozen_config.spinner.update)
+   spinner_cb.handle = 
self._event_loop.call_soon(spinner_cb)
return self._select_files(args)
finally:
-   if spinner_id is not None:
-   self._event_loop.source_remove(spinner_id)
+   if spinner_cb.handle is not None:
+   spinner_cb.handle.cancel()
 
def _select_files(self, myfiles):
"""Given a list of .tbz2s, .ebuilds sets, and deps, populate
diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index c421dc5..7c8f150 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -1369,32 +1369,28 @@ class vardbapi(dbapi):
global_event_loop() or EventLoop(main=False))
root = self._vardb._eroot
 
-   def search_pkg(cpv):
+   def search_pkg(cpv, search_future):
dblnk = self._vardb._dblink(cpv)
+   results = []
for path, name, is_basename in path_info_list:
if is_basename:
for p in dblnk._contents.keys():
if os.path.basename(p) 
== name:
-   
search_pkg.results.append((dblnk,
+   
results.append((dblnk,

dblnk._contents.unmap_key(

p)[len(root):]))
else:
key = 
dblnk._match_contents(path)
if key is not False:
-   
search_pkg.results.append(
+   results.append(
(dblnk, 
key[len(root):]))
-   search_pkg.complete = True
-   return False
-
-   search_pkg.results = []
+   search_future.set_result(results)
 
for cpv in self._vardb.cpv_all():
-   del search_pkg.results[:]
-   search_pkg.complete = False
-   event_loop.idle_add(search_pkg, cpv)
-   while not search_pkg.complete:
-   event_loop.iteration()
-   for result in search_pkg.results:
+   search_future = event_loop.create_future()
+   event_loop.call_soon(search_pkg, cpv, 
search_future)
+   event_loop.run_until_complete(search_future)
+   for result in search_future.result():
yield result
 
 class vartree(object):
-- 
2.10.2

Re: [gentoo-portage-dev] [PATCH] Future: implement add_done_callback for asyncio compat (bug 591760)

[Zac Medico]

On Sun, Mar 26, 2017 at 12:07 PM, Brian Dolbec <dol...@gentoo.org> wrote:
> On Sun, 26 Mar 2017 03:13:11 -0700
> Zac Medico <zmed...@gentoo.org> wrote:
>
>> Implement the add_done_callback and remove_done_callback methods,
>> since they are required in order to make further progress toward
>> asyncio compatibility.
>>
>> Also implement the AbstractEventLoop create_future method for the
>> EventLoop class, so that it returns an instance of _EventLoopFuture.
>> EventLoop currently does not implement some of the
>> asyncio.AbstractEventLoop methods that asyncio.Future requires for
>> its add_done_callback implementation, and the create_future method
>> conveniently solves this problem.
>>
>> X-Gentoo-bug: 591760
>> X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=591760
>> ---
>>  pym/portage/tests/ebuild/test_ipc_daemon.py|  3 +-
>>  .../tests/util/eventloop/test_call_soon_fifo.py|  6 +-
>>  pym/portage/tests/util/futures/__init__.py |  0
>>  pym/portage/tests/util/futures/__test__.py |  0
>>  .../tests/util/futures/test_done_callback.py   | 35 +
>>  pym/portage/util/_async/SchedulerInterface.py  |  3 +-
>>  pym/portage/util/_eventloop/EventLoop.py   | 14 
>>  pym/portage/util/futures/futures.py| 82
>> -- 8 files changed, 132 insertions(+), 11
>> deletions(-) create mode 100644
>> pym/portage/tests/util/futures/__init__.py create mode 100644
>> pym/portage/tests/util/futures/__test__.py create mode 100644
>> pym/portage/tests/util/futures/test_done_callback.py
>>
>
> looks fine...  /me ignoring the lack of parameters descriptions in the
> docstrings

Yeah, I skipped parameter docstrings because the interfaces are
compatible with the python standard libraries.

Pushed:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=4b12ed04ec6b99f5a948e0eea5778a4fac502740
-- 
Thanks,
Zac

[gentoo-portage-dev] [PATCH] Future: implement add_done_callback for asyncio compat (bug 591760)

[Zac Medico]

Implement the add_done_callback and remove_done_callback methods, since
they are required in order to make further progress toward asyncio
compatibility.

Also implement the AbstractEventLoop create_future method for the
EventLoop class, so that it returns an instance of _EventLoopFuture.
EventLoop currently does not implement some of the
asyncio.AbstractEventLoop methods that asyncio.Future requires for
its add_done_callback implementation, and the create_future method
conveniently solves this problem.

X-Gentoo-bug: 591760
X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=591760
---
 pym/portage/tests/ebuild/test_ipc_daemon.py|  3 +-
 .../tests/util/eventloop/test_call_soon_fifo.py|  6 +-
 pym/portage/tests/util/futures/__init__.py |  0
 pym/portage/tests/util/futures/__test__.py |  0
 .../tests/util/futures/test_done_callback.py   | 35 +
 pym/portage/util/_async/SchedulerInterface.py  |  3 +-
 pym/portage/util/_eventloop/EventLoop.py   | 14 
 pym/portage/util/futures/futures.py| 82 --
 8 files changed, 132 insertions(+), 11 deletions(-)
 create mode 100644 pym/portage/tests/util/futures/__init__.py
 create mode 100644 pym/portage/tests/util/futures/__test__.py
 create mode 100644 pym/portage/tests/util/futures/test_done_callback.py

diff --git a/pym/portage/tests/ebuild/test_ipc_daemon.py 
b/pym/portage/tests/ebuild/test_ipc_daemon.py
index 68f139a..fc79165 100644
--- a/pym/portage/tests/ebuild/test_ipc_daemon.py
+++ b/pym/portage/tests/ebuild/test_ipc_daemon.py
@@ -16,7 +16,6 @@ from portage.util import ensure_dirs
 from portage.util._async.ForkProcess import ForkProcess
 from portage.util._async.TaskScheduler import TaskScheduler
 from portage.util._eventloop.global_event_loop import global_event_loop
-from portage.util.futures.futures import Future
 from _emerge.SpawnProcess import SpawnProcess
 from _emerge.EbuildBuildDir import EbuildBuildDir
 from _emerge.EbuildIpcDaemon import EbuildIpcDaemon
@@ -150,7 +149,7 @@ class IpcDaemonTestCase(TestCase):
self._run_done.set_result(True)
 
def _run(self, event_loop, task_scheduler, timeout):
-   self._run_done = Future()
+   self._run_done = event_loop.create_future()
timeout_id = event_loop.timeout_add(timeout,
self._timeout_callback, task_scheduler)
task_scheduler.addExitListener(self._exit_callback)
diff --git a/pym/portage/tests/util/eventloop/test_call_soon_fifo.py 
b/pym/portage/tests/util/eventloop/test_call_soon_fifo.py
index 5ecc13f..f970c67 100644
--- a/pym/portage/tests/util/eventloop/test_call_soon_fifo.py
+++ b/pym/portage/tests/util/eventloop/test_call_soon_fifo.py
@@ -7,22 +7,22 @@ import random
 from portage import os
 from portage.tests import TestCase
 from portage.util._eventloop.global_event_loop import global_event_loop
-from portage.util.futures.futures import Future
+
 
 class CallSoonFifoTestCase(TestCase):
 
def testCallSoonFifo(self):
 
+   event_loop = global_event_loop()
inputs = [random.random() for index in range(10)]
outputs = []
-   finished = Future()
+   finished = event_loop.create_future()
 
def add_output(value):
outputs.append(value)
if len(outputs) == len(inputs):
finished.set_result(True)
 
-   event_loop = global_event_loop()
for value in inputs:
event_loop.call_soon(functools.partial(add_output, 
value))
 
diff --git a/pym/portage/tests/util/futures/__init__.py 
b/pym/portage/tests/util/futures/__init__.py
new file mode 100644
index 000..e69de29
diff --git a/pym/portage/tests/util/futures/__test__.py 
b/pym/portage/tests/util/futures/__test__.py
new file mode 100644
index 000..e69de29
diff --git a/pym/portage/tests/util/futures/test_done_callback.py 
b/pym/portage/tests/util/futures/test_done_callback.py
new file mode 100644
index 000..76b727b
--- /dev/null
+++ b/pym/portage/tests/util/futures/test_done_callback.py
@@ -0,0 +1,35 @@
+# Copyright 2017 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from portage.tests import TestCase
+from portage.util._eventloop.global_event_loop import global_event_loop
+
+
+class FutureDoneCallbackTestCase(TestCase):
+
+   def testFutureDoneCallback(self):
+
+   event_loop = global_event_loop()
+
+   def done_callback(finished):
+   done_callback_called.set_result(True)
+
+   done_callback_called = event_loop.create_future()
+   finished = event_loop.create_future()
+   finished.add_done_callback(done_callback)
+   event_loop.call_soon(finished.set_result, True)
+   

Re: [gentoo-portage-dev] [PATCH] phase-functions: Fix cleaning fake FILESDIR, reported by Arfrever

[Zac Medico]

On Sun, Mar 26, 2017 at 12:30 AM, Michał Górny  wrote:
> ---
>  bin/phase-functions.sh | 1 +
>  1 file changed, 1 insertion(+)
>
> diff --git a/bin/phase-functions.sh b/bin/phase-functions.sh
> index 50f0fdb9b..dfd8733c8 100644
> --- a/bin/phase-functions.sh
> +++ b/bin/phase-functions.sh
> @@ -283,6 +283,7 @@ __dyn_clean() {
>
> rm -rf "${PORTAGE_BUILDDIR}/build-info"
> rm -rf "${WORKDIR}"
> +   rm -f "${PORTAGE_BUILDDIR}/files"
> fi
>
> if [ -f "${PORTAGE_BUILDDIR}/.unpacked" ]; then
> --
> 2.12.1
>
>

Looks good.
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH 5/5] FetchIterator: add terminate method

[Zac Medico]

On Fri, Mar 24, 2017 at 1:05 PM, Brian Dolbec <dol...@gentoo.org> wrote:
> On Thu, 23 Mar 2017 19:55:00 -0700
> Zac Medico <zmed...@gentoo.org> wrote:
>
>> Add a terminate method to FetchIterator so that it doesn't
>> delay termination of emirrordist via SIGINT. Otherwise it
>> it is possible for the __iter__ method to loop for a very
>> long time after SIGINT has been delivered. For example,
>> this could happen if there are many ebuilds with stale
>> cache and RESTRICT=mirror. This issue was discovered
>> during testing of changes to the MirrorDistTask.terminate
>> implementation.
>> ---
>>  pym/portage/_emirrordist/FetchIterator.py  | 21 +
>>  pym/portage/_emirrordist/MirrorDistTask.py |  6 +-
>>  2 files changed, 26 insertions(+), 1 deletion(-)
>>
>
> Not that I know enough about all this that I could say "your doing it
> wrong"
>
> But the series looks OK, I didn't see any obvious goofs ;)
>
> Thanks

Thanks, pushed:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=04b1012594bfad1be719547e2c88a2dcf1051dc1
https://gitweb.gentoo.org/proj/portage.git/commit/?id=86400e9f864e86f8f677ccda9ce4103d6d02ef87
https://gitweb.gentoo.org/proj/portage.git/commit/?id=61878e4fbdfef5f8512b34640089e954a14e6d12
https://gitweb.gentoo.org/proj/portage.git/commit/?id=7defd54354c17afce7f36f53431260c1909481be
https://gitweb.gentoo.org/proj/portage.git/commit/?id=f70db92cba82535c8f65d385bd08a40207e24ec1
-- 
Thanks,
Zac

[gentoo-portage-dev] [PATCH 4/5] emirrordist: add debug SIGUSR1 handler

[Zac Medico]

This is handy for debugging issues with SIGTERM/SIGINT
handling.
---
 bin/emirrordist | 8 
 1 file changed, 8 insertions(+)

diff --git a/bin/emirrordist b/bin/emirrordist
index 0368eee..17f99f5 100755
--- a/bin/emirrordist
+++ b/bin/emirrordist
@@ -2,6 +2,7 @@
 # Copyright 2013-2014 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
+import signal
 import sys
 
 import portage
@@ -10,4 +11,11 @@ portage._disable_legacy_globals()
 from portage._emirrordist.main import emirrordist_main
 
 if __name__ == "__main__":
+
+   def debug_signal(_signum, _frame):
+   import pdb
+   pdb.set_trace()
+
+   signal.signal(signal.SIGUSR1, debug_signal)
+
sys.exit(emirrordist_main(sys.argv[1:]))
-- 
2.10.2

[gentoo-portage-dev] [PATCH 5/5] FetchIterator: add terminate method

[Zac Medico]

Add a terminate method to FetchIterator so that it doesn't
delay termination of emirrordist via SIGINT. Otherwise it
it is possible for the __iter__ method to loop for a very
long time after SIGINT has been delivered. For example,
this could happen if there are many ebuilds with stale
cache and RESTRICT=mirror. This issue was discovered
during testing of changes to the MirrorDistTask.terminate
implementation.
---
 pym/portage/_emirrordist/FetchIterator.py  | 21 +
 pym/portage/_emirrordist/MirrorDistTask.py |  6 +-
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/pym/portage/_emirrordist/FetchIterator.py 
b/pym/portage/_emirrordist/FetchIterator.py
index 16a0b04..3841979 100644
--- a/pym/portage/_emirrordist/FetchIterator.py
+++ b/pym/portage/_emirrordist/FetchIterator.py
@@ -1,6 +1,8 @@
 # Copyright 2013 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
+import threading
+
 from portage import os
 from portage.checksum import (_apply_hash_filter,
_filter_unaccelarated_hashes, _hash_filter)
@@ -13,6 +15,19 @@ class FetchIterator(object):
def __init__(self, config):
self._config = config
self._log_failure = config.log_failure
+   self._terminated = threading.Event()
+
+   def terminate(self):
+   """
+   Schedules early termination of the __iter__ method, which is
+   useful because under some conditions it's possible for __iter__
+   to loop for a long time without yielding to the caller. For
+   example, it's useful when there are many ebuilds with stale
+   cache and RESTRICT=mirror.
+
+   This method is thread-safe (and safe for signal handlers).
+   """
+   self._terminated.set()
 
def _iter_every_cp(self):
# List categories individually, in order to start yielding 
quicker,
@@ -37,6 +52,9 @@ class FetchIterator(object):
 
for cp in self._iter_every_cp():
 
+   if self._terminated.is_set():
+   return
+
for tree in portdb.porttrees:
 
# Reset state so the Manifest is pulled once
@@ -46,6 +64,9 @@ class FetchIterator(object):
 
for cpv in portdb.cp_list(cp, mytree=tree):
 
+   if self._terminated.is_set():
+   return
+
try:
restrict, = portdb.aux_get(cpv, 
("RESTRICT",),
mytree=tree)
diff --git a/pym/portage/_emirrordist/MirrorDistTask.py 
b/pym/portage/_emirrordist/MirrorDistTask.py
index 0702eb1..8da9f06 100644
--- a/pym/portage/_emirrordist/MirrorDistTask.py
+++ b/pym/portage/_emirrordist/MirrorDistTask.py
@@ -32,9 +32,11 @@ class MirrorDistTask(CompositeTask):
self._config = config
self._term_rlock = threading.RLock()
self._term_callback_handle = None
+   self._fetch_iterator = None
 
def _start(self):
-   fetch = TaskScheduler(iter(FetchIterator(self._config)),
+   self._fetch_iterator = FetchIterator(self._config)
+   fetch = TaskScheduler(iter(self._fetch_iterator),
max_jobs=self._config.options.jobs,
max_load=self._config.options.load_average,
event_loop=self._config.event_loop)
@@ -226,6 +228,8 @@ class MirrorDistTask(CompositeTask):
self._term_callback)
 
def _term_callback(self):
+   if self._fetch_iterator is not None:
+   self._fetch_iterator.terminate()
self.cancel()
self.wait()
 
-- 
2.10.2

[gentoo-portage-dev] [PATCH 2/5] PollScheduler: terminate via call_soon for asyncio compat

[Zac Medico]

Use call_soon to schedule the _termination_check callback when needed.
The previous idle_add usage was relatively inefficient, because it
scheduled the _termination_check callback to be called in every
iteration of the event loop.

Add a _cleanup method to handle cleanup of callbacks registered with
the global event loop. Since the terminate method is thread safe and it
interacts with self._term_callback_handle, use this variable only while
holding a lock.
---
 pym/_emerge/PollScheduler.py  | 57 +++
 pym/_emerge/Scheduler.py  |  7 ++--
 pym/portage/util/_async/AsyncScheduler.py | 16 -
 3 files changed, 54 insertions(+), 26 deletions(-)

diff --git a/pym/_emerge/PollScheduler.py b/pym/_emerge/PollScheduler.py
index b118ac1..569879b 100644
--- a/pym/_emerge/PollScheduler.py
+++ b/pym/_emerge/PollScheduler.py
@@ -25,8 +25,10 @@ class PollScheduler(object):
a non-main thread)
@type main: bool
"""
+   self._term_rlock = threading.RLock()
self._terminated = threading.Event()
self._terminated_tasks = False
+   self._term_check_handle = None
self._max_jobs = 1
self._max_load = None
self._scheduling = False
@@ -44,6 +46,21 @@ class PollScheduler(object):
def _is_background(self):
return self._background
 
+   def _cleanup(self):
+   """
+   Cleanup any callbacks that have been registered with the global
+   event loop.
+   """
+   # The self._term_check_handle attribute requires locking
+   # since it's modified by the thread safe terminate method.
+   with self._term_rlock:
+   if self._term_check_handle not in (None, False):
+   self._term_check_handle.cancel()
+   # This prevents the terminate method from scheduling
+   # any more callbacks (since _cleanup must eliminate all
+   # callbacks in order to ensure complete cleanup).
+   self._term_check_handle = False
+
def terminate(self):
"""
Schedules asynchronous, graceful termination of the scheduler
@@ -51,26 +68,36 @@ class PollScheduler(object):
 
This method is thread-safe (and safe for signal handlers).
"""
-   self._terminated.set()
+   with self._term_rlock:
+   if self._term_check_handle is None:
+   self._terminated.set()
+   self._term_check_handle = 
self._event_loop.call_soon_threadsafe(
+   self._termination_check, True)
 
-   def _termination_check(self):
+   def _termination_check(self, retry=False):
"""
Calls _terminate_tasks() if appropriate. It's guaranteed not to
-   call it while _schedule_tasks() is being called. The check 
should
-   be executed for each iteration of the event loop, for response 
to
-   termination signals at the earliest opportunity. It always 
returns
-   True, for continuous scheduling via idle_add.
+   call it while _schedule_tasks() is being called. This method 
must
+   only be called via the event loop thread.
+
+   @param retry: If True then reschedule if scheduling state 
prevents
+   immediate termination.
+   @type retry: bool
"""
-   if not self._scheduling and \
-   self._terminated.is_set() and \
+   if self._terminated.is_set() and \
not self._terminated_tasks:
-   self._scheduling = True
-   try:
-   self._terminated_tasks = True
-   self._terminate_tasks()
-   finally:
-   self._scheduling = False
-   return True
+   if not self._scheduling:
+   self._scheduling = True
+   try:
+   self._terminated_tasks = True
+   self._terminate_tasks()
+   finally:
+   self._scheduling = False
+
+   elif retry:
+   with self._term_rlock:
+   self._term_check_handle = 
self._event_loop.call_soon(
+   self._termination_check, True)
 
def _terminate_tasks(self):
"""
diff --git a/pym/_emerge/Scheduler.py 

Re: [gentoo-portage-dev] [PATCH] phase-helpers.sh: Loop over A rather than SRC_URI in __eapi0_pkg_nofetch.

[Zac Medico]

On Thu, Mar 23, 2017 at 2:55 AM, Ulrich Müller  wrote:
> Looping over SRC_URI also outputs non-filename elements (e.g, use
> conditionals) which is avoided when looping over A.
>
> Gentoo-Bug: 613132
> ---
>  bin/phase-helpers.sh | 8 
>  1 file changed, 4 insertions(+), 4 deletions(-)

Looks good.
-- 
Thanks,
Zac

[gentoo-portage-dev] [PATCH] pkg_use_display: show masked/forced state of USE_EXPAND flags (bug 490562)

[Zac Medico]

Fix pkg_use_display to test if the prefixed flag is in use.force or
use.mask, rather than the unprefixed flag.

X-Gentoo-bug: 490562
X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=490562
---
 pym/_emerge/UseFlagDisplay.py | 19 +--
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/pym/_emerge/UseFlagDisplay.py b/pym/_emerge/UseFlagDisplay.py
index f460474..12820e9 100644
--- a/pym/_emerge/UseFlagDisplay.py
+++ b/pym/_emerge/UseFlagDisplay.py
@@ -3,6 +3,7 @@
 
 from __future__ import unicode_literals
 
+import collections
 from itertools import chain
 import sys
 
@@ -60,6 +61,10 @@ class UseFlagDisplay(object):
sort_separated = cmp_sort_key(_cmp_separated)
del _cmp_separated
 
+
+_flag_info = collections.namedtuple('_flag_info', ('flag', 'display'))
+
+
 def pkg_use_display(pkg, opts, modified_use=None):
settings = pkg.root_config.settings
use_expand = pkg.use.expand
@@ -81,27 +86,29 @@ def pkg_use_display(pkg, opts, modified_use=None):
if f.startswith(flag_prefix):
use_expand_flags.add(f)
use_enabled.setdefault(
-   varname.upper(), 
[]).append(f[len(flag_prefix):])
+   varname.upper(), []).append(
+   _flag_info(f, 
f[len(flag_prefix):]))
 
for f in pkg.iuse.all:
if f.startswith(flag_prefix):
use_expand_flags.add(f)
if f not in use:
use_disabled.setdefault(
-   varname.upper(), 
[]).append(f[len(flag_prefix):])
+   varname.upper(), []).append(
+   _flag_info(f, 
f[len(flag_prefix):]))
 
var_order = set(use_enabled)
var_order.update(use_disabled)
var_order = sorted(var_order)
var_order.insert(0, 'USE')
use.difference_update(use_expand_flags)
-   use_enabled['USE'] = list(use)
+   use_enabled['USE'] = list(_flag_info(f, f) for f in use)
use_disabled['USE'] = []
 
for f in pkg.iuse.all:
if f not in use and \
f not in use_expand_flags:
-   use_disabled['USE'].append(f)
+   use_disabled['USE'].append(_flag_info(f, f))
 
flag_displays = []
for varname in var_order:
@@ -109,9 +116,9 @@ def pkg_use_display(pkg, opts, modified_use=None):
continue
flags = []
for f in use_enabled.get(varname, []):
-   flags.append(UseFlagDisplay(f, True, f in forced_flags))
+   flags.append(UseFlagDisplay(f.display, True, f.flag in 
forced_flags))
for f in use_disabled.get(varname, []):
-   flags.append(UseFlagDisplay(f, False, f in 
forced_flags))
+   flags.append(UseFlagDisplay(f.display, False, f.flag in 
forced_flags))
if alphabetical_use:
flags.sort(key=UseFlagDisplay.sort_combined)
else:
-- 
2.10.2

Re: [gentoo-portage-dev] [PATCH v2] emerge: fix --usepkg when ebuild is not available (bug 613360)

[Zac Medico]

On Tue, Mar 21, 2017 at 4:21 PM, Brian Dolbec <dol...@gentoo.org> wrote:
> On Mon, 20 Mar 2017 18:22:40 -0700
> Zac Medico <zmed...@gentoo.org> wrote:
>
>> Fix emerge --usepkg to use a binary package when the corresponding
>> ebuild is not available (and --use-ebuild-visibility is not enabled),
>> in cases when no other package is available to satisfy the dependency.
>> This reverts an unintended behavior change from commit
>> e309323f156528a8a79a1f755e1326e8880346b7.
>>
>> Fixes: e309323f1565 ("emerge: fix --use-ebuild-visibility to reject
>> binary packages (bug 612960)") X-Gentoo-bug: 613360
>> X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=613360
>> ---
>> [PATCH v2] moves the conditional logic earier, for more backward
>> compatiblity
>>
>>  pym/_emerge/depgraph.py|  3 ++-
>>  .../resolver/test_binary_pkg_ebuild_visibility.py  | 26
>> ++ 2 files changed, 28 insertions(+), 1
>> deletion(-)
>>
>
> looks good

Thanks, merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=82bfd91325b052a4c9250a04939641c15b3d2a20
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH v3] portage.package.ebuild: Use a fake FILESDIR to catch invalid accesses

[Zac Medico]

On 03/21/2017 12:46 AM, Michał Górny wrote:
> Use a model of fake FILESDIR path to ensure that invalid accesses to
> FILESDIR will result in failures rather than being silently allowed by
> Portage. This mostly involves accesses in the global scope and pkg_*
> phases, although the current model does not cover the latter completely
> (i.e. does not guarantee that the directory is removed post src_*).
> 
> This model aims to follow PMS wording quite precisely. The value of
> FILESDIR is meant to be stable throughout the build process, and it is
> reliably set to a temporary directory path. However, since the path is
> not guaranteed to be present outside src_*, the directory symlink is not
> actually created before src_* phases.
> ---
>  man/ebuild.5 |  6 +++---
>  pym/_emerge/EbuildPhase.py   |  6 +-
>  pym/portage/package/ebuild/config.py |  3 ---
>  pym/portage/package/ebuild/doebuild.py   |  2 +-
>  pym/portage/package/ebuild/prepare_build_dirs.py | 13 +
>  5 files changed, 22 insertions(+), 8 deletions(-)

Looks good now. Thanks!
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH v2] portage.package.ebuild: Use a fake FILESDIR to catch invalid accesses

[Zac Medico]

On 03/18/2017 10:26 AM, Michał Górny wrote:
> @@ -129,6 +130,9 @@ class EbuildPhase(CompositeTask):
>   # If the environment.bz2 doesn't exist, then ebuild.sh 
> will
>   # source the ebuild as a fallback.
>  
> + if self.phase == "unpack":
> + _prepare_fake_filesdir(self.settings)
> +
>   self._start_lock()

The _prepare_fake_filesdir call should not execute until the build
directory lock has been acquired (initiated by the _start_lock call).

The beginning of the _start_ebuild method is a good place.
-- 
Thanks,
Zac

[gentoo-portage-dev] [PATCH v2] emerge: fix --usepkg when ebuild is not available (bug 613360)

[Zac Medico]

Fix emerge --usepkg to use a binary package when the corresponding
ebuild is not available (and --use-ebuild-visibility is not enabled),
in cases when no other package is available to satisfy the dependency.
This reverts an unintended behavior change from commit
e309323f156528a8a79a1f755e1326e8880346b7.

Fixes: e309323f1565 ("emerge: fix --use-ebuild-visibility to reject binary 
packages (bug 612960)")
X-Gentoo-bug: 613360
X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=613360
---
[PATCH v2] moves the conditional logic earier, for more backward compatiblity

 pym/_emerge/depgraph.py|  3 ++-
 .../resolver/test_binary_pkg_ebuild_visibility.py  | 26 ++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
index 543f4dc..7c9130b 100644
--- a/pym/_emerge/depgraph.py
+++ b/pym/_emerge/depgraph.py
@@ -6062,7 +6062,8 @@ class depgraph(object):

identical_binary = True

break
 
-   if not identical_binary and 
pkg.built:
+   if (not identical_binary and 
pkg.built and
+   (use_ebuild_visibility 
or matched_packages)):
# If the ebuild 
no longer exists or it's
# keywords have 
been dropped, reject built
# instances 
(installed or binary).
diff --git a/pym/portage/tests/resolver/test_binary_pkg_ebuild_visibility.py 
b/pym/portage/tests/resolver/test_binary_pkg_ebuild_visibility.py
index ea65abd..0d01d06 100644
--- a/pym/portage/tests/resolver/test_binary_pkg_ebuild_visibility.py
+++ b/pym/portage/tests/resolver/test_binary_pkg_ebuild_visibility.py
@@ -104,6 +104,32 @@ class BinaryPkgEbuildVisibilityTestCase(TestCase):
'[binary]app-misc/foo-3',
],
),
+
+   # The default behavior is to enforce ebuild visibility 
as
+   # long as a visible package is available to satisfy the
+   # current atom. In the following test case, ebuild 
visibility
+   # is ignored in order to satisfy the =app-misc/foo-3 
atom.
+   ResolverPlaygroundTestCase(
+   ["=app-misc/foo-3"],
+   options = {
+   "--usepkg": True,
+   },
+   success = True,
+   mergelist = [
+   '[binary]app-misc/foo-3',
+   ],
+   ),
+
+   # Verify that --use-ebuild-visibility works with 
--usepkg
+   # when no other visible package is available.
+   ResolverPlaygroundTestCase(
+   ["=app-misc/foo-3"],
+   options = {
+   "--use-ebuild-visibility": "y",
+   "--usepkg": True,
+   },
+   success = False,
+   ),
)
 
playground = ResolverPlayground(binpkgs=binpkgs, 
ebuilds=ebuilds,
-- 
2.10.2

Re: [gentoo-portage-dev] [PATCH 3/4] portage.package.ebuild.doebuild: Override DISTDIR unconditionally, #612972

[Zac Medico]

On 03/18/2017 02:16 PM, Michał Górny wrote:
> On sob, 2017-03-18 at 13:45 -0700, Zac Medico wrote:
>> On Sat, Mar 18, 2017 at 1:12 PM, Zac Medico <zmed...@gentoo.org> wrote:
>>> On Sat, Mar 18, 2017 at 12:57 PM, Zac Medico <zmed...@gentoo.org> wrote:
>>>> On Sat, Mar 18, 2017 at 12:04 PM, Michał Górny <mgo...@gentoo.org> wrote:
>>>>> Ensure that DISTDIR is always defined to the path to the shadow
>>>>> directory. This ensures that PMS rules for consistent value are
>>>>> followed, and that no global scope calls should be able to access
>>>>> the distfile directory. This also ensures that global-scope assignments
>>>>> (e.g. in PATCHES) do not work around the shadow directory.
>>>>> ---
>>>>>  pym/portage/package/ebuild/doebuild.py   | 9 +
>>>>>  pym/portage/package/ebuild/prepare_build_dirs.py | 6 ++
>>>>>  2 files changed, 7 insertions(+), 8 deletions(-)
>>>>>
>>>>> diff --git a/pym/portage/package/ebuild/doebuild.py 
>>>>> b/pym/portage/package/ebuild/doebuild.py
>>>>> index 15e4abb48..8efc08334 100644
>>>>> --- a/pym/portage/package/ebuild/doebuild.py
>>>>> +++ b/pym/portage/package/ebuild/doebuild.py
>>>>> @@ -348,7 +348,8 @@ def doebuild_environment(myebuild, mydo, myroot=None, 
>>>>> settings=None,
>>>>>
>>>>> mysettings["PORTDIR"] = os.path.realpath(mysettings["PORTDIR"])
>>>>> mysettings.pop("PORTDIR_OVERLAY", None)
>>>>> -   mysettings["DISTDIR"] = os.path.realpath(mysettings["DISTDIR"])
>>>>> +   if "PORTAGE_ACTUAL_DISTDIR" not in mysettings:
>>>>> +   mysettings["PORTAGE_ACTUAL_DISTDIR"] = 
>>>>> os.path.realpath(mysettings["DISTDIR"])
>>>>> mysettings["RPMDIR"]  = os.path.realpath(mysettings["RPMDIR"])
>>>>>
>>>>> mysettings["ECLASSDIR"]   = mysettings["PORTDIR"]+"/eclass"
>>>>> @@ -390,6 +391,7 @@ def doebuild_environment(myebuild, mydo, myroot=None, 
>>>>> settings=None,
>>>>> mysettings["WORKDIR"] = 
>>>>> os.path.join(mysettings["PORTAGE_BUILDDIR"], "work")
>>>>> mysettings["D"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], 
>>>>> "image") + os.sep
>>>>> mysettings["T"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], 
>>>>> "temp")
>>>>> +   mysettings["DISTDIR"] = 
>>>>> os.path.join(settings["PORTAGE_BUILDDIR"], "distdir")
>>>>> mysettings["FILESDIR"] = 
>>>>> os.path.join(settings["PORTAGE_BUILDDIR"], "files")
>>>>
>>>> The EbuildFetcher already_fetched and _prefetch_size_ok methods use
>>>> DISTDIR after doebuild_environment has been called, so they need to be
>>>> updated to use PORTAGE_ACTUAL_DISTDIR.
>>>
>>> Also, this code in doebuild uses DISTDIR before _prepare_fake_distdir
>>> gets called called:
>>>
>>> for x in alist:
>>> writemsg_stdout(">>> Checking %s's mtime...\n" % x)
>>> try:
>>> x_st = os.stat(os.path.join(
>>> mysettings["DISTDIR"], x))
>>>
>>> Also, bin/ebuild calls doebuild_environment before doebuild, which
>>> means that this doebuild code also needs to use PORTAGE_ACTUAL_DISTDIR
>>> if it's in mysettings:
>>>
>>> mf = repo_config.load_manifest(pkgdir, mysettings["DISTDIR"])
>>
>> We should probably implement the fake DISTDIR setting inside the
>> config environ method, so that we can eliminate all of this
>> error-prone DISTDIR/PORTAGE_ACTUAL_DISTDIR stuff.
> 
> You're the expert here. I'm just the person who hammers it till it seems
> to work semi-reliably, and curse on whoever designed it ;-P. Would you
> be able to take it from here? I barely understand what you're talking
> about.

Every ebuild phase gets its environment from the config environ method, so
adding this code to the end of the environ method will perform the
necessary translation for all ebuild phases:

diff --git a/pym/portage/package/ebuild/config.py 
b/pym/portage/package/ebuild/config.py
index ef29afe..4c642a7 100644
--- a/pym/portage/package/ebuild/config.py
+++ b/pym/portage/package/ebuild/config.py
@@ -2816,6 +2816,15 @@ class config(object):
else:
raise AssertionError("C locale did not 
pass the test!")
 
+   try:
+   builddir = mydict["PORTAGE_BUILDDIR"]
+   distdir = mydict["DISTDIR"]
+   except KeyError:
+   pass
+   else:
+   mydict["PORTAGE_ACTUAL_DISTDIR"] = distdir
+   mydict["DISTDIR"] = os.path.join(builddir, "distdir")
+
return mydict
 
def thirdpartymirrors(self):


-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH 3/4] portage.package.ebuild.doebuild: Override DISTDIR unconditionally, #612972

[Zac Medico]

On Sat, Mar 18, 2017 at 1:12 PM, Zac Medico <zmed...@gentoo.org> wrote:
> On Sat, Mar 18, 2017 at 12:57 PM, Zac Medico <zmed...@gentoo.org> wrote:
>> On Sat, Mar 18, 2017 at 12:04 PM, Michał Górny <mgo...@gentoo.org> wrote:
>>> Ensure that DISTDIR is always defined to the path to the shadow
>>> directory. This ensures that PMS rules for consistent value are
>>> followed, and that no global scope calls should be able to access
>>> the distfile directory. This also ensures that global-scope assignments
>>> (e.g. in PATCHES) do not work around the shadow directory.
>>> ---
>>>  pym/portage/package/ebuild/doebuild.py   | 9 +
>>>  pym/portage/package/ebuild/prepare_build_dirs.py | 6 ++
>>>  2 files changed, 7 insertions(+), 8 deletions(-)
>>>
>>> diff --git a/pym/portage/package/ebuild/doebuild.py 
>>> b/pym/portage/package/ebuild/doebuild.py
>>> index 15e4abb48..8efc08334 100644
>>> --- a/pym/portage/package/ebuild/doebuild.py
>>> +++ b/pym/portage/package/ebuild/doebuild.py
>>> @@ -348,7 +348,8 @@ def doebuild_environment(myebuild, mydo, myroot=None, 
>>> settings=None,
>>>
>>> mysettings["PORTDIR"] = os.path.realpath(mysettings["PORTDIR"])
>>> mysettings.pop("PORTDIR_OVERLAY", None)
>>> -   mysettings["DISTDIR"] = os.path.realpath(mysettings["DISTDIR"])
>>> +   if "PORTAGE_ACTUAL_DISTDIR" not in mysettings:
>>> +   mysettings["PORTAGE_ACTUAL_DISTDIR"] = 
>>> os.path.realpath(mysettings["DISTDIR"])
>>> mysettings["RPMDIR"]  = os.path.realpath(mysettings["RPMDIR"])
>>>
>>> mysettings["ECLASSDIR"]   = mysettings["PORTDIR"]+"/eclass"
>>> @@ -390,6 +391,7 @@ def doebuild_environment(myebuild, mydo, myroot=None, 
>>> settings=None,
>>> mysettings["WORKDIR"] = 
>>> os.path.join(mysettings["PORTAGE_BUILDDIR"], "work")
>>> mysettings["D"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], 
>>> "image") + os.sep
>>> mysettings["T"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], 
>>> "temp")
>>> +   mysettings["DISTDIR"] = os.path.join(settings["PORTAGE_BUILDDIR"], 
>>> "distdir")
>>> mysettings["FILESDIR"] = os.path.join(settings["PORTAGE_BUILDDIR"], 
>>> "files")
>>
>> The EbuildFetcher already_fetched and _prefetch_size_ok methods use
>> DISTDIR after doebuild_environment has been called, so they need to be
>> updated to use PORTAGE_ACTUAL_DISTDIR.
>
> Also, this code in doebuild uses DISTDIR before _prepare_fake_distdir
> gets called called:
>
> for x in alist:
> writemsg_stdout(">>> Checking %s's mtime...\n" % x)
> try:
> x_st = os.stat(os.path.join(
> mysettings["DISTDIR"], x))
>
> Also, bin/ebuild calls doebuild_environment before doebuild, which
> means that this doebuild code also needs to use PORTAGE_ACTUAL_DISTDIR
> if it's in mysettings:
>
> mf = repo_config.load_manifest(pkgdir, mysettings["DISTDIR"])

We should probably implement the fake DISTDIR setting inside the
config environ method, so that we can eliminate all of this
error-prone DISTDIR/PORTAGE_ACTUAL_DISTDIR stuff.
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH 3/4] portage.package.ebuild.doebuild: Override DISTDIR unconditionally, #612972

[Zac Medico]

On Sat, Mar 18, 2017 at 12:57 PM, Zac Medico <zmed...@gentoo.org> wrote:
> On Sat, Mar 18, 2017 at 12:04 PM, Michał Górny <mgo...@gentoo.org> wrote:
>> Ensure that DISTDIR is always defined to the path to the shadow
>> directory. This ensures that PMS rules for consistent value are
>> followed, and that no global scope calls should be able to access
>> the distfile directory. This also ensures that global-scope assignments
>> (e.g. in PATCHES) do not work around the shadow directory.
>> ---
>>  pym/portage/package/ebuild/doebuild.py   | 9 +
>>  pym/portage/package/ebuild/prepare_build_dirs.py | 6 ++
>>  2 files changed, 7 insertions(+), 8 deletions(-)
>>
>> diff --git a/pym/portage/package/ebuild/doebuild.py 
>> b/pym/portage/package/ebuild/doebuild.py
>> index 15e4abb48..8efc08334 100644
>> --- a/pym/portage/package/ebuild/doebuild.py
>> +++ b/pym/portage/package/ebuild/doebuild.py
>> @@ -348,7 +348,8 @@ def doebuild_environment(myebuild, mydo, myroot=None, 
>> settings=None,
>>
>> mysettings["PORTDIR"] = os.path.realpath(mysettings["PORTDIR"])
>> mysettings.pop("PORTDIR_OVERLAY", None)
>> -   mysettings["DISTDIR"] = os.path.realpath(mysettings["DISTDIR"])
>> +   if "PORTAGE_ACTUAL_DISTDIR" not in mysettings:
>> +   mysettings["PORTAGE_ACTUAL_DISTDIR"] = 
>> os.path.realpath(mysettings["DISTDIR"])
>> mysettings["RPMDIR"]  = os.path.realpath(mysettings["RPMDIR"])
>>
>> mysettings["ECLASSDIR"]   = mysettings["PORTDIR"]+"/eclass"
>> @@ -390,6 +391,7 @@ def doebuild_environment(myebuild, mydo, myroot=None, 
>> settings=None,
>> mysettings["WORKDIR"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], 
>> "work")
>> mysettings["D"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], 
>> "image") + os.sep
>> mysettings["T"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], 
>> "temp")
>> +   mysettings["DISTDIR"] = os.path.join(settings["PORTAGE_BUILDDIR"], 
>> "distdir")
>> mysettings["FILESDIR"] = os.path.join(settings["PORTAGE_BUILDDIR"], 
>> "files")
>
> The EbuildFetcher already_fetched and _prefetch_size_ok methods use
> DISTDIR after doebuild_environment has been called, so they need to be
> updated to use PORTAGE_ACTUAL_DISTDIR.

Also, this code in doebuild uses DISTDIR before _prepare_fake_distdir
gets called called:

for x in alist:
writemsg_stdout(">>> Checking %s's mtime...\n" % x)
try:
x_st = os.stat(os.path.join(
mysettings["DISTDIR"], x))

Also, bin/ebuild calls doebuild_environment before doebuild, which
means that this doebuild code also needs to use PORTAGE_ACTUAL_DISTDIR
if it's in mysettings:

mf = repo_config.load_manifest(pkgdir, mysettings["DISTDIR"])
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH 3/4] portage.package.ebuild.doebuild: Override DISTDIR unconditionally, #612972

[Zac Medico]

On Sat, Mar 18, 2017 at 12:04 PM, Michał Górny  wrote:
> Ensure that DISTDIR is always defined to the path to the shadow
> directory. This ensures that PMS rules for consistent value are
> followed, and that no global scope calls should be able to access
> the distfile directory. This also ensures that global-scope assignments
> (e.g. in PATCHES) do not work around the shadow directory.
> ---
>  pym/portage/package/ebuild/doebuild.py   | 9 +
>  pym/portage/package/ebuild/prepare_build_dirs.py | 6 ++
>  2 files changed, 7 insertions(+), 8 deletions(-)
>
> diff --git a/pym/portage/package/ebuild/doebuild.py 
> b/pym/portage/package/ebuild/doebuild.py
> index 15e4abb48..8efc08334 100644
> --- a/pym/portage/package/ebuild/doebuild.py
> +++ b/pym/portage/package/ebuild/doebuild.py
> @@ -348,7 +348,8 @@ def doebuild_environment(myebuild, mydo, myroot=None, 
> settings=None,
>
> mysettings["PORTDIR"] = os.path.realpath(mysettings["PORTDIR"])
> mysettings.pop("PORTDIR_OVERLAY", None)
> -   mysettings["DISTDIR"] = os.path.realpath(mysettings["DISTDIR"])
> +   if "PORTAGE_ACTUAL_DISTDIR" not in mysettings:
> +   mysettings["PORTAGE_ACTUAL_DISTDIR"] = 
> os.path.realpath(mysettings["DISTDIR"])
> mysettings["RPMDIR"]  = os.path.realpath(mysettings["RPMDIR"])
>
> mysettings["ECLASSDIR"]   = mysettings["PORTDIR"]+"/eclass"
> @@ -390,6 +391,7 @@ def doebuild_environment(myebuild, mydo, myroot=None, 
> settings=None,
> mysettings["WORKDIR"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], 
> "work")
> mysettings["D"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], 
> "image") + os.sep
> mysettings["T"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], "temp")
> +   mysettings["DISTDIR"] = os.path.join(settings["PORTAGE_BUILDDIR"], 
> "distdir")
> mysettings["FILESDIR"] = os.path.join(settings["PORTAGE_BUILDDIR"], 
> "files")

The EbuildFetcher already_fetched and _prefetch_size_ok methods use
DISTDIR after doebuild_environment has been called, so they need to be
updated to use PORTAGE_ACTUAL_DISTDIR.
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH] emerge: fix --use-ebuild-visibility to reject binary packages (bug 612960)

[Zac Medico]

On Sat, Mar 18, 2017 at 12:10 AM, Brian Dolbec <dol...@gentoo.org> wrote:
> On Fri, 17 Mar 2017 20:48:22 -0700
> Zac Medico <zmed...@gentoo.org> wrote:
>
>> Fix the --use-ebuild-visibility option to reject binary packages for
>> which ebuilds are either masked or unavailable. The included test case
>> fails with out this fix.
>>
>> X-Gentoo-bug: 612960
>> X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=612960
>> ---
>>  pym/_emerge/depgraph.py| 32 +++-
>>  .../resolver/test_binary_pkg_ebuild_visibility.py  | 94
>> ++ 2 files changed, 123 insertions(+), 3
>> deletions(-) create mode 100644
>
> Looks fine

Thanks, pushed:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=e309323f156528a8a79a1f755e1326e8880346b7

-- 
Thanks,
Zac

[gentoo-portage-dev] [PATCH] emerge: fix --use-ebuild-visibility to reject binary packages (bug 612960)

[Zac Medico]

Fix the --use-ebuild-visibility option to reject binary packages for
which ebuilds are either masked or unavailable. The included test case
fails with out this fix.

X-Gentoo-bug: 612960
X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=612960
---
 pym/_emerge/depgraph.py| 32 +++-
 .../resolver/test_binary_pkg_ebuild_visibility.py  | 94 ++
 2 files changed, 123 insertions(+), 3 deletions(-)
 create mode 100644 
pym/portage/tests/resolver/test_binary_pkg_ebuild_visibility.py

diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
index e94b96c..3834983 100644
--- a/pym/_emerge/depgraph.py
+++ b/pym/_emerge/depgraph.py
@@ -4889,6 +4889,9 @@ class depgraph(object):
vardb = self._frozen_config.roots[root].trees["vartree"].dbapi
bindb = self._frozen_config.roots[root].trees["bintree"].dbapi
dbs = self._dynamic_config._filtered_trees[root]["dbs"]
+   use_ebuild_visibility = self._frozen_config.myopts.get(
+   '--use-ebuild-visibility', 'n') != 'n'
+
for db, pkg_type, built, installed, db_keys in dbs:
if installed:
continue
@@ -4975,6 +4978,7 @@ class depgraph(object):
eapi=pkg.eapi):

required_use_unsatisfied.append(pkg)
continue
+
root_slot = (pkg.root, 
pkg.slot_atom)
if pkg.built and root_slot in 
self._rebuild.rebuild_list:
mreasons = ["need to 
rebuild from source"]
@@ -4988,6 +4992,19 @@ class depgraph(object):

self._dynamic_config.ignored_binaries.get(
pkg, 
{}).get("changed_deps")):
mreasons = ["changed 
deps"]
+   elif (pkg.built and 
use_ebuild_visibility and
+   not 
self._equiv_ebuild_visible(pkg)):
+   equiv_ebuild = 
self._equiv_ebuild(pkg)
+   if equiv_ebuild is None:
+   mreasons = 
["ebuild not unavailable"]
+   elif not mreasons:
+   metadata, 
mreasons = get_mask_info(
+   
root_config, equiv_ebuild.cpv, pkgsettings,
+   portdb, 
equiv_ebuild.pkg_type,
+   
equiv_ebuild.built, equiv_ebuild.installed,
+   
db_keys, myrepo=equiv_ebuild.repo,
+   
_pkg_use_enabled=self._pkg_use_enabled)
+
masked_packages.append(
(root_config, pkgsettings, cpv, 
repo, metadata, mreasons))
 
@@ -5548,6 +5565,14 @@ class depgraph(object):
"""
return depth + n if isinstance(depth, int) else depth
 
+   def _equiv_ebuild(self, pkg):
+   try:
+   return self._pkg(
+   pkg.cpv, "ebuild", pkg.root_config, 
myrepo=pkg.repo)
+   except portage.exception.PackageNotFound:
+   return next(self._iter_match_pkgs(pkg.root_config,
+   "ebuild", Atom("=%s" % (pkg.cpv,))), None)
+
def _equiv_ebuild_visible(self, pkg, autounmask_level=None):
try:
pkg_eb = self._pkg(
@@ -6021,11 +6046,11 @@ class depgraph(object):
# reinstall the same exact 
version only due
# to a KEYWORDS mask. See bug 
#252167.
 
+   identical_binary = False
if pkg.type_name != "ebuild" 
and matched_packages:
# Don't re-install a 
binary package that is
# identical to the 
currently installed package
# (see bug #354441).
-   identical_binary = False
   

Re: [gentoo-portage-dev] [PATCH] depgraph: avoid missed update with slot operator and circ dep (bug 612874)

[Zac Medico]

On Fri, Mar 17, 2017 at 8:37 AM, Brian Dolbec <dol...@gentoo.org> wrote:
> On Fri, 17 Mar 2017 01:18:19 -0700
> Zac Medico <zmed...@gentoo.org> wrote:
>
>> Fix check_reverse_dependencies to ignore direct circular dependencies,
>> since these dependencies tend to prevent updates of packages. This
>> solves a missed update from llvm:0 to llvm:4 when clang is not in the
>> world file, as demonstrated by the included test case.
>>
>> X-Gentoo-bug: 612874
>> X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=612874
>> ---
>>  pym/_emerge/depgraph.py| 31
>> - .../resolver/test_slot_operator_exclusive_slots.py
>> | 39 ++
>> pym/portage/util/digraph.py|  6  3 files
>> changed, 68 insertions(+), 8 deletions(-)
>
> Looks good
>
> --
> Brian Dolbec 
>

Thanks, pushed:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=35208d80a55228a50ea4ac64904465ce4651e381
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH v2] depgraph: fix missed atom_not_selected initialization (bug 612846)

[Zac Medico]

On Thu, Mar 16, 2017 at 3:33 PM, Brian Dolbec <dol...@gentoo.org> wrote:
> On Thu, 16 Mar 2017 14:19:07 -0700
> Zac Medico <zmed...@gentoo.org> wrote:
>
>> Fix the _slot_operator_update_probe method to ensure that the
>> atom_not_selected variable is always properly initialized. This
>> problem prevented the method from identifying slot operator rebuild
>> candidates, leading to dependency conflicts and/or missed updates.
>> For example, this may have triggered the missed llvm update reported
>> in bug 611742, since these dependencies from the mesa-17.0.1 ebuild
>> are capable of triggering the problem, when atom_not_selected is not
>> properly set to True for the second atom:
>>
>>   || (
>>   sys-devel/llvm:4[${MULTILIB_USEDEP}]
>>   >=sys-devel/llvm-3.6.0:0[${MULTILIB_USEDEP}]
>>   )
>>
>> X-Gentoo-bug: 612846
>> X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=612846
>> ---
>> [PATCH v2] fixes it to handle soname atoms correctly
>>
>>  pym/_emerge/depgraph.py | 17 ++---
>>  1 file changed, 14 insertions(+), 3 deletions(-)
>>
>> diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
>> index ad94fb7..511944c 100644
>> --- a/pym/_emerge/depgraph.py
>> +++ b/pym/_emerge/depgraph.py
>> @@ -1895,7 +1895,9 @@ class depgraph(object):
>>   all_candidate_pkgs = None
>>
>>   for atom in atoms:
>> - atom_not_selected = False
>> + # The _select_atoms_probe method is
>> expensive, so initialization
>> + # of this variable is only
>> permformed on demand.
>> + atom_not_selected = None
>>
>>   if not atom.package:
>>   unevaluated_atom = None
>> @@ -1977,8 +1979,8 @@ class depgraph(object):
>>   if selected_atoms is
>> None: selected_atoms = self._select_atoms_probe(
>>   dep.child.root,
>> replacement_parent)
>> - if unevaluated_atom
>> not in selected_atoms:
>> -
>> atom_not_selected = True
>> + atom_not_selected =
>> unevaluated_atom not in selected_atoms
>> + if atom_not_selected:
>>   break
>>
>>   if not insignificant and \
>> @@ -1989,6 +1991,15 @@ class depgraph(object):
>>   (pkg,
>> unevaluated_atom or atom)) candidate_pkgs.append(pkg)
>>
>> + # When unevaluated_atom is None, it
>> means that atom is
>> + # an soname atom which is
>> unconditionally selected, and
>> + # _select_atoms_probe is not
>> applicable.
>> + if atom_not_selected is None and
>> unevaluated_atom is not None:
>> + if selected_atoms is None:
>> + selected_atoms =
>> self._select_atoms_probe(
>> +
>> dep.child.root, replacement_parent)
>> + atom_not_selected =
>> unevaluated_atom not in selected_atoms +
>>   if atom_not_selected:
>>   continue
>>   
>> replacement_candidates.append(candidate_pkg_atoms)
>
> Looks good other than 2 typos permfomed and "# an soname"  -n

Thanks, merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=e5d638cf855656afab141da8d7b4d7f66cad3450
-- 
Thanks,
Zac

[gentoo-portage-dev] [PATCH] depgraph: fix missed atom_not_selected initialization (bug 612846)

[Zac Medico]

Fix the _slot_operator_update_probe method to ensure that the
atom_not_selected variable is always properly initialized. This
problem prevented the method from identifying slot operator rebuild
candidates, leading to dependency conflicts and/or missed updates.
For example, this may have triggered the missed llvm update reported
in bug 611742, since these dependencies from the mesa-17.0.1 ebuild
are capable of triggering the problem, when atom_not_selected is not
properly set to True for the second atom:

|| (
sys-devel/llvm:4[${MULTILIB_USEDEP}]
>=sys-devel/llvm-3.6.0:0[${MULTILIB_USEDEP}]
)

X-Gentoo-bug: 612846
X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=612846
---
 pym/_emerge/depgraph.py | 14 +++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
index ad94fb7..832b472 100644
--- a/pym/_emerge/depgraph.py
+++ b/pym/_emerge/depgraph.py
@@ -1895,7 +1895,9 @@ class depgraph(object):
all_candidate_pkgs = None
 
for atom in atoms:
-   atom_not_selected = False
+   # The _select_atoms_probe method is expensive, 
so initialization
+   # of this variable is only permformed on demand.
+   atom_not_selected = None
 
if not atom.package:
unevaluated_atom = None
@@ -1977,8 +1979,8 @@ class depgraph(object):
if selected_atoms is None:
selected_atoms = 
self._select_atoms_probe(
dep.child.root, 
replacement_parent)
-   if unevaluated_atom not in 
selected_atoms:
-   atom_not_selected = True
+   atom_not_selected = 
unevaluated_atom not in selected_atoms
+   if atom_not_selected:
break
 
if not insignificant and \
@@ -1989,6 +1991,12 @@ class depgraph(object):
(pkg, unevaluated_atom 
or atom))
candidate_pkgs.append(pkg)
 
+   if atom_not_selected is None:
+   if selected_atoms is None:
+   selected_atoms = 
self._select_atoms_probe(
+   dep.child.root, 
replacement_parent)
+   atom_not_selected = unevaluated_atom 
not in selected_atoms
+
if atom_not_selected:
continue

replacement_candidates.append(candidate_pkg_atoms)
-- 
2.10.2

Re: [gentoo-portage-dev] [PATCH] depgraph: fix slot operator rebuild for llvm:0 to llvm:4 upgrade (bug 612772)

[Zac Medico]

On Wed, Mar 15, 2017 at 9:39 PM, Brian Dolbec <dol...@gentoo.org> wrote:
> On Wed, 15 Mar 2017 20:23:43 -0700
> Zac Medico <zmed...@gentoo.org> wrote:
>
>> Fix check_reverse_dependencies to ignore dependencies of parent
>> packages that could be uninstalled in order to solve a blocker
>> conflict. This case is similar to the one from bug 584626, except
>> that the relevant parent package is in an older slot which is blocked
>> by a newer slot. In this case, the _upgrade_available method returns
>> False, because the package in the older slot is the highest version
>> version available for its slot. Therefore, a new _in_blocker_conflict
>> method is needed to detect parent packages that cold be uninstalled.
>> The included unit test fails without this fix.
>>
>> Since the _in_blocker_conflict method requires information that is
>> collected by the _validate_blockers method, the _validate_blockers
>> method now has to be called before the _process_slot_conflict and
>> _slot_operator_trigger_reinstalls methods.
>>
>> X-Gentoo-bug: 612772
>> X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=612772
>> ---
>>  pym/_emerge/depgraph.py|  59 ---
>>  .../resolver/test_slot_operator_exclusive_slots.py | 109
>> + 2 files changed, 155 insertions(+), 13
>> deletions(-) create mode 100644
>> pym/portage/tests/resolver/test_slot_operator_exclusive_slots.py
>>
>> diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
>> index 1379b05..96b6d5f 100644
>> --- a/pym/_emerge/depgraph.py
>> +++ b/pym/_emerge/depgraph.py
>> @@ -387,7 +387,10 @@ class _dynamic_depgraph_config(object):
>>   # Contains only unsolvable Package -> Blocker edges
>>   self._unsolvable_blockers = digraph()
>>   # Contains all Blocker -> Blocked Package edges
>> - self._blocked_pkgs = digraph()
>> + # Do not initialized this until the depgraph
>   /
>   typo  -d
>
>
> Otherwise looks good :)

Thanks, merged with typo fix:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=15e67f5516e0779d2cba37704c15b42193808197
-- 
Thanks,
Zac

[gentoo-portage-dev] [PATCH] depgraph: fix slot operator rebuild for llvm:0 to llvm:4 upgrade (bug 612772)

[Zac Medico]

Fix check_reverse_dependencies to ignore dependencies of parent packages
that could be uninstalled in order to solve a blocker conflict. This case
is similar to the one from bug 584626, except that the relevant parent
package is in an older slot which is blocked by a newer slot. In this
case, the _upgrade_available method returns False, because the package
in the older slot is the highest version version available for its
slot. Therefore, a new _in_blocker_conflict method is needed to detect
parent packages that cold be uninstalled. The included unit test fails
without this fix.

Since the _in_blocker_conflict method requires information that is
collected by the _validate_blockers method, the _validate_blockers
method now has to be called before the _process_slot_conflict and
_slot_operator_trigger_reinstalls methods.

X-Gentoo-bug: 612772
X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=612772
---
 pym/_emerge/depgraph.py|  59 ---
 .../resolver/test_slot_operator_exclusive_slots.py | 109 +
 2 files changed, 155 insertions(+), 13 deletions(-)
 create mode 100644 
pym/portage/tests/resolver/test_slot_operator_exclusive_slots.py

diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
index 1379b05..96b6d5f 100644
--- a/pym/_emerge/depgraph.py
+++ b/pym/_emerge/depgraph.py
@@ -387,7 +387,10 @@ class _dynamic_depgraph_config(object):
# Contains only unsolvable Package -> Blocker edges
self._unsolvable_blockers = digraph()
# Contains all Blocker -> Blocked Package edges
-   self._blocked_pkgs = digraph()
+   # Do not initialized this until the depgraph _validate_blockers
+   # method is called, so that the _in_blocker_conflict method can
+   # assert that _validate_blockers has been called first.
+   self._blocked_pkgs = None
# Contains world packages that have been protected from
# uninstallation but may not have been added to the graph
# if the graph is not complete yet.
@@ -1466,9 +1469,22 @@ class depgraph(object):
 
self._solve_non_slot_operator_slot_conflicts()
 
+   if not self._validate_blockers():
+   # Blockers don't trigger the _skip_restart flag, since
+   # backtracking may solve blockers when it solves slot
+   # conflicts (or by blind luck).
+   raise self._unknown_internal_error()
+
+   # Both _process_slot_conflict and 
_slot_operator_trigger_reinstalls
+   # can call _slot_operator_update_probe, which requires that
+   # self._dynamic_config._blocked_pkgs has been initialized by a
+   # call to the _validate_blockers method.
for conflict in 
self._dynamic_config._package_tracker.slot_conflicts():
self._process_slot_conflict(conflict)
 
+   if self._dynamic_config._allow_backtracking:
+   self._slot_operator_trigger_reinstalls()
+
def _process_slot_conflict(self, conflict):
"""
Process slot conflict data to identify specific atoms which
@@ -1829,9 +1845,12 @@ class depgraph(object):
not 
self._frozen_config.excluded_pkgs.
findAtomForPackage(parent,

modified_use=self._pkg_use_enabled(parent)) and
-   
self._upgrade_available(parent)):
+   
(self._upgrade_available(parent) or
+   (parent.installed and 
self._in_blocker_conflict(parent:
# This parent may be 
irrelevant, since an
-   # update is available (see bug 
584626).
+   # update is available (see bug 
584626), or
+   # it could be uninstalled in 
order to solve
+   # a blocker conflict (bug 
612772).
continue
 
atom_set = 
InternalPackageSet(initial_atoms=(atom,),
@@ -2125,6 +2144,24 @@ class depgraph(object):
 
self._dynamic_config._need_restart = True
 
+   def _in_blocker_conflict(self, pkg):
+   """
+   Check if pkg is involved in a blocker conflict. This method
+   only works after the _validate_blockers method has been called.
+   """
+
+   if self._dynamic_config._blocked_pkgs is None:
+   raise AssertionError(
+   '_in_blocker_conflict called 

Re: [gentoo-portage-dev] [PATCH v3] movefile: support in-kernel file copying on Linux (bug 607868)

[Zac Medico]

On Wed, Mar 15, 2017 at 3:25 PM, Brian Dolbec <dol...@gentoo.org> wrote:
> On Fri,  3 Mar 2017 18:18:50 -0800
> Zac Medico <zmed...@gentoo.org> wrote:
>
>> Perform in-kernel file copying when possible, and also support
>> reflinks and sparse files. If the optimized implementation
>> fails at runtime, gracefully fallback to a plain read/write
>> loop.
>>
>> Compile-time and run-time fallbacks are implemented, so that
>> any incompatiblities will be handled gracefully. For example,
>> if the code is compiled on a system that supports the
>> copy_file_range syscall, but at run-time an older kernel that
>> does not support this syscall is detected, it will be handled
>> gracefully. There are similar fallbacks for lack of lseek
>> SEEK_DATA and sendfile support.
>>
>> X-Gentoo-Bug: 607868
>> X-Gentoo-Bug-Url: https://bugs.gentoo.org/show_bug.cgi?id=607868
>> ---
>> [PATCH v3] Changes:
>>  * Add function documentation comments
>>  * Rename do_lseek function to do_lseek_data
>>  * Fix do_lseek_data to handle sparse blocks at EOF, and
>>fix _reflink_linux_file_copy to call ftruncate
>>  * Eliminate indirection in python copyfile function
>>  * Cleaned up error variable handling
>>  * Added lseek calls to ensure correct revovery from EINTR
>>  * Added buf != NULL call before free(buf)
>>
>>  pym/portage/tests/util/file_copy/__init__.py  |   0
>>  pym/portage/tests/util/file_copy/__test__.py  |   0
>>  pym/portage/tests/util/file_copy/test_copyfile.py |  71 
>>  pym/portage/util/file_copy/__init__.py|  36 ++
>>  pym/portage/util/movefile.py  |   4 +-
>>  setup.py  |   9 +
>>  src/portage_util_file_copy_reflink_linux.c| 385
>> ++ 7 files changed, 504 insertions(+), 1
>> deletion(-) create mode 100644
>
>
> I'd say go for it, merge.  There's been no additional feedback or review

Thanks, merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=8ab5c8835931fd9ec098dbf4c5f416eb32e4a3a4
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH] portage.checksum: Store supported hash types in a frozenset

[Zac Medico]

On Mon, Mar 13, 2017 at 7:45 AM, Michał Górny  wrote:
> Copy the list of supported hash types (hashfunc_map dict) into
> a frozenset to support efficient access in the public API.
> ---
>  pym/portage/checksum.py | 25 ++---
>  1 file changed, 14 insertions(+), 11 deletions(-)

Looks good. Thanks!
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCHES] portage.checksum hacking, pt. 4

[Zac Medico]

On Sun, Mar 12, 2017 at 11:59 AM, Michał Górny  wrote:
> Hi,
>
> Here's a huge batch of patches for portage.checksum and relevant stuff.
> It's not a complete rewrite as I have originally planned, and it's still
> not the code I'd like to see but functionally it has been improved a lot.

The series looks good to me, except we should change the
get_valid_checksum_keys function to return a frozenset.
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH 01/14] Use public API: hashfunc_map -> get_valid_checksum_keys()

[Zac Medico]

On Sun, Mar 12, 2017 at 2:13 PM, Michał Górny <mgo...@gentoo.org> wrote:
> W dniu 12.03.2017, nie o godzinie 13∶36 -0700, użytkownik Zac Medico
> napisał:
>> On Sun, Mar 12, 2017 at 11:59 AM, Michał Górny <mgo...@gentoo.org> wrote:
>> > Use get_valid_checksum_keys() function instead of accessing hashfunc_map
>> > directly throughout the Portage.
>> > ---
>> >  pym/portage/_emirrordist/FetchTask.py |  2 +-
>> >  pym/portage/dbapi/bintree.py  |  4 ++--
>> >  pym/portage/eclass_cache.py   |  2 +-
>> >  pym/portage/manifest.py   |  4 ++--
>> >  pym/portage/package/ebuild/fetch.py   | 10 +-
>> >  5 files changed, 11 insertions(+), 11 deletions(-)
>> >
>> > diff --git a/pym/portage/_emirrordist/FetchTask.py 
>> > b/pym/portage/_emirrordist/FetchTask.py
>> > index 64de67582..203b8c213 100644
>> > --- a/pym/portage/_emirrordist/FetchTask.py
>> > +++ b/pym/portage/_emirrordist/FetchTask.py
>> > @@ -574,7 +574,7 @@ class FetchTask(CompositeTask):
>> > else:
>> > for hash_name in self.digests:
>> > if hash_name != "size" and \
>> > -   hash_name in 
>> > portage.checksum.hashfunc_map:
>> > +   hash_name in 
>> > portage.checksum.get_valid_checksum_keys():
>>
>> The get_valid_checksum_keys function creates a list for each call, so
>> this is a lot less efficient that dict hash table lookup that it was
>> before.
>
> Does it really matter though? AFAICS this is just a matter of optimizing
> the function, e.g. putting the keys in a frozenset.

It's going to be most convenient if get_valid_checksum_keys returns a frozenset.
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH 01/14] Use public API: hashfunc_map -> get_valid_checksum_keys()

[Zac Medico]

On Sun, Mar 12, 2017 at 11:59 AM, Michał Górny  wrote:
> Use get_valid_checksum_keys() function instead of accessing hashfunc_map
> directly throughout the Portage.
> ---
>  pym/portage/_emirrordist/FetchTask.py |  2 +-
>  pym/portage/dbapi/bintree.py  |  4 ++--
>  pym/portage/eclass_cache.py   |  2 +-
>  pym/portage/manifest.py   |  4 ++--
>  pym/portage/package/ebuild/fetch.py   | 10 +-
>  5 files changed, 11 insertions(+), 11 deletions(-)
>
> diff --git a/pym/portage/_emirrordist/FetchTask.py 
> b/pym/portage/_emirrordist/FetchTask.py
> index 64de67582..203b8c213 100644
> --- a/pym/portage/_emirrordist/FetchTask.py
> +++ b/pym/portage/_emirrordist/FetchTask.py
> @@ -574,7 +574,7 @@ class FetchTask(CompositeTask):
> else:
> for hash_name in self.digests:
> if hash_name != "size" and \
> -   hash_name in 
> portage.checksum.hashfunc_map:
> +   hash_name in 
> portage.checksum.get_valid_checksum_keys():

The get_valid_checksum_keys function creates a list for each call, so
this is a lot less efficient that dict hash table lookup that it was
before.
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH] config.setcpv: fix handling of IUSE changes (bug 611896)

[Zac Medico]

On Thu, Mar 9, 2017 at 5:33 PM, Brian Dolbec <dol...@gentoo.org> wrote:
> On Thu,  9 Mar 2017 13:38:04 -0800
> Zac Medico <zmed...@gentoo.org> wrote:
>
>> Fix setcpv to correctly regenerate USE when the IUSE differs from
>> the previous setcpv call. Changes in IUSE affect USE_EXPAND
>> wildcard expansion in the regenerate method.
>>
>> X-Gentoo-bug: 611896
>> X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=611896
>> ---
>>  pym/portage/package/ebuild/config.py | 7 +--
>>  1 file changed, 5 insertions(+), 2 deletions(-)
>>
>> diff --git a/pym/portage/package/ebuild/config.py
>> b/pym/portage/package/ebuild/config.py index 4f7e5c9..ef29afe 100644
>> --- a/pym/portage/package/ebuild/config.py
>> +++ b/pym/portage/package/ebuild/config.py
>> @@ -1623,6 +1623,10 @@ class config(object):
>>   elif previous_penv:
>>   has_changed = True
>>
>> + if not (previous_iuse == iuse and
>> + previous_iuse_effective is not None ==
>> eapi_attrs.iuse_effective):
>> + has_changed = True
>> +
>>   if has_changed:
>>   self.reset(keeping_pkg=1)
>>
>> @@ -1645,8 +1649,7 @@ class config(object):
>>
>>   # If reset() has not been called, it's safe to return
>>   # early if IUSE has not changed.
>> - if not has_changed and previous_iuse == iuse and \
>> - (previous_iuse_effective is not None ==
>> eapi_attrs.iuse_effective):
>> + if not has_changed:
>>   return
>>
>>   # Filter out USE flags that aren't part of IUSE.
>> This has to
>
> looks good :)
>
> --
> Brian Dolbec 
>
>

Thanks, merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=3ccd41702be7ed3f5bdbe123ec04349749b40c08

-- 
Thanks,
Zac

[gentoo-portage-dev] [PATCH] config.setcpv: fix handling of IUSE changes (bug 611896)

[Zac Medico]

Fix setcpv to correctly regenerate USE when the IUSE differs from
the previous setcpv call. Changes in IUSE affect USE_EXPAND
wildcard expansion in the regenerate method.

X-Gentoo-bug: 611896
X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=611896
---
 pym/portage/package/ebuild/config.py | 7 +--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/pym/portage/package/ebuild/config.py 
b/pym/portage/package/ebuild/config.py
index 4f7e5c9..ef29afe 100644
--- a/pym/portage/package/ebuild/config.py
+++ b/pym/portage/package/ebuild/config.py
@@ -1623,6 +1623,10 @@ class config(object):
elif previous_penv:
has_changed = True
 
+   if not (previous_iuse == iuse and
+   previous_iuse_effective is not None == 
eapi_attrs.iuse_effective):
+   has_changed = True
+
if has_changed:
self.reset(keeping_pkg=1)
 
@@ -1645,8 +1649,7 @@ class config(object):
 
# If reset() has not been called, it's safe to return
# early if IUSE has not changed.
-   if not has_changed and previous_iuse == iuse and \
-   (previous_iuse_effective is not None == 
eapi_attrs.iuse_effective):
+   if not has_changed:
return
 
# Filter out USE flags that aren't part of IUSE. This has to
-- 
2.10.2

Re: [gentoo-portage-dev] [PATCH] depgraph: fix runtime package mask interaction with slot operator rebuilds (bug 612094)

[Zac Medico]

On Thu, Mar 9, 2017 at 11:28 AM, Brian Dolbec <dol...@gentoo.org> wrote:
> On Wed,  8 Mar 2017 22:11:45 -0800
> Zac Medico <zmed...@gentoo.org> wrote:
>
>> In some cases the backtracking runtime package mask can interact badly
>> with slot operator rebuilds, preventing a solution from being found.
>> This patch fixes the problem, which is demonstrated by the included
>> unit test.
>>
>> X-Gentoo-bug: 612094
>> X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=612094
>> ---
>>  pym/_emerge/depgraph.py|  28 +++--
>>  .../test_slot_operator_runtime_pkg_mask.py | 136
>> + 2 files changed, 152 insertions(+), 12
>
>
> yeah, looks fine
> --
> Brian Dolbec 
>
>

Thanks, merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=c01f3fbd23def329eb1d1b0fc8f79959119a8a82
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH] depgraph: fix backtracking for slot operator rebuilds (bug 612042)

[Zac Medico]

On Wed, Mar 8, 2017 at 10:56 AM, Brian Dolbec  wrote:
>
> looks good :)
>
> --
> Brian Dolbec 
>
>

Thanks, merged:


https://gitweb.gentoo.org/proj/portage.git/commit/?id=7c01a73c5a3ff969ddaa43e91730a7372a8a10c8
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH] use_reduce: reserve missing_white_space_check for invalid tokens (bug 611838)

[Zac Medico]

On Wed, Mar 8, 2017 at 10:50 AM, Brian Dolbec <dol...@gentoo.org> wrote:
> On Mon,  6 Mar 2017 12:51:08 -0800
> Zac Medico <zmed...@gentoo.org> wrote:
>
>> Since it's possible for a URI to contain parenthesis, only call
>> missing_white_space_check for tokens that fail to validate with
>> token_class. The missing_white_space_check function only serves
>> to clarify exception messages, so it must not be allowed to
>> reject valid tokens.
>> ---
>>  pym/portage/dep/__init__.py | 4 ++--
>>  1 file changed, 2 insertions(+), 2 deletions(-)
>>
>> diff --git a/pym/portage/dep/__init__.py b/pym/portage/dep/__init__.py
>> index 968ff5b..6ff6adc 100644
>> --- a/pym/portage/dep/__init__.py
>> +++ b/pym/portage/dep/__init__.py
>> @@ -677,8 +677,6 @@ def use_reduce(depstr, uselist=[], masklist=[],
>> matchall=False, excludeall=[], i need_simple_token = True
>>   stack[level].append(token)
>>   else:
>> - missing_white_space_check(token, pos)
>> -
>>   if need_bracket:
>>   raise InvalidDependString(
>>   _("expected: '(', got: '%s',
>> token %s") % (token, pos+1)) @@ -698,12 +696,14 @@ def
>> use_reduce(depstr, uselist=[], masklist=[], matchall=False,
>> excludeall=[], i token = token_class(token, eapi=eapi,
>> is_valid_flag=is_valid_flag) except InvalidAtom as e:
>> +
>> missing_white_space_check(token, pos) raise InvalidDependString(
>>   _("Invalid
>> atom (%s), token %s") \ % (e, pos+1), errors=(e,))
>>   except SystemExit:
>>   raise
>>   except Exception as e:
>> +
>> missing_white_space_check(token, pos) raise InvalidDependString(
>>   _("Invalid
>> token '%s', token %s") % (token, pos+1))
>
> looks good :)

Thanks, merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=c4447175e57a8311f254cfc9fa32646467e1830f
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH v2] emerge: auto-enable --with-bdeps if --usepkg is not enabled (bug 598444)

[Zac Medico]

On Wed, Mar 8, 2017 at 10:48 AM, Brian Dolbec <dol...@gentoo.org> wrote:
> On Sun,  5 Mar 2017 00:40:25 -0800
> Zac Medico <zmed...@gentoo.org> wrote:
>
>> It's useful to automatically enable --with-bdeps so that @world
>> updates will update all packages that are not eligible for removal by
>> emerge --depclean. However, many users of binary packages do not want
>> unnecessary build time dependencies installed, therefore do not
>> auto-enable --with-bdeps for installation actions when the --usepkg
>> option is enabled.
>>
>> A new --with-bdeps-auto=<y|n> option is provided, making it possible
>> to enable or disable the program logic that causes --with-bdeps to be
>> automatically enabled. Use --with-bdeps-auto=n to prevent --with-bdeps
>> from being automatically enabled for installation actions. This is
>> useful for some rare cases in which --with-bdeps triggers unsolvable
>> dependency conflicts (and putting --with-bdeps=n in
>> EMERGE_DEFAULT_OPTS would cause undesirable --depclean behavior).
>>
>> X-Gentoo-bug: 598444
>> X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=598444
>> ---
>> [PATCH v2] adds a --with-bdeps-auto=<y|n> option which is useful for
>> some rare cases in which --with-bdeps triggers unsolvable dependency
>> conflicts
>>
>>  man/emerge.1 |  37 +++-
>>  pym/_emerge/create_depgraph_params.py|   5 +
>>  pym/_emerge/depgraph.py  |   4 +-
>>  pym/_emerge/main.py  |   5 +
>>  pym/portage/tests/resolver/ResolverPlayground.py |   5 +
>>  pym/portage/tests/resolver/test_bdeps.py | 215
>> +++ 6 files changed, 266 insertions(+), 5
>> deletions(-) create mode 100644
>
> Looks fine, merge please :)

Thanks merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=852c729bdef3d4c2e2d459a43dc21f0a05dfa2ba

-- 
Thanks,
Zac

[gentoo-portage-dev] [PATCH] use_reduce: reserve missing_white_space_check for invalid tokens (bug 611838)

[Zac Medico]

Since it's possible for a URI to contain parenthesis, only call
missing_white_space_check for tokens that fail to validate with
token_class. The missing_white_space_check function only serves
to clarify exception messages, so it must not be allowed to
reject valid tokens.
---
 pym/portage/dep/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pym/portage/dep/__init__.py b/pym/portage/dep/__init__.py
index 968ff5b..6ff6adc 100644
--- a/pym/portage/dep/__init__.py
+++ b/pym/portage/dep/__init__.py
@@ -677,8 +677,6 @@ def use_reduce(depstr, uselist=[], masklist=[], 
matchall=False, excludeall=[], i
need_simple_token = True
stack[level].append(token)  
else:
-   missing_white_space_check(token, pos)
-
if need_bracket:
raise InvalidDependString(
_("expected: '(', got: '%s', token %s") 
% (token, pos+1))
@@ -698,12 +696,14 @@ def use_reduce(depstr, uselist=[], masklist=[], 
matchall=False, excludeall=[], i
token = token_class(token, 
eapi=eapi,

is_valid_flag=is_valid_flag)
except InvalidAtom as e:
+   
missing_white_space_check(token, pos)
raise InvalidDependString(
_("Invalid atom (%s), 
token %s") \
% (e, pos+1), 
errors=(e,))
except SystemExit:
raise
except Exception as e:
+   
missing_white_space_check(token, pos)
raise InvalidDependString(
_("Invalid token '%s', 
token %s") % (token, pos+1))
 
-- 
2.10.2

Re: [gentoo-portage-dev] [PATCH v2] emerge: auto-enable --with-bdeps if --usepkg is not enabled (bug 598444)

[Zac Medico]

On 03/05/2017 11:44 AM, Michael Orlitzky wrote:
> On 03/05/2017 02:12 PM, Zac Medico wrote:
>>
>> Incorrect.
>>
>> ...
>>
>> Incorrect.
>>
> 
> I see my mistakes, but maintain that this is confusing =)

It could be worse, and I think it's questionable that we could do any
better, given the nature of the problem.

>> The --with-bdeps-auto option results in desirable behavior by default,
>> and it's also backward compatible with existing --with-bdeps and
>> --usepkg usage. It just does the right thing, minimizing the impact to
>> existing emerge usage.
> 
> I was hoping that since nothing breaks with --update-bdeps=y and
> --clean-bdeps=n (the new defaults), we could just disable --with-bdeps
> immediately and emit a warning when it's given.

Breaking backward compatibility is too disruptive, unless we allow for a
transition period where --with-bdeps is deprecated.

>> There some problems with --update-bdeps/--clean-bdeps:
>>
>> * The program logic will be more complicated, since --with-bdeps will
>> have to override both options for backward-compatibility with existing
>> --with-bdeps usage.
> 
> Not applicable if --with-bdeps goes away.

We don't have a justification to break compatibility without a
transition period.

>> * The meaning of --clean-bdeps is confusing. Does --clean-bdeps=y mean
>> to clean build time deps, or does it mean to pull build time deps into
>> the dependency graph for removal operations?
> 
> --clean-bdeps means clean the bdeps.

It's confusing to have an option with inverted meaning relative to
--with-bdeps. Cleaning the bdeps is a consequence of excluding them from
the dependency graph, not an action in itself. --clean-bdeps sounds like
an action.

> I totally agree that the worst option of all is to keep --with-bdeps AND
> introduce the other two.
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH v2] emerge: auto-enable --with-bdeps if --usepkg is not enabled (bug 598444)

[Zac Medico]

On 03/05/2017 07:08 AM, Michael Orlitzky wrote:
> On 03/05/2017 03:40 AM, Zac Medico wrote:
>>
>> A new --with-bdeps-auto=<y|n> option is provided, making it possible to
>> enable or disable the program logic that causes --with-bdeps to be
>> automatically enabled. Use --with-bdeps-auto=n to prevent --with-bdeps
>> from being automatically enabled for installation actions. This is useful
>> for some rare cases in which --with-bdeps triggers unsolvable dependency
>> conflicts (and putting --with-bdeps=n in EMERGE_DEFAULT_OPTS would cause
>> undesirable --depclean behavior).
>>
> 
> If I understand correctly, the end result of this is two --flags that
> combine in a (rather complicated) way to let the user control the
> default bdeps behavior of both "emerge --update" and "emerge --depclean"
> separately. I'll try to summarize my understanding:
> 
>I. emerge --update
> 
>  I.a. Some people want to --update the build dependencies they have
>   installed for e.g. security purposes.
> 
>  I.b. Others don't want build deps pulled in by "emerge --update",
>   because they're using binary packages or because it causes
>   conflicts in the resolver (llvm).
> 
>   II. emerge --depclean
> 
> II.a. The default behavior is to not remove build-only dependencies
>   because, generally, they will just have to rebuilt again.
> 
> II.b. However, some people want to remove the build-only deps that
>   aren't strictly in use -- particularly if those build-only
>   deps are not being updated by emerge --update.
> 
> 
> To choose between those four options...
> 
>   i.   --with-bdeps=n and --with-bdeps-auto=y gives you I.a. + II.b.

Incorrect.

These options give you I.b. + II.b. because --with-bdeps=n overrides
--with-bdeps-auto settings (the override is backward-compatible with
existing --with-bdeps usage).

Since I.a. + II.b. is a contradictory combination, there is no
combination of EMERGE_DEFAULT_OPTS that will give you I.a. + II.b.

Specifying --with-bdeps-auto=y has no effect, since that's the default.

>   ii.  --with-bdeps=n and --with-bdeps-auto=n gives you I.b. + II.b.

Correct.

Specifying --with-bdeps-auto=n has no effect, since --with-bdeps=n
overrides --with-bdeps-auto settings (the override is
backward-compatible with existing --with-bdeps usage).

>   iii. --with-bdeps=y and --with-bdeps-auto=y gives you I.a. + II.a.

Correct.

Specifying --with-bdeps-auto=y has no effect for 2 reasons:

* --with-bdeps-auto=y is the default
* --with-bdeps=y overrides --with-bdeps-auto settings (the override is
backward-compatible with existing --with-bdeps usage).

>   iv.  --with-bdeps=y and --with-bdeps-auto=n gives you I.a. + II.a.

Correct.

Specifying --with-bdeps-auto=n has no effect, since --with-bdeps=y
overrides --with-bdeps-auto settings (the override is
backward-compatible with existing --with-bdeps usage).

> That only gets you three out of the four options. You have to read the
> fine print to get the other:
> 
>   v.   passing no flags explicitly gives you I.b. and II.a.

Incorrect.

Since --with-bdeps-auto=y is the the default setting, you get I.a. +
II.a. unless --usepkg is enabled. If --usepkg is enabled, you get I.b. +
II.a., which is backward-compatible with existing --usepkg usage.

> If there's going to be two flags, can't we do better? Why not just name
> the flags after what we want them to do:
> 
>   --update-bdeps=<y|n> (default: y)
> 
>   --clean-bdeps=<y|n>  (default: n)

The --with-bdeps-auto option results in desirable behavior by default,
and it's also backward compatible with existing --with-bdeps and
--usepkg usage. It just does the right thing, minimizing the impact to
existing emerge usage.

There some problems with --update-bdeps/--clean-bdeps:

* The program logic will be more complicated, since --with-bdeps will
have to override both options for backward-compatibility with existing
--with-bdeps usage.

* The meaning of --clean-bdeps is confusing. Does --clean-bdeps=y mean
to clean build time deps, or does it mean to pull build time deps into
the dependency graph for removal operations?

> With those two options, it's easy to tell portage exactly what you want.
> If I don't like the defaults, I can turn one of them off without
> affecting the other.

No, there are problems, as described above.

> But what about the --usepkg magic? I think the workaround can be left
> as-is. When --usepkg is enabled, switch the default for --update-bdeps
> to "n" unless it is explicitly set.

That would work, but introducing 2 new options is no really appealing,
because it will lead to more complicated logic that will be more
difficult to explain and for people to comprehend. I really don't

[gentoo-portage-dev] [PATCH v2] emerge: auto-enable --with-bdeps if --usepkg is not enabled (bug 598444)

[Zac Medico]

It's useful to automatically enable --with-bdeps so that @world updates
will update all packages that are not eligible for removal by
emerge --depclean. However, many users of binary packages do not want
unnecessary build time dependencies installed, therefore do not
auto-enable --with-bdeps for installation actions when the --usepkg
option is enabled.

A new --with-bdeps-auto= option is provided, making it possible to
enable or disable the program logic that causes --with-bdeps to be
automatically enabled. Use --with-bdeps-auto=n to prevent --with-bdeps
from being automatically enabled for installation actions. This is useful
for some rare cases in which --with-bdeps triggers unsolvable dependency
conflicts (and putting --with-bdeps=n in EMERGE_DEFAULT_OPTS would cause
undesirable --depclean behavior).

X-Gentoo-bug: 598444
X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=598444
---
[PATCH v2] adds a --with-bdeps-auto= option which is useful for some
rare cases in which --with-bdeps triggers unsolvable dependency conflicts

 man/emerge.1 |  37 +++-
 pym/_emerge/create_depgraph_params.py|   5 +
 pym/_emerge/depgraph.py  |   4 +-
 pym/_emerge/main.py  |   5 +
 pym/portage/tests/resolver/ResolverPlayground.py |   5 +
 pym/portage/tests/resolver/test_bdeps.py | 215 +++
 6 files changed, 266 insertions(+), 5 deletions(-)
 create mode 100644 pym/portage/tests/resolver/test_bdeps.py

diff --git a/man/emerge.1 b/man/emerge.1
index 5b61220..7db4271 100644
--- a/man/emerge.1
+++ b/man/emerge.1
@@ -986,13 +986,44 @@ The default is set to "y" (on).
 .TP
 .BR "\-\-with\-bdeps < y | n >"
 In dependency calculations, pull in build time dependencies
-that are not strictly required. This defaults to \'n\' for
-installation actions, meaning they will not be installed, and
-\'y\' for the \fB\-\-depclean\fR action, meaning they will not be removed.
+that are not strictly required. This option is automatically enabled for
+installation actions, meaning they will be installed, and defaults to
+\(aqy\(aq for the \fB\-\-depclean\fR action, meaning they will not be
+removed. In order to prevent the \fB\-\-with\-bdeps\fR option from being
+automatically enabled for installation actions, specify
+\fB\-\-with\-bdeps\-auto=n\fR in either the command line or
+\fBEMERGE_DEFAULT_OPTS\fR.
+
+Since many users of binary packages do not want unnecessary build time
+dependencies installed, this option is not automatically enabled for
+installation actions when the \fB\-\-usepkg\fR option is enabled. In
+order to pull in build time dependencies for binary packages with
+\fB\-\-usepkg\fR, \fB\-\-with\-bdeps=y\fR must be specified explicitly.
+This also applies to options that enable the \fB\-\-usepkg\fR option
+implicitly, such as \fB\-\-getbinpkg\fR.
+
 This setting can be added to
 \fBEMERGE_DEFAULT_OPTS\fR (see make.conf(5)) and later overridden via the
 command line.
 .TP
+.BR "\-\-with\-bdeps\-auto < y | n >"
+This option is used to enable or disable the program logic that causes
+\fB\-\-with\-bdeps\fR is to be automatically enabled for installation
+actions. This option is enabled by default. Use
+\fB\-\-with\-bdeps\-auto=n\fR to prevent \fB\-\-with\-bdeps\fR from
+being automatically enabled for installation actions. This setting can
+be added to \fBEMERGE_DEFAULT_OPTS\fR (see make.conf(5)) and later
+overridden via the command line.
+
+\fBNOTE:\fR The program logic that causes \fB\-\-with\-bdeps\fR to be
+automatically enabled for installation actions does not affect removal
+actions such as the \fB\-\-depclean\fR action. Therefore, when
+\fB\-\-with\-bdeps\-auto=n\fR is specified in \fBEMERGE_DEFAULT_OPTS\fR,
+it does not affect the default \fB\-\-with\-bdeps=y\fR setting that
+applies to the \fB\-\-depclean\fR action. The default
+\fB\-\-with\-bdeps=y\fR setting that applies to the \fB\-\-depclean\fR
+action can be overridden only by specifying \fB\-\-with\-bdeps=n\fR.
+.TP
 .BR "\-\-with\-test\-deps [ y | n ]"
 For packages matched by arguments, this option will pull in dependencies
 that are conditional on the "test" USE flag, even if "test" is not
diff --git a/pym/_emerge/create_depgraph_params.py 
b/pym/_emerge/create_depgraph_params.py
index 2c64928..cdea029 100644
--- a/pym/_emerge/create_depgraph_params.py
+++ b/pym/_emerge/create_depgraph_params.py
@@ -13,6 +13,8 @@ def create_depgraph_params(myopts, myaction):
# deep:  go into the dependencies of already merged packages
# empty: pretend nothing is merged
# complete:  completely account for all known dependencies
+   # bdeps: satisfy build time dependencies of packages that are
+   #   already built, even though they are not strictly required
# remove:build graph for use in removing packages
# rebuilt_binaries: replace installed packages with rebuilt binaries

Re: [gentoo-portage-dev] [PATCH] emerge: auto-enable --with-bdeps if --usepkg is not enabled (bug 598444)

[Zac Medico]

On 03/04/2017 02:30 AM, Zac Medico wrote:
> On 03/04/2017 01:59 AM, Michał Górny wrote:
>> W dniu 04.03.2017, sob o godzinie 01∶34 -0800, użytkownik Zac Medico
>> napisał:
>>> It's useful to enable --with-bdeps by default so that @world updates
>>> will update all packages that are not eligible for removal by
>>> emerge --depclean. However, many users of binary packages do not want
>>> unnecessary build time dependencies installed, therefore do not
>>> auto-enable --with-bdeps for installation actions when the --usepkg
>>> option is enabled.
>>>
>>> X-Gentoo-bug: 598444
>>> X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=598444
>>> ---
>>>  man/emerge.1 |  18 ++-
>>>  pym/_emerge/create_depgraph_params.py|   4 +
>>>  pym/_emerge/depgraph.py  |   4 +-
>>>  pym/portage/tests/resolver/ResolverPlayground.py |   5 +
>>>  pym/portage/tests/resolver/test_bdeps.py | 197 
>>> +++
>>>  5 files changed, 223 insertions(+), 5 deletions(-)
>>>  create mode 100644 pym/portage/tests/resolver/test_bdeps.py
>>
>> Just to be clear, this will break stuff for people who have more than
>> one slot of LLVM enabled since different slots build-depend on colliding
>> packages.
> 
> Hmm, that's unfortunate, because putting --with-bdeps=n in
> EMERGE_DEFAULT_OPTS is not good for --depclean. I'll have to think about
> a better way to handle this case. We've already got a bug open for this
> sort of situation here:
> 
> https://bugs.gentoo.org/show_bug.cgi?id=427938

For people who want --with-bdeps disabled for installation actions, and
--with-bdeps enabled for removal actions like --depclean, putting
--with-bdeps=n in EMERGE_DEFAULT_OPTS is going to do the wrong thing for
removal actions. So, we'll need to introduce some kind of flag that can
be used to disable the automatic --with-bdeps behavior. For example,
when can introduce --with-bdeps-auto=n as a means to disable the
automatic --with-bdeps behavior.
-- 
Thanks,
Zac

[gentoo-portage-dev] [PATCH] emerge: auto-enable --with-bdeps if --usepkg is not enabled (bug 598444)

[Zac Medico]

It's useful to enable --with-bdeps by default so that @world updates
will update all packages that are not eligible for removal by
emerge --depclean. However, many users of binary packages do not want
unnecessary build time dependencies installed, therefore do not
auto-enable --with-bdeps for installation actions when the --usepkg
option is enabled.

X-Gentoo-bug: 598444
X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=598444
---
 man/emerge.1 |  18 ++-
 pym/_emerge/create_depgraph_params.py|   4 +
 pym/_emerge/depgraph.py  |   4 +-
 pym/portage/tests/resolver/ResolverPlayground.py |   5 +
 pym/portage/tests/resolver/test_bdeps.py | 197 +++
 5 files changed, 223 insertions(+), 5 deletions(-)
 create mode 100644 pym/portage/tests/resolver/test_bdeps.py

diff --git a/man/emerge.1 b/man/emerge.1
index 5b61220..970bb5b 100644
--- a/man/emerge.1
+++ b/man/emerge.1
@@ -986,9 +986,21 @@ The default is set to "y" (on).
 .TP
 .BR "\-\-with\-bdeps < y | n >"
 In dependency calculations, pull in build time dependencies
-that are not strictly required. This defaults to \'n\' for
-installation actions, meaning they will not be installed, and
-\'y\' for the \fB\-\-depclean\fR action, meaning they will not be removed.
+that are not strictly required. This option is enabled automatically for
+installation actions, meaning they will be installed, and is also enabled
+automatically for the \fB\-\-depclean\fR action, meaning they will not be
+removed. In order to prevent this option from being enabled automatically,
+\fB\-\-with-bdeps=n\fR must be specified either in the command line or
+via \fBEMERGE_DEFAULT_OPTS\fR.
+
+Since many users of binary packages do not want unnecessary build time
+dependencies installed, this option is not enabled automatically for
+installation actions when the \fB\-\-usepkg\fR option is enabled. In
+order to pull in build time dependencies for binary packages with
+\fB\-\-usepkg\fR, \fB\-\-with-bdeps=y\fR must be specified explicitly.
+This also applies to options that enable the \fB\-\-usepkg\fR option
+implicitly, such as \fB\-\-getbinpkg\fR.
+
 This setting can be added to
 \fBEMERGE_DEFAULT_OPTS\fR (see make.conf(5)) and later overridden via the
 command line.
diff --git a/pym/_emerge/create_depgraph_params.py 
b/pym/_emerge/create_depgraph_params.py
index 2c64928..676aac5 100644
--- a/pym/_emerge/create_depgraph_params.py
+++ b/pym/_emerge/create_depgraph_params.py
@@ -13,6 +13,8 @@ def create_depgraph_params(myopts, myaction):
# deep:  go into the dependencies of already merged packages
# empty: pretend nothing is merged
# complete:  completely account for all known dependencies
+   # bdeps: satisfy build time dependencies of packages that are
+   #   already built, even though they are not strictly required
# remove:build graph for use in removing packages
# rebuilt_binaries: replace installed packages with rebuilt binaries
# rebuild_if_new_slot: rebuild or reinstall packages when
@@ -32,6 +34,8 @@ def create_depgraph_params(myopts, myaction):
bdeps = myopts.get("--with-bdeps")
if bdeps is not None:
myparams["bdeps"] = bdeps
+   elif myaction == "remove" or "--usepkg" not in myopts:
+   myparams["bdeps"] = "auto"
 
ignore_built_slot_operator_deps = 
myopts.get("--ignore-built-slot-operator-deps")
if ignore_built_slot_operator_deps is not None:
diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
index ce0fde1..02a3226 100644
--- a/pym/_emerge/depgraph.py
+++ b/pym/_emerge/depgraph.py
@@ -2362,7 +2362,7 @@ class depgraph(object):
if ebuild is None:
changed = False
else:
-   if self._dynamic_config.myparams.get("bdeps", "n") == 
"y":
+   if self._dynamic_config.myparams.get("bdeps") in ("y", 
"auto"):
depvars = Package._dep_keys
else:
depvars = Package._runtime_keys
@@ -2998,7 +2998,7 @@ class depgraph(object):
 
ignore_build_time_deps = False
if pkg.built and not removal_action:
-   if self._dynamic_config.myparams.get("bdeps", "n") == 
"y":
+   if self._dynamic_config.myparams.get("bdeps") in ("y", 
"auto"):
# Pull in build time deps as requested, but 
marked them as
# "optional" since they are not strictly 
required. This allows
# more freedom in the merge order calculation 
for solving
diff --git a/pym/portage/tests/resolver/ResolverPlayground.py 
b/pym/portage/tests/resolver/ResolverPlayground.py
index d1434f7..d803719 100644
--- a/pym/portage/tests/resolver/ResolverPlayground.py
+++ 

Re: [gentoo-portage-dev] [PATCH v2] movefile: support in-kernel file copying on Linux (bug 607868)

[Zac Medico]

On 03/03/2017 12:45 PM, Michał Górny wrote:
> W dniu 02.03.2017, czw o godzinie 17∶09 -0800, użytkownik Zac Medico
>> +def copyfile(src, dst):
>> +"""
>> +Copy the contents (no metadata) of the file named src to a file
>> +named dst.
>> +
>> +If possible, copying is done within the kernel, and uses
>> +"copy acceleration" techniques (such as reflinks). This also
>> +supports sparse files.
>> +
>> +@param src: path of source file
>> +@type src: str
>> +@param dst: path of destination file
>> +@type dst: str
>> +"""
>> +global _copyfile
>> +
>> +if _copyfile is None:
>> +if _file_copy is None:
>> +_copyfile = shutil.copyfile
>> +else:
>> +_copyfile = _optimized_copyfile
> 
> I dare say the caching logic takes more time than the actual logic.
> However, this could be made even simpler -- you could just check
> the condition in global scope and set _copyfile value there.

In v3 it's like this:

if _file_copy is None:
copyfile = shutil.copyfile
else:
copyfile = _optimized_copyfile

>> +static off_t
>> +do_lseek(int fd_out, int fd_in, loff_t *off_out) {
> 
> Maybe name it do_lseek_data() to make the goal clearer? Some function
> docs in comments would also be helpful.

Renamed to do_lseek_data v3.

>> +
>> +if (!(error && errno == EINTR && PyErr_CheckSignals() == 0))
>> +eintr_retry = 0;
> 
> To be honest, I can't do it but it'd be really a good idea if someone
> could analyze all the code paths where the code above could give 'error
> = 1' and ensure that:
> 
> a. there is no case when we error=1 and leave errno unmodified/unset
> (i.e. can accidentally leave earlier EINTR there),

In v3, I set error = errno immediately after every function call that
fails. That makes it really easy to see exactly which function calls the
errno value is picked up from.

> b. we can correctly recover from any EINTR, i.e. if EINTR causes some
> function to be interrupted in the middle, our next iteration starts
> safely.

The state is mostly tracked by the offset_out variable, which makes it
hard to screw up. We just have to watch that variable carefully, and
make sure we don't corrupt its state. I added in a couple of extra lseek
calls to ensure that the fd_in file offset is always in sync with the
offset_out variable when we resume from EINTR.

>> +}
>> +
>> +free(buf);
> 
> if (buf != NULL) ?

The free(3) documentation says we're allowed to pass in a null pointer,
but I've added a buf != NULL check for readability.
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH] movefile: support in-kernel file copying on Linux (bug 607868)

[Zac Medico]

On 03/02/2017 08:24 AM, Michał Górny wrote:
> W dniu 01.03.2017, śro o godzinie 19∶03 -0800, użytkownik Zac Medico
> napisał:
>> +def _test_optimized_copyfile():
>> +"""
>> +Test if _optimized_copyfile works. It will fail for Linux versions
>> +from 2.6.0 to 2.6.32, because sendfile does not support writing
>> +to regular files. It will also fail for Linux versions less than
>> +3.1 if portage was compiled 3.1 or later, due to missing support
>> +for lseek SEEK_DATA/SEEK_HOLE.
> 
> I don't really like the idea of relying on one-time attempt to determine
> whether a complex mechanism lacking proper fallback will work. But I
> guess you're only aiming at catching corner cases here.

In v2, there's a complete fallback mechanism in the native code, and the
_test_optimized_copyfile function has been eliminated.

>> +/* Revert SEEK_HOLE offset change, since we're going
>> + * to copy the data that comes before the hole.
>> + */
>> +if (lseek(fd_in, offset_out, SEEK_SET) < 0) {
>> +error = 1;
>> +break;
>> +}
>> +
>> +copyfunc_ret = copyfunc(fd_out,
>> +fd_in,
>> +_out,
>> +offset_hole - offset_data);
>> +
>> +if (copyfunc_ret < 0) {
>> +if ((errno == EXDEV || errno == ENOSYS) &&
>> +copyfunc == cfr_wrapper) {
>> +/* Use sendfile instead of copy_file_range for
>> + * cross-device copies, or when the copy_file_range
>> + * syscall is not available (less than Linux 4.5).
>> + */
>> +copyfunc = sendfile;
>> +copyfunc_ret = copyfunc(fd_out,
>> +fd_in,
>> +_out,
>> +offset_hole - offset_data);
>> +
>> +if (copyfunc_ret < 0) {
> 
> I think you still should have a proper fallback for sendfile() refusing
> to do its job.

In the lseek loop, sendfile should always work if lseek SEEK_DATA works,
so there should be no need to fallback for sendfile failure here. In v2,
I've added a comment about this.

>> +while (eintr_retry) {
>> +
>> +error = 0;
>> +
>> +Py_BEGIN_ALLOW_THREADS
>> +
>> +if (!stat_acquired && fstat(fd_in, ) < 0) {
>> +error = 1;
>> +}
>> +else {
>> +stat_acquired = 1;
>> +while (offset_out < sb.st_size) {
>> +copyfunc_ret = sendfile(fd_out,
>> +fd_in,
>> +_out,
>> +sb.st_size - offset_out);
>> +
>> +if (copyfunc_ret < 0) {
> 
> Likewise, especially that old kernels may refuse file-to-file copy here.

In v2, it will fallback to a plain read/write loop.
-- 
Thanks,
Zac

[gentoo-portage-dev] [PATCH v2] movefile: support in-kernel file copying on Linux (bug 607868)

[Zac Medico]

Perform in-kernel file copying when possible, and also support
reflinks and sparse files. If the optimized implementation
fails at runtime, gracefully fallback to a plain read/write
loop.

Compile-time and run-time fallbacks are implemented, so that
any incompatiblities will be handled gracefully. For example,
if the code is compiled on a system that supports the
copy_file_range syscall, but at run-time an older kernel that
does not support this syscall is detected, it will be handled
gracefully. There are similar fallbacks for lack of lseek
SEEK_DATA and sendfile support.

X-Gentoo-Bug: 607868
X-Gentoo-Bug-Url: https://bugs.gentoo.org/show_bug.cgi?id=607868
---
[PATCH v2] adds a native fallback that will work on any kernel,
using a plain read/write loop.

 pym/portage/tests/util/file_copy/__init__.py  |   0
 pym/portage/tests/util/file_copy/__test__.py  |   0
 pym/portage/tests/util/file_copy/test_copyfile.py |  68 +
 pym/portage/util/file_copy/__init__.py|  45 
 pym/portage/util/movefile.py  |   4 +-
 setup.py  |   9 +
 src/portage_util_file_copy_reflink_linux.c| 298 ++
 7 files changed, 423 insertions(+), 1 deletion(-)
 create mode 100644 pym/portage/tests/util/file_copy/__init__.py
 create mode 100644 pym/portage/tests/util/file_copy/__test__.py
 create mode 100644 pym/portage/tests/util/file_copy/test_copyfile.py
 create mode 100644 pym/portage/util/file_copy/__init__.py
 create mode 100644 src/portage_util_file_copy_reflink_linux.c

diff --git a/pym/portage/tests/util/file_copy/__init__.py 
b/pym/portage/tests/util/file_copy/__init__.py
new file mode 100644
index 000..e69de29
diff --git a/pym/portage/tests/util/file_copy/__test__.py 
b/pym/portage/tests/util/file_copy/__test__.py
new file mode 100644
index 000..e69de29
diff --git a/pym/portage/tests/util/file_copy/test_copyfile.py 
b/pym/portage/tests/util/file_copy/test_copyfile.py
new file mode 100644
index 000..94fb4d7
--- /dev/null
+++ b/pym/portage/tests/util/file_copy/test_copyfile.py
@@ -0,0 +1,68 @@
+# Copyright 2017 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import shutil
+import tempfile
+
+from portage import os
+from portage.tests import TestCase
+from portage.checksum import perform_md5
+from portage.util.file_copy import copyfile
+
+
+class CopyFileTestCase(TestCase):
+
+   def testCopyFile(self):
+
+   tempdir = tempfile.mkdtemp()
+   try:
+   src_path = os.path.join(tempdir, 'src')
+   dest_path = os.path.join(tempdir, 'dest')
+   content = b'foo'
+
+   with open(src_path, 'wb') as f:
+   f.write(content)
+
+   copyfile(src_path, dest_path)
+
+   self.assertEqual(perform_md5(src_path), 
perform_md5(dest_path))
+   finally:
+   shutil.rmtree(tempdir)
+
+
+class CopyFileSparseTestCase(TestCase):
+
+   def testCopyFileSparse(self):
+
+   tempdir = tempfile.mkdtemp()
+   try:
+   src_path = os.path.join(tempdir, 'src')
+   dest_path = os.path.join(tempdir, 'dest')
+   content = b'foo'
+
+   # Use seek to create some sparse blocks. Don't make 
these
+   # files too big, in case the filesystem doesn't support
+   # sparse files.
+   with open(src_path, 'wb') as f:
+   f.write(content)
+   f.seek(2**18, 1)
+   f.write(content)
+   f.seek(2**19, 1)
+   f.write(content)
+
+   copyfile(src_path, dest_path)
+
+   self.assertEqual(perform_md5(src_path), 
perform_md5(dest_path))
+
+   # This last part of the test is expected to fail when 
sparse
+   # copy is not implemented, so set the todo flag in order
+   # to tolerate failures.
+   self.todo = True
+
+   # If sparse blocks were preserved, then both files 
should
+   # consume the same number of blocks.
+   self.assertEqual(
+   os.stat(src_path).st_blocks,
+   os.stat(dest_path).st_blocks)
+   finally:
+   shutil.rmtree(tempdir)
diff --git a/pym/portage/util/file_copy/__init__.py 
b/pym/portage/util/file_copy/__init__.py
new file mode 100644
index 000..34c46ad
--- /dev/null
+++ b/pym/portage/util/file_copy/__init__.py
@@ -0,0 +1,45 @@
+# Copyright 2017 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import 

Re: [gentoo-portage-dev] [PATCH] movefile: support in-kernel file copying on Linux (bug 607868)

[Zac Medico]

On 03/02/2017 08:24 AM, Michał Górny wrote:
>> +if (copyfunc_ret < 0) {
>> +if ((errno == EXDEV || errno == ENOSYS) &&
>> +copyfunc == cfr_wrapper) {
>> +/* Use sendfile instead of copy_file_range for
>> + * cross-device copies, or when the copy_file_range
>> + * syscall is not available (less than Linux 4.5).
>> + */
>> +copyfunc = sendfile;
>> +copyfunc_ret = copyfunc(fd_out,
>> +fd_in,
>> +_out,
>> +offset_hole - offset_data);
>> +
>> +if (copyfunc_ret < 0) {
> 
> I think you still should have a proper fallback for sendfile() refusing
> to do its job.

Yeah, that case seems easy enough to handle, so I'll look into adding a
fallback.
-- 
Thanks,
Zac

[gentoo-portage-dev] [PATCH] movefile: support in-kernel file copying on Linux (bug 607868)

[Zac Medico]

Perform in-kernel file copying when possible, and also support
reflinks and sparse files. If the optimized implementation
fails at runtime, gracefully fallback to shutil.copyfile.

Compile-time and run-time fallbacks are implemented, so that
any incompatiblities will be handled gracefully. For example,
if the code is compiled on a system that supports the
copy_file_range syscall, but at run-time an older kernel that
does not support this syscall is detected, it will be handled
gracefully.

X-Gentoo-Bug: 607868
X-Gentoo-Bug-Url: https://bugs.gentoo.org/show_bug.cgi?id=607868
---
 pym/portage/tests/util/file_copy/__init__.py  |   0
 pym/portage/tests/util/file_copy/__test__.py  |   0
 pym/portage/tests/util/file_copy/test_copyfile.py |  68 +++
 pym/portage/util/file_copy/__init__.py|  78 
 pym/portage/util/movefile.py  |   4 +-
 setup.py  |   9 +
 src/portage_util_file_copy_reflink_linux.c| 225 ++
 7 files changed, 383 insertions(+), 1 deletion(-)
 create mode 100644 pym/portage/tests/util/file_copy/__init__.py
 create mode 100644 pym/portage/tests/util/file_copy/__test__.py
 create mode 100644 pym/portage/tests/util/file_copy/test_copyfile.py
 create mode 100644 pym/portage/util/file_copy/__init__.py
 create mode 100644 src/portage_util_file_copy_reflink_linux.c

diff --git a/pym/portage/tests/util/file_copy/__init__.py 
b/pym/portage/tests/util/file_copy/__init__.py
new file mode 100644
index 000..e69de29
diff --git a/pym/portage/tests/util/file_copy/__test__.py 
b/pym/portage/tests/util/file_copy/__test__.py
new file mode 100644
index 000..e69de29
diff --git a/pym/portage/tests/util/file_copy/test_copyfile.py 
b/pym/portage/tests/util/file_copy/test_copyfile.py
new file mode 100644
index 000..987a701
--- /dev/null
+++ b/pym/portage/tests/util/file_copy/test_copyfile.py
@@ -0,0 +1,68 @@
+# Copyright 2017 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import shutil
+import tempfile
+
+from portage import os
+from portage.tests import TestCase
+from portage.checksum import perform_md5
+from portage.util.file_copy import copyfile
+
+
+class CopyFileTestCase(TestCase):
+
+   def testCopyFile(self):
+
+   tempdir = tempfile.mkdtemp()
+   try:
+   src_path = os.path.join(tempdir, 'src')
+   dest_path = os.path.join(tempdir, 'dest')
+   content = b'foo'
+
+   with open(src_path, 'wb') as f:
+   f.write(content)
+
+   copyfile(src_path, dest_path)
+
+   self.assertEqual(perform_md5(src_path), 
perform_md5(dest_path))
+   finally:
+   shutil.rmtree(tempdir)
+
+
+class CopyFileSparseTestCase(TestCase):
+
+   def testCopyFileSparse(self):
+
+   # This test is expected to fail on platforms where we have
+   # not implemented sparse copy, so set the todo flag in order
+   # to tolerate failures.
+   self.todo = True
+
+   tempdir = tempfile.mkdtemp()
+   try:
+   src_path = os.path.join(tempdir, 'src')
+   dest_path = os.path.join(tempdir, 'dest')
+   content = b'foo'
+
+   # Use seek to create some sparse blocks. Don't make 
these
+   # files too big, in case the filesystem doesn't support
+   # sparse files.
+   with open(src_path, 'wb') as f:
+   f.write(content)
+   f.seek(2**18, 1)
+   f.write(content)
+   f.seek(2**19, 1)
+   f.write(content)
+
+   copyfile(src_path, dest_path)
+
+   self.assertEqual(perform_md5(src_path), 
perform_md5(dest_path))
+
+   # If sparse blocks were preserved, then both files 
should
+   # consume the same number of blocks.
+   self.assertEqual(
+   os.stat(src_path).st_blocks,
+   os.stat(dest_path).st_blocks)
+   finally:
+   shutil.rmtree(tempdir)
diff --git a/pym/portage/util/file_copy/__init__.py 
b/pym/portage/util/file_copy/__init__.py
new file mode 100644
index 000..5c7aff1
--- /dev/null
+++ b/pym/portage/util/file_copy/__init__.py
@@ -0,0 +1,78 @@
+# Copyright 2017 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+import os
+import shutil
+import tempfile
+
+try:
+   from portage.util.file_copy.reflink_linux import file_copy as _file_copy
+except ImportError:
+   _file_copy = None
+
+
+_copyfile = None
+
+
+def 

Re: [gentoo-portage-dev] [PATCH 1/2] checksum: Fix overriding fallbacks on broken pycrypto

[Zac Medico]

On 02/28/2017 11:34 PM, Michał Górny wrote:
> The pycrypto override used the same variables as actual hash functions
> before determining whether its functions are useful. As a result, if
> pycrypto had a broken module and no hash function was generated,
> the possible previous implementation was replaced by None.
> ---
>  pym/portage/checksum.py | 12 ++--
>  1 file changed, 6 insertions(+), 6 deletions(-)
> 
> diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py
> index a46b820af..fc38417a7 100644
> --- a/pym/portage/checksum.py
> +++ b/pym/portage/checksum.py
> @@ -105,14 +105,14 @@ except ImportError:
>  # is broken somehow.
>  try:
>   from Crypto.Hash import SHA256, RIPEMD
> - sha256hash = getattr(SHA256, 'new', None)
> - if sha256hash is not None:
> + sha256hash_ = getattr(SHA256, 'new', None)
> + if sha256hash_ is not None:
>   sha256hash = _generate_hash_function("SHA256",
> - sha256hash, origin="pycrypto")
> - rmd160hash = getattr(RIPEMD, 'new', None)
> - if rmd160hash is not None:
> + sha256hash_, origin="pycrypto")
> + rmd160hash_ = getattr(RIPEMD, 'new', None)
> + if rmd160hash_ is not None:
>   rmd160hash = _generate_hash_function("RMD160",
> - rmd160hash, origin="pycrypto")
> + rmd160hash_, origin="pycrypto")
>  except ImportError:
>   pass
>  
> 

Looks good.
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH 2/2] checksum: Add pycryptodome fallbacks for SHA3 and BLAKE2

[Zac Medico]

On 02/28/2017 11:34 PM, Michał Górny wrote:
> ---
>  pym/portage/checksum.py | 22 ++
>  1 file changed, 22 insertions(+)
> 
> diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py
> index fc38417a7..042a0a745 100644
> --- a/pym/portage/checksum.py
> +++ b/pym/portage/checksum.py
> @@ -116,6 +116,28 @@ try:
>  except ImportError:
>   pass
>  
> +try:
> + # added in pycryptodome
> + from Crypto.Hash import BLAKE2b, BLAKE2s, SHA3_256, SHA3_512
> + blake2bhash_ = getattr(BLAKE2b, 'new', None)
> + if blake2bhash_ is not None:
> + blake2bhash = _generate_hash_function("BLAKE2B",
> + blake2bhash_, origin="pycrypto")
> + blake2shash_ = getattr(BLAKE2s, 'new', None)
> + if blake2shash_ is not None:
> + blake2shash = _generate_hash_function("BLAKE2S",
> + blake2shash_, origin="pycrypto")
> + sha3_256hash_ = getattr(SHA3_256, 'new', None)
> + if sha3_256hash_ is not None:
> + sha3_256hash = _generate_hash_function("SHA3_256",
> + sha3_256hash_, origin="pycrypto")
> + sha3_512hash_ = getattr(SHA3_512, 'new', None)
> + if sha3_512hash_ is not None:
> + sha3_512hash = _generate_hash_function("SHA3_512",
> + sha3_512hash_, origin="pycrypto")
> +except ImportError:
> + pass
> +
>  # Use hashlib from python-2.5 if available and prefer it over pycrypto and 
> internal fallbacks.
>  # Need special handling for RMD160/WHIRLPOOL as they may not always be 
> provided by hashlib.
>  try:
> 

Looks good.
-- 
Thanks,
Zac

[gentoo-portage-dev] Re: [PATCH] GitSync: fix spurious "sync-depth is deprecated" messages (bug 610708)

[Zac Medico]

On 02/24/2017 04:02 PM, Zac Medico wrote:
> The CheckGitConfig._check_depth method must not set the sync_depth
> attribute, or else it will trigger "sync-depth is deprecated" messages.
> 
> Fixes: b3f6297a791a ("repos.conf: rename sync-depth option to clone-depth")
> X-Gentoo-Bug: 610708
> X-Gentoo-Bug-Url: https://bugs.gentoo.org/show_bug.cgi?id=610708

Merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=58e8b280794af7396115ea76747f0cc5fc5ab013
-- 
Thanks,
Zac

[gentoo-portage-dev] [PATCH] GitSync: fix spurious "sync-depth is deprecated" messages (bug 610708)

[Zac Medico]

The CheckGitConfig._check_depth method must not set the sync_depth
attribute, or else it will trigger "sync-depth is deprecated" messages.

Fixes: b3f6297a791a ("repos.conf: rename sync-depth option to clone-depth")
X-Gentoo-Bug: 610708
X-Gentoo-Bug-Url: https://bugs.gentoo.org/show_bug.cgi?id=610708
---
 pym/portage/sync/modules/git/__init__.py | 4 
 pym/portage/sync/modules/git/git.py  | 9 +++--
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/pym/portage/sync/modules/git/__init__.py 
b/pym/portage/sync/modules/git/__init__.py
index 2df60e3..60b7395 100644
--- a/pym/portage/sync/modules/git/__init__.py
+++ b/pym/portage/sync/modules/git/__init__.py
@@ -21,8 +21,6 @@ class CheckGitConfig(CheckSyncConfig):
 
def _check_depth(self, attr):
d = getattr(self.repo, attr)
-   # default
-   setattr(self.repo, attr, 1)
 
if d is not None:
try:
@@ -33,8 +31,6 @@ class CheckGitConfig(CheckSyncConfig):
% (attr.replace('_', '-'), d),
level=self.logger.ERROR, noiselevel=-1)
else:
-   if d == 0:
-   d = None
setattr(self.repo, attr, d)
 
 
diff --git a/pym/portage/sync/modules/git/git.py 
b/pym/portage/sync/modules/git/git.py
index d5400d5..d432886 100644
--- a/pym/portage/sync/modules/git/git.py
+++ b/pym/portage/sync/modules/git/git.py
@@ -53,9 +53,14 @@ class GitSync(NewBase):
if self.settings.get("PORTAGE_QUIET") == "1":
git_cmd_opts += " --quiet"
if self.repo.clone_depth is not None:
-   git_cmd_opts += " --depth %d" % self.repo.clone_depth
+   if self.repo.clone_depth != 0:
+   git_cmd_opts += " --depth %d" % 
self.repo.clone_depth
elif self.repo.sync_depth is not None:
-   git_cmd_opts += " --depth %d" % self.repo.sync_depth
+   if self.repo.sync_depth != 0:
+   git_cmd_opts += " --depth %d" % 
self.repo.sync_depth
+   else:
+   # default
+   git_cmd_opts += " --depth 1"
if 
self.repo.module_specific_options.get('sync-git-clone-extra-opts'):
git_cmd_opts += " %s" % 
self.repo.module_specific_options['sync-git-clone-extra-opts']
git_cmd = "%s clone%s %s ." % (self.bin_command, git_cmd_opts,
-- 
2.10.2

Re: [gentoo-portage-dev] [PATCH] grabfile_package: support -* in profile "packages" files (bug 610670)

[Zac Medico]

On 02/23/2017 09:03 AM, Joakim Tjernlund wrote:
> On Thu, 2017-02-23 at 07:52 -0800, Brian Dolbec wrote:
>> On Thu, 23 Feb 2017 11:53:15 +
>> Joakim Tjernlund <joakim.tjernl...@infinera.com> wrote:
>>
>>> On Thu, 2017-02-23 at 02:52 -0800, Zac Medico wrote:
>>>> Support -* in order to make it easier to create profiles for
>>>> minimal systems (especially those built entirely from binary
>>>> packages).  
>>>
>>> Would be nice, but I don't get what the "packages" file is?
>>
>>
>> That would be the 'packages' file (list of required packages)that are
>> required for that specific profile.   This patch would allow to
>> override that packages file  to build an image that only contained the
>> minimum runtime pkgs required to perform the tasks it is suppose to.
>> The idea being you would not need gcc, automake, ... or even portage or
>> possibly python.  The built image could of course be considered more
>> secure not having a compiler, etc... not to mention smaller.
> 
> Many of the packages files in /usr/portage/profiles already contain -* so
> I guess this must be another packages file?
> in /etc/portage or possibly /usr/portage/packges/ ?
> 
>   Jocke
> 

Here's an example:

https://gitweb.gentoo.org/repo/gentoo.git/tree/profiles/base/packages

Files like that define the @system set.
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH] grabfile_package: support -* in profile "packages" files (bug 610670)

[Zac Medico]

On 02/23/2017 07:52 AM, Brian Dolbec wrote:
> On Thu, 23 Feb 2017 11:53:15 +
> Joakim Tjernlund <joakim.tjernl...@infinera.com> wrote:
> 
>> On Thu, 2017-02-23 at 02:52 -0800, Zac Medico wrote:
>>> Support -* in order to make it easier to create profiles for
>>> minimal systems (especially those built entirely from binary
>>> packages).  
>>
>> Would be nice, but I don't get what the "packages" file is?
> 
> 
> That would be the 'packages' file (list of required packages)that are
> required for that specific profile.   This patch would allow to
> override that packages file  to build an image that only contained the
> minimum runtime pkgs required to perform the tasks it is suppose to.
> The idea being you would not need gcc, automake, ... or even portage or
> possibly python.  The built image could of course be considered more
> secure not having a compiler, etc... not to mention smaller.
> 
> 
> Zac, looks fine to me.

Thanks, merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=f4b0714010f1237280fd48dcb65989679917d20a
-- 
Thanks,
Zac

[gentoo-portage-dev] [PATCH] grabfile_package: support -* in profile "packages" files (bug 610670)

[Zac Medico]

Support -* in order to make it easier to create profiles for
minimal systems (especially those built entirely from binary
packages).

X-Gentoo-Bug: 610670
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=610670
---
 pym/portage/util/__init__.py | 9 -
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/pym/portage/util/__init__.py b/pym/portage/util/__init__.py
index c2c871f..45710ba 100644
--- a/pym/portage/util/__init__.py
+++ b/pym/portage/util/__init__.py
@@ -478,13 +478,20 @@ def grabfile_package(myfilename, compatlevel=0, 
recursive=0,
eapi = read_corresponding_eapi_file(
myfilename, default=eapi_default)
mybasename = os.path.basename(myfilename)
+   is_packages_file = mybasename == 'packages'
atoms = []
for pkg, source_file in pkgs:
pkg_orig = pkg
# for packages and package.mask files
if pkg[:1] == "-":
+   if is_packages_file and pkg == '-*':
+   if remember_source_file:
+   atoms.append((pkg, source_file))
+   else:
+   atoms.append(pkg)
+   continue
pkg = pkg[1:]
-   if pkg[:1] == '*' and mybasename == 'packages':
+   if pkg[:1] == '*' and is_packages_file:
pkg = pkg[1:]
try:
pkg = Atom(pkg, allow_wildcard=allow_wildcard,
-- 
2.10.2

Re: [gentoo-portage-dev] [PATCH] etc-update: fix hang when using_editor is set, bug #544440

[Zac Medico]

On 02/22/2017 11:10 AM, Fabian Groffen wrote:
> Don't try to use an interactive editor to obtain the difference between
> files.  Fall back to plain `diff -q` in this case.
> ---
>  bin/etc-update | 19 +++
>  1 file changed, 15 insertions(+), 4 deletions(-)
> 
> diff --git a/bin/etc-update b/bin/etc-update
> index e0f7224dc..ea69f1478 100755
> --- a/bin/etc-update
> +++ b/bin/etc-update
> @@ -649,10 +649,21 @@ do_distconf() {
>   elif [[ -L ${efile} || -L ${file} ]] ; then
>   # not the same file types
>   continue
> - elif diff_command "${file}" "${efile}" &> /dev/null; then
> - # replace identical copy
> - mv "${file}" "${efile}"
> - break
> + else
> + local ret=
> + if [[ ${using_editor} == 0 ]] ; then
> + diff_command "${file}" "${efile}" &> /dev/null
> + ret=$?
> + else
> + # fall back to plain diff
> + diff -q "${file}" "${efile}" &> /dev/null
> + ret=$?
> + fi
> + if [[ ${ret} == 0 ]] ; then
> + # replace identical copy
> + mv "${file}" "${efile}"
> + break
> + fi
>   fi
>   done
>  }
> 

Looks good. Merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=246373bbe52c55e912381d8555cceb70db0ec41b
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH] repos.conf: rename sync-depth option to clone-depth

[Zac Medico]

On 02/21/2017 07:38 PM, Brian Dolbec wrote:
> hmm, I might be out of sync, but if this isn't merged already, looks
> fine.

Thanks, merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=b3f6297a791a748a4bca370283705576568a20c2
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH] repoman: use regular expression to detect line continuations

[Zac Medico]

On 02/21/2017 07:29 PM, Brian Dolbec wrote:
> Code seems fine to me, I trust you ;)

Thanks, merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=3e82d2e09f124f3b77d1b913c945b93341ee4053
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH 3/3] repoman: Deprecate confutils.eclass

[Zac Medico]

This whole series looks good.
-- 
Thanks,
Zac

[gentoo-portage-dev] [PATCH] repoman: use regular expression to detect line continuations

[Zac Medico]

Use a regular expression to detect line continuations, instead
of the unicode_escape codec, since the unicode_escape codec is
not really intended to be used this way.

This solves an issue with python3.6, where a DeprecationWarning
is triggered by ebuilds containing escape sequences, like this
warning triggered by a sed expression in the dev-db/sqlite
ebuilds:

DeprecationWarning: invalid escape sequence '\['
---
 repoman/pym/repoman/modules/scan/ebuild/checks.py | 28 +++
 1 file changed, 8 insertions(+), 20 deletions(-)

diff --git a/repoman/pym/repoman/modules/scan/ebuild/checks.py 
b/repoman/pym/repoman/modules/scan/ebuild/checks.py
index 15e2251..d21bf0c 100644
--- a/repoman/pym/repoman/modules/scan/ebuild/checks.py
+++ b/repoman/pym/repoman/modules/scan/ebuild/checks.py
@@ -8,8 +8,8 @@ and correctness of an ebuild."""
 
 from __future__ import unicode_literals
 
-import codecs
 from itertools import chain
+import operator
 import re
 import time
 
@@ -923,11 +923,10 @@ def checks_init(experimental_inherit=False):
 
 _here_doc_re = re.compile(r'.*<<[-]?(\w+)\s*(>\s*\S+\s*)?$')
 _ignore_comment_re = re.compile(r'^\s*#')
+_continuation_re = re.compile(r'(\\)*$')
 
 
 def run_checks(contents, pkg):
-   unicode_escape_codec = codecs.lookup('unicode_escape')
-   unicode_escape = lambda x: unicode_escape_codec.decode(x)[0]
if _constant_checks is None:
checks_init()
checks = _constant_checks
@@ -957,32 +956,21 @@ def run_checks(contents, pkg):
#   cow
# This will merge these lines like so:
#   inherit foo bar moo cow
-   try:
-   # A normal line will end in the two bytes: <\> <\n>.  
So decoding
-   # that will result in python thinking the <\n> is being 
escaped
-   # and eat the single <\> which makes it hard for us to 
detect.
-   # Instead, strip the newline (which we know all lines 
have), and
-   # append a <0>.  Then when python escapes it, if the 
line ended
-   # in a <\>, we'll end up with a <\0> marker to key off 
of.  This
-   # shouldn't be a problem with any valid ebuild ...
-   line_escaped = unicode_escape(line.rstrip('\n') + '0')
-   except SystemExit:
-   raise
-   except:
-   # Who knows what kind of crazy crap an ebuild will have
-   # in it -- don't allow it to kill us.
-   line_escaped = line
+   # A line ending with an even number of backslashes does not 
count,
+   # because the last backslash is escaped. Therefore, search for 
an
+   # odd number of backslashes.
+   line_escaped = 
operator.sub(*_continuation_re.search(line).span()) % 2 == 1
if multiline:
# Chop off the \ and \n bytes from the previous line.
multiline = multiline[:-2] + line
-   if not line_escaped.endswith('\0'):
+   if not line_escaped:
line = multiline
num = multinum
multiline = None
else:
continue
else:
-   if line_escaped.endswith('\0'):
+   if line_escaped:
multinum = num
multiline = line
continue
-- 
2.10.2

Re: [gentoo-portage-dev] [PATCH] emerge: sync given repos even if auto-sync is false

[Zac Medico]

On 02/20/2017 09:53 AM, Alexandru Elisei wrote:
> ---
> #
> # This will be followed by a series of patches to ignore repos with sync-type
> # unset, as per the man page, fail when sync-uri is not specified and an 
> update
> # to the emerge.1 man page to reflect the change in behavior.
> #
>  pym/_emerge/actions.py  |  8 +--
>  pym/portage/emaint/modules/sync/sync.py | 40 
> -
>  2 files changed, 10 insertions(+), 38 deletions(-)
> 
> diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py
> index 71e362e..cc0269d 100644
> --- a/pym/_emerge/actions.py
> +++ b/pym/_emerge/actions.py
> @@ -1999,9 +1999,13 @@ def action_sync(emerge_config, 
> trees=DeprecationWarning,
>   action=action, args=[], trees=trees, opts=opts)
>  
>   syncer = SyncRepos(emerge_config)
> -
>   return_messages = "--quiet" not in emerge_config.opts
> - success, msgs = syncer.auto_sync(options={'return-messages': 
> return_messages})
> + options = {'return-messages' : return_messages}
> + if emerge_config.args:
> + options['repo'] = emerge_config.args
> + success, msgs = syncer.repo(options=options)
> + else:
> + success, msgs = syncer.auto_sync(options=options)
>   if return_messages:
>   print_results(msgs)
>  

The above change works fine by itself, so I've merged it in this commit:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=75f0936b8866b143643abcd6c302cd72fc71eef3

The changes to sync-type and sync-uri validation can go in separately.
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH V2] sync.py: extend the checks in _get_repos() and fail when necessary (bugs 567478, 576272, 601054)

[Zac Medico]

On 02/19/2017 01:02 PM, Alexandru Elisei wrote:
>   if auto_sync_only:
> - return self._filter_auto(selected_repos)
> - return selected_repos
> + selected_repos = self._filter_auto(selected_repos)
> + #print("_get_repos(), selected =", selected_repos)
> + if emerge_repos:
> + skipped_repos = set(emerge_repos) - set(selected_repos)
> + if skipped_repos:
> + msgs.append(warn(" * ") + "auto-sync is 
> disabled for repo(s): %s" %
> + " ".join(repo.name for repo in 
> skipped_repos) + "\n")
> + return (False, selected_repos, msgs)
> + return (True, selected_repos, msgs)

I feel like it should be possible to use emerge_repos arguments to sync
repos that have auto-sync disabled, but that seems to be disallowed
here, because _filter_auto will have filtered out those repos that have
auto-sync disabled.
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH V2] sync.py: extend the checks in _get_repos() and fail when necessary (bugs 567478, 576272, 601054)

[Zac Medico]

On 02/19/2017 01:02 PM, Alexandru Elisei wrote:
> + valid_repos = []
> + missing_sync_type = []
> + for repo in selected_repos:
> + if repo.sync_type is None:
> + missing_sync_type.append(repo.name)
> + else:
> + valid_repos.append(repo)
> + if missing_sync_type:
> + msgs.append(warn(" * ") + "Missing sync-type for 
> repo(s): %s" %
> + " ".join(missing_sync_type) + "\n")
> + return (False, valid_repos, msgs)
> +
>   if auto_sync_only:
> - return self._filter_auto(selected_repos)
> - return selected_repos
> + selected_repos = self._filter_auto(selected_repos)

Do we support local repos that don't have a sync-uri? If so, what
sync-type should be set for such a repo? Is it also necessary to set
auto-sync = no, in order to avoid an error for such a repo?
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH] sync.py: extend the checks in _get_repos() and fail when necessary (bugs 567478, 576272, 601054)

[Zac Medico]

On Mon, Feb 13, 2017 at 2:14 AM, Alexandru Elisei
 wrote:
> @@ -204,14 +231,7 @@ class SyncRepos(object):
> k = "--" + k.replace("_", "-")
> self.emerge_config.opts[k] = v
>
> -   selected_repos = [repo for repo in selected_repos if 
> repo.sync_type
> is not None]

The mail client wrapped the line here, which causes git am to fail.

[gentoo-portage-dev] [PATCH] repos.conf: rename sync-depth option to clone-depth

[Zac Medico]

Since sync-depth actually controls clone depth, rename it
to clone-depth, and show a warning message when the sync-depth
option has been specified:

UserWarning: repos.conf: sync-depth is deprecated, use clone-depth instead

This makes it feasible to change the meaning of sync-depth in
the future (it could be used to control git pull depth).

X-Gentoo-Bug: 552814
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=552814
---
 man/portage.5|  7 +--
 pym/portage/repository/config.py | 16 
 pym/portage/sync/modules/git/__init__.py | 14 +-
 pym/portage/sync/modules/git/git.py  |  4 +++-
 4 files changed, 29 insertions(+), 12 deletions(-)

diff --git a/man/portage.5 b/man/portage.5
index 415817a..82e979e 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -925,6 +925,10 @@ Valid values: yes, no, true, false.
 If unset, the repo will be treated as set
 yes, true.
 .TP
+.B clone\-depth
+Specifies clone depth to use for DVCS repositories. Defaults to 1 (only
+the newest commit). If set to 0, the depth is unlimited.
+.TP
 .B eclass\-overrides
 Makes given repository inherit eclasses from specified repositories.
 .br
@@ -972,8 +976,7 @@ Valid values: true, false.
 Specifies CVS repository.
 .TP
 .B sync\-depth
-Specifies clone depth to use for DVCS repositories. Defaults to 1 (only
-the newest commit). If set to 0, the depth is unlimited.
+This is a deprecated alias for the \fBclone\-depth\fR option.
 .TP
 .B sync\-git\-clone\-extra\-opts
 Extra options to give to git when cloning repository (git clone).
diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py
index 67c717d..b65ed97 100644
--- a/pym/portage/repository/config.py
+++ b/pym/portage/repository/config.py
@@ -75,7 +75,8 @@ class RepoConfig(object):
"""Stores config of one repository"""
 
__slots__ = ('aliases', 'allow_missing_manifest', 
'allow_provide_virtual',
-   'auto_sync', 'cache_formats', 'create_manifest', 
'disable_manifest',
+   'auto_sync', 'cache_formats', 'clone_depth',
+   'create_manifest', 'disable_manifest',
'eapi', 'eclass_db', 'eclass_locations', 'eclass_overrides',
'find_invalid_path_char', 'force', 'format', 'local_config', 
'location',
'main_repo', 'manifest_hashes', 'masters', 'missing_repo_name',
@@ -168,7 +169,13 @@ class RepoConfig(object):
auto_sync = auto_sync.strip().lower()
self.auto_sync = auto_sync
 
+   self.clone_depth = repo_opts.get('clone-depth')
self.sync_depth = repo_opts.get('sync-depth')
+
+   if self.sync_depth is not None:
+   warnings.warn(_("repos.conf: sync-depth is deprecated,"
+   " use clone-depth instead"))
+
self.sync_hooks_only_on_change = repo_opts.get(
'sync-hooks-only-on-change', 'false').lower() == 'true'
 
@@ -505,7 +512,8 @@ class RepoConfigLoader(object):
if repos_conf_opts is not None:
# Selectively copy only the 
attributes which
# repos.conf is allowed to 
override.
-   for k in ('aliases', 
'auto_sync', 'eclass_overrides',
+   for k in ('aliases', 
'auto_sync',
+   'clone_depth', 
'eclass_overrides',
'force', 'masters', 
'priority', 'strict_misc_digests',
'sync_depth', 
'sync_hooks_only_on_change',
'sync_type', 
'sync_umask', 'sync_uri', 'sync_user',
@@ -929,8 +937,8 @@ class RepoConfigLoader(object):
 
def config_string(self):
bool_keys = ("strict_misc_digests",)
-   str_or_int_keys = ("auto_sync", "format", "location",
-   "main_repo", "priority",
+   str_or_int_keys = ("auto_sync", "clone_depth", "format", 
"location",
+   "main_repo", "priority", "sync_depth",
"sync_type", "sync_umask", "sync_uri", 'sync_user')
str_tuple_keys = ("aliases", "eclass_overrides", "force")
repo_config_tuple_keys = ("masters",)
diff --git a/pym/portage/sync/modules/git/__init__.py 
b/pym/portage/sync/modules/git/__init__.py
index d5eb5c6..2df60e3 100644
--- a/pym/portage/sync/modules/git/__init__.py
+++ b/pym/portage/sync/modules/git/__init__.py
@@ -16,22 +16,26 @@ class CheckGitConfig(CheckSyncConfig):
self.checks.append('check_depth')
 
def check_depth(self):
-   d = self.repo.sync_depth
+   for attr in ('clone_depth', 

Re: [gentoo-portage-dev] [PATCH] emerge: make emerge --sync print messages from SyncRepos.auto_sync()

[Zac Medico]

On 02/04/2017 09:49 AM, Alexandru Elisei wrote:
> Emerge will only display messages if --quiet is not set.
> 
> The word 'emaint' has been removed from two messages to avoid confusion.
> ---
>  pym/_emerge/actions.py  | 7 +--
>  pym/portage/emaint/modules/sync/sync.py | 7 +++
>  2 files changed, 8 insertions(+), 6 deletions(-)

Looks good, thanks! Merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=f57ae38b978069e50c2b1bf2d41bffed23ecc11d

-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH] repoman: Check for empty files in filesdir.

[Zac Medico]

On 02/12/2017 08:48 AM, Ulrich Mueller wrote:
> This checks for files with zero size in filesdir. The QA script at
> https://qa-reports.gentoo.org/output/find-binary-files.txt reports a
> couple of them which at least in part are blunders.
> 
> Should be harmless enough not to need a discussion about policy in
> gentoo-dev. Patch included below.
> 
> Ulrich

Thanks, merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=04e5f8dee2130901386f4f1b65328bbf0b8104c1
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH 08/18] Add tentative support for EAPI6 eapply_user function

[Zac Medico]

On 12/01/2014 01:28 PM, Michał Górny wrote:
> Add support for the user patch applying function.
> ---
>  bin/eapi.sh  |  4 
>  bin/phase-helpers.sh | 22 ++
>  2 files changed, 26 insertions(+)
> 
> diff --git a/bin/eapi.sh b/bin/eapi.sh
> index 8bb..6e78750 100644
> --- a/bin/eapi.sh
> +++ b/bin/eapi.sh
> @@ -76,6 +76,10 @@ ___eapi_has_eapply() {
>   [[ ! ${1-${EAPI}} =~ 
> ^(0|1|2|3|4|4-python|4-slot-abi|5|5-hdepend|5-progress)$ ]]
>  }
>  
> +___eapi_has_eapply_user() {
> + [[ ! ${1-${EAPI}} =~ 
> ^(0|1|2|3|4|4-python|4-slot-abi|5|5-hdepend|5-progress)$ ]]
> +}
> +
>  ___eapi_has_master_repositories() {
>   [[ ${1-${EAPI}} =~ ^(5-progress)$ ]]
>  }
> diff --git a/bin/phase-helpers.sh b/bin/phase-helpers.sh
> index e9fbbb4..f4b64ee 100644
> --- a/bin/phase-helpers.sh
> +++ b/bin/phase-helpers.sh
> @@ -986,6 +986,28 @@ if ___eapi_has_eapply; then
>   }
>  fi
>  
> +if ___eapi_has_eapply_user; then
> + eapply_user() {
> + local basedir=${PORTAGE_CONFIGROOT%/}/etc/portage/patches
> +
> + local d applied
> + # possibilities:
> + # 1. ${CATEGORY}/${P}-${PR} (note: -r0 desired to avoid applying
> + #${P} twice)
> + # 2. ${CATEGORY}/${P}
> + # 3. ${CATEGORY}/${PN}
> + # all of the above may be optionally followed by a slot
> + for d in 
> "${basedir}"/${CATEGORY}/{${P}-${PR},${P},${PN}}{,:${SLOT%/*}}; do
> + if [[ -d ${d} ]]; then
> + eapply "${d}"
> + applied=1

I think it should break out of the loop here, like epatch_user does.

It doesn't make sense to apply more-specific patches before
less-specific patches, does it?

Maybe we can just treat this as a bug fix? Is anyone relying on the
multiple directory usage?

> + fi
> + done
> +
> + [[ -n ${applied} ]] && ewarn "User patches applied."
> + }
> +fi
> +
>  if ___eapi_has_master_repositories; then
>   master_repositories() {
>   local output repository=$1 retval
> 


-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH] sys-apps/portage: add native-extensions USE flag (bug 571444)

[Zac Medico]

On 02/01/2017 01:03 PM, Michał Górny wrote:
> W dniu 01.02.2017, śro o godzinie 09∶06 -0800, użytkownik Zac Medico
> napisał:
>> The native-extensions USE flag will enable building of the
>> libc bindings. This is not enabled by default because it does
>> not support cross compilation.
>>
>> X-Gentoo-bug: 571444
>> X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=571444
>> ---
>>  sys-apps/portage/metadata.xml| 1 +
>>  sys-apps/portage/portage-.ebuild | 7 ++-
>>  2 files changed, 7 insertions(+), 1 deletion(-)
>>
>> diff --git a/sys-apps/portage/metadata.xml b/sys-
>> apps/portage/metadata.xml
>> index e032ea5..882d3ba 100644
>> --- a/sys-apps/portage/metadata.xml
>> +++ b/sys-apps/portage/metadata.xml
>> @@ -12,6 +12,7 @@
>>
>>  Build html API documentation with
>> epydoc.
>>  Use inter-process communication between portage
>> and running ebuilds.
>> +Build native extensions. Cross-
>> compilation is not supported.
>>  Preserve extended attributes (filesystem-
>> stored metadata) when installing files. Usually only required for
>> hardened systems.
>>
>>  
>> diff --git a/sys-apps/portage/portage-.ebuild b/sys-
>> apps/portage/portage-.ebuild
>> index 6a6f515..981db26 100644
>> --- a/sys-apps/portage/portage-.ebuild
>> +++ b/sys-apps/portage/portage-.ebuild
>> @@ -19,7 +19,7 @@ HOMEPAGE="https://wiki.gentoo.org/wiki/Project:Porta
>> ge"
>>  LICENSE="GPL-2"
>>  KEYWORDS=""
>>  SLOT="0"
>> -IUSE="build doc epydoc +ipc linguas_ru selinux xattr"
>> +IUSE="build doc epydoc +ipc linguas_ru native-extensions selinux
>> xattr"
> 
> Wouldn't it be better to enable it by default?

Yeah, that should be fine. We may get a few duplicates of bug 594744
reported, but that's no reason not to enable it by default.
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH] sys-apps/portage: add native-extensions USE flag (bug 571444)

[Zac Medico]

On 02/01/2017 09:57 AM, Brian Dolbec wrote:
> On Wed,  1 Feb 2017 09:06:37 -0800
> Zac Medico <zmed...@gentoo.org> wrote:
> 
>> The native-extensions USE flag will enable building of the
>> libc bindings. This is not enabled by default because it does
>> not support cross compilation.
>>
>> X-Gentoo-bug: 571444
>> X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=571444
>> ---
>>  sys-apps/portage/metadata.xml| 1 +
>>  sys-apps/portage/portage-.ebuild | 7 ++-
>>  2 files changed, 7 insertions(+), 1 deletion(-)
>>
>> diff --git a/sys-apps/portage/metadata.xml
>> b/sys-apps/portage/metadata.xml index e032ea5..882d3ba 100644
>> --- a/sys-apps/portage/metadata.xml
>> +++ b/sys-apps/portage/metadata.xml
>> @@ -12,6 +12,7 @@
>>
>>  Build html API documentation with
>> epydoc. Use inter-process communication
>> between portage and running ebuilds.
>> +Build native extensions.
>> Cross-compilation is not supported. > name="xattr">Preserve extended attributes (filesystem-stored
>> metadata) when installing files. Usually only required for hardened
>> systems.   diff --git
>> a/sys-apps/portage/portage-.ebuild
>> b/sys-apps/portage/portage-.ebuild index 6a6f515..981db26 100644
>> --- a/sys-apps/portage/portage-.ebuild +++
>> b/sys-apps/portage/portage-.ebuild @@ -19,7 +19,7 @@
>> HOMEPAGE="https://wiki.gentoo.org/wiki/Project:Portage;
>> LICENSE="GPL-2" KEYWORDS=""
>>  SLOT="0"
>> -IUSE="build doc epydoc +ipc linguas_ru selinux xattr"
>> +IUSE="build doc epydoc +ipc linguas_ru native-extensions selinux
>> xattr" 
>>  DEPEND="!build? ( $(python_gen_impl_dep 'ssl(+)') )
>>  >=app-arch/tar-1.27  
>> @@ -85,6 +85,11 @@ pkg_setup() {
>>  python_prepare_all() {
>>  distutils-r1_python_prepare_all
>>  
>> +if use native-extensions; then
>> +printf "[build_ext]\nportage-ext-modules=true" >> \
>> +setup.cfg || die
>> +fi
>> +
>>  if ! use ipc ; then
>>  einfo "Disabling ipc..."
>>  sed -e "s:_enable_ipc_daemon =
>> True:_enable_ipc_daemon = False:" \
> 
> looks good :)
> 

Thanks, pushed:

https://gitweb.gentoo.org/repo/gentoo.git/commit/?id=dbe0bd44461aff3097fde6c64147f1a61e30816d
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH] sync.py: set returncode when task failed in SyncScheduler._task_exit()

[Zac Medico]

On 01/26/2017 11:00 AM, Alexandru Elisei wrote:
> If task.returncode is not os.EX_OK then returncode isn't defined when
> trying to append the (repo, returncode) tuple to self.retvals. This will
> raise an UnboundLocalError exception.
> 
> This reverts the changes to the function made by commit f143e58.
> ---
>  pym/portage/emaint/modules/sync/sync.py | 1 +
>  1 file changed, 1 insertion(+)
> 
> diff --git a/pym/portage/emaint/modules/sync/sync.py
> b/pym/portage/emaint/modules/sync/sync.py
> index b190b3c..076297a 100644
> --- a/pym/portage/emaint/modules/sync/sync.py
> +++ b/pym/portage/emaint/modules/sync/sync.py
> @@ -362,6 +362,7 @@ class SyncScheduler(AsyncScheduler):
>   # that hooks will be called in a backward-compatible manner
>   # even if all sync tasks have failed.
>   hooks_enabled = True
> + returncode = task.returncode
>   if task.returncode == os.EX_OK:
>   returncode, message, updatecache_flg, hooks_enabled = 
> task.result
>   if message:
> 

Thanks, applied:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=9305bccfb29807f54a9dc514d3aea70639eb2c49
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH] emerge: fix error handling for clean_logs

[Zac Medico]

On 01/25/2017 07:16 PM, Mike Gilbert wrote:
> Commit f143e58dd changed the return value of CleanLogs.clean() to a
> tuple (returncode, messages).
> 
> X-Gentoo-Bug: 607236
> X-Gentoo-Bug-URL: https://bugs.gentoo.org/607236
> ---
>  pym/_emerge/post_emerge.py | 6 +++---
>  1 file changed, 3 insertions(+), 3 deletions(-)
> 
> diff --git a/pym/_emerge/post_emerge.py b/pym/_emerge/post_emerge.py
> index 0cb533cf8..7e6063c52 100644
> --- a/pym/_emerge/post_emerge.py
> +++ b/pym/_emerge/post_emerge.py
> @@ -29,10 +29,10 @@ def clean_logs(settings):
>   return
>  
>   cleanlogs = CleanLogs()
> - errors = cleanlogs.clean(settings=settings)
> - if errors:
> + returncode, msgs = cleanlogs.clean(settings=settings)
> + if not returncode:
>   out = portage.output.EOutput()
> - for msg in errors:
> + for msg in msgs:
>   out.eerror(msg)
>  
>  def display_news_notification(root_config, myopts):
> 

I like Fixes: tags [1]. Otherwise, looks good.

[1]
https://kernel.org/doc/html/latest/process/submitting-patches.html#using-reported-by-tested-by-reviewed-by-suggested-by-and-fixes
-- 
Thanks,
Zac

[gentoo-portage-dev] [PATCH] spawn: instantiate userpriv_groups before fork (bug 582098)

[Zac Medico]

Make spawn force instantiation of portage.data.userpriv_groups in the
main process, in order to avoid redundant instantiation in child
processes. This mitigates the impact of "Bad file descriptor" errors
reported in bug 582098, by avoiding redundant instantiation of
userpriv_groups in child processes. It may even solve the problem
completely, if the "Bad file descriptor" errors are triggered by
interactions between garbage collection and the file descriptor
operations performed in the _exec function by the _setup_pipes call.

X-Gentoo-Bug: 582098
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=582098
---
 pym/portage/process.py | 4 
 1 file changed, 4 insertions(+)

diff --git a/pym/portage/process.py b/pym/portage/process.py
index ba41ea8..bc4efb5 100644
--- a/pym/portage/process.py
+++ b/pym/portage/process.py
@@ -305,6 +305,10 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, 
returnpid=False,
if unshare_net or unshare_ipc:
find_library("c")
 
+   # Force instantiation of portage.data.userpriv_groups before the
+   # fork, so that the result is cached in the main process.
+   bool(groups)
+
parent_pid = os.getpid()
pid = None
try:
-- 
2.10.2

Re: [gentoo-portage-dev] Re: [PATCH V2] emerge: check correctly for --sync success (bug 606588)

[Zac Medico]

On 01/20/2017 11:23 AM, Alexandru Elisei wrote:
> The class SyncRepos will keep a list of (repo, returncode) tuples.
> If sync is unsuccessful emerge will use this list to return the first
> failed repository return code.

I've pushed a minimal fix:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=23f239134b0290cbe0435e13d8f1b1757c6d8524

It just returns 1 on failure. Would it really be useful to preserve the
returncode from the first failed repo? Probably not.
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH v2] repoman: always add a newline if the message body is one line

[Zac Medico]

On 01/14/2017 11:00 AM, Mike Gilbert wrote:
> The second line in a git commit message is supposed to be blank.
> Also, use a regex to more exactly match a Field: value line.
> 
> Before:
>   dev-libs/libfoo: fix another bug
>   Package-Manager: Portage-2.3.3, Repoman-2.3.1
> 
> After:
>   dev-libs/libfoo: fix another bug
> 
>   Package-Manager: Portage-2.3.3, Repoman-2.3.1
> ---
>  repoman/pym/repoman/actions.py | 6 --
>  1 file changed, 4 insertions(+), 2 deletions(-)
> 
> diff --git a/repoman/pym/repoman/actions.py b/repoman/pym/repoman/actions.py
> index 58b00d494..6b0c6459d 100644
> --- a/repoman/pym/repoman/actions.py
> +++ b/repoman/pym/repoman/actions.py
> @@ -6,6 +6,7 @@ import errno
>  import io
>  import logging
>  import platform
> +import re
>  import signal
>  import sys
>  import tempfile
> @@ -128,8 +129,9 @@ class Actions(object):
>   myupdates, mymanifests, myremoved, 
> mychanged, myautoadd,
>   mynew, commitmessage)
>  
> - lastline = commitmessage.splitlines()[-1]
> - if not ':' in lastline:
> + lines = commitmessage.splitlines()
> + lastline = lines[-1]
> + if len(lines) == 1 or re.match(r'^\S+:\s', lastline) is None:
>   commitmessage += '\n'
>  
>   commit_footer = self.get_commit_footer()
> 

LGTM. Thanks!
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH] repoman: always add a newline if the message body is one line

[Zac Medico]

On 01/14/2017 08:58 AM, Mike Gilbert wrote:
> The second line in a git commit message is supposed to be blank.
> 
> Before:
>   dev-libs/libfoo: fix another bug
>   Package-Manager: Portage-2.3.3, Repoman-2.3.1
> 
> After:
>   dev-libs/libfoo: fix another bug
> 
>   Package-Manager: Portage-2.3.3, Repoman-2.3.1
> ---
>  repoman/pym/repoman/actions.py | 5 +++--
>  1 file changed, 3 insertions(+), 2 deletions(-)
> 
> diff --git a/repoman/pym/repoman/actions.py b/repoman/pym/repoman/actions.py
> index 58b00d494..82c76b2fb 100644
> --- a/repoman/pym/repoman/actions.py
> +++ b/repoman/pym/repoman/actions.py
> @@ -128,8 +128,9 @@ class Actions(object):
>   myupdates, mymanifests, myremoved, 
> mychanged, myautoadd,
>   mynew, commitmessage)
>  
> - lastline = commitmessage.splitlines()[-1]
> - if not ':' in lastline:
> + lines = commitmessage.splitlines()
> + lastline = lines[-1]
> + if len(lines) == 1 or not ':' in lastline:

Maybe a regular expression would be better, like this:

   if len(lines) == 1 or re.match(r'^\S+:\s', lastline) is None:


>   commitmessage += '\n'
>  
>   commit_footer = self.get_commit_footer()
> 


-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH] repoman: add HOMEPAGE.missingurischeme check

[Zac Medico]

On 01/09/2017 02:07 AM, Wim Muskee wrote:
> thx Zac, I would never have figured that out myself
> 
> the updated patch:
> 
> From c703875bebd82271921d7e94b7cd61936b87d32b Mon Sep 17 00:00:00 2001
> From: Wim Muskee >
> Date: Sat, 7 Jan 2017 12:02:41 +0100
> Subject: [PATCH] repoman: add HOMEPAGE.missingurischeme check

Thanks, merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=55dedaa865334543e51693838700dcf5e72754d2
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH] bin/socks5-server.py: convert address from bytes to str (bug 604474)

[Zac Medico]

On 01/12/2017 03:10 PM, Brian Dolbec wrote:
> On Thu, 12 Jan 2017 09:36:36 -0800
> Zac Medico <zmed...@gentoo.org> wrote:
> 
>> Use the idna codec to decode the destination address that is read
>> from the client. This fixes a "TypeError: must be str, not bytes"
>> exception raised from getaddrinfo with Python 3.4.5.
>>
>> X-Gentoo-Bug: 604474
>> X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=604474
>> ---
>>  bin/socks5-server.py | 5 +
>>  1 file changed, 5 insertions(+)
>>
>> diff --git a/bin/socks5-server.py b/bin/socks5-server.py
>> index cfe3ece..d46cf53 100644
>> --- a/bin/socks5-server.py
>> +++ b/bin/socks5-server.py
>> @@ -83,6 +83,11 @@ class Socks5Server(object):
>>  data = yield from
>> reader.readexactly(1) addr_len, = struct.unpack('!B', data)
>>  addr = yield from
>> reader.readexactly(addr_len)
>> +try:
>> +addr = addr.decode('idna')
>> +except UnicodeDecodeError:
>> +rpl = 0x04  # host
>> unreachable +
>>  elif atyp == 0x04:  # IPv6
>>  data = yield from
>> reader.readexactly(16) addr = socket.inet_ntop(socket.AF_INET6, data)
> 
> 
> looks good
> 

Thanks, pushed:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=0d32c30b1db848e9eac6fe2e216e6373846467a2
-- 
Thanks,
Zac

[gentoo-portage-dev] [PATCH] bin/socks5-server.py: convert address from bytes to str (bug 604474)

[Zac Medico]

Use the idna codec to decode the destination address that is read
from the client. This fixes a "TypeError: must be str, not bytes"
exception raised from getaddrinfo with Python 3.4.5.

X-Gentoo-Bug: 604474
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=604474
---
 bin/socks5-server.py | 5 +
 1 file changed, 5 insertions(+)

diff --git a/bin/socks5-server.py b/bin/socks5-server.py
index cfe3ece..d46cf53 100644
--- a/bin/socks5-server.py
+++ b/bin/socks5-server.py
@@ -83,6 +83,11 @@ class Socks5Server(object):
data = yield from reader.readexactly(1)
addr_len, = struct.unpack('!B', data)
addr = yield from reader.readexactly(addr_len)
+   try:
+   addr = addr.decode('idna')
+   except UnicodeDecodeError:
+   rpl = 0x04  # host unreachable
+
elif atyp == 0x04:  # IPv6
data = yield from reader.readexactly(16)
addr = socket.inet_ntop(socket.AF_INET6, data)
-- 
2.7.4

Re: [gentoo-portage-dev] [PATCH] repoman: add HOMEPAGE.missingurischeme check

[Zac Medico]

On 01/07/2017 05:32 AM, Michael Orlitzky wrote:
> On 01/07/2017 06:08 AM, Wim Muskee wrote:
>>
>> URISCHEME_RE = re.compile(r'^[a-z\-]+://')
>>
>> ...
>>
>> URISCHEME_RE.match(ebuild.metadata.get("HOMEPAGE")) is None:
>>
> 
> The PMS allows some weird stuff in HOMEPAGE:
> 
>   https://dev.gentoo.org/~ulm/pms/head/pms.html#x1-760008
> 
> Specifically,
> 
>   In addition, SRC_URI, HOMEPAGE, RESTRICT, PROPERTIES, LICENSE and
>   REQUIRED_USE use dependency-style specifications to specify their
>   values.
> 
> That means that something like,
> 
>   HOMEPAGE="branding? ( https://www.mozilla.org/ )
>!branding? ( https://www.gentoo.org/ )"
> 
> would be valid. It's a little crazy, but there it is.
> 
> If you can figure out a way to parse a dependency spec (this has to
> exist somewhere in repoman/portage), then you can run your check against
> the URLs at the leaf nodes. At that point, it should be relatively easy
> to update the regex to match the RFC =)
> 
>   https://tools.ietf.org/html/rfc3986#section-3.1

This will return a flat list:

portage.dep.use_reduce(ebuild.metadata["HOMEPAGE"], matchall=True,
flat=True)
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH] man/emaint.1: Add sync to synopsis and fix its in sync --auto

[Zac Medico]

On 01/07/2017 09:24 AM, Chris Mayo wrote:
> Signed-off-by: Chris Mayo 
> ---
>  man/emaint.1 | 4 ++--
>  1 file changed, 2 insertions(+), 2 deletions(-)
> 
> diff --git a/man/emaint.1 b/man/emaint.1
> index 24e4744..4617ef8 100644
> --- a/man/emaint.1
> +++ b/man/emaint.1
> @@ -5,7 +5,7 @@ emaint \- performs package management related system health 
> checks and maintenan
>  .BR emaint
>  [\fIoptions\fR]
>  [\fBall\fR | \fBbinhost\fR | \fBcleanresume\fR | \
> -\fBmerges\fR | \fBmovebin\fR | \fBmoveinst\fR | \fBworld\fR]
> +\fBmerges\fR | \fBmovebin\fR | \fBmoveinst\fR | \fBsync\fR | \fBworld\fR]
>  .SH DESCRIPTION
>  The emaint program provides a command line interface to package
>  management health checks and maintenance.
> @@ -85,7 +85,7 @@ deleted.
>  .SH OPTIONS sync command only
>  .TP
>  .B \-a, \-\-auto
> -Sync repositories which have its auto\-sync setting set yes, true.
> +Sync repositories which have their auto\-sync setting set yes, true.
>  .TP
>  .B \-A, \-\-allrepos
>  Sync all repositories which have a sync\-uri specified.
> 

Thanks, applied:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=e567328c17d7b03d9f773661d03673b76054570d
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH] _dep_check_composite_db: select highest in slot conflict (bug 554070)

[Zac Medico]

On 01/04/2017 02:28 PM, Brian Dolbec wrote:
> On Tue,  3 Jan 2017 17:13:25 -0800
> Zac Medico <zmed...@gentoo.org> wrote:
> 
>> Fix the _dep_check_composite_db._visible method to select the highest
>> version involved in a slot conflict, for consistency with the change
>> in ab07ac68fa1e04ed64e2e0f6c753ff169a32d517. The included unit test
>> fails without this fix.
>>
>> Fixes: ab07ac68fa1e ("depgraph: select highest version involved in
>> slot conflict (bug 554070)") X-Gentoo-Bug: 554070
>> X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=554070
>> ---
>>  pym/_emerge/depgraph.py|   8 +-
>>  .../resolver/test_imagemagick_graphicsmagick.py| 104
>> + 2 files changed, 110 insertions(+), 2
>> deletions(-) create mode 100644
>> pym/portage/tests/resolver/test_imagemagick_graphicsmagick.py
>>
> 
> 
> LGTM
> 

Thanks, merged:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=033477d6a4cc1adcabed966b8aa82b51b42af33c
-- 
Thanks,
Zac

[gentoo-portage-dev] [PATCH] _dep_check_composite_db: select highest in slot conflict (bug 554070)

[Zac Medico]

Fix the _dep_check_composite_db._visible method to select the highest
version involved in a slot conflict, for consistency with the change
in ab07ac68fa1e04ed64e2e0f6c753ff169a32d517. The included unit test
fails without this fix.

Fixes: ab07ac68fa1e ("depgraph: select highest version involved in slot 
conflict (bug 554070)")
X-Gentoo-Bug: 554070
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=554070
---
 pym/_emerge/depgraph.py|   8 +-
 .../resolver/test_imagemagick_graphicsmagick.py| 104 +
 2 files changed, 110 insertions(+), 2 deletions(-)
 create mode 100644 
pym/portage/tests/resolver/test_imagemagick_graphicsmagick.py

diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
index e298337..b387bfd 100644
--- a/pym/_emerge/depgraph.py
+++ b/pym/_emerge/depgraph.py
@@ -9130,8 +9130,12 @@ class _dep_check_composite_db(dbapi):
# into the same slot.
return True
 
-   in_graph = 
next(self._depgraph._dynamic_config._package_tracker.match(
-   self._root, pkg.slot_atom, installed=False), None)
+   # Use reversed iteration in order to get descending order here,
+   # so that the highest version involved in a slot conflict is
+   # selected (see bug 554070).
+   in_graph = next(reversed(list(
+   self._depgraph._dynamic_config._package_tracker.match(
+   self._root, pkg.slot_atom, installed=False))), None)
 
if in_graph is None:
# Mask choices for packages which are not the highest 
visible
diff --git a/pym/portage/tests/resolver/test_imagemagick_graphicsmagick.py 
b/pym/portage/tests/resolver/test_imagemagick_graphicsmagick.py
new file mode 100644
index 000..9e52249
--- /dev/null
+++ b/pym/portage/tests/resolver/test_imagemagick_graphicsmagick.py
@@ -0,0 +1,104 @@
+# Copyright 2016 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from portage.tests import TestCase
+from portage.tests.resolver.ResolverPlayground import (
+   ResolverPlayground,
+   ResolverPlaygroundTestCase,
+)
+
+class ImageMagickGraphicsMagickTestCase(TestCase):
+
+   def testImageMagickUpdate(self):
+
+   ebuilds = {
+   "media-gfx/imagemagick-6.9.7.0" : {
+   "EAPI": "6",
+   "SLOT": "0/6.9.7.0",
+   },
+
+   "media-gfx/imagemagick-6.9.6.6" : {
+   "EAPI": "6",
+   "SLOT": "0/6.9.6.6",
+   },
+
+   "media-gfx/inkscape-0.91-r3" : {
+   "EAPI": "6",
+   "DEPEND": "media-gfx/imagemagick:=",
+   "RDEPEND": "media-gfx/imagemagick:=",
+   },
+
+   "media-video/dvdrip-0.98.11-r3" : {
+   "EAPI": "6",
+   "DEPEND": "|| ( 
media-gfx/graphicsmagick[imagemagick] media-gfx/imagemagick )",
+   "RDEPEND": "|| ( 
media-gfx/graphicsmagick[imagemagick] media-gfx/imagemagick )",
+   },
+
+   "media-gfx/graphicsmagick-1.3.25" : {
+   "EAPI": "6",
+   "SLOT": "0/1.3",
+   "IUSE": "imagemagick",
+   "RDEPEND": "imagemagick? ( 
!media-gfx/imagemagick )",
+   },
+   }
+
+   installed = {
+   "media-gfx/imagemagick-6.9.6.6" : {
+   "EAPI": "6",
+   "SLOT": "0/6.9.6.6",
+   },
+
+   "media-gfx/inkscape-0.91-r3" : {
+   "EAPI": "6",
+   "DEPEND": "media-gfx/imagemagick:0/6.9.6.6=",
+   "RDEPEND": "media-gfx/imagemagick:0/6.9.6.6=",
+   },
+
+   "media-video/dvdrip-0.98.11-r3" : {
+   "EAPI": "6",
+   "DEPEND": "|| ( 
media-gfx/graphicsmagick[imagemagick] media-gfx/imagemagick )",
+   "RDEPEND": "|| ( 
media-gfx/graphicsmagick[imagemagick] media-gfx/imagemagick )",
+   },
+
+   "media-gfx/graphicsmagick-1.3.25" : {
+   "EAPI": "6",
+   "SLOT": "0/1.3",
+   "IUSE": "imagemagick",
+   "USE": "",
+   "RDEPEND": "imagemagick? ( 
!media-gfx/imagemagick )",
+   },
+   }
+
+   world = (
+   

Re: [gentoo-portage-dev] [PATCH v2] portageq: allow disabling regex matching of maintainer emails #604164

[Zac Medico]

On 12/30/2016 01:43 PM, Göktürk Yüksek wrote:
> When the email address of a maintainer contains unescaped
> regex-special characters (such as '+'), the maintainer-email match may
> return undesirable results.
> 
> Add a command line option '--no-regex' to use re.escape() with list
> comprehension on maintainer emails when constructing the matcher
> regex. This way, an exact string match can be made rather than a regex
> match.
> ---
>  bin/portageq | 10 +-
>  1 file changed, 9 insertions(+), 1 deletion(-)
> 
> diff --git a/bin/portageq b/bin/portageq
> index d645635..63867d3 100755
> --- a/bin/portageq
> +++ b/bin/portageq
> @@ -1082,7 +1082,10 @@ def pquery(parser, opts, args):
>   maintainer_emails = []
>   for x in opts.maintainer_email:
>   maintainer_emails.extend(x.split(","))
> - xml_matchers.append(MaintainerEmailMatcher(maintainer_emails))
> + if opts.no_regex: # Escape regex-special characters for an 
> exact match
> + 
> xml_matchers.append(MaintainerEmailMatcher([re.escape(m) for m in 
> maintainer_emails]))
> + else:
> + 
> xml_matchers.append(MaintainerEmailMatcher(maintainer_emails))
>   if opts.orphaned:
>   xml_matchers.append(match_orphaned)
>  
> @@ -1240,6 +1243,11 @@ def add_pquery_arguments(parser):
>   "help": "comma-separated list of 
> maintainer email regexes to search for"
>   },
>   {
> + "longopt": "--no-regex",
> + "action": "store_true",
> + "help": "Use exact matching instead of 
> regex matching for --maintainer-email"
> + },
> + {
>   "longopt": "--orphaned",
>   "action": "store_true",
>   "help": "match only orphaned 
> (maintainer-needed) packages"
> 

Thanks, pushed:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=aa57d60d9c77a46f542475dcf448c83af40e73e1

-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH v1] portageq: escape regex-special characters in maintainer emails #604164

[Zac Medico]

On 12/30/2016 10:02 AM, Göktürk Yüksek wrote:
> When the email address of a maintainer contains regex-special
> characters (such as '+'), re.match() returns incorrect results. Use
> re.escape() with list comprehension on maintainer emails when
> constructing the matcher regex.

I've commented on bug 604164 that we should add a new option rather than
drop regex support from the existing option.
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH] depgraph: clarify "update has been skipped" message (bug 602854)

[Zac Medico]

On 12/28/2016 05:25 PM, Brian Dolbec wrote:
> On Wed, 28 Dec 2016 16:01:04 -0800
> Zac Medico <zmed...@gentoo.org> wrote:
> 
>> When an update has been skipped, clearly display both the selected and
>> skipped packages.
>>
>> X-Gentoo-bug: 602854
>> X-Gentoo-bug-url: https://bugs.gentoo.org/602854
>> ---
>>  pym/_emerge/depgraph.py | 9 -
>>  1 file changed, 8 insertions(+), 1 deletion(-)
>>
>> diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
>> index cb12b05..e298337 100644
>> --- a/pym/_emerge/depgraph.py
>> +++ b/pym/_emerge/depgraph.py
>> @@ -1044,7 +1044,14 @@ class depgraph(object):
>>  writemsg(str(pkg.slot_atom), noiselevel=-1)
>>  if pkg.root_config.settings["ROOT"] != "/":
>>  writemsg(" for %s" % (pkg.root,),
>> noiselevel=-1)
>> -writemsg("\n", noiselevel=-1)
>> +writemsg("\n\n", noiselevel=-1)
>> +
>> +selected_pkg =
>> next(self._dynamic_config._package_tracker.match(
>> +pkg.root, pkg.slot_atom), None)
>> +
>> +writemsg("  selected: %s\n" %
>> (selected_pkg,), noiselevel=-1)
>> +writemsg("  skipped: %s (see unsatisfied
>> dependency below)\n"
>> +% (pkg,), noiselevel=-1)
>>  
>>  for parent, root, atom in parent_atoms:
>>  self._show_unsatisfied_dep(root,
>> atom, myparent=parent)
> 
> 
> looks good :)
> 

Thanks, pushed:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=7dc23bc66b410b5a2f9640dc93ca65f5b2678670
-- 
Thanks,
Zac

[gentoo-portage-dev] [PATCH] depgraph: clarify "update has been skipped" message (bug 602854)

[Zac Medico]

When an update has been skipped, clearly display both the selected and
skipped packages.

X-Gentoo-bug: 602854
X-Gentoo-bug-url: https://bugs.gentoo.org/602854
---
 pym/_emerge/depgraph.py | 9 -
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
index cb12b05..e298337 100644
--- a/pym/_emerge/depgraph.py
+++ b/pym/_emerge/depgraph.py
@@ -1044,7 +1044,14 @@ class depgraph(object):
writemsg(str(pkg.slot_atom), noiselevel=-1)
if pkg.root_config.settings["ROOT"] != "/":
writemsg(" for %s" % (pkg.root,), noiselevel=-1)
-   writemsg("\n", noiselevel=-1)
+   writemsg("\n\n", noiselevel=-1)
+
+   selected_pkg = 
next(self._dynamic_config._package_tracker.match(
+   pkg.root, pkg.slot_atom), None)
+
+   writemsg("  selected: %s\n" % (selected_pkg,), 
noiselevel=-1)
+   writemsg("  skipped: %s (see unsatisfied dependency 
below)\n"
+   % (pkg,), noiselevel=-1)
 
for parent, root, atom in parent_atoms:
self._show_unsatisfied_dep(root, atom, 
myparent=parent)
-- 
2.7.4

Re: [gentoo-portage-dev] [PATCH] binarytree._read_metadata: return empty strings for undefine values (bug 603826)

[Zac Medico]

On 12/27/2016 03:39 PM, Brian Dolbec wrote:
> On Tue, 27 Dec 2016 13:33:55 -0800
> Zac Medico <zmed...@gentoo.org> wrote:
> 
>> Fix the binarytree._read_metadata method to return empty strings for
>> undefined metadata values, in order to fulfill a contract with the
>> _pkg_str class. This prevents an AttributeError triggered by old
>> binary packages which have undefined repository metadata, as reported
>> in bug 603826.
>>
>> X-Gentoo-bug: 603826
>> X-Gentoo-bug-url: https://bugs.gentoo.org/603826
>> ---
>>  pym/portage/dbapi/bintree.py | 31 +--
>>  pym/portage/versions.py  |  8 +++-
>>  2 files changed, 32 insertions(+), 7 deletions(-)
>>
>> diff --git a/pym/portage/dbapi/bintree.py
>> b/pym/portage/dbapi/bintree.py index f483059..3341889 100644
>> --- a/pym/portage/dbapi/bintree.py
>> +++ b/pym/portage/dbapi/bintree.py
>> @@ -1,4 +1,4 @@
>> -# Copyright 1998-2014 Gentoo Foundation
>> +# Copyright 1998-2016 Gentoo Foundation
>>  # Distributed under the terms of the GNU General Public License v2
>>  
>>  from __future__ import unicode_literals
>> @@ -1041,12 +1041,12 @@ class binarytree(object):
>>  noiselevel=-1)
>>  return
>>  metadata = self._read_metadata(full_path, s)
>> -slot = metadata.get("SLOT")
>> +invalid_depend = False
>>  try:
>>  self._eval_use_flags(cpv, metadata)
>>  except portage.exception.InvalidDependString:
>> -slot = None
>> -if slot is None:
>> +invalid_depend = True
>> +if invalid_depend or not metadata.get("SLOT"):
>>  writemsg(_("!!! Invalid binary package:
>> '%s'\n") % full_path, noiselevel=-1)
>>  return
>> @@ -1126,6 +1126,21 @@ class binarytree(object):
>>  return cpv
>>  
>>  def _read_metadata(self, filename, st, keys=None):
>> +"""
>> +Read metadata from a binary package. The returned
>> metadata
>> +dictionary will contain empty strings for any values
>> that
>> +are undefined (this is important because the
>> _pkg_str class
>> +distinguishes between missing and undefined values).
>> +
>> +@param filename: File path of the binary package
>> +@type filename: string
>> +@param st: stat result for the binary package
>> +@type st: os.stat_result
>> +@param keys: optional list of specific metadata keys
>> to retrieve
>> +@type keys: iterable
>> +@rtype: dict
>> +@return: package metadata
>> +"""
>>  if keys is None:
>>  keys = self.dbapi._aux_cache_keys
>>  metadata = self.dbapi._aux_cache_slot_dict()
>> @@ -1139,10 +1154,14 @@ class binarytree(object):
>>  metadata[k] = _unicode(st.st_size)
>>  else:
>>  v =
>> binary_metadata.get(_unicode_encode(k))
>> -if v is not None:
>> +if v is None:
>> +if k == "EAPI":
>> +metadata[k] = "0"
>> +else:
>> +metadata[k] = ""
>> +else:
>>  v = _unicode_decode(v)
>>  metadata[k] = "
>> ".join(v.split())
>> -metadata.setdefault("EAPI", "0")
>>  return metadata
>>  
>>  def _inject_file(self, pkgindex, cpv, filename):
>> diff --git a/pym/portage/versions.py b/pym/portage/versions.py
>> index a028d93..adfb1c3 100644
>> --- a/pym/portage/versions.py
>> +++ b/pym/portage/versions.py
>> @@ -1,5 +1,5 @@
>>  # versions.py -- core Portage functionality
>> -# Copyright 1998-2014 Gentoo Foundation
>> +# Copyright 1998-2016 Gentoo Foundation
>>  # Distributed under the terms of the GNU General Public License v2
>>  
>>  from __future__ import unicode_literals
>> @@ -359,6 +359,12 @@ class _pkg_str(_unicode):
>>  Instan

Re: [gentoo-portage-dev] [PATCH] LinkageMapELF: compute mulilib category for preserved libs (bug 598080)

[Zac Medico]

On 12/27/2016 01:23 PM, Brian Dolbec wrote:
> On Mon, 26 Dec 2016 20:37:39 -0800
> Zac Medico <zmed...@gentoo.org> wrote:
> 
>> When support for parsing ELF headers in order to compute multilib
>> category was added in commit f1c1b8a77eebf7713b32e5f9945690f60f4f46de,
>> the LinkageMapELF class was not updated to do the same for preserved
>> libraries. This has resulted in incorrect preserve-libs handling
>> as reported in bug 598080, for ABIs including x32 where the
>> _approx_multilib_categories mapping is insufficient. This patch fixes
>> LinkageMapELF to compute the multilib category for each preserved
>> library, in the same way as the _post_src_install_soname_symlinks
>> function, so that the LinkageMapELF.findConsumers method will operate
>> correctly with preserved libraries of all ABIs.
>>
>> Fixes: f1c1b8a77eeb ("Generate soname dependency metadata (bug
>> 282639)") X-Gentoo-bug: 598080
>> X-Gentoo-bug-url: https://bugs.gentoo.org/598080
>> ---
>>  pym/portage/util/_dyn_libs/LinkageMapELF.py | 33
>> +++-- 1 file changed, 26 insertions(+), 7
>> deletions(-)
>>
>> diff --git a/pym/portage/util/_dyn_libs/LinkageMapELF.py
>> b/pym/portage/util/_dyn_libs/LinkageMapELF.py index 0b09fe5..a063621
>> 100644 --- a/pym/portage/util/_dyn_libs/LinkageMapELF.py
>> +++ b/pym/portage/util/_dyn_libs/LinkageMapELF.py
>> @@ -4,6 +4,7 @@
>>  import errno
>>  import logging
>>  import subprocess
>> +import sys
>>  
>>  import portage
>>  from portage import _encodings
>> @@ -12,6 +13,7 @@ from portage import _unicode_decode
>>  from portage import _unicode_encode
>>  from portage.cache.mappings import slot_dict_class
>>  from portage.const import EPREFIX
>> +from portage.dep.soname.multilib_category import
>> compute_multilib_category from portage.exception import
>> CommandNotFound, InvalidData from portage.localization import _
>>  from portage.util import getlibpaths
>> @@ -20,6 +22,12 @@ from portage.util import normalize_path
>>  from portage.util import varexpand
>>  from portage.util import writemsg_level
>>  from portage.util._dyn_libs.NeededEntry import NeededEntry
>> +from portage.util.elf.header import ELFHeader
>> +
>> +if sys.hexversion >= 0x300:
>> +_unicode = str
>> +else:
>> +_unicode = unicode
>>  
>>  # Map ELF e_machine values from NEEDED.ELF.2 to approximate multilib
>>  # categories. This approximation will produce incorrect results on
>> x32 @@ -283,15 +291,26 @@ class LinkageMapELF(object):
>>  l = l[3:].rstrip("\n")
>>  if not l:
>>  continue
>> -fields = l.split(";")
>> -if len(fields) < 5:
>> -
>> writemsg_level(_("\nWrong number of fields " \
>> -"returned
>> from scanelf: %s\n\n") % (l,),
>> +try:
>> +entry =
>> NeededEntry.parse("scanelf", l)
>> +except InvalidData as e:
>> +
>> writemsg_level("\n%s\n\n" % (e,), level=logging.ERROR, noiselevel=-1)
>>  continue
>> -fields[1] =
>> fields[1][root_len:]
>> -owner = plibs.pop(fields[1],
>> None)
>> -lines.append((owner,
>> "scanelf", ";".join(fields)))
>> +try:
>> +with
>> open(_unicode_encode(entry.filename,
>> +
>> encoding=_encodings['fs'],
>> +
>> errors='strict'), 'rb') as f:
>> +elf_header =
>> ELFHeader.read(f)
>> +except EnvironmentError as e:
>> +if e.errno !=
>> errno.ENOENT:
>> +raise
>> +# File removed
>> concurrently.
>> +continue
>> +entry.multilib_category =
>> compute_multilib_category(elf_header)
>> +entry.filename =
>> entry.filename[root_len:]
>> +owner =
>> plibs.pop(entry.filename, None)
>> +lines.append((owner,
>> "scanelf", _unicode(entry))) proc.wait()
>>  proc.stdout.close()
>>  
> 
> looks fine
> 

Thanks, pushed:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=cbaee3c3b28f52947c142da8df24d6b0817962f9

-- 
Thanks,
Zac

[gentoo-portage-dev] [PATCH] binarytree._read_metadata: return empty strings for undefine values (bug 603826)

[Zac Medico]

Fix the binarytree._read_metadata method to return empty strings for
undefined metadata values, in order to fulfill a contract with the
_pkg_str class. This prevents an AttributeError triggered by old
binary packages which have undefined repository metadata, as reported
in bug 603826.

X-Gentoo-bug: 603826
X-Gentoo-bug-url: https://bugs.gentoo.org/603826
---
 pym/portage/dbapi/bintree.py | 31 +--
 pym/portage/versions.py  |  8 +++-
 2 files changed, 32 insertions(+), 7 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index f483059..3341889 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2014 Gentoo Foundation
+# Copyright 1998-2016 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 from __future__ import unicode_literals
@@ -1041,12 +1041,12 @@ class binarytree(object):
noiselevel=-1)
return
metadata = self._read_metadata(full_path, s)
-   slot = metadata.get("SLOT")
+   invalid_depend = False
try:
self._eval_use_flags(cpv, metadata)
except portage.exception.InvalidDependString:
-   slot = None
-   if slot is None:
+   invalid_depend = True
+   if invalid_depend or not metadata.get("SLOT"):
writemsg(_("!!! Invalid binary package: '%s'\n") % 
full_path,
noiselevel=-1)
return
@@ -1126,6 +1126,21 @@ class binarytree(object):
return cpv
 
def _read_metadata(self, filename, st, keys=None):
+   """
+   Read metadata from a binary package. The returned metadata
+   dictionary will contain empty strings for any values that
+   are undefined (this is important because the _pkg_str class
+   distinguishes between missing and undefined values).
+
+   @param filename: File path of the binary package
+   @type filename: string
+   @param st: stat result for the binary package
+   @type st: os.stat_result
+   @param keys: optional list of specific metadata keys to retrieve
+   @type keys: iterable
+   @rtype: dict
+   @return: package metadata
+   """
if keys is None:
keys = self.dbapi._aux_cache_keys
metadata = self.dbapi._aux_cache_slot_dict()
@@ -1139,10 +1154,14 @@ class binarytree(object):
metadata[k] = _unicode(st.st_size)
else:
v = binary_metadata.get(_unicode_encode(k))
-   if v is not None:
+   if v is None:
+   if k == "EAPI":
+   metadata[k] = "0"
+   else:
+   metadata[k] = ""
+   else:
v = _unicode_decode(v)
metadata[k] = " ".join(v.split())
-   metadata.setdefault("EAPI", "0")
return metadata
 
def _inject_file(self, pkgindex, cpv, filename):
diff --git a/pym/portage/versions.py b/pym/portage/versions.py
index a028d93..adfb1c3 100644
--- a/pym/portage/versions.py
+++ b/pym/portage/versions.py
@@ -1,5 +1,5 @@
 # versions.py -- core Portage functionality
-# Copyright 1998-2014 Gentoo Foundation
+# Copyright 1998-2016 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 from __future__ import unicode_literals
@@ -359,6 +359,12 @@ class _pkg_str(_unicode):
Instances are typically created in dbapi.cp_list() or the Atom 
contructor,
and propagate from there. Generally, code that pickles these objects 
will
manually convert them to a plain unicode object first.
+
+   Instances of this class will have missing attributes for metadata that
+   has not been passed into the constructor. The missing attributes are
+   used to distinguish missing metadata values from undefined metadata 
values.
+   For example, the repo attribute will be missing if the 'repository' key
+   is missing from the metadata dictionary.
"""
 
def __new__(cls, cpv, metadata=None, settings=None, eapi=None,
-- 
2.7.4

Re: [gentoo-portage-dev] [PATCH] slot_conflict_handler: report packages that can't be rebuilt (bug 602964)

[Zac Medico]

On 12/18/2016 07:44 AM, Brian Dolbec wrote:
> On Sun, 18 Dec 2016 05:58:45 -0800
> Zac Medico <zmed...@gentoo.org> wrote:
> 
>> Report packages that need to be rebuilt in order to solve slot
>> conflicts, but cannot be rebuilt for some reason. The following
>> reasons will be detected:
>>
>> * matched by --exclude argument
>> * matched by --useoldpkg-atoms argument
>> * ebuild is masked or unavailable
>>
>> Example output:
>>
>> !!! The slot conflict(s) shown above involve package(s) which may
>> need to !!! be rebuilt in order to solve the conflict(s). However,
>> the following !!! package(s) cannot be rebuilt for the reasons shown:
>>
>>   (sys-apps/less-480:0/0::gentoo, installed): ebuild is masked or
>> unavailable
>>
>> X-Gentoo-bug: 602964
>> X-Gentoo-bug-url: https://bugs.gentoo.org/602964
>> ---
>>  pym/_emerge/resolver/slot_collision.py | 37
>> -- 1 file changed, 35 insertions(+),
>> 2 deletions(-)
>>
>> diff --git a/pym/_emerge/resolver/slot_collision.py
>> b/pym/_emerge/resolver/slot_collision.py index cfb5885..64147c9 100644
>> --- a/pym/_emerge/resolver/slot_collision.py
>> +++ b/pym/_emerge/resolver/slot_collision.py
>> @@ -241,6 +241,8 @@ class slot_conflict_handler(object):
>>  Print all slot conflicts in a human readable way.
>>  """
>>  _pkg_use_enabled = self.depgraph._pkg_use_enabled
>> +usepkgonly = "--usepkgonly" in self.myopts
>> +need_rebuild = {}
>>  verboseconflicts = "--verbose-conflicts" in
>> self.myopts any_omitted_parents = False
>>  msg = self.conflict_msg
>> @@ -394,6 +396,29 @@ class slot_conflict_handler(object):
>>  
>> selected_for_display.update(
>>  
>> best_matches.values())
>>  elif type in
>> ("soname", "slot"):
>> +# Check for
>> packages that might need to
>> +# be
>> rebuilt, but cannot be rebuilt for
>> +# some
>> reason.
>> +for ppkg,
>> atom, other_pkg in parents:
>> +if
>> not ppkg.installed:
>> +
>> continue
>> +if
>> not (atom.soname or atom.slot_operator_built):
>> +
>> continue
>> +if
>> self.depgraph._frozen_config.excluded_pkgs.findAtomForPackage(ppkg,
>> +
>> modified_use=self.depgraph._pkg_use_enabled(ppkg)):
>> +
>> selected_for_display.add((ppkg, atom))
>> +
>> need_rebuild[ppkg] = 'matched by --exclude argument'
>> +elif
>> self.depgraph._frozen_config.useoldpkg_atoms.findAtomForPackage(ppkg,
>> +
>> modified_use=self.depgraph._pkg_use_enabled(ppkg)):
>> +
>> selected_for_display.add((ppkg, atom))
>> +
>> need_rebuild[ppkg] = 'matched by --useoldpkg-atoms argument'
>> +elif
>> usepkgonly:
>> +
>> # This case is tricky, so keep quiet in order to avoid
>> false-positives.
>> +
>> pass
>> +elif
>> not self.depgraph._equiv_ebuild_visible(ppkg):
>> +
>> selected_for_display.add((ppkg, atom))
>> +
>> need_rebuild[ppkg] = 'ebuild is masked or unavailable' +
>>  for ppkg,
>> atom, other_pkg in parents: selected_for_display.add((ppkg, atom))
>>  if
>> not verboseconflicts: @@ -611,10 +636,18 @@ class
>> slot_conflict_handler(object): msg.append(colorize("INFORM",
>>  "NOTE: Use the '--verbose-conflicts'"
>>  " option to display parents omitted
>> above"))
>> -msg.append("\n\n")
>> -else:
>>  msg.append("\n")
>>  
>> +if need_rebuild:
>> +

[gentoo-portage-dev] [PATCH] slot_conflict_handler: report packages that can't be rebuilt (bug 602964)

[Zac Medico]

Report packages that need to be rebuilt in order to solve slot
conflicts, but cannot be rebuilt for some reason. The following
reasons will be detected:

* matched by --exclude argument
* matched by --useoldpkg-atoms argument
* ebuild is masked or unavailable

Example output:

!!! The slot conflict(s) shown above involve package(s) which may need to
!!! be rebuilt in order to solve the conflict(s). However, the following
!!! package(s) cannot be rebuilt for the reasons shown:

  (sys-apps/less-480:0/0::gentoo, installed): ebuild is masked or unavailable

X-Gentoo-bug: 602964
X-Gentoo-bug-url: https://bugs.gentoo.org/602964
---
 pym/_emerge/resolver/slot_collision.py | 37 --
 1 file changed, 35 insertions(+), 2 deletions(-)

diff --git a/pym/_emerge/resolver/slot_collision.py 
b/pym/_emerge/resolver/slot_collision.py
index cfb5885..64147c9 100644
--- a/pym/_emerge/resolver/slot_collision.py
+++ b/pym/_emerge/resolver/slot_collision.py
@@ -241,6 +241,8 @@ class slot_conflict_handler(object):
Print all slot conflicts in a human readable way.
"""
_pkg_use_enabled = self.depgraph._pkg_use_enabled
+   usepkgonly = "--usepkgonly" in self.myopts
+   need_rebuild = {}
verboseconflicts = "--verbose-conflicts" in self.myopts
any_omitted_parents = False
msg = self.conflict_msg
@@ -394,6 +396,29 @@ class slot_conflict_handler(object):

selected_for_display.update(

best_matches.values())
elif type in ("soname", "slot"):
+   # Check for packages 
that might need to
+   # be rebuilt, but 
cannot be rebuilt for
+   # some reason.
+   for ppkg, atom, 
other_pkg in parents:
+   if not 
ppkg.installed:
+   continue
+   if not 
(atom.soname or atom.slot_operator_built):
+   continue
+   if 
self.depgraph._frozen_config.excluded_pkgs.findAtomForPackage(ppkg,
+   
modified_use=self.depgraph._pkg_use_enabled(ppkg)):
+   
selected_for_display.add((ppkg, atom))
+   
need_rebuild[ppkg] = 'matched by --exclude argument'
+   elif 
self.depgraph._frozen_config.useoldpkg_atoms.findAtomForPackage(ppkg,
+   
modified_use=self.depgraph._pkg_use_enabled(ppkg)):
+   
selected_for_display.add((ppkg, atom))
+   
need_rebuild[ppkg] = 'matched by --useoldpkg-atoms argument'
+   elif usepkgonly:
+   # This 
case is tricky, so keep quiet in order to avoid false-positives.
+   pass
+   elif not 
self.depgraph._equiv_ebuild_visible(ppkg):
+   
selected_for_display.add((ppkg, atom))
+   
need_rebuild[ppkg] = 'ebuild is masked or unavailable'
+
for ppkg, atom, 
other_pkg in parents:

selected_for_display.add((ppkg, atom))
if not 
verboseconflicts:
@@ -611,10 +636,18 @@ class slot_conflict_handler(object):
msg.append(colorize("INFORM",
"NOTE: Use the '--verbose-conflicts'"
" option to display parents omitted above"))
-   msg.append("\n\n")
-   else:
msg.append("\n")
 
+   if need_rebuild:
+   msg.append("\n!!! The slot conflict(s) shown above 
involve package(s) which may need to\n")
+   

Re: [gentoo-portage-dev] [PATCH] man: emaint typo fix, #575178

[Zac Medico]

On 12/14/2016 11:19 AM, Wim Muskee wrote:
> From 74d266404ec157bf97791f308a4858712028d607 Mon Sep 17 00:00:00 2001
> From: Wim Muskee >
> Date: Mon, 12 Dec 2016 19:59:08 +0100
> Subject: [PATCH] man: emaint typo fix, #575178
> 
> ---
>  man/emaint.1 | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/man/emaint.1 b/man/emaint.1
> index cfdadc8..24e4744 100644
> --- a/man/emaint.1
> +++ b/man/emaint.1
> @@ -1,6 +1,6 @@
>  .TH "EMAINT" "1" "May 2015" "Portage VERSION" "Portage"
>  .SH NAME
> -emaint \- performs package managment related system health checks and 
> maintenance
> +emaint \- performs package management related system health checks and 
> maintenance
>  .SH SYNOPSIS
>  .BR emaint
>  [\fIoptions\fR]
> 

Thanks, applied:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=a29847f77d57f2c423c30b931fa40e65523795c4
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH] repoman: Fix versioning system

[Zac Medico]

On 12/05/2016 12:32 PM, Brian Dolbec wrote:
> On Mon, 5 Dec 2016 11:42:10 -0800
> Brian Dolbec  wrote:
> 
>> From 1586f1a25c41fb6036a24b47cfa58e3e818b8a58 Mon Sep 17 00:00:00 2001
>> From: Brian Dolbec 
>> Date: Mon, 5 Dec 2016 11:27:15 -0800
>> Subject: [PATCH] repoman: Fix versioning system
>>
>> Repoman had been showing the portage version.  Which was the same for
>> the last release. Copy the live versions code from portage, Modify as
>> needed to get the correct tag info. Add portage version to --version
>> output. ---
>>  repoman/pym/repoman/__init__.py | 70
>> +
> 
> For some reason, without the single quoting the match string, "git describe 
> --match portage-*" 
> would fail while "git describe --match repoman-*" would without the single 
> quotes.
> 
> Anyway, I updated the repoman patch to single quote it.

Yeah, it definitely needs to be quoted, in order to prevent shell
expansion. Apparently you had a file or directory patching portage-* in
your working directory.
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH] depgraph: select highest version involved in slot conflict (bug 554070)

[Zac Medico]

On 12/04/2016 07:43 PM, Brian Dolbec wrote:
> On Sun,  4 Dec 2016 18:54:37 -0800
> Zac Medico <zmed...@gentoo.org> wrote:
> 
>> Fix depgraph's package selection logic to choose the highest version
>> involved in a slot conflict, for correct operation of
>> conflict_downgrade logic in the dep_zapdeps function which was
>> introduced in commit a9064d08ef4c92a5d0d1bfb3dc8a01b7850812b0. This
>> will prevent incorrect re-ordering of || deps in dep_zapdeps, as
>> reported in bug 554070.
>>
>> X-Gentoo-Bug: 554070
>> X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=554070
>> Fixes a9064d08ef4c ("Solve more slot-operator conflicts (531656)")
>> ---
>>  pym/_emerge/depgraph.py | 11 +--
>>  1 file changed, 9 insertions(+), 2 deletions(-)
>>
>> diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
>> index 9161914..ee6cf68 100644
>> --- a/pym/_emerge/depgraph.py
>> +++ b/pym/_emerge/depgraph.py
>> @@ -6071,8 +6071,15 @@ class depgraph(object):
>>  # will always end with a
>> break statement below # this point.
>>  if find_existing_node:
>> -e_pkg =
>> next(self._dynamic_config._package_tracker.match(
>> -root,
>> pkg.slot_atom, installed=False), None)
>> +# Use reversed
>> iteration in order to get
>> +# descending order
>> here, so that the highest
>> +# version involved
>> in a slot conflict is
>> +# selected. This is
>> needed for correct operation
>> +# of
>> conflict_downgrade logic in the dep_zapdeps
>> +# function (see bug
>> 554070).
>> +e_pkg =
>> next(reversed(list(
>> +
>> self._dynamic_config._package_tracker.match(
>> +root,
>> pkg.slot_atom, installed=False))), None) 
>>  if not e_pkg:
>>  break
> 
> LGTM
> 

Pushed:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=ab07ac68fa1e04ed64e2e0f6c753ff169a32d517
-- 
Thanks,
Zac

[gentoo-portage-dev] [PATCH] depgraph: select highest version involved in slot conflict (bug 554070)

[Zac Medico]

Fix depgraph's package selection logic to choose the highest version
involved in a slot conflict, for correct operation of conflict_downgrade
logic in the dep_zapdeps function which was introduced in commit
a9064d08ef4c92a5d0d1bfb3dc8a01b7850812b0. This will prevent incorrect
re-ordering of || deps in dep_zapdeps, as reported in bug 554070.

X-Gentoo-Bug: 554070
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=554070
Fixes a9064d08ef4c ("Solve more slot-operator conflicts (531656)")
---
 pym/_emerge/depgraph.py | 11 +--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
index 9161914..ee6cf68 100644
--- a/pym/_emerge/depgraph.py
+++ b/pym/_emerge/depgraph.py
@@ -6071,8 +6071,15 @@ class depgraph(object):
# will always end with a break 
statement below
# this point.
if find_existing_node:
-   e_pkg = 
next(self._dynamic_config._package_tracker.match(
-   root, pkg.slot_atom, 
installed=False), None)
+   # Use reversed iteration in 
order to get
+   # descending order here, so 
that the highest
+   # version involved in a slot 
conflict is
+   # selected. This is needed for 
correct operation
+   # of conflict_downgrade logic 
in the dep_zapdeps
+   # function (see bug 554070).
+   e_pkg = next(reversed(list(
+   
self._dynamic_config._package_tracker.match(
+   root, pkg.slot_atom, 
installed=False))), None)
 
if not e_pkg:
break
-- 
2.7.4

Re: [gentoo-portage-dev] [PATCH] _emerge/depgraph.py: Autounmask-write fails when there isn't a file in package.*/ bug 598116

[Zac Medico]

On 12/03/2016 01:48 PM, Brian Dolbec wrote:
> 
> From c4a61bebca1cfeb0833cefb2c64be6156bdb8e8d Mon Sep 17 00:00:00 2001
> From: "hackers.terabit" 
> Date: Thu, 27 Oct 2016 03:29:16 +
> Subject: [PATCH] _emerge/depgraph.py: Autounmask-write fails when there isn't
>  a file in package.*/ bug 598116
> 
> Instead of outputting "!!! No file to write for ..." error message,
> Use a sane default filename.
> Add zz- prefix to ensure it remains the last file in sorted order.
> 
> X-Gentoo-bug: 598116
> X-Gentoo-bug-url: https://bugs.gentoo.org/598116
> ---
>  pym/_emerge/depgraph.py | 6 ++
>  1 file changed, 6 insertions(+)
> 
> diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
> index 9161914..7dc1f55 100644
> --- a/pym/_emerge/depgraph.py
> +++ b/pym/_emerge/depgraph.py
> @@ -8240,6 +8240,12 @@ class depgraph(object):
>   
> child.endswith("~"):
>   continue
>   
> stack.append(os.path.join(p, child))
> + # If the directory is empty add a file with name
> + # pattern file_name.default
> + if last_file_path == None:
> + last_file_path=os.path.join(file_path, 
> file_path, "zz-autounmask")
> + with open(last_file_path,"a+") as default:
> + default.write("# " + file_name)
>  
>   return last_file_path
>  
> 

Patch looks good except it could use these style tweaks:

-if last_file_path == None:
+if last_file_path is None:

-last_file_path=os.path.join(file_path, file_path, "zz-autounmask")
+last_file_path = os.path.join(file_path, file_path, "zz-autounmask")

-with open(last_file_path,"a+") as default:
+with open(last_file_path, "a+") as default:
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH] egencache: Migrate _special_filename class to portage.utils.changelog for api use

[Zac Medico]

On 12/03/2016 01:49 PM, Brian Dolbec wrote:
> + if self.file_type_lt(self, other):
> + return True
> + elif self.file_type_lt(other, self):
> + return False

Patch looks good, except there are 2 missed renames from file_type_lt to
_file_type_lt above.
-- 
Thanks,
Zac

Re: [gentoo-portage-dev] [PATCH] bin/ebuild: fix EBUILD_FORCE_TEST / RESTRICT interaction (bug 601466)

[Zac Medico]

On 12/03/2016 07:30 AM, Brian Dolbec wrote:
> On Fri,  2 Dec 2016 22:07:39 -0800
> Zac Medico <zmed...@gentoo.org> wrote:
> 
>> Ensure that config adjustments are applied to all relevant
>> config instances, in order to prevent inconsistencies like
>> the one that triggered bug 601466.
>>
>> X-Gentoo-Bug: 601466
>> X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=601466
>> ---
>>  bin/ebuild | 23 ---
>>  1 file changed, 16 insertions(+), 7 deletions(-)
>>
>> diff --git a/bin/ebuild b/bin/ebuild
>> index 1f99177..e8b8bae 100755
>> --- a/bin/ebuild
>> +++ b/bin/ebuild
>> @@ -244,7 +244,11 @@ if mytree == "porttree" and
>> build_dir_phases.intersection(pargs): ebuild_changed = \
>>  portage.portdb._pull_valid_cache(cpv, ebuild,
>> ebuild_portdir)[0] is None 
>> -tmpsettings = portage.config(clone=portage.settings)
>> +# Make configuration adjustments to portage.portdb.doebuild_settings,
>> +# in order to enforce consistency for EBUILD_FORCE_TEST support
>> +# (see bug 601466).
>> +tmpsettings = portage.portdb.doebuild_settings
>> +
>>  tmpsettings["PORTAGE_VERBOSE"] = "1"
>>  tmpsettings.backup_changes("PORTAGE_VERBOSE")
>>  
>> @@ -272,6 +276,17 @@ if "merge" in pargs and "noauto" in
>> tmpsettings.features: print("Disabling noauto in features... merge
>> disables it. (qmerge doesn't)") tmpsettings.features.discard("noauto")
>>  
>> +if 'digest' in tmpsettings.features:
>> +if pargs and pargs[0] not in ("digest", "manifest"):
>> +pargs = ['digest'] + pargs
>> +# We only need to build digests on the first pass.
>> +tmpsettings.features.discard('digest')
>> +
>> +# Now that configuration adjustments are complete, create a clone of
>> +# tmpsettings. The current instance refers to
>> portdb.doebuild_settings, +# and we want to avoid the possibility of
>> unintended side-effects. +tmpsettings =
>> portage.config(clone=tmpsettings) +
>>  try:
>>  metadata = dict(zip(Package.metadata_keys,
>>  portage.db[portage.settings['EROOT']][mytree].dbapi.aux_get(
>> @@ -315,12 +330,6 @@ def stale_env_warning():
>>  
>> open(os.path.join(tmpsettings['PORTAGE_BUILDDIR'],
>>  '.ebuild_changed'),
>> 'w').close() 
>> -if 'digest' in tmpsettings.features:
>> -if pargs and pargs[0] not in ("digest", "manifest"):
>> -pargs = ['digest'] + pargs
>> -# We only need to build digests on the first pass.
>> -tmpsettings.features.discard('digest')
>> -
>>  checked_for_stale_env = False
>>  
>>  for arg in pargs:
> 
> looks fine to me
> 

Thanks, pushed (with fixup to eliminate duplicate "Forcing Test" messages):

https://gitweb.gentoo.org/proj/portage.git/commit/?id=0c06ff5f8f3e86592bbaeb38f274797505c45b2a
-- 
Thanks,
Zac

[gentoo-portage-dev] [PATCH] bin/ebuild: fix EBUILD_FORCE_TEST / RESTRICT interaction (bug 601466)

[Zac Medico]

Ensure that config adjustments are applied to all relevant
config instances, in order to prevent inconsistencies like
the one that triggered bug 601466.

X-Gentoo-Bug: 601466
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=601466
---
 bin/ebuild | 23 ---
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/bin/ebuild b/bin/ebuild
index 1f99177..e8b8bae 100755
--- a/bin/ebuild
+++ b/bin/ebuild
@@ -244,7 +244,11 @@ if mytree == "porttree" and 
build_dir_phases.intersection(pargs):
ebuild_changed = \
portage.portdb._pull_valid_cache(cpv, ebuild, 
ebuild_portdir)[0] is None
 
-tmpsettings = portage.config(clone=portage.settings)
+# Make configuration adjustments to portage.portdb.doebuild_settings,
+# in order to enforce consistency for EBUILD_FORCE_TEST support
+# (see bug 601466).
+tmpsettings = portage.portdb.doebuild_settings
+
 tmpsettings["PORTAGE_VERBOSE"] = "1"
 tmpsettings.backup_changes("PORTAGE_VERBOSE")
 
@@ -272,6 +276,17 @@ if "merge" in pargs and "noauto" in tmpsettings.features:
print("Disabling noauto in features... merge disables it. (qmerge 
doesn't)")
tmpsettings.features.discard("noauto")
 
+if 'digest' in tmpsettings.features:
+   if pargs and pargs[0] not in ("digest", "manifest"):
+   pargs = ['digest'] + pargs
+   # We only need to build digests on the first pass.
+   tmpsettings.features.discard('digest')
+
+# Now that configuration adjustments are complete, create a clone of
+# tmpsettings. The current instance refers to portdb.doebuild_settings,
+# and we want to avoid the possibility of unintended side-effects.
+tmpsettings = portage.config(clone=tmpsettings)
+
 try:
metadata = dict(zip(Package.metadata_keys,
portage.db[portage.settings['EROOT']][mytree].dbapi.aux_get(
@@ -315,12 +330,6 @@ def stale_env_warning():

open(os.path.join(tmpsettings['PORTAGE_BUILDDIR'],
'.ebuild_changed'), 'w').close()
 
-if 'digest' in tmpsettings.features:
-   if pargs and pargs[0] not in ("digest", "manifest"):
-   pargs = ['digest'] + pargs
-   # We only need to build digests on the first pass.
-   tmpsettings.features.discard('digest')
-
 checked_for_stale_env = False
 
 for arg in pargs:
-- 
2.7.4

Re: [gentoo-portage-dev] [PATCH] _post_src_install_uid_fix: allow files with portage group permissions (bug 600804)

[Zac Medico]

On 11/29/2016 03:26 PM, Brian Dolbec wrote:
> On Tue, 29 Nov 2016 12:43:16 -0800
> Zac Medico <zmed...@gentoo.org> wrote:
> 
>> Allow ebuilds to install files with portage group permissions, as
>> a means to restrict access to package manager resources.
>>
>> X-Gentoo-Bug: 600804
>> X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=600804
>> ---
>>  pym/portage/package/ebuild/doebuild.py | 9 ++---
>>  1 file changed, 6 insertions(+), 3 deletions(-)
>>
>> diff --git a/pym/portage/package/ebuild/doebuild.py
>> b/pym/portage/package/ebuild/doebuild.py index 52dbf8b..4baae17 100644
>> --- a/pym/portage/package/ebuild/doebuild.py
>> +++ b/pym/portage/package/ebuild/doebuild.py
>> @@ -2008,7 +2008,7 @@ def _postinst_bsdflags(mysettings):
>>  def _post_src_install_uid_fix(mysettings, out):
>>  """
>>  Files in $D with user and group bits that match the "portage"
>> -user or group are automatically mapped to PORTAGE_INST_UID
>> and
>> +user and group are automatically mapped to PORTAGE_INST_UID
>> and PORTAGE_INST_GID if necessary. The chown system call may clear
>>  S_ISUID and S_ISGID bits, so those bits are restored if
>>  necessary.
>> @@ -2154,8 +2154,11 @@ def _post_src_install_uid_fix(mysettings, out):
>>  mystat.st_ino not in
>> counted_inodes: counted_inodes.add(mystat.st_ino)
>>  size += mystat.st_size
>> -if mystat.st_uid != portage_uid and \
>> -mystat.st_gid != portage_gid:
>> +
>> +# Only remap the UID/GID if both
>> match the portage user,
>> +# in order to avoid interference
>> with ebuilds that install
>> +# files with portage group
>> permissions (see bug 600804).
>> +if (mystat.st_uid, mystat.st_gid) !=
>> (portage_uid, portage_gid): continue
>>  myuid = -1
>>  mygid = -1
> 
> looks good to me
> 

Thanks, pushed:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=f479a4cdcac5db92231f489f232f10eb934c6f12
-- 
Thanks,
Zac