[gentoo-portage-dev] [PATCH] Add @changed-subslot package set

2021-01-15 Thread Matt Turner
This set is the upgradable packages for which the highest visible
version has a different subslot than the currently installed version.

The primary purpose of this feature is for use in catalyst builds. We
update the "seed" stage3 before using it to build a new stage1.

Updating the entire stage is expensive and unnecessary (since we're
going to build the latest packages in stage1 and then rebuild everything
in stage3).

What we definitely do need to update in the original stage3 however, is
any package that would trigger a subslot rebuild.

For example: gcc links with libmpfr.so from dev-libs/mpfr. mpfr's SONAME
changes from libmpfr.so.4 (SLOT="0/4") to libmpfr.so.6 (SLOT="0/6"). If
the seed stage's dev-libs/mpfr is not updated before emerging gcc, gcc
will link with libmpfr.so.4, but the latest version of dev-libs/mpfr
will be built and libmpfr.so.6 included into the stage1. Since the old
libmpfr.so.4 is not included in the stage1, gcc will not work, breaking
subsequent stage builds.

Our current options to update the seed are too large a hammer (e.g.,
"--update --deep --newuse @world" or "--update --deep --newuse
--complete-graph --rebuild-if-new-ver gcc") and spend too much time
updating seed stages for no gain beyond updating only packages for whom
the subslot has changed.

With this set, catalyst will likely use

emerge @changed-subslot --ignore-built-slot-operator-deps y

to update the seed stage.

Thank you to Zac Medico for showing me how to do this.

Bug: https://bugs.gentoo.org/739004
Signed-off-by: Matt Turner 
---
 cnf/sets/portage.conf  |  5 +
 lib/portage/_sets/dbapi.py | 39 +-
 2 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/cnf/sets/portage.conf b/cnf/sets/portage.conf
index 22f0fa3a5..5651a9c53 100644
--- a/cnf/sets/portage.conf
+++ b/cnf/sets/portage.conf
@@ -84,6 +84,11 @@ exclude-files = /usr/bin/Xorg
 [rebuilt-binaries]
 class = portage.sets.dbapi.RebuiltBinaries
 
+# Installed packages for which the subslot of the highest visible ebuild
+# version is different than the currently installed version.
+[changed-subslot]
+class = portage.sets.dbapi.SubslotChangedSet
+
 # Installed packages for which the highest visible ebuild
 # version is lower than the currently installed version.
 [downgrade]
diff --git a/lib/portage/_sets/dbapi.py b/lib/portage/_sets/dbapi.py
index 52367c4a6..46ba5c17d 100644
--- a/lib/portage/_sets/dbapi.py
+++ b/lib/portage/_sets/dbapi.py
@@ -15,7 +15,7 @@ from portage._sets import SetConfigError, get_boolean
 import portage
 
 __all__ = ["CategorySet", "ChangedDepsSet", "DowngradeSet",
-   "EverythingSet", "OwnerSet", "VariableSet"]
+   "EverythingSet", "OwnerSet", "SubslotChangedSet", "VariableSet"]
 
 class EverythingSet(PackageSet):
_operations = ["merge"]
@@ -167,6 +167,43 @@ class VariableSet(EverythingSet):
 
singleBuilder = classmethod(singleBuilder)
 
+class SubslotChangedSet(PackageSet):
+
+   _operations = ["merge", "unmerge"]
+
+   description = "Package set which contains all packages " + \
+   "for which the subslot of the highest visible ebuild is " + \
+   "different than the currently installed version."
+
+   def __init__(self, portdb=None, vardb=None):
+   super(SubslotChangedSet, self).__init__()
+   self._portdb = portdb
+   self._vardb = vardb
+
+   def load(self):
+   atoms = []
+   xmatch = self._portdb.xmatch
+   xmatch_level = "bestmatch-visible"
+   cp_list = self._vardb.cp_list
+   pkg_str = self._vardb._pkg_str
+   for cp in self._vardb.cp_all():
+   for cpv in cp_list(cp):
+   pkg = pkg_str(cpv, None)
+   slot_atom = "%s:%s" % (pkg.cp, pkg.slot)
+   ebuild = xmatch(xmatch_level, slot_atom)
+   if not ebuild:
+   continue
+   if pkg.sub_slot != ebuild.sub_slot:
+   atoms.append(slot_atom)
+
+   self._setAtoms(atoms)
+
+   def singleBuilder(cls, options, settings, trees):
+   return cls(portdb=trees["porttree"].dbapi,
+   vardb=trees["vartree"].dbapi)
+
+   singleBuilder = classmethod(singleBuilder)
+
 class DowngradeSet(PackageSet):
 
_operations = ["merge", "unmerge"]
-- 
2.26.2




[gentoo-portage-dev] [PATCH gentoolkit] equery: Remove 'changes' subcommand

2021-01-04 Thread Matt Turner
ChangeLogs have been gone from gentoo.git since the beginning, and
Council agreed in 2016 to allow Infra to decide whether to distribute
them through rsync, which they have decided not to do [1].

[1] https://projects.gentoo.org/council/meeting-logs/20160410-summary.txt

Signed-off-by: Matt Turner 
---
 pym/gentoolkit/equery/__init__.py   |   1 -
 pym/gentoolkit/equery/changes.py| 184 
 pym/gentoolkit/helpers.py   | 173 --
 pym/gentoolkit/test/equery/test_init.py |   1 -
 pym/gentoolkit/test/test_helpers.py |  47 --
 5 files changed, 406 deletions(-)
 delete mode 100644 pym/gentoolkit/equery/changes.py

diff --git a/pym/gentoolkit/equery/__init__.py 
b/pym/gentoolkit/equery/__init__.py
index 4640086..677f534 100644
--- a/pym/gentoolkit/equery/__init__.py
+++ b/pym/gentoolkit/equery/__init__.py
@@ -42,7 +42,6 @@ __authors__ = (
 
 NAME_MAP = {
'b': 'belongs',
-   'c': 'changes',
'k': 'check',
'd': 'depends',
'g': 'depgraph',
diff --git a/pym/gentoolkit/equery/changes.py b/pym/gentoolkit/equery/changes.py
deleted file mode 100644
index f234ecb..000
--- a/pym/gentoolkit/equery/changes.py
+++ /dev/null
@@ -1,184 +0,0 @@
-# Copyright(c) 2009, Gentoo Foundation
-#
-# Licensed under the GNU General Public License, v2 or higher
-
-"""Displays the ChangeLog entry for the latest installable version of an 
atom"""
-
-__docformat__ = 'epytext'
-
-# ===
-# Imports
-# ===
-
-import sys
-import os
-from getopt import gnu_getopt, GetoptError
-
-import gentoolkit.pprinter as pp
-from gentoolkit.atom import Atom
-from gentoolkit.equery import format_options, mod_usage
-from gentoolkit.helpers import ChangeLog
-from gentoolkit.query import Query
-
-# ===
-# Globals
-# ===
-
-QUERY_OPTS = {
-   'only_latest': False,
-   'show_full_log': False,
-   'limit': None,
-   'from': None,
-   'to': None
-}
-
-# =
-# Functions
-# =
-
-def print_help(with_description=True):
-   """Print description, usage and a detailed help message.
-
-   @type with_description: bool
-   @param with_description: if true, print module's __doc__ string
-   """
-
-   if with_description:
-   print(__doc__.strip())
-   print()
-   print(mod_usage(mod_name="changes"))
-   print()
-   print(pp.emph("examples"))
-   print (" c portage# show latest visible 
"
-  "version's entry")
-   print(" c portage --full --limit=3   # show 3 latest 
entries")
-   print(" c '=sys-apps/portage-2.1.6*' # use atom syntax")
-   print(" c portage --from=2.2_rc60 --to=2.2_rc70  # use version ranges")
-   print()
-   print(pp.command("options"))
-   print(format_options((
-   (" -h, --help", "display this help message"),
-   (" -l, --latest", "display only the latest ChangeLog entry"),
-   (" -f, --full", "display the full ChangeLog"),
-   (" --limit=NUM",
-   "limit the number of entries displayed (with --full)"),
-   (" --from=VER", "set which version to display from"),
-   (" --to=VER", "set which version to display to"),
-   )))
-
-
-def parse_module_options(module_opts):
-   """Parse module options and update QUERY_OPTS"""
-
-   opts = (x[0] for x in module_opts)
-   posargs = (x[1] for x in module_opts)
-   for opt, posarg in zip(opts, posargs):
-   if opt in ('-h', '--help'):
-   print_help()
-   sys.exit(0)
-   elif opt in ('-f', '--full'):
-   QUERY_OPTS['show_full_log'] = True
-   elif opt in ('-l', '--latest'):
-   QUERY_OPTS['only_latest'] = True
-   elif opt in ('--limit',):
-   set_limit(posarg)
-   elif opt in ('--from',):
-   QUERY_OPTS['from'] = posarg
-   elif opt in ('--to',):
-   QUERY_OPTS['to'] = posarg
-
-
-def print_entries(entries):
-   """Print entries and strip trailing whitespace from the last entry."""
-
-   len_entries = len(entries)
-   for i, entry in enumerate(entries, start=1):
-   if i < len_entries:
-   pp.uprint(entry)
-   else:
-   pp.uprint(entry.strip())
-
-
-def set_limit(posarg):
-   """Set a limit in QUERY_OPTS on how many ChangeLog entries to display.
-
-   Die if

[gentoo-portage-dev] [PATCH gentoolkit] gentoolkit: Remove gentoolkit.test.cmp

2021-01-04 Thread Matt Turner
Signed-off-by: Matt Turner 
---
Seems to only be used by duplicated unit tests? I guess this might have
been useful when Python 2 was still supported?

 pym/gentoolkit/test/__init__.py  | 23 ---
 pym/gentoolkit/test/test_atom.py | 15 ---
 pym/gentoolkit/test/test_cpv.py  | 15 ---
 3 files changed, 53 deletions(-)

diff --git a/pym/gentoolkit/test/__init__.py b/pym/gentoolkit/test/__init__.py
index 9e41686..e69de29 100644
--- a/pym/gentoolkit/test/__init__.py
+++ b/pym/gentoolkit/test/__init__.py
@@ -1,23 +0,0 @@
-#!/usr/bin/python
-# Copyright 2009 Gentoo Foundation
-#
-# Distributed under the terms of the GNU General Public License v2
-
-__all__ = ['cmp']
-
-# py3k doesn't have cmp emulate it in order to keep testing cmp
-# in python-2.x
-#XXX: not sure if this is the best place for this
-try:
-   cmp = cmp
-except NameError:
-   def cmp(a, b):
-   if a == b:
-   return 0
-   elif a < b:
-   return -1
-   elif a > b:
-   return 1
-   # just to be safe, __lt__/ __gt__ above should have thrown
-   # something like this already
-   raise TypeError("Comparison between unorderable types")
diff --git a/pym/gentoolkit/test/test_atom.py b/pym/gentoolkit/test/test_atom.py
index 664bf40..6df52a9 100644
--- a/pym/gentoolkit/test/test_atom.py
+++ b/pym/gentoolkit/test/test_atom.py
@@ -7,7 +7,6 @@ import unittest
 
 from gentoolkit.atom import Atom
 from gentoolkit.cpv import CPV
-from gentoolkit.test import cmp
 
 """Atom test suite (verbatim) from pkgcore."""
 
@@ -17,26 +16,12 @@ class TestGentoolkitAtom(unittest.TestCase):
# logic bugs hidden behind short circuiting comparisons for 
metadata
# is why we test the comparison *both* ways.
self.assertEqual(o1, o2)
-   c = cmp(o1, o2)
-   self.assertEqual(c, 0,
-   msg="checking cmp for %r, %r, aren't equal: got %i" % 
(o1, o2, c))
self.assertEqual(o2, o1)
-   c = cmp(o2, o1)
-   self.assertEqual(c, 0,
-   msg="checking cmp for %r, %r,aren't equal: got %i" % 
(o2, o1, c))
 
def assertNotEqual2(self, o1, o2):
# is why we test the comparison *both* ways.
self.assertNotEqual(o1, o2)
-   c = cmp(o1, o2)
-   self.assertNotEqual(c, 0,
-   msg="checking cmp for %r, %r, not supposed to be equal, 
got %i"
-   % (o1, o2, c))
self.assertNotEqual(o2, o1)
-   c = cmp(o2, o1)
-   self.assertNotEqual(c, 0,
-   msg="checking cmp for %r, %r, not supposed to be equal, 
got %i"
-   % (o2, o1, c))
 
def test_comparison(self):
self.assertEqual2(Atom('cat/pkg'), Atom('cat/pkg'))
diff --git a/pym/gentoolkit/test/test_cpv.py b/pym/gentoolkit/test/test_cpv.py
index 3817e9f..92ffba5 100644
--- a/pym/gentoolkit/test/test_cpv.py
+++ b/pym/gentoolkit/test/test_cpv.py
@@ -7,7 +7,6 @@
 import unittest
 
 from gentoolkit.cpv import CPV, compare_strs
-from gentoolkit.test import cmp
 
 class TestGentoolkitCPV(unittest.TestCase):
 
@@ -15,26 +14,12 @@ class TestGentoolkitCPV(unittest.TestCase):
# logic bugs hidden behind short circuiting comparisons for 
metadata
# is why we test the comparison *both* ways.
self.assertEqual(o1, o2)
-   c = cmp(o1, o2)
-   self.assertEqual(c, 0,
-   msg="checking cmp for %r, %r, aren't equal: got %i" % 
(o1, o2, c))
self.assertEqual(o2, o1)
-   c = cmp(o2, o1)
-   self.assertEqual(c, 0,
-   msg="checking cmp for %r, %r,aren't equal: got %i" % 
(o2, o1, c))
 
def assertNotEqual2(self, o1, o2):
# is why we test the comparison *both* ways.
self.assertNotEqual(o1, o2)
-   c = cmp(o1, o2)
-   self.assertNotEqual(c, 0,
-   msg="checking cmp for %r, %r, not supposed to be equal, 
got %i"
-   % (o1, o2, c))
self.assertNotEqual(o2, o1)
-   c = cmp(o2, o1)
-   self.assertNotEqual(c, 0,
-   msg="checking cmp for %r, %r, not supposed to be equal, 
got %i"
-   % (o2, o1, c))
 
def test_comparison(self):
self.assertEqual2(CPV('pkg'), CPV('pkg'))
-- 
2.26.2




[gentoo-portage-dev] [PATCH gentoolkit] eclean: Add --changed-iuse flag

2021-01-02 Thread Matt Turner
Allows binpkgs to be deleted if they are not usable due to IUSE changes.
---
Just kind of spitballing. I'm not sure about what USE flags we should
ignore or whether it should be configurable, etc. On one hand, deleting
binpkgs that don't have a newly added PYTHON_TARGET option might make
sense if your binhost is configured to rebuild the package. On the
other, you probably don't want to throw out amd64 binpkgs because
abi_riscv_* was added.

 pym/gentoolkit/eclean/cli.py|  8 -
 pym/gentoolkit/eclean/search.py | 62 +
 2 files changed, 54 insertions(+), 16 deletions(-)

diff --git a/pym/gentoolkit/eclean/cli.py b/pym/gentoolkit/eclean/cli.py
index e31fde9..910296b 100644
--- a/pym/gentoolkit/eclean/cli.py
+++ b/pym/gentoolkit/eclean/cli.py
@@ -146,6 +146,8 @@ def printUsage(_error=None, help=None):
green("packages"),"action:", file=out)
print( yellow(" --changed-deps")+
"   - delete packages for which ebuild 
dependencies have changed", file=out)
+   print( yellow(" --changed-iuse")+
+   "   - delete packages for which IUSE have 
changed", file=out)
print( yellow(" -i, --ignore-failure")+
" - ignore failure to locate PKGDIR", 
file=out)
print( file=out)
@@ -264,6 +266,8 @@ def parseArgs(options={}):
options['verbose'] = True
elif o in ("--changed-deps"):
options['changed-deps'] = True
+   elif o in ("--changed-iuse"):
+   options['changed-iuse'] = True
elif o in ("-i", "--ignore-failure"):
options['ignore-failure'] = True
else:
@@ -291,7 +295,8 @@ def parseArgs(options={}):
getopt_options['short']['distfiles'] = "fs:"
getopt_options['long']['distfiles'] = ["fetch-restricted", 
"size-limit="]
getopt_options['short']['packages'] = "i"
-   getopt_options['long']['packages'] = ["ignore-failure", "changed-deps"]
+   getopt_options['long']['packages'] = ["ignore-failure", "changed-deps",
+ "changed-iuse"]
# set default options, except 'nocolor', which is set in main()
options['interactive'] = False
options['pretend'] = False
@@ -305,6 +310,7 @@ def parseArgs(options={}):
options['size-limit'] = 0
options['verbose'] = False
options['changed-deps'] = False
+   options['changed-iuse'] = False
options['ignore-failure'] = False
# if called by a well-named symlink, set the action accordingly:
action = None
diff --git a/pym/gentoolkit/eclean/search.py b/pym/gentoolkit/eclean/search.py
index 8f6e52f..8344e15 100644
--- a/pym/gentoolkit/eclean/search.py
+++ b/pym/gentoolkit/eclean/search.py
@@ -498,6 +498,47 @@ def _deps_equal(deps_a, eapi_a, deps_b, eapi_b, 
uselist=None):
return deps_a == deps_b
 
 
+def _changed_deps(cpv, options, port_dbapi, bin_dbapi):
+   if not options['changed-deps']:
+   return False
+
+   dep_keys = ('RDEPEND', 'PDEPEND')
+   keys = ('EAPI', 'USE') + dep_keys
+   binpkg_metadata = dict(zip(keys, bin_dbapi.aux_get(cpv, keys)))
+   ebuild_metadata = dict(zip(keys, port_dbapi.aux_get(cpv, keys)))
+
+   return _deps_equal(' '.join(binpkg_metadata[key] for key in dep_keys),
+  binpkg_metadata['EAPI'], ' 
'.join(ebuild_metadata[key] for key in dep_keys),
+  ebuild_metadata['EAPI'],
+  
frozenset(binpkg_metadata['USE'].split()))
+
+
+# IUSE flags to be ignored for purposes of invalidating binpkgs
+def ignore(iuse):
+   return iuse.startswith('abi_')
+
+
+# Prune the + characters from USE flags (e.g. '+cxx' -> 'cxx')
+def prune_plus(iuse):
+   if iuse.startswith('+'):
+   return iuse[1:]
+   return iuse
+
+
+def _changed_iuse(cpv, options, port_dbapi, bin_dbapi):
+   if not options['changed-iuse']:
+   return False
+
+   keys = ('IUSE',)
+   binpkg_metadata = [prune_plus(x) for x in  bin_dbapi.aux_get(cpv, 
keys)[0].split(' ') if not ignore(x)]
+   ebuild_metadata = [prune_plus(x) for x in port_dbapi.aux_get(cpv, 
keys)[0].split(' ') if not ignore(x)]
+
+   if binpkg_metadata == ['']: binpkg_metadata = []
+   if ebuild_metadata == ['']: ebuild_metadata = []
+
+   return binpkg_metadata != ebuild_metadata
+
+
 def findPackages(
options,
exclude=None,
@@ -570,21 +611,6 @@ def findPackages(
if mtime >= time_limit:
continue
 
-   # Exclude if binpkg exists in the porttree and 

Re: [gentoo-portage-dev] [PATCH gentoolkit] bin: Add merge-driver-ekeyword

2020-12-31 Thread Matt Turner
On Mon, Dec 28, 2020 at 8:09 PM Zac Medico  wrote:
>
> On 12/28/20 3:15 PM, Matt Turner wrote:
> > +def apply_keyword_changes(ebuild: str, pathname: str,
> > +  changes: List[Tuple[Optional[str],
> > +  Optional[str]]]) -> int:
> > +result: int = 0
> > +
> > +# ekeyword will only modify files named *.ebuild, so make a symlink
> > +ebuild_symlink: str = os.path.basename(pathname)
> > +os.symlink(ebuild, ebuild_symlink)
>
> Are we sure that the current working directory is an entirely safe place
> to create this symlink? A simple fix would be to use
> tempfile.TemporaryDirectory to create a temporary directory to hold the
> symlink. Or, we could change ekeyword to assume that an argument is an
> ebuild if os.path.isfile(arg) succeeds.

Thanks, this is a good question. I've sent a v3 patch using
tempfile.TemporaryDirectory. I think that's better than passing
".merge_file_SEd3R8" to ekeyword since the filename is printed and
it's nice to see what file is being modified during the rebase.



[gentoo-portage-dev] [PATCH gentoolkit] bin: Add merge-driver-ekeyword

2020-12-31 Thread Matt Turner
Since the KEYWORDS=... assignment is a single line, git struggles to
handle conflicts. When rebasing a series of commits that modify the
KEYWORDS=... it's usually easier to throw them away and reapply on the
new tree than it is to manually handle conflicts during the rebase.

git allows a 'merge driver' program to handle conflicts; this program
handles conflicts in the KEYWORDS=... assignment. E.g., given an ebuild
with these keywords:

KEYWORDS="~alpha amd64 arm arm64 ~hppa ppc ppc64 x86"

One developer drops the ~alpha keyword and pushes to gentoo.git, and
another developer stabilizes hppa. Without this merge driver, git
requires the second developer to manually resolve the conflict which is
tedious and prone to mistakes when rebasing a long series of patches.
With the custom merge driver, it automatically resolves the conflict.

To use the merge driver, configure your gentoo.git as such:

gentoo.git/.git/config:

[merge "keywords"]
name = KEYWORDS merge driver
driver = merge-driver-ekeyword %O %A %B %P

gentoo.git/.git/info/attributes:

*.ebuild merge=keywords

Signed-off-by: Matt Turner 
---
v3: Address Zac's feedback: use tempfile.TemporaryDirectory

Since ekeyword prints the name of the file modified, I think using a
symlink with the name of the original file is better than having it
print .merge_file_SEd3R8.

 bin/merge-driver-ekeyword | 132 ++
 1 file changed, 132 insertions(+)
 create mode 100755 bin/merge-driver-ekeyword

diff --git a/bin/merge-driver-ekeyword b/bin/merge-driver-ekeyword
new file mode 100755
index 000..2df83fc
--- /dev/null
+++ b/bin/merge-driver-ekeyword
@@ -0,0 +1,132 @@
+#!/usr/bin/python
+#
+# Copyright 2020 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2 or later
+
+"""
+Custom git merge driver for handling conflicts in KEYWORDS assignments
+
+See https://git-scm.com/docs/gitattributes#_defining_a_custom_merge_driver
+"""
+
+import difflib
+import os
+import sys
+import tempfile
+
+from typing import List, Optional, Tuple
+
+from gentoolkit.ekeyword import ekeyword
+
+
+def keyword_array(keyword_line: str) -> List[str]:
+# Find indices of string inside the double-quotes
+i1: int = keyword_line.find('"') + 1
+i2: int = keyword_line.rfind('"')
+
+# Split into array of KEYWORDS
+return keyword_line[i1:i2].split(' ')
+
+
+def keyword_line_changes(old: str, new: str) -> List[Tuple[Optional[str],
+   Optional[str]]]:
+a: List[str] = keyword_array(old)
+b: List[str] = keyword_array(new)
+
+s = difflib.SequenceMatcher(a=a, b=b)
+
+changes = []
+for tag, i1, i2, j1, j2 in s.get_opcodes():
+if tag == 'replace':
+changes.append((a[i1:i2], b[j1:j2]),)
+elif tag == 'delete':
+changes.append((a[i1:i2], None),)
+elif tag == 'insert':
+changes.append((None, b[j1:j2]),)
+else:
+assert tag == 'equal'
+return changes
+
+
+def keyword_changes(ebuild1: str, ebuild2: str) -> List[Tuple[Optional[str],
+  Optional[str]]]:
+with open(ebuild1) as e1, open(ebuild2) as e2:
+lines1 = e1.readlines()
+lines2 = e2.readlines()
+
+diff = difflib.unified_diff(lines1, lines2, n=0)
+assert next(diff) == '--- \n'
+assert next(diff) == '+++ \n'
+
+hunk: int = 0
+old: str = ''
+new: str = ''
+
+for line in diff:
+if line.startswith('@@ '):
+if hunk > 0:
+break
+hunk += 1
+elif line.startswith('-'):
+if old or new:
+break
+old = line
+elif line.startswith('+'):
+if not old or new:
+break
+new = line
+else:
+if 'KEYWORDS=' in old and 'KEYWORDS=' in new:
+return keyword_line_changes(old, new)
+return None
+
+
+def apply_keyword_changes(ebuild: str, pathname: str,
+  changes: List[Tuple[Optional[str],
+  Optional[str]]]) -> int:
+result: int = 0
+
+with tempfile.TemporaryDirectory() as tmpdir:
+# ekeyword will only modify files named *.ebuild, so make a symlink
+ebuild_symlink: str = os.path.join(tmpdir, os.path.basename(pathname))
+os.symlink(os.path.join(os.getcwd(), ebuild), ebuild_symlink)
+
+for removals, additions in changes:
+args = []
+for rem in removals:
+# Drop leading '~' and '-' characters and prepend '^'
+i = 1 if rem[0] in ('~', '-') else 0
+args.append('^' + rem[i:])
+if 

[gentoo-portage-dev] [PATCH gentoolkit] bin: Add merge-driver-ekeyword

2020-12-28 Thread Matt Turner
Since the KEYWORDS=... assignment is a single line, git struggles to
handle conflicts. When rebasing a series of commits that modify the
KEYWORDS=... it's usually easier to throw them away and reapply on the
new tree than it is to manually handle conflicts during the rebase.

git allows a 'merge driver' program to handle conflicts; this program
handles conflicts in the KEYWORDS=... assignment. E.g., given an ebuild
with these keywords:

KEYWORDS="~alpha amd64 arm arm64 ~hppa ppc ppc64 x86"

One developer drops the ~alpha keyword and pushes to gentoo.git, and
another developer stabilizes hppa. Without this merge driver, git
requires the second developer to manually resolve the conflict which is
tedious and prone to mistakes when rebasing a long series of patches.
With the custom merge driver, it automatically resolves the conflict.

To use the merge driver, configure your gentoo.git as such:

gentoo.git/.git/config:

[merge "keywords"]
name = KEYWORDS merge driver
driver = merge-driver-ekeyword %O %A %B %P

gentoo.git/.git/info/attributes:

*.ebuild merge=keywords

Signed-off-by: Matt Turner 
---
 bin/merge-driver-ekeyword | 131 ++
 1 file changed, 131 insertions(+)
 create mode 100755 bin/merge-driver-ekeyword

diff --git a/bin/merge-driver-ekeyword b/bin/merge-driver-ekeyword
new file mode 100755
index 000..2142dc8
--- /dev/null
+++ b/bin/merge-driver-ekeyword
@@ -0,0 +1,131 @@
+#!/usr/bin/python
+#
+# Copyright 2020 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2 or later
+
+"""
+Custom git merge driver for handling conflicts in KEYWORDS assignments
+
+See https://git-scm.com/docs/gitattributes#_defining_a_custom_merge_driver
+"""
+
+import difflib
+import os
+import sys
+
+from typing import List, Optional, Tuple
+
+from gentoolkit.ekeyword import ekeyword
+
+
+def keyword_array(keyword_line: str) -> List[str]:
+# Find indices of string inside the double-quotes
+i1: int = keyword_line.find('"') + 1
+i2: int = keyword_line.rfind('"')
+
+# Split into array of KEYWORDS
+return keyword_line[i1:i2].split(' ')
+
+
+def keyword_line_changes(old: str, new: str) -> List[Tuple[Optional[str],
+   Optional[str]]]:
+a: List[str] = keyword_array(old)
+b: List[str] = keyword_array(new)
+
+s = difflib.SequenceMatcher(a=a, b=b)
+
+changes = []
+for tag, i1, i2, j1, j2 in s.get_opcodes():
+if tag == 'replace':
+changes.append((a[i1:i2], b[j1:j2]),)
+elif tag == 'delete':
+changes.append((a[i1:i2], None),)
+elif tag == 'insert':
+changes.append((None, b[j1:j2]),)
+else:
+assert tag == 'equal'
+return changes
+
+
+def keyword_changes(ebuild1: str, ebuild2: str) -> List[Tuple[Optional[str],
+  Optional[str]]]:
+with open(ebuild1) as e1, open(ebuild2) as e2:
+lines1 = e1.readlines()
+lines2 = e2.readlines()
+
+diff = difflib.unified_diff(lines1, lines2, n=0)
+assert next(diff) == '--- \n'
+assert next(diff) == '+++ \n'
+
+hunk: int = 0
+old: str = ''
+new: str = ''
+
+for line in diff:
+if line.startswith('@@ '):
+if hunk > 0:
+break
+hunk += 1
+elif line.startswith('-'):
+if old or new:
+break
+old = line
+elif line.startswith('+'):
+if not old or new:
+break
+new = line
+else:
+if 'KEYWORDS=' in old and 'KEYWORDS=' in new:
+return keyword_line_changes(old, new)
+return None
+
+
+def apply_keyword_changes(ebuild: str, pathname: str,
+  changes: List[Tuple[Optional[str],
+  Optional[str]]]) -> int:
+result: int = 0
+
+# ekeyword will only modify files named *.ebuild, so make a symlink
+ebuild_symlink: str = os.path.basename(pathname)
+os.symlink(ebuild, ebuild_symlink)
+
+for removals, additions in changes:
+args = []
+for rem in removals:
+# Drop leading '~' and '-' characters and prepend '^'
+i = 1 if rem[0] in ('~', '-') else 0
+args.append('^' + rem[i:])
+if additions:
+args.extend(additions)
+args.append(ebuild_symlink)
+
+result = ekeyword.main(args)
+if result != 0:
+break
+
+os.remove(ebuild_symlink)
+return result
+
+
+def main(argv):
+if len(argv) != 5:
+sys.exit(-1)
+
+O = argv[1] # %O - filename of original
+A = argv[2] # %A - filename of our current ve

Re: [gentoo-portage-dev] [PATCH] Drop Python 2 comatibility in extension modules

2020-12-24 Thread Matt Turner
Typo in commit title: comatibility



[gentoo-portage-dev] Re: [RFC PATCH gentoolkit] bin: Add merge-driver-ekeyword

2020-12-23 Thread Matt Turner
On Wed, Dec 23, 2020 at 2:46 PM Junio C Hamano  wrote:
>
> Matt Turner  writes:
>
> > I want to handle conflicts automatically on lines like
> >
> >> KEYWORDS="~alpha ~amd64 ~arm ~arm64 ~hppa ~ia64 ~mips ~ppc ~ppc64 ~riscv 
> >> ~s390 ~sparc ~x86"
> >
> > where conflicts frequently happen by adding/removing ~ before the
> > architecture names or adding/removing whole architectures. I don't
> > know if I should use a custom git merge driver or a custom git merge
> > strategy.
>
> A merge strategy is about how the changes at the tree level are
> handled.  A merge driver is given three blobs (original, your
> version, and their version) and comes up with a merged blob.
>
> In your case, you'd want a custom merge driver if you want to handle
> word changes on a single line, because the default text merge driver
> is pretty much line oriented.

Thanks, that makes sense. The merge driver I've written seems to work
great for handling the KEYWORDS=... line.

If users could more simply opt into using it (e.g., on the command
line rather than enabling it via ~/.gitattributes) I think it would be
fine to use. Better yet, is there a way git can be configured to
fallback to another merge driver if the first returns a non-zero
status due to unresolved conflicts? For example, if there are changes
to other lines, how can I fall back to another merge driver?

Thank you for your advice!



[gentoo-portage-dev] Re: [RFC PATCH gentoolkit] bin: Add merge-driver-ekeyword

2020-12-22 Thread Matt Turner
tl;dr:

I want to handle conflicts automatically on lines like

> KEYWORDS="~alpha ~amd64 ~arm ~arm64 ~hppa ~ia64 ~mips ~ppc ~ppc64 ~riscv 
> ~s390 ~sparc ~x86"

where conflicts frequently happen by adding/removing ~ before the
architecture names or adding/removing whole architectures. I don't
know if I should use a custom git merge driver or a custom git merge
strategy.


So the program in the patch below works, but it's not ideal, because
it rejects any hunks that don't touch the KEYWORDS=... assignment.

As I understand it, a custom git merge driver is intended to be used
to merge whole file formats, like JSON. As a result, you configure it
via gitattributes on a per-extension basis.

I really just want to make the default recursive git merge handle
KEYWORDS=... conflicts automatically, and I don't expect to be able to
make a git merge driver that can handle arbitrary conflicts in
*.ebuild files. If the merge driver returns non-zero if it was unable
to resolve the conflicts, but when it does so git evidently doesn't
fallback and insert the typical <<< HEAD ... === ... >>> markers.
Maybe I could make my merge driver insert those like git normally
does? Seems like git's  logic is probably a bit better about handling
some conflicts than my tool would be.

So... is a git merge strategy the thing I want? I don't know. There
doesn't seem to really be any documentation on writing git merge
strategies. I've only found [1] and [2].

Cc'ing g...@vger.kernel.org, since I expect that's where the experts
are. Hopefully they have suggestions.


[1] 
https://stackoverflow.com/questions/23140240/git-how-do-i-add-a-custom-merge-strategy
[2] 
https://stackoverflow.com/questions/54528824/any-documentation-for-writing-a-custom-git-merge-strategy


On Sun, Dec 20, 2020 at 10:44 PM Matt Turner  wrote:
>
> Since the KEYWORDS=... assignment is a single line, git struggles to
> handle conflicts. When rebasing a series of commits that modify the
> KEYWORDS=... it's usually easier to throw them away and reapply on the
> new tree than it is to manually handle conflicts during the rebase.
>
> git allows a 'merge driver' program to handle conflicts; this program
> handles conflicts in the KEYWORDS=... assignment. E.g., given an ebuild
> with these keywords:
>
> KEYWORDS="~alpha ~amd64 ~arm ~arm64 ~hppa ~ia64 ~mips ~ppc ~ppc64 ~riscv 
> ~s390 ~sparc ~x86"
>
> One developer drops the ~alpha keyword and pushes to gentoo.git, and
> another developer stabilizes hppa. Without this merge driver, git
> requires the second developer to manually resolve the conflict.  With
> the custom merge driver, it automatically resolves the conflict.
>
> gentoo.git/.git/config:
>
> [core]
> ...
> attributesfile = ~/.gitattributes
> [merge "keywords"]
> name = KEYWORDS merge driver
>         driver = merge-driver-ekeyword %O %A %B
>
>  ~/.gitattributes:
>
> *.ebuild merge=keywords
>
> Signed-off-by: Matt Turner 
> ---
> One annoying wart in the program is due to the fact that ekeyword
> won't work on any file not named *.ebuild. I make a symlink (and set up
> an atexit handler to remove it) to work around this. I'm not sure we
> could make ekeyword handle arbitrary filenames given its complex multi-
> argument parameter support. git merge files are named .merge_file_X
> according to git-unpack-file(1), so we could allow those. Thoughts?
>
>  bin/merge-driver-ekeyword | 125 ++
>  1 file changed, 125 insertions(+)
>  create mode 100755 bin/merge-driver-ekeyword
>
> diff --git a/bin/merge-driver-ekeyword b/bin/merge-driver-ekeyword
> new file mode 100755
> index 000..6e645a9
> --- /dev/null
> +++ b/bin/merge-driver-ekeyword
> @@ -0,0 +1,125 @@
> +#!/usr/bin/python
> +#
> +# Copyright 2020 Gentoo Authors
> +# Distributed under the terms of the GNU General Public License v2 or later
> +
> +"""
> +Custom git merge driver for handling conflicts in KEYWORDS assignments
> +
> +See https://git-scm.com/docs/gitattributes#_defining_a_custom_merge_driver
> +"""
> +
> +import atexit
> +import difflib
> +import os
> +import shutil
> +import sys
> +
> +from typing import List, Optional, Tuple
> +
> +from gentoolkit.ekeyword import ekeyword
> +
> +
> +def keyword_array(keyword_line: str) -> List[str]:
> +# Find indices of string inside the double-quotes
> +i1: int = keyword_line.find('"') + 1
> +i2: int = keyword_line.rfind('"')
> +
> +# Split into array of KEYWORDS
> +return keyword_line[i1:i2].split(' ')
> +
> +
> +def keyword_line_changes(old: str, new: str) -> List[Tuple[Optional[st

[gentoo-portage-dev] [RFC PATCH gentoolkit] bin: Add merge-driver-ekeyword

2020-12-20 Thread Matt Turner
Since the KEYWORDS=... assignment is a single line, git struggles to
handle conflicts. When rebasing a series of commits that modify the
KEYWORDS=... it's usually easier to throw them away and reapply on the
new tree than it is to manually handle conflicts during the rebase.

git allows a 'merge driver' program to handle conflicts; this program
handles conflicts in the KEYWORDS=... assignment. E.g., given an ebuild
with these keywords:

KEYWORDS="~alpha ~amd64 ~arm ~arm64 ~hppa ~ia64 ~mips ~ppc ~ppc64 ~riscv ~s390 
~sparc ~x86"

One developer drops the ~alpha keyword and pushes to gentoo.git, and
another developer stabilizes hppa. Without this merge driver, git
requires the second developer to manually resolve the conflict.  With
the custom merge driver, it automatically resolves the conflict.

gentoo.git/.git/config:

[core]
...
attributesfile = ~/.gitattributes
[merge "keywords"]
name = KEYWORDS merge driver
driver = merge-driver-ekeyword %O %A %B

 ~/.gitattributes:

*.ebuild merge=keywords

Signed-off-by: Matt Turner 
---
One annoying wart in the program is due to the fact that ekeyword
won't work on any file not named *.ebuild. I make a symlink (and set up
an atexit handler to remove it) to work around this. I'm not sure we
could make ekeyword handle arbitrary filenames given its complex multi-
argument parameter support. git merge files are named .merge_file_X
according to git-unpack-file(1), so we could allow those. Thoughts?

 bin/merge-driver-ekeyword | 125 ++
 1 file changed, 125 insertions(+)
 create mode 100755 bin/merge-driver-ekeyword

diff --git a/bin/merge-driver-ekeyword b/bin/merge-driver-ekeyword
new file mode 100755
index 000..6e645a9
--- /dev/null
+++ b/bin/merge-driver-ekeyword
@@ -0,0 +1,125 @@
+#!/usr/bin/python
+#
+# Copyright 2020 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2 or later
+
+"""
+Custom git merge driver for handling conflicts in KEYWORDS assignments
+
+See https://git-scm.com/docs/gitattributes#_defining_a_custom_merge_driver
+"""
+
+import atexit
+import difflib
+import os
+import shutil
+import sys
+
+from typing import List, Optional, Tuple
+
+from gentoolkit.ekeyword import ekeyword
+
+
+def keyword_array(keyword_line: str) -> List[str]:
+# Find indices of string inside the double-quotes
+i1: int = keyword_line.find('"') + 1
+i2: int = keyword_line.rfind('"')
+
+# Split into array of KEYWORDS
+return keyword_line[i1:i2].split(' ')
+
+
+def keyword_line_changes(old: str, new: str) -> List[Tuple[Optional[str],
+   Optional[str]]]:
+a: List[str] = keyword_array(old)
+b: List[str] = keyword_array(new)
+
+s = difflib.SequenceMatcher(a=a, b=b)
+
+changes = []
+for tag, i1, i2, j1, j2 in s.opcodes():
+if tag == 'replace':
+changes.append((a[i1:i2], b[j1:j2]),)
+elif tag == 'delete':
+changes.append((a[i1:i2], None),)
+elif tag == 'insert':
+changes.append((None, b[j1:j2]),)
+else:
+assert tag == 'equal'
+return changes
+
+
+def keyword_changes(ebuild1: str, ebuild2: str) -> List[Tuple[Optional[str],
+  Optional[str]]]:
+with open(ebuild1) as e1, open(ebuild2) as e2:
+lines1 = e1.readlines()
+lines2 = e2.readlines()
+
+diff = difflib.unified_diff(lines1, lines2, n=0)
+assert next(diff) == '--- \n'
+assert next(diff) == '+++ \n'
+
+hunk: int = 0
+old: str = ''
+new: str = ''
+
+for line in diff:
+if line.startswith('@@ '):
+if hunk > 0: break
+hunk += 1
+elif line.startswith('-'):
+if old or new: break
+old = line
+elif line.startswith('+'):
+if not old or new: break
+new = line
+else:
+if 'KEYWORDS=' in old and 'KEYWORDS=' in new:
+return keyword_line_changes(old, new)
+return None
+
+
+def apply_keyword_changes(ebuild: str,
+  changes: List[Tuple[Optional[str],
+  Optional[str]]]) -> int:
+# ekeyword will only modify files named *.ebuild, so make a symlink
+ebuild_symlink = ebuild + '.ebuild'
+os.symlink(ebuild, ebuild_symlink)
+atexit.register(lambda: os.remove(ebuild_symlink))
+
+for removals, additions in changes:
+args = []
+for rem in removals:
+# Drop leading '~' and '-' characters and prepend '^'
+i = 1 if rem[0] in ('~', '-') else 0
+args.append('^' + rem[i:])
+if additions:
+ar

[gentoo-portage-dev] [PATCH gentoolkit 3/3] eclean: Remove unneeded __init__ to enable pytest

2020-12-20 Thread Matt Turner
Prevented the unit test from running:

pym/gentoolkit/test/eclean/distsupport.py:435: PytestCollectionWarning:
cannot collect test class 'TestDisfiles' because it has a __init__
constructor (from: pym/gentoolkit/test/eclean/test_search.py)

Signed-off-by: Matt Turner 
---
 pym/gentoolkit/test/eclean/distsupport.py | 6 --
 1 file changed, 6 deletions(-)

diff --git a/pym/gentoolkit/test/eclean/distsupport.py 
b/pym/gentoolkit/test/eclean/distsupport.py
index 0fc2db8..da7cdbb 100644
--- a/pym/gentoolkit/test/eclean/distsupport.py
+++ b/pym/gentoolkit/test/eclean/distsupport.py
@@ -434,12 +434,6 @@ class OutputSimulator:
 
 class TestDisfiles:
 
-   def __init__(self):
-   self.workdir = None
-   self.target_file = None
-   self.target_symlink = None
-   self.test_filepaths = None
-
def setUp(self):
# create the dist dir
self.tmpdir = mkdtemp()
-- 
2.26.2




[gentoo-portage-dev] [PATCH gentoolkit 2/3] ekeyword: Fix unit test

2020-12-20 Thread Matt Turner
arch_status changed from a Dict[str] to Dict[Tuple[str, str]], but these
bits of test data were not updated. Update the examples while we're here
(e.g. arm64 is stable with stable profiles now).

Fixes: 459cfba47d25 (ekeyword: Use now-common load_profile_data() from eshowkw)
Signed-off-by: Matt Turner 
---
 pym/gentoolkit/ekeyword/test_ekeyword.py | 38 +++-
 1 file changed, 18 insertions(+), 20 deletions(-)

diff --git a/pym/gentoolkit/ekeyword/test_ekeyword.py 
b/pym/gentoolkit/ekeyword/test_ekeyword.py
index 3d23585..12a0b54 100755
--- a/pym/gentoolkit/ekeyword/test_ekeyword.py
+++ b/pym/gentoolkit/ekeyword/test_ekeyword.py
@@ -165,28 +165,26 @@ class TestProcessKeywords(unittest.TestCase):
ekeyword.Op(None, 'all', None),
)
arch_status = {
-   'alpha': None,
-   'arm': 'stable',
-   'arm64': 'exp',
-   'm68k': 'dev',
+   'alpha': ('stable', '~arch'),
+   'arm':   ('stable', 'arch'),
+   'm68k':  ('exp', '~arch'),
+   's390':  ('exp', 'arch'),
}
-   self._test('* ~alpha ~arm ~arm64 ~m68k ~mips ~arm-linux', ops,
-  '* ~alpha arm ~arm64 ~m68k ~mips ~arm-linux', 
arch_status)
+   self._test('* ~alpha ~arm ~m68k ~mips ~s390 ~arm-linux', ops,
+  '* ~alpha arm ~m68k ~mips s390 ~arm-linux', 
arch_status)
 
def testAllUnstable(self):
ops = (
ekeyword.Op('~', 'all', None),
)
arch_status = {
-   'alpha': None,
-   'arm': 'stable',
-   'arm64': 'exp',
-   'm68k': 'dev',
-   's390': 'dev',
-   'sh': 'dev',
+   'alpha': ('stable', '~arch'),
+   'arm':   ('stable', 'arch'),
+   'm68k':  ('exp', '~arch'),
+   's390':  ('exp', 'arch'),
}
-   self._test('-* ~* * alpha arm arm64 m68k arm-linux', ops,
-  '-* ~* * ~alpha ~arm ~arm64 ~m68k ~arm-linux', 
arch_status)
+   self._test('-* ~* * alpha arm m68k s390 arm-linux', ops,
+  '-* ~* * ~alpha ~arm ~m68k ~s390 ~arm-linux', 
arch_status)
 
def testAllMultiUnstableStable(self):
ops = (
@@ -194,13 +192,13 @@ class TestProcessKeywords(unittest.TestCase):
ekeyword.Op(None, 'all', None),
)
arch_status = {
-   'alpha': None,
-   'arm': 'stable',
-   'arm64': 'exp',
-   'm68k': 'dev',
+   'alpha': ('stable', '~arch'),
+   'arm':   ('stable', 'arch'),
+   'm68k':  ('exp', '~arch'),
+   's390':  ('exp', 'arch'),
}
-   self._test('-* ~* * alpha arm arm64 m68k', ops,
-  '-* ~* * ~alpha arm ~arm64 ~m68k', arch_status)
+   self._test('-* ~* * alpha arm m68k s390', ops,
+  '-* ~* * ~alpha arm ~m68k s390', arch_status)
 
def testAllDisabled(self):
"""Make sure ~all does not change -arch to ~arch"""
-- 
2.26.2




[gentoo-portage-dev] [PATCH gentoolkit 1/3] ekeyword: Rename unit test so that it runs

2020-12-20 Thread Matt Turner
unittests should be named test_*.py so that they are discoverable.

Signed-off-by: Matt Turner 
---
 pym/gentoolkit/ekeyword/pytest.ini  | 2 +-
 .../ekeyword/{ekeyword_unittest.py => test_ekeyword.py} | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename pym/gentoolkit/ekeyword/{ekeyword_unittest.py => test_ekeyword.py} (99%)

diff --git a/pym/gentoolkit/ekeyword/pytest.ini 
b/pym/gentoolkit/ekeyword/pytest.ini
index 622c9d8..7e21bec 100644
--- a/pym/gentoolkit/ekeyword/pytest.ini
+++ b/pym/gentoolkit/ekeyword/pytest.ini
@@ -1,3 +1,3 @@
 [pytest]
 addopts = --cov
-python_files = *_unittest.py
+python_files = test_*.py
diff --git a/pym/gentoolkit/ekeyword/ekeyword_unittest.py 
b/pym/gentoolkit/ekeyword/test_ekeyword.py
similarity index 99%
rename from pym/gentoolkit/ekeyword/ekeyword_unittest.py
rename to pym/gentoolkit/ekeyword/test_ekeyword.py
index ef2e256..3d23585 100755
--- a/pym/gentoolkit/ekeyword/ekeyword_unittest.py
+++ b/pym/gentoolkit/ekeyword/test_ekeyword.py
@@ -12,7 +12,7 @@ import unittest
 
 import mock
 
-import ekeyword
+from gentoolkit.ekeyword import ekeyword
 
 
 TESTDIR = os.path.join(os.path.dirname(__file__), 'tests')
-- 
2.26.2




[gentoo-portage-dev] [PATCH gentoolkit 5/4] Remove unused sys imports

2020-12-20 Thread Matt Turner
Reported by flake8.

Signed-off-by: Matt Turner 
---
 pym/gentoolkit/eclean/exclude.py | 1 -
 pym/gentoolkit/enalyze/rebuild.py| 1 -
 pym/gentoolkit/helpers.py| 1 -
 pym/gentoolkit/revdep_rebuild/analyse.py | 1 -
 pym/gentoolkit/revdep_rebuild/cache.py   | 1 -
 pym/gentoolkit/revdep_rebuild/collect.py | 1 -
 pym/gentoolkit/test/test_keyword.py  | 1 -
 7 files changed, 7 deletions(-)

diff --git a/pym/gentoolkit/eclean/exclude.py b/pym/gentoolkit/eclean/exclude.py
index 45ecd52..1da9523 100644
--- a/pym/gentoolkit/eclean/exclude.py
+++ b/pym/gentoolkit/eclean/exclude.py
@@ -5,7 +5,6 @@
 
 
 import os
-import sys
 import re
 import portage
 from portage import _encodings, _unicode_encode
diff --git a/pym/gentoolkit/enalyze/rebuild.py 
b/pym/gentoolkit/enalyze/rebuild.py
index 3f3f4fd..c0ac8cb 100644
--- a/pym/gentoolkit/enalyze/rebuild.py
+++ b/pym/gentoolkit/enalyze/rebuild.py
@@ -11,7 +11,6 @@ what packages according to the Installed package database"""
 
 
 import os
-import sys
 
 import gentoolkit
 from gentoolkit.module_base import ModuleBase
diff --git a/pym/gentoolkit/helpers.py b/pym/gentoolkit/helpers.py
index a979ca7..e7185c3 100644
--- a/pym/gentoolkit/helpers.py
+++ b/pym/gentoolkit/helpers.py
@@ -24,7 +24,6 @@ __docformat__ = 'epytext'
 # ===
 
 import os
-import sys
 import re
 from functools import partial
 from itertools import chain
diff --git a/pym/gentoolkit/revdep_rebuild/analyse.py 
b/pym/gentoolkit/revdep_rebuild/analyse.py
index 3e46a51..bdd8306 100644
--- a/pym/gentoolkit/revdep_rebuild/analyse.py
+++ b/pym/gentoolkit/revdep_rebuild/analyse.py
@@ -5,7 +5,6 @@
 import os
 import re
 import time
-import sys
 
 from portage import _encodings, _unicode_encode
 from portage.output import bold, blue, yellow, green
diff --git a/pym/gentoolkit/revdep_rebuild/cache.py 
b/pym/gentoolkit/revdep_rebuild/cache.py
index f8b7841..ab0b7d7 100644
--- a/pym/gentoolkit/revdep_rebuild/cache.py
+++ b/pym/gentoolkit/revdep_rebuild/cache.py
@@ -5,7 +5,6 @@ Functions for reading, saving and verifying the data caches
 
 from portage import os
 import time
-import sys
 
 from portage import _encodings, _unicode_encode
 from portage.output import red
diff --git a/pym/gentoolkit/revdep_rebuild/collect.py 
b/pym/gentoolkit/revdep_rebuild/collect.py
index 74a44f7..38ff48e 100644
--- a/pym/gentoolkit/revdep_rebuild/collect.py
+++ b/pym/gentoolkit/revdep_rebuild/collect.py
@@ -6,7 +6,6 @@ import re
 from portage import os
 import glob
 import stat
-import sys
 
 import portage
 from portage import _encodings, _unicode_encode
diff --git a/pym/gentoolkit/test/test_keyword.py 
b/pym/gentoolkit/test/test_keyword.py
index 99a79b5..8ba5e30 100644
--- a/pym/gentoolkit/test/test_keyword.py
+++ b/pym/gentoolkit/test/test_keyword.py
@@ -1,4 +1,3 @@
-import sys
 import unittest
 
 from gentoolkit import keyword
-- 
2.26.2




[gentoo-portage-dev] [PATCH gentoolkit 4/4] Remove inherits from object

2020-12-20 Thread Matt Turner
Signed-off-by: Matt Turner 
---
 pym/gentoolkit/cpv.py | 2 +-
 pym/gentoolkit/eclean/clean.py| 2 +-
 pym/gentoolkit/eclean/output.py   | 2 +-
 pym/gentoolkit/eclean/pkgindex.py | 2 +-
 pym/gentoolkit/eclean/search.py   | 2 +-
 pym/gentoolkit/enalyze/lib.py | 4 ++--
 pym/gentoolkit/equery/belongs.py  | 2 +-
 pym/gentoolkit/equery/check.py| 2 +-
 pym/gentoolkit/equery/depends.py  | 2 +-
 pym/gentoolkit/formatters.py  | 2 +-
 pym/gentoolkit/helpers.py | 4 ++--
 pym/gentoolkit/keyword.py | 2 +-
 pym/gentoolkit/metadata.py| 8 
 pym/gentoolkit/module_base.py | 2 +-
 pym/gentoolkit/package.py | 2 +-
 pym/gentoolkit/revdep_rebuild/analyse.py  | 2 +-
 pym/gentoolkit/revdep_rebuild/assign.py   | 2 +-
 pym/gentoolkit/test/eclean/creator.py | 2 +-
 pym/gentoolkit/test/eclean/distsupport.py | 6 +++---
 pym/gentoolkit/test/eclean/test_clean.py  | 2 +-
 pym/gentoolkit/versionmatch.py| 2 +-
 21 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/pym/gentoolkit/cpv.py b/pym/gentoolkit/cpv.py
index 563dab8..5238e24 100644
--- a/pym/gentoolkit/cpv.py
+++ b/pym/gentoolkit/cpv.py
@@ -37,7 +37,7 @@ isvalid_rev_re = re.compile(r'(\d+|0\d+\.\d+)')
 # Classes
 # ===
 
-class CPV(object):
+class CPV:
"""Provides methods on a category/package-version string.
 
Will also correctly split just a package or package-version string.
diff --git a/pym/gentoolkit/eclean/clean.py b/pym/gentoolkit/eclean/clean.py
index 2b750e3..3f6fe45 100644
--- a/pym/gentoolkit/eclean/clean.py
+++ b/pym/gentoolkit/eclean/clean.py
@@ -11,7 +11,7 @@ import gentoolkit.pprinter as pp
 from gentoolkit.eclean.pkgindex import PkgIndex
 
 
-class CleanUp(object):
+class CleanUp:
"""Performs all cleaning actions to distfiles or package directories.
 
@param controller: a progress output/user interaction controller 
function
diff --git a/pym/gentoolkit/eclean/output.py b/pym/gentoolkit/eclean/output.py
index 122d550..e2ed221 100644
--- a/pym/gentoolkit/eclean/output.py
+++ b/pym/gentoolkit/eclean/output.py
@@ -9,7 +9,7 @@ from portage.output import blue, yellow, teal, green, red
 from gentoolkit.pprinter import cpv, number
 
 
-class OutputControl(object):
+class OutputControl:
"""Outputs data according to predetermined options and handles any user
interaction.
 
diff --git a/pym/gentoolkit/eclean/pkgindex.py 
b/pym/gentoolkit/eclean/pkgindex.py
index 6cf9000..46c734d 100644
--- a/pym/gentoolkit/eclean/pkgindex.py
+++ b/pym/gentoolkit/eclean/pkgindex.py
@@ -13,7 +13,7 @@ from gentoolkit.eprefix import EPREFIX
 import portage
 
 
-class PkgIndex(object):
+class PkgIndex:
"""Handle the cleaning of the binpkg Package
Index file
 
diff --git a/pym/gentoolkit/eclean/search.py b/pym/gentoolkit/eclean/search.py
index 992949c..8f6e52f 100644
--- a/pym/gentoolkit/eclean/search.py
+++ b/pym/gentoolkit/eclean/search.py
@@ -50,7 +50,7 @@ def get_distdir():
 distdir = get_distdir()
 
 
-class DistfilesSearch(object):
+class DistfilesSearch:
"""
 
@param output: verbose output method or (lambda x: None) to 
turn off
diff --git a/pym/gentoolkit/enalyze/lib.py b/pym/gentoolkit/enalyze/lib.py
index 8580bb7..50c7d11 100644
--- a/pym/gentoolkit/enalyze/lib.py
+++ b/pym/gentoolkit/enalyze/lib.py
@@ -17,7 +17,7 @@ from gentoolkit.flag import (reduce_flags, get_flags, 
get_all_cpv_use,
 import portage
 
 
-class FlagAnalyzer(object):
+class FlagAnalyzer:
"""Specialty functions for analysing an installed package's
USE flags.  Can be used for single or mulitple use without
needing to be reset unless the system USE flags are changed.
@@ -142,7 +142,7 @@ class FlagAnalyzer(object):
return _flags
 
 
-class KeywordAnalyser(object):
+class KeywordAnalyser:
"""Specialty functions for analysing the installed package db for
keyword useage and the packages that used them.
 
diff --git a/pym/gentoolkit/equery/belongs.py b/pym/gentoolkit/equery/belongs.py
index 8289366..1c9ab13 100644
--- a/pym/gentoolkit/equery/belongs.py
+++ b/pym/gentoolkit/equery/belongs.py
@@ -36,7 +36,7 @@ QUERY_OPTS = {
 # Classes
 # ===
 
-class BelongsPrinter(object):
+class BelongsPrinter:
"""Outputs a formatted list of packages that claim to own a files."""
 
def __init__(self, verbose=True, name_only=False):
diff --git a/pym/gentoolkit/equery/check.py b/pym/gentoolkit/equery/check.py
index e965164..47eba98 100644
--- a/pym/gentoolkit/equery/check.py
+++ b/pym/gentoolkit/equery/check.py
@@ -41,7 +41,7 @@ QUERY_OPTS = {
 # Classes
 # ===
 
-class VerifyContents(object):
+class VerifyContents:
&q

[gentoo-portage-dev] [PATCH gentoolkit 3/4] Remove Python < 3 version checks

2020-12-20 Thread Matt Turner
Signed-off-by: Matt Turner 
---
 pym/gentoolkit/eclean/exclude.py  |  2 --
 pym/gentoolkit/enalyze/rebuild.py |  2 --
 pym/gentoolkit/equery/uses.py |  2 --
 pym/gentoolkit/equery/which.py|  2 --
 pym/gentoolkit/helpers.py |  2 --
 pym/gentoolkit/profile.py |  3 ---
 pym/gentoolkit/revdep_rebuild/analyse.py  |  2 --
 pym/gentoolkit/revdep_rebuild/assign.py   |  6 --
 pym/gentoolkit/revdep_rebuild/cache.py|  7 +--
 pym/gentoolkit/revdep_rebuild/collect.py  | 10 +-
 pym/gentoolkit/revdep_rebuild/settings.py |  9 +
 pym/gentoolkit/test/eclean/creator.py |  2 --
 pym/gentoolkit/test/test_keyword.py   |  2 --
 13 files changed, 3 insertions(+), 48 deletions(-)

diff --git a/pym/gentoolkit/eclean/exclude.py b/pym/gentoolkit/eclean/exclude.py
index ee5e1e1..45ecd52 100644
--- a/pym/gentoolkit/eclean/exclude.py
+++ b/pym/gentoolkit/eclean/exclude.py
@@ -6,8 +6,6 @@
 
 import os
 import sys
-if sys.hexversion < 0x300:
-   from io import open
 import re
 import portage
 from portage import _encodings, _unicode_encode
diff --git a/pym/gentoolkit/enalyze/rebuild.py 
b/pym/gentoolkit/enalyze/rebuild.py
index 4ad571f..3f3f4fd 100644
--- a/pym/gentoolkit/enalyze/rebuild.py
+++ b/pym/gentoolkit/enalyze/rebuild.py
@@ -12,8 +12,6 @@ what packages according to the Installed package database"""
 
 import os
 import sys
-if sys.hexversion < 0x300:
-   from io import open
 
 import gentoolkit
 from gentoolkit.module_base import ModuleBase
diff --git a/pym/gentoolkit/equery/uses.py b/pym/gentoolkit/equery/uses.py
index 1260f56..dfb6f31 100644
--- a/pym/gentoolkit/equery/uses.py
+++ b/pym/gentoolkit/equery/uses.py
@@ -12,8 +12,6 @@ __docformat__ = 'epytext'
 
 import os
 import sys
-if sys.hexversion < 0x300:
-   from io import open
 
 from functools import partial
 from getopt import gnu_getopt, GetoptError
diff --git a/pym/gentoolkit/equery/which.py b/pym/gentoolkit/equery/which.py
index 326e692..c7fabd7 100644
--- a/pym/gentoolkit/equery/which.py
+++ b/pym/gentoolkit/equery/which.py
@@ -14,8 +14,6 @@ __docformat__ = 'epytext'
 
 import os
 import sys
-if sys.hexversion < 0x300:
-   from io import open
 from getopt import gnu_getopt, GetoptError
 
 
diff --git a/pym/gentoolkit/helpers.py b/pym/gentoolkit/helpers.py
index b4cc795..02ac20e 100644
--- a/pym/gentoolkit/helpers.py
+++ b/pym/gentoolkit/helpers.py
@@ -25,8 +25,6 @@ __docformat__ = 'epytext'
 
 import os
 import sys
-if sys.hexversion < 0x300:
-   from io import open
 import re
 from functools import partial
 from itertools import chain
diff --git a/pym/gentoolkit/profile.py b/pym/gentoolkit/profile.py
index 01f823a..dcd02cc 100644
--- a/pym/gentoolkit/profile.py
+++ b/pym/gentoolkit/profile.py
@@ -15,9 +15,6 @@ import os.path
 import portage
 import sys
 
-if sys.hexversion < 0x300:
-   from io import open
-
 from portage import _encodings, _unicode_encode
 
 
diff --git a/pym/gentoolkit/revdep_rebuild/analyse.py 
b/pym/gentoolkit/revdep_rebuild/analyse.py
index 4269ade..6ce1568 100644
--- a/pym/gentoolkit/revdep_rebuild/analyse.py
+++ b/pym/gentoolkit/revdep_rebuild/analyse.py
@@ -6,8 +6,6 @@ import os
 import re
 import time
 import sys
-if sys.hexversion < 0x300:
-   from io import open
 
 from portage import _encodings, _unicode_encode
 from portage.output import bold, blue, yellow, green
diff --git a/pym/gentoolkit/revdep_rebuild/assign.py 
b/pym/gentoolkit/revdep_rebuild/assign.py
index 1e5f3e9..3fa9299 100644
--- a/pym/gentoolkit/revdep_rebuild/assign.py
+++ b/pym/gentoolkit/revdep_rebuild/assign.py
@@ -15,12 +15,6 @@ import portage
 from portage import portdb
 from portage.output import bold, red, yellow, green
 
-# Make all str conversions unicode
-try:
-   str = unicode
-except NameError:
-   pass
-
 
 class _file_matcher(object):
"""
diff --git a/pym/gentoolkit/revdep_rebuild/cache.py 
b/pym/gentoolkit/revdep_rebuild/cache.py
index 66fbd9d..f8b7841 100644
--- a/pym/gentoolkit/revdep_rebuild/cache.py
+++ b/pym/gentoolkit/revdep_rebuild/cache.py
@@ -6,11 +6,6 @@ Functions for reading, saving and verifying the data caches
 from portage import os
 import time
 import sys
-if sys.hexversion < 0x300:
-   from io import open
-   _unicode = unicode  # noqa
-else:
-   _unicode = str
 
 from portage import _encodings, _unicode_encode
 from portage.output import red
@@ -59,7 +54,7 @@ def save_cache(logger, to_save={}, 
temp_path=DEFAULTS['DEFAULT_TMP_DIR']):
try:
_file = open(_unicode_encode(os.path.join(temp_path, 
'timestamp'),
encoding=_encodings['fs']), mode='w', 
encoding=_encodings['content'])
-   _file.write(_unicode(int(time.time(
+   _file.write(str(int(time.time(
_file.close()
 
for key,val in to_sa

[gentoo-portage-dev] [PATCH gentoolkit 2/4] Remove $Header$

2020-12-20 Thread Matt Turner
Signed-off-by: Matt Turner 
---
 bin/ekeyword  | 2 --
 bin/enalyze   | 2 --
 bin/epkginfo  | 2 --
 bin/equery| 2 --
 bin/euse  | 2 --
 bin/imlate| 2 --
 bin/revdep-rebuild| 2 --
 pym/gentoolkit/__init__.py| 2 --
 pym/gentoolkit/atom.py| 2 --
 pym/gentoolkit/base.py| 2 --
 pym/gentoolkit/cpv.py | 2 --
 pym/gentoolkit/dbapi.py   | 2 --
 pym/gentoolkit/dependencies.py| 2 --
 pym/gentoolkit/ekeyword/Makefile  | 2 --
 pym/gentoolkit/enalyze/__init__.py| 2 --
 pym/gentoolkit/equery/__init__.py | 2 --
 pym/gentoolkit/equery/belongs.py  | 2 --
 pym/gentoolkit/equery/changes.py  | 2 --
 pym/gentoolkit/equery/check.py| 2 --
 pym/gentoolkit/equery/depends.py  | 2 --
 pym/gentoolkit/equery/depgraph.py | 2 --
 pym/gentoolkit/equery/files.py| 2 --
 pym/gentoolkit/equery/has.py  | 2 --
 pym/gentoolkit/equery/hasuse.py   | 2 --
 pym/gentoolkit/equery/list_.py| 2 --
 pym/gentoolkit/equery/meta.py | 2 --
 pym/gentoolkit/equery/size.py | 2 --
 pym/gentoolkit/equery/uses.py | 2 --
 pym/gentoolkit/equery/which.py| 2 --
 pym/gentoolkit/formatters.py  | 2 --
 pym/gentoolkit/helpers.py | 2 --
 pym/gentoolkit/keyword.py | 2 --
 pym/gentoolkit/metadata.py| 2 --
 pym/gentoolkit/module_base.py | 2 --
 pym/gentoolkit/package.py | 2 --
 pym/gentoolkit/pprinter.py| 2 --
 pym/gentoolkit/query.py   | 2 --
 pym/gentoolkit/sets.py| 2 --
 pym/gentoolkit/test/__init__.py   | 2 --
 pym/gentoolkit/test/eclean/__init__.py| 2 --
 pym/gentoolkit/test/eclean/creator.py | 2 --
 pym/gentoolkit/test/eclean/distsupport.py | 2 --
 pym/gentoolkit/test/eclean/test_clean.py  | 2 --
 pym/gentoolkit/test/eclean/test_search.py | 2 --
 pym/gentoolkit/test/equery/__init__.py| 2 --
 pym/gentoolkit/test/test_atom.py  | 2 --
 pym/gentoolkit/test/test_cpv.py   | 2 --
 pym/gentoolkit/versionmatch.py| 2 --
 48 files changed, 96 deletions(-)

diff --git a/bin/ekeyword b/bin/ekeyword
index 2e3c78e..8767fe3 100755
--- a/bin/ekeyword
+++ b/bin/ekeyword
@@ -2,8 +2,6 @@
 #
 # Copyright 2002-2017 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2 or later
-#
-# $Header$
 
 """Manage KEYWORDS in ebuilds easily.
 
diff --git a/bin/enalyze b/bin/enalyze
index a0bb29a..9e27bed 100755
--- a/bin/enalyze
+++ b/bin/enalyze
@@ -3,8 +3,6 @@
 # Copyright 2010 Brian Dolbec 
 # Copyright 2002-2010 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2 or later
-#
-# $Header$
 
 """'enalyze' is a flexible utility for Gentoo linux which can display various
 information about installed packages, such as the USE flags used and the
diff --git a/bin/epkginfo b/bin/epkginfo
index 4f87176..5d3aab2 100755
--- a/bin/epkginfo
+++ b/bin/epkginfo
@@ -2,8 +2,6 @@
 #
 # Copyright 2009 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2 or later
-#
-# $Header$
 
 """Shortcut to equery meta"""
 
diff --git a/bin/equery b/bin/equery
index 77371d1..386194d 100755
--- a/bin/equery
+++ b/bin/equery
@@ -2,8 +2,6 @@
 #
 # Copyright 2002-2009 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2 or later
-#
-# $Header$
 
 """equery is a flexible utility for Gentoo linux which can display various
 information about packages, such as the files they own, their USE flags,
diff --git a/bin/euse b/bin/euse
index 263d0f2..c3b6638 100755
--- a/bin/euse
+++ b/bin/euse
@@ -2,8 +2,6 @@
 # Disable globbing because "-*" and such is valid as a use flag.
 set -f
 
-# $Header$
-
 # bash replacement for the original euse by Arun Bhanu
 # Author: Marius Mauch 
 # Jared Hancock (Signigicant rewrite for package.use support)
diff --git a/bin/imlate b/bin/imlate
index 040d19e..cd4f7ab 100755
--- a/bin/imlate
+++ b/bin/imlate
@@ -2,8 +2,6 @@
 #
 # Copyright 2002-2017 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2 or later
-#
-# $Header$
 
 """Manage KEYWORDS in ebuilds easily.
 
diff --git a/bin/revdep-rebuild b/bin/revdep-rebuild
index 24d349e..51783c5 100755
--- a/bin/revdep-rebuild
+++ b/bin/revdep-rebuild
@@ -3,8 +3,6 @@
 # Copyright 2010 Brian Dolbec 
 # Copyright 2002-2010 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2 or later
-#
-# $Header$
 
 """'r

[gentoo-portage-dev] [PATCH gentoolkit 1/4] Remove imports from __future__

2020-12-20 Thread Matt Turner
gentoolkit supports only Python 3.6+ now, so these are not used.

Signed-off-by: Matt Turner 
---
 bin/eclean   | 3 ---
 bin/eclean-dist  | 3 ---
 bin/eclean-pkg   | 3 ---
 bin/ekeyword | 2 --
 bin/enalyze  | 2 --
 bin/epkginfo | 2 --
 bin/equery   | 2 --
 bin/euse | 8 
 bin/imlate   | 2 --
 bin/revdep-rebuild   | 2 --
 pym/gentoolkit/base.py   | 2 --
 pym/gentoolkit/dbapi.py  | 2 --
 pym/gentoolkit/eclean/clean.py   | 3 ---
 pym/gentoolkit/eclean/cli.py | 3 ---
 pym/gentoolkit/eclean/exclude.py | 3 ---
 pym/gentoolkit/eclean/output.py  | 3 ---
 pym/gentoolkit/eclean/pkgindex.py| 3 ---
 pym/gentoolkit/eclean/search.py  | 3 ---
 pym/gentoolkit/ekeyword/ekeyword.py  | 2 --
 pym/gentoolkit/ekeyword/ekeyword_unittest.py | 2 --
 pym/gentoolkit/enalyze/analyze.py| 2 --
 pym/gentoolkit/enalyze/output.py | 2 --
 pym/gentoolkit/enalyze/rebuild.py| 3 ---
 pym/gentoolkit/eprefix.py| 2 --
 pym/gentoolkit/equery/__init__.py| 2 --
 pym/gentoolkit/equery/belongs.py | 2 --
 pym/gentoolkit/equery/changes.py | 2 --
 pym/gentoolkit/equery/check.py   | 2 --
 pym/gentoolkit/equery/depends.py | 2 --
 pym/gentoolkit/equery/depgraph.py| 2 --
 pym/gentoolkit/equery/files.py   | 2 --
 pym/gentoolkit/equery/has.py | 2 --
 pym/gentoolkit/equery/hasuse.py  | 2 --
 pym/gentoolkit/equery/list_.py   | 2 --
 pym/gentoolkit/equery/meta.py| 2 --
 pym/gentoolkit/equery/size.py| 2 --
 pym/gentoolkit/equery/uses.py| 2 --
 pym/gentoolkit/equery/which.py   | 2 --
 pym/gentoolkit/eshowkw/keywords_header.py| 2 --
 pym/gentoolkit/imlate/imlate.py  | 3 ---
 pym/gentoolkit/module_base.py| 2 --
 pym/gentoolkit/revdep_rebuild/analyse.py | 2 --
 pym/gentoolkit/revdep_rebuild/assign.py  | 2 --
 pym/gentoolkit/revdep_rebuild/cache.py   | 2 --
 pym/gentoolkit/revdep_rebuild/collect.py | 2 --
 pym/gentoolkit/revdep_rebuild/rebuild.py | 2 --
 pym/gentoolkit/revdep_rebuild/settings.py| 2 --
 pym/gentoolkit/revdep_rebuild/stuff.py   | 2 --
 pym/gentoolkit/test/eclean/creator.py| 3 ---
 pym/gentoolkit/test/eclean/distsupport.py| 2 --
 pym/gentoolkit/test/eclean/test_clean.py | 2 --
 pym/gentoolkit/test/eclean/test_search.py| 3 ---
 setup.py | 3 ---
 53 files changed, 4 insertions(+), 122 deletions(-)

diff --git a/bin/eclean b/bin/eclean
index 715787b..90f9e55 100755
--- a/bin/eclean
+++ b/bin/eclean
@@ -4,9 +4,6 @@
 Distributed under the terms of the GNU General Public License v2
 """
 
-from __future__ import print_function
-
-
 # Meta:
 __author__ = "Thomas de Grenier de Latour (tgl), " + \
"modular re-write by: Brian Dolbec (dol-sen)"
diff --git a/bin/eclean-dist b/bin/eclean-dist
index 715787b..90f9e55 100755
--- a/bin/eclean-dist
+++ b/bin/eclean-dist
@@ -4,9 +4,6 @@
 Distributed under the terms of the GNU General Public License v2
 """
 
-from __future__ import print_function
-
-
 # Meta:
 __author__ = "Thomas de Grenier de Latour (tgl), " + \
"modular re-write by: Brian Dolbec (dol-sen)"
diff --git a/bin/eclean-pkg b/bin/eclean-pkg
index 715787b..90f9e55 100755
--- a/bin/eclean-pkg
+++ b/bin/eclean-pkg
@@ -4,9 +4,6 @@
 Distributed under the terms of the GNU General Public License v2
 """
 
-from __future__ import print_function
-
-
 # Meta:
 __author__ = "Thomas de Grenier de Latour (tgl), " + \
"modular re-write by: Brian Dolbec (dol-sen)"
diff --git a/bin/ekeyword b/bin/ekeyword
index 72b3f95..2e3c78e 100755
--- a/bin/ekeyword
+++ b/bin/ekeyword
@@ -13,8 +13,6 @@ the current list as they appear, and ebuilds are processed as 
they appear.
 
 """
 
-from __future__ import print_function
-
 import os
 import sys
 # This block ensures that ^C interrupts are handled quietly.
diff --git a/bin/enalyze b/bin/enalyze
index 5991f60..a0bb29a 100755
--- a/bin/enalyze
+++ b/bin/enalyze
@@ -12,8 +12,6 @@ packages that use them.  It can also be used to help rebuild 
/etc/portage/packag
 files in the event of corruption, and possibly more.
 """
 
-from __future__ import print_function
-
 import sys
 # This block ensures that ^C interrupts are handled quietly.
 try:
diff --git a/bin/epkginfo b/bin/epkginfo
index 75a2f3f..4f87176

Re: [gentoo-portage-dev] [PATCH 0/2] emerge: Add short -l option for --load-average (bug 699256)

2020-08-15 Thread Matt Turner
On Sat, Aug 15, 2020 at 12:27 PM Zac Medico  wrote:
>
> Add a short -l option for --load-average just like make(1) has.

That's awesome, thank you. That'll make some catalyst work easier for me.



Re: [gentoo-portage-dev] [PATCH] Change BINPKG_COMPRESS default from bzip2 to xz

2020-04-26 Thread Matt Turner
On Sun, Apr 26, 2020 at 12:29 PM Michał Górny  wrote:
>
> On Sun, 2020-04-26 at 09:55 -0700, Matt Turner wrote:
> > Bug: https://bugs.gentoo.org/715108
> > Signed-off-by: Matt Turner 
> > ---
> > Strawman patch. Bikeshed away.
> >
>
> xz is generally slow and doesn't do parallel good.  If we want to change
> this, we should go for something cool like zstd that scales better.

That works for me.



[gentoo-portage-dev] [PATCH] Change BINPKG_COMPRESS default from bzip2 to xz

2020-04-26 Thread Matt Turner
Bug: https://bugs.gentoo.org/715108
Signed-off-by: Matt Turner 
---
Strawman patch. Bikeshed away.

 bin/quickpkg   | 2 +-
 lib/portage/package/ebuild/doebuild.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/quickpkg b/bin/quickpkg
index df8c1a8e8..1feb01fa1 100755
--- a/bin/quickpkg
+++ b/bin/quickpkg
@@ -115,7 +115,7 @@ def quickpkg_atom(options, infos, arg, eout):
binpkg_tmpfile = os.path.join(bintree.pkgdir,
cpv + ".tbz2." + str(os.getpid()))
ensure_dirs(os.path.dirname(binpkg_tmpfile))
-   binpkg_compression = settings.get("BINPKG_COMPRESS", 
"bzip2")
+   binpkg_compression = settings.get("BINPKG_COMPRESS", 
"xz")
try:
compression = _compressors[binpkg_compression]
except KeyError as e:
diff --git a/lib/portage/package/ebuild/doebuild.py 
b/lib/portage/package/ebuild/doebuild.py
index 2bff94cb1..5020b46fe 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -539,7 +539,7 @@ def doebuild_environment(myebuild, mydo, myroot=None, 
settings=None,
mysettings["KV"] = ""
mysettings.backup_changes("KV")
 
-   binpkg_compression = mysettings.get("BINPKG_COMPRESS", "bzip2")
+   binpkg_compression = mysettings.get("BINPKG_COMPRESS", "xz")
try:
compression = _compressors[binpkg_compression]
except KeyError as e:
-- 
2.25.3




[gentoo-portage-dev] [PATCH] man/make.conf.5: Mention eclean in PKGDIR section

2020-04-26 Thread Matt Turner
And remove obsolete text from binpkg-multi-instance. I believe this was
obsolete even before my gentoolkit patches that add the --changed-deps
flag to eclean packages.

Signed-off-by: Matt Turner 
---
 man/make.conf.5 | 7 +++
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/man/make.conf.5 b/man/make.conf.5
index f82fed65a..a0aa5f129 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -310,10 +310,6 @@ running older portage, since the file format is identical, 
the
 per\-package PATH attribute in the 'Packages' index directs them to
 download the file from the correct URI, and they automatically use
 BUILD_TIME metadata to select the latest builds.
-
-There is currently no automated way to prune old builds from PKGDIR,
-although it is possible to remove packages manually, and then run
-\(aqemaint \-\-fix binhost' to update the ${PKGDIR}/Packages index.
 .TP
 .B buildpkg
 Binary packages will be created for all packages that are merged. Also see
@@ -838,6 +834,9 @@ to it's category. However, for backward compatibility with 
the layout
 used by older versions of portage, if the \fI${PKGDIR}/All\fR directory
 exists then all packages will be stored inside of it and symlinks to
 the packages will be created in the category subdirectories.
+
+If you would like to selectively prune obsolete files from this directory, see
+\fBeclean\fR(1) from the gentoolkit package.
 .br
 Defaults to /var/cache/binpkgs.
 .TP
-- 
2.25.3




Re: [gentoo-portage-dev] [PATCH v2 gentoolkit 2/2] eclean: Add option to delete binpkgs with changed deps

2020-03-11 Thread Matt Turner
On Wed, Mar 11, 2020 at 10:23 PM Zac Medico  wrote:
> The coupling with --destructive logic complicates matters. It raises the
> question, why isn't --time-limit logic also coupled with --destructive
> logic? I think "in order to preserve the status quo" is a reasonable
> answer to this question.

Yeah :(

It's clear to me that these options were added in a very ad hoc
manner. And... named badly if I do say so. E.g., destructive and
package_names do not correspond, at least in my mind, to the
operations they do.

I'd like to clean those up as a follow on.



Re: [gentoo-portage-dev] [PATCH v2 gentoolkit 2/2] eclean: Add option to delete binpkgs with changed deps

2020-03-11 Thread Matt Turner
On Wed, Mar 11, 2020 at 9:36 PM Matt Turner  wrote:
>
> On Wed, Mar 11, 2020 at 9:31 PM Zac Medico  wrote:
> >
> > On 3/6/20 10:11 PM, Matt Turner wrote:
> > > Signed-off-by: Matt Turner 
> > > ---
> > >  pym/gentoolkit/eclean/cli.py|  7 ++-
> > >  pym/gentoolkit/eclean/search.py | 24 +++-
> > >  2 files changed, 29 insertions(+), 2 deletions(-)
> > >
> > > diff --git a/pym/gentoolkit/eclean/cli.py b/pym/gentoolkit/eclean/cli.py
> > > index 1a99b3e..39aafd3 100644
> > > --- a/pym/gentoolkit/eclean/cli.py
> > > +++ b/pym/gentoolkit/eclean/cli.py
> > > @@ -147,6 +147,8 @@ def printUsage(_error=None, help=None):
> > >   or help in ('all','packages'):
> > >   print( "Available", yellow("options"),"for the",
> > >   green("packages"),"action:", file=out)
> > > + print( yellow(" --changed-deps")+
> > > + "   - delete packages for which ebuild 
> > > dependencies have changed", file=out)
> > >   print( yellow(" -i, --ignore-failure")+
> > >   " - ignore failure to locate PKGDIR", 
> > > file=out)
> > >   print( file=out)
> > > @@ -263,6 +265,8 @@ def parseArgs(options={}):
> > >   options['size-limit'] = parseSize(a)
> > >   elif o in ("-v", "--verbose") and not 
> > > options['quiet']:
> > >   options['verbose'] = True
> > > + elif o in ("--changed-deps"):
> > > + options['changed-deps'] = True
> > >   elif o in ("-i", "--ignore-failure"):
> > >   options['ignore-failure'] = True
> > >   else:
> > > @@ -290,7 +294,7 @@ def parseArgs(options={}):
> > >   getopt_options['short']['distfiles'] = "fs:"
> > >   getopt_options['long']['distfiles'] = ["fetch-restricted", 
> > > "size-limit="]
> > >   getopt_options['short']['packages'] = "i"
> > > - getopt_options['long']['packages'] = ["ignore-failure"]
> > > + getopt_options['long']['packages'] = ["ignore-failure", 
> > > "changed-deps"]
> > >   # set default options, except 'nocolor', which is set in main()
> > >   options['interactive'] = False
> > >   options['pretend'] = False
> > > @@ -303,6 +307,7 @@ def parseArgs(options={}):
> > >   options['fetch-restricted'] = False
> > >   options['size-limit'] = 0
> > >   options['verbose'] = False
> > > + options['changed-deps'] = False
> > >   options['ignore-failure'] = False
> > >   # if called by a well-named symlink, set the action accordingly:
> > >   action = None
> > > diff --git a/pym/gentoolkit/eclean/search.py 
> > > b/pym/gentoolkit/eclean/search.py
> > > index 0efefdb..17655cb 100644
> > > --- a/pym/gentoolkit/eclean/search.py
> > > +++ b/pym/gentoolkit/eclean/search.py
> > > @@ -13,6 +13,8 @@ import sys
> > >  from functools import partial
> > >
> > >  import portage
> > > +from portage.dep import Atom, use_reduce
> > > +from portage.dep._slot_operator import strip_slots
> > >
> > >  import gentoolkit.pprinter as pp
> > >  from gentoolkit.eclean.exclude import (exclDictMatchCP, exclDictExpand,
> > > @@ -488,6 +490,17 @@ class DistfilesSearch(object):
> > >   return clean_me, saved
> > >
> > >
> > > +def _deps_equal(deps_a, deps_b, eapi, uselist=None):
> > > + """Compare two dependency lists given a set of USE flags"""
> > > + if deps_a == deps_b: return True
> > > +
> > > + deps_a = use_reduce(deps_a, uselist=uselist, eapi=eapi, 
> > > token_class=Atom)
> > > + deps_b = use_reduce(deps_b, uselist=uselist, eapi=eapi, 
> > > token_class=Atom)
> > > + strip_slots(deps_a)
> > > + strip_slots(deps_b)
> > > + return deps_a == deps_b
> > > +
> > > +
> > >  def findPackages(
> > >   options,
> > >   exclude=None,
> &g

Re: [gentoo-portage-dev] [PATCH v2 gentoolkit 2/2] eclean: Add option to delete binpkgs with changed deps

2020-03-11 Thread Matt Turner
On Wed, Mar 11, 2020 at 9:31 PM Zac Medico  wrote:
>
> On 3/6/20 10:11 PM, Matt Turner wrote:
> > Signed-off-by: Matt Turner 
> > ---
> >  pym/gentoolkit/eclean/cli.py|  7 ++-
> >  pym/gentoolkit/eclean/search.py | 24 +++-
> >  2 files changed, 29 insertions(+), 2 deletions(-)
> >
> > diff --git a/pym/gentoolkit/eclean/cli.py b/pym/gentoolkit/eclean/cli.py
> > index 1a99b3e..39aafd3 100644
> > --- a/pym/gentoolkit/eclean/cli.py
> > +++ b/pym/gentoolkit/eclean/cli.py
> > @@ -147,6 +147,8 @@ def printUsage(_error=None, help=None):
> >   or help in ('all','packages'):
> >   print( "Available", yellow("options"),"for the",
> >   green("packages"),"action:", file=out)
> > + print( yellow(" --changed-deps")+
> > + "   - delete packages for which ebuild 
> > dependencies have changed", file=out)
> >   print( yellow(" -i, --ignore-failure")+
> >   " - ignore failure to locate PKGDIR", 
> > file=out)
> >   print( file=out)
> > @@ -263,6 +265,8 @@ def parseArgs(options={}):
> >   options['size-limit'] = parseSize(a)
> >   elif o in ("-v", "--verbose") and not 
> > options['quiet']:
> >   options['verbose'] = True
> > + elif o in ("--changed-deps"):
> > + options['changed-deps'] = True
> >   elif o in ("-i", "--ignore-failure"):
> >   options['ignore-failure'] = True
> >   else:
> > @@ -290,7 +294,7 @@ def parseArgs(options={}):
> >   getopt_options['short']['distfiles'] = "fs:"
> >   getopt_options['long']['distfiles'] = ["fetch-restricted", 
> > "size-limit="]
> >   getopt_options['short']['packages'] = "i"
> > - getopt_options['long']['packages'] = ["ignore-failure"]
> > + getopt_options['long']['packages'] = ["ignore-failure", 
> > "changed-deps"]
> >   # set default options, except 'nocolor', which is set in main()
> >   options['interactive'] = False
> >   options['pretend'] = False
> > @@ -303,6 +307,7 @@ def parseArgs(options={}):
> >   options['fetch-restricted'] = False
> >   options['size-limit'] = 0
> >   options['verbose'] = False
> > + options['changed-deps'] = False
> >   options['ignore-failure'] = False
> >   # if called by a well-named symlink, set the action accordingly:
> >   action = None
> > diff --git a/pym/gentoolkit/eclean/search.py 
> > b/pym/gentoolkit/eclean/search.py
> > index 0efefdb..17655cb 100644
> > --- a/pym/gentoolkit/eclean/search.py
> > +++ b/pym/gentoolkit/eclean/search.py
> > @@ -13,6 +13,8 @@ import sys
> >  from functools import partial
> >
> >  import portage
> > +from portage.dep import Atom, use_reduce
> > +from portage.dep._slot_operator import strip_slots
> >
> >  import gentoolkit.pprinter as pp
> >  from gentoolkit.eclean.exclude import (exclDictMatchCP, exclDictExpand,
> > @@ -488,6 +490,17 @@ class DistfilesSearch(object):
> >   return clean_me, saved
> >
> >
> > +def _deps_equal(deps_a, deps_b, eapi, uselist=None):
> > + """Compare two dependency lists given a set of USE flags"""
> > + if deps_a == deps_b: return True
> > +
> > + deps_a = use_reduce(deps_a, uselist=uselist, eapi=eapi, 
> > token_class=Atom)
> > + deps_b = use_reduce(deps_b, uselist=uselist, eapi=eapi, 
> > token_class=Atom)
> > + strip_slots(deps_a)
> > + strip_slots(deps_b)
> > + return deps_a == deps_b
> > +
> > +
> >  def findPackages(
> >   options,
> >   exclude=None,
> > @@ -562,7 +575,16 @@ def findPackages(
> >
> >   # Exclude if binpkg exists in the porttree and not --deep
> >   if not destructive and port_dbapi.cpv_exists(cpv):
> > - continue
> > + if not options['changed-deps']:
> > + continue
>
> We can't can't continue above, since that will skip all of the filters
> that occur later in the loop. So, we have to nest the below changed-deps
> code under if options['changed-deps']:

I'm happy to make that change, but I don't think it's necessary,
strictly speaking, since this is inside an 'if not destructive'
conditional and the only filter afterwards is 'if destructive'.

In case we add more filters in the future, I'll make the change you suggested.

Thanks a bunch for your reviews!



Re: [gentoo-portage-dev] [PATCH v2 gentoolkit 2/2] eclean: Add option to delete binpkgs with changed deps

2020-03-11 Thread Matt Turner
On Tue, Mar 10, 2020 at 8:30 PM Zac Medico  wrote:
>
> On 3/6/20 10:11 PM, Matt Turner wrote:
> > +def _deps_equal(deps_a, deps_b, eapi, uselist=None):
> > + """Compare two dependency lists given a set of USE flags"""
> > + if deps_a == deps_b: return True
> > +
> > + deps_a = use_reduce(deps_a, uselist=uselist, eapi=eapi, 
> > token_class=Atom)
> > + deps_b = use_reduce(deps_b, uselist=uselist, eapi=eapi, 
> > token_class=Atom)
>
> It's pure luck that passing a list of depstrings to use_reduce works
> here, so it will be more future-proof to use ' '.join(depstr) instead.
> The relevant code in use_reduce looks like this:
>
> if isinstance(depstr, list):
> if portage._internal_caller:
> warnings.warn(_("Passing paren_reduced dep arrays to %s is 
> deprecated. " + \
> "Pass the original dep string instead.") % \
> ('portage.dep.use_reduce',), DeprecationWarning, 
> stacklevel=2)
> depstr = paren_enclose(depstr)

Okay, thank you. I've fixed this with:

- binpkg_deps = bin_dbapi.aux_get(cpv, keys)
- ebuild_deps = port_dbapi.aux_get(cpv, keys)
+ binpkg_deps = ' '.join(bin_dbapi.aux_get(cpv, keys))
+ ebuild_deps = ' '.join(port_dbapi.aux_get(cpv, keys))

With that fixed, do the patches look good to you?



[gentoo-portage-dev] [PATCH v2 gentoolkit 2/2] eclean: Add option to delete binpkgs with changed deps

2020-03-06 Thread Matt Turner
Signed-off-by: Matt Turner 
---
 pym/gentoolkit/eclean/cli.py|  7 ++-
 pym/gentoolkit/eclean/search.py | 24 +++-
 2 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/pym/gentoolkit/eclean/cli.py b/pym/gentoolkit/eclean/cli.py
index 1a99b3e..39aafd3 100644
--- a/pym/gentoolkit/eclean/cli.py
+++ b/pym/gentoolkit/eclean/cli.py
@@ -147,6 +147,8 @@ def printUsage(_error=None, help=None):
or help in ('all','packages'):
print( "Available", yellow("options"),"for the",
green("packages"),"action:", file=out)
+   print( yellow(" --changed-deps")+
+   "   - delete packages for which ebuild 
dependencies have changed", file=out)
print( yellow(" -i, --ignore-failure")+
" - ignore failure to locate PKGDIR", 
file=out)
print( file=out)
@@ -263,6 +265,8 @@ def parseArgs(options={}):
options['size-limit'] = parseSize(a)
elif o in ("-v", "--verbose") and not options['quiet']:
options['verbose'] = True
+   elif o in ("--changed-deps"):
+   options['changed-deps'] = True
elif o in ("-i", "--ignore-failure"):
options['ignore-failure'] = True
else:
@@ -290,7 +294,7 @@ def parseArgs(options={}):
getopt_options['short']['distfiles'] = "fs:"
getopt_options['long']['distfiles'] = ["fetch-restricted", 
"size-limit="]
getopt_options['short']['packages'] = "i"
-   getopt_options['long']['packages'] = ["ignore-failure"]
+   getopt_options['long']['packages'] = ["ignore-failure", "changed-deps"]
# set default options, except 'nocolor', which is set in main()
options['interactive'] = False
options['pretend'] = False
@@ -303,6 +307,7 @@ def parseArgs(options={}):
options['fetch-restricted'] = False
options['size-limit'] = 0
options['verbose'] = False
+   options['changed-deps'] = False
options['ignore-failure'] = False
# if called by a well-named symlink, set the action accordingly:
action = None
diff --git a/pym/gentoolkit/eclean/search.py b/pym/gentoolkit/eclean/search.py
index 0efefdb..17655cb 100644
--- a/pym/gentoolkit/eclean/search.py
+++ b/pym/gentoolkit/eclean/search.py
@@ -13,6 +13,8 @@ import sys
 from functools import partial
 
 import portage
+from portage.dep import Atom, use_reduce
+from portage.dep._slot_operator import strip_slots
 
 import gentoolkit.pprinter as pp
 from gentoolkit.eclean.exclude import (exclDictMatchCP, exclDictExpand,
@@ -488,6 +490,17 @@ class DistfilesSearch(object):
return clean_me, saved
 
 
+def _deps_equal(deps_a, deps_b, eapi, uselist=None):
+   """Compare two dependency lists given a set of USE flags"""
+   if deps_a == deps_b: return True
+
+   deps_a = use_reduce(deps_a, uselist=uselist, eapi=eapi, 
token_class=Atom)
+   deps_b = use_reduce(deps_b, uselist=uselist, eapi=eapi, 
token_class=Atom)
+   strip_slots(deps_a)
+   strip_slots(deps_b)
+   return deps_a == deps_b
+
+
 def findPackages(
options,
exclude=None,
@@ -562,7 +575,16 @@ def findPackages(
 
# Exclude if binpkg exists in the porttree and not --deep
if not destructive and port_dbapi.cpv_exists(cpv):
-   continue
+   if not options['changed-deps']:
+   continue
+
+   keys = ('RDEPEND', 'PDEPEND')
+   binpkg_deps = bin_dbapi.aux_get(cpv, keys)
+   ebuild_deps = port_dbapi.aux_get(cpv, keys)
+   uselist = bin_dbapi.aux_get(cpv, ['USE'])[0].split()
+
+   if _deps_equal(binpkg_deps, ebuild_deps, cpv.eapi, 
uselist):
+   continue
 
if destructive and var_dbapi.cpv_exists(cpv):
# Exclude if an instance of the package is installed 
due to
-- 
2.24.1




[gentoo-portage-dev] [PATCH v2 gentoolkit 1/2] eclean: Rewrite findPackages()

2020-03-06 Thread Matt Turner
I found the original code to be nearly incomprehensible. Instead of
populating a dict of potential binpkgs to remove and then removing from
the to-be-removed list, just selectively add to-be-removed packages.

Signed-off-by: Matt Turner 
---
 pym/gentoolkit/eclean/search.py | 113 
 1 file changed, 55 insertions(+), 58 deletions(-)

diff --git a/pym/gentoolkit/eclean/search.py b/pym/gentoolkit/eclean/search.py
index 58bd97e..0efefdb 100644
--- a/pym/gentoolkit/eclean/search.py
+++ b/pym/gentoolkit/eclean/search.py
@@ -498,89 +498,86 @@ def findPackages(
port_dbapi=portage.db[portage.root]["porttree"].dbapi,
var_dbapi=portage.db[portage.root]["vartree"].dbapi
):
-   """Find all obsolete binary packages.
-
-   XXX: packages are found only by symlinks.
-   Maybe i should also return .tbz2 files from All/ that have
-   no corresponding symlinks.
+   """Find obsolete binary packages.
 
@param options: dict of options determined at runtime
-   @param exclude: an exclusion dict as defined in
-   exclude.parseExcludeFile class.
-   @param destructive: boolean, defaults to False
-   @param time_limit: integer time value as returned by parseTime()
-   @param package_names: boolean, defaults to False.
-   used only if destructive=True
-   @param pkgdir: path to the binary package dir being checked
+   @type  options: dict
+   @param exclude: exclusion dict (as defined in the 
exclude.parseExcludeFile class)
+   @type  exclude: dict, optional
+   @param destructive: binpkg is obsolete if not installed (default: 
`False`)
+   @type  destructive: bool, optional
+   @param time_limit: exclude binpkg if newer than time value as returned 
by parseTime()
+   @type  time_limit: int, optional
+   @param package_names: exclude all binpkg versions if package is 
installed
+ (used with 
`destructive=True`) (default: `False`)
+   @type  package_names: bool, optional
+   @param pkgdir: path to the binpkg cache (PKGDIR)
+   @type  pkgdir: str
@param port_dbapi: defaults to 
portage.db[portage.root]["porttree"].dbapi
-   can be overridden for tests.
-   @param var_dbapi: defaults to portage.db[portage.root]["vartree"].dbapi
-   can be overridden for tests.
+  Can be overridden for tests.
+   @param  var_dbapi: defaults to portage.db[portage.root]["vartree"].dbapi
+  Can be overridden for tests.
 
+   @return binary packages to remove. e.g. {'cat/pkg-ver': [filepath]}
@rtype: dict
-   @return clean_me i.e. {'cat/pkg-ver.tbz2': [filepath],}
"""
if exclude is None:
exclude = {}
-   clean_me = {}
-   # create a full package dictionary
 
-   # now do an access test, os.walk does not error for "no read permission"
+   # Access test, os.walk does not error for "no read permission"
try:
test = os.listdir(pkgdir)
del test
except EnvironmentError as er:
if options['ignore-failure']:
exit(0)
-   print( pp.error("Error accessing PKGDIR." ), file=sys.stderr)
-   print( pp.error("(Check your make.conf file and 
environment)."), file=sys.stderr)
-   print( pp.error("Error: %s" %str(er)), file=sys.stderr)
+   print(pp.error("Error accessing PKGDIR."), file=sys.stderr)
+   print(pp.error("(Check your make.conf file and environment)."), 
file=sys.stderr)
+   print(pp.error("Error: %s" % str(er)), file=sys.stderr)
exit(1)
 
-   # if portage supports FEATURES=binpkg-multi-instance, then
-   # cpv_all can return multiple instances per cpv, where
-   # instances are distinguishable by some extra attributes
-   # provided by portage's _pkg_str class
+   # Create a dictionary of all installed packages
+   if destructive and package_names:
+   installed = dict.fromkeys(var_dbapi.cp_all())
+   else:
+   installed = {}
+
+   # Dictionary of binary packages to clean. Organized as cpv->[pkgs] in 
order
+   # to support FEATURES=binpkg-multi-instance.
+   dead_binpkgs = {}
+
bin_dbapi = portage.binarytree(pkgdir=pkgdir, 
settings=var_dbapi.settings).dbapi
for cpv in bin_dbapi.cpv_all():
-   mtime = int(bin_dbapi.aux_get(cpv, ['_mtime_'])[0])
-   if time_limit and mtime >= time_limit:
-   # time-limit exclusio

Re: [gentoo-portage-dev] [PATCH gentoolkit 2/2] eclean: Add option to delete binpkgs with changed deps

2020-03-06 Thread Matt Turner
On Mon, Mar 2, 2020 at 9:15 PM Zac Medico  wrote:
>
> On 3/2/20 1:11 PM, Matt Turner wrote:
> > On Sun, Mar 1, 2020 at 10:39 PM Zac Medico  wrote:
> >>
> >> On 2/20/20 9:29 PM, Matt Turner wrote:
> >>> @@ -564,7 +577,22 @@ def findPackages(
> >>>
> >>>  # Exclude if binpkg exists in the porttree and not --deep
> >>>  if not destructive and port_dbapi.cpv_exists(cpv):
> >>> -continue
> >>> +if not options['changed-deps']:
> >>> +continue
> >>> +
> >>> +uselist = bin_dbapi.aux_get(cpv, ['USE'])[0].split()
> >>> +all_equal = True
> >>> +
> >>> +for k in ('RDEPEND', 'PDEPEND'):
> >>> +binpkg_deps = bin_dbapi.aux_get(cpv, [k])
> >>> +ebuild_deps = port_dbapi.aux_get(cpv, [k])
> >>> +
> >>> +if not _deps_equal(binpkg_deps, ebuild_deps, cpv.eapi, 
> >>> uselist):
> >>> +all_equal = False
> >>> +break
> >>> +
> >>> +if all_equal:
> >>> +continue
> >>
> >> If all_equal is True, then none of the other filters have an opportunity
> >> to add this package to the dead_binpkgs set. That's not good is it?
> >
> > There are four cases we skip including packages: 1) exclude list, 2)
> > time limit, 3) non-destructive and package still exists (and now
> > optionally runtime deps haven't changed), 4) destructive and package
> > is installed. Cases (3) and (4) are non-overlapping.
> >
> > If none of those cases are true, then we add the package to the
> > dead_binpkgs set. The logic looks right to me.
> >
> > Maybe I'm not understanding.
>
> What I imagine is that you could have some old packages that you
> probably want to delete because they're so old, even though their deps
> have not changed. Meanwhile you have some packages that are
> relatively recent and you'd like to delete them if they have changed deps.
>
> Given the current logic, I guess you'd have to do separate passes, one
> to delete packages based on age and another to delete packages based on
> changed deps. Maybe it's fine to require separate passes for this kind
> of thing. I supposed the alternative would be to add an --or flag that
> would allow you to say that you want to delete packages if they are at
> least a certain age or they have changed deps.

Oh, I think I understand now.

I was confused about this. I expected that --time-limit=2w to mean
that eclean should delete everything older than two weeks, but it's
actually the opposite, more or less. It actually means to *keep*
everything with less than two weeks old. Surprised me...

>  -t, --time-limit=   - don't delete files modified since 

So, with that surprising behavior I think my patch is doing the right
thing, but with the wrong comment. I'll fix those and send v2 patches
with the tabs restored, etc.

Thanks a bunch for the review.



Re: [gentoo-portage-dev] [PATCH gentoolkit 2/2] eclean: Add option to delete binpkgs with changed deps

2020-03-02 Thread Matt Turner
On Sun, Mar 1, 2020 at 10:39 PM Zac Medico  wrote:
>
> On 2/20/20 9:29 PM, Matt Turner wrote:
> > @@ -564,7 +577,22 @@ def findPackages(
> >
> >  # Exclude if binpkg exists in the porttree and not --deep
> >  if not destructive and port_dbapi.cpv_exists(cpv):
> > -continue
> > +if not options['changed-deps']:
> > +continue
> > +
> > +uselist = bin_dbapi.aux_get(cpv, ['USE'])[0].split()
> > +all_equal = True
> > +
> > +for k in ('RDEPEND', 'PDEPEND'):
> > +binpkg_deps = bin_dbapi.aux_get(cpv, [k])
> > +ebuild_deps = port_dbapi.aux_get(cpv, [k])
> > +
> > +if not _deps_equal(binpkg_deps, ebuild_deps, cpv.eapi, 
> > uselist):
> > +all_equal = False
> > +break
> > +
> > +if all_equal:
> > +continue
>
> If all_equal is True, then none of the other filters have an opportunity
> to add this package to the dead_binpkgs set. That's not good is it?

There are four cases we skip including packages: 1) exclude list, 2)
time limit, 3) non-destructive and package still exists (and now
optionally runtime deps haven't changed), 4) destructive and package
is installed. Cases (3) and (4) are non-overlapping.

If none of those cases are true, then we add the package to the
dead_binpkgs set. The logic looks right to me.

Maybe I'm not understanding.

With your other suggestion in place, the code looks like this, which
is hopefully clearer.

# Exclude if binpkg exists in the porttree and not --deep
if not destructive and port_dbapi.cpv_exists(cpv):
if not options['changed-deps']:
continue

keys = ('RDEPEND', 'PDEPEND')
binpkg_deps = bin_dbapi.aux_get(cpv, keys)
ebuild_deps = port_dbapi.aux_get(cpv, keys)
uselist = bin_dbapi.aux_get(cpv, ['USE'])[0].split()

if _deps_equal(binpkg_deps, ebuild_deps, cpv.eapi, uselist):
continue

Unfortunately I don't have any packages with changed-deps at the
moment for testing :(



Re: [gentoo-portage-dev] [PATCH gentoolkit 2/2] eclean: Add option to delete binpkgs with changed deps

2020-03-02 Thread Matt Turner
On Mon, Mar 2, 2020 at 12:40 PM Matt Turner  wrote:
>
> On Sun, Mar 1, 2020 at 10:32 PM Zac Medico  wrote:
> >
> > On 2/20/20 9:29 PM, Matt Turner wrote:
> > > +
> > >  def findPackages(
> > >  options,
> > >  exclude=None,
> > > @@ -564,7 +577,22 @@ def findPackages(
> > >
> > >  # Exclude if binpkg exists in the porttree and not --deep
> > >  if not destructive and port_dbapi.cpv_exists(cpv):
> > > -continue
> > > +if not options['changed-deps']:
> > > +continue
> > > +
> > > +uselist = bin_dbapi.aux_get(cpv, ['USE'])[0].split()
> > > +all_equal = True
> > > +
> > > +for k in ('RDEPEND', 'PDEPEND'):
> > > +binpkg_deps = bin_dbapi.aux_get(cpv, [k])
> > > +ebuild_deps = port_dbapi.aux_get(cpv, [k])
> > > +
> > > +if not _deps_equal(binpkg_deps, ebuild_deps, cpv.eapi, 
> > > uselist):
> > > +all_equal = False
> > > +break
> > > +
> > > +if all_equal:
> > > +continue
> > >
> > >  if destructive and var_dbapi.cpv_exists(cpv):
> > >  # Exclude if an instance of the package is installed due to
> > >
> >
> > The aux_get calls are expensive, so it's more efficient to get multiple
> > values with each call like:
> > keys = ('RDEPEND', 'PDEPEND')
> > binpkg_deps = dict(zip(keys, bin_dbapi.aux_get(cpv, keys))
> > ebuild_deps = dict(zip(keys, port_dbapi.aux_get(cpv, keys))
> >
> > Otherwise, looks good.
>
> Thanks, that makes the code a lot nicer too.

Actually, use_reduce wants a list (it calls .split). Wrapping those in
list() looks like it works, but I suspect that's not as you intended.
What does the zip add over just doing this?

  binpkg_deps = bin_dbapi.aux_get(cpv, keys)
  ebuild_deps = port_dbapi.aux_get(cpv, keys)



Re: [gentoo-portage-dev] [PATCH gentoolkit 2/2] eclean: Add option to delete binpkgs with changed deps

2020-03-02 Thread Matt Turner
On Sun, Mar 1, 2020 at 10:32 PM Zac Medico  wrote:
>
> On 2/20/20 9:29 PM, Matt Turner wrote:
> > +
> >  def findPackages(
> >  options,
> >  exclude=None,
> > @@ -564,7 +577,22 @@ def findPackages(
> >
> >  # Exclude if binpkg exists in the porttree and not --deep
> >  if not destructive and port_dbapi.cpv_exists(cpv):
> > -continue
> > +if not options['changed-deps']:
> > +continue
> > +
> > +uselist = bin_dbapi.aux_get(cpv, ['USE'])[0].split()
> > +all_equal = True
> > +
> > +for k in ('RDEPEND', 'PDEPEND'):
> > +binpkg_deps = bin_dbapi.aux_get(cpv, [k])
> > +ebuild_deps = port_dbapi.aux_get(cpv, [k])
> > +
> > +if not _deps_equal(binpkg_deps, ebuild_deps, cpv.eapi, 
> > uselist):
> > +all_equal = False
> > +break
> > +
> > +if all_equal:
> > +continue
> >
> >  if destructive and var_dbapi.cpv_exists(cpv):
> >  # Exclude if an instance of the package is installed due to
> >
>
> The aux_get calls are expensive, so it's more efficient to get multiple
> values with each call like:
> keys = ('RDEPEND', 'PDEPEND')
> binpkg_deps = dict(zip(keys, bin_dbapi.aux_get(cpv, keys))
> ebuild_deps = dict(zip(keys, port_dbapi.aux_get(cpv, keys))
>
> Otherwise, looks good.

Thanks, that makes the code a lot nicer too.



[gentoo-portage-dev] Re: [PATCH gentoolkit 1/2] eclean: Rewrite findPackages()

2020-02-20 Thread Matt Turner
On Thu, Feb 20, 2020 at 9:29 PM Matt Turner  wrote:
>
> I found the original code to be nearly incomprehensible. Instead of
> populating a dict of potential binpkgs to remove and then removing from
> the to-be-removed list, just selectively add to-be-removed packages.
>
> Signed-off-by: Matt Turner 
> ---
> I switched from tabs to spaces in the process. I can revert back if
> desired.
>
>  pym/gentoolkit/eclean/search.py | 189 
>  1 file changed, 94 insertions(+), 95 deletions(-)
>
> diff --git a/pym/gentoolkit/eclean/search.py b/pym/gentoolkit/eclean/search.py
> index 58bd97e..831ba39 100644
> --- a/pym/gentoolkit/eclean/search.py
> +++ b/pym/gentoolkit/eclean/search.py
> @@ -489,98 +489,97 @@ class DistfilesSearch(object):
>
>
>  def findPackages(
> -   options,
> -   exclude=None,
> -   destructive=False,
> -   time_limit=0,
> -   package_names=False,
> -   pkgdir=None,
> -   port_dbapi=portage.db[portage.root]["porttree"].dbapi,
> -   var_dbapi=portage.db[portage.root]["vartree"].dbapi
> -   ):
> -   """Find all obsolete binary packages.
> -
> -   XXX: packages are found only by symlinks.
> -   Maybe i should also return .tbz2 files from All/ that have
> -   no corresponding symlinks.
> -
> -   @param options: dict of options determined at runtime
> -   @param exclude: an exclusion dict as defined in
> -   exclude.parseExcludeFile class.
> -   @param destructive: boolean, defaults to False
> -   @param time_limit: integer time value as returned by parseTime()
> -   @param package_names: boolean, defaults to False.
> -   used only if destructive=True
> -   @param pkgdir: path to the binary package dir being checked
> -   @param port_dbapi: defaults to 
> portage.db[portage.root]["porttree"].dbapi
> -   can be overridden for tests.
> -   @param var_dbapi: defaults to 
> portage.db[portage.root]["vartree"].dbapi
> -   can be overridden for tests.
> -
> -   @rtype: dict
> -   @return clean_me i.e. {'cat/pkg-ver.tbz2': [filepath],}
> -   """
> -   if exclude is None:
> -   exclude = {}
> -   clean_me = {}
> -   # create a full package dictionary
> -
> -   # now do an access test, os.walk does not error for "no read 
> permission"
> -   try:
> -   test = os.listdir(pkgdir)
> -   del test
> -   except EnvironmentError as er:
> -   if options['ignore-failure']:
> -   exit(0)
> -   print( pp.error("Error accessing PKGDIR." ), file=sys.stderr)
> -   print( pp.error("(Check your make.conf file and 
> environment)."), file=sys.stderr)
> -   print( pp.error("Error: %s" %str(er)), file=sys.stderr)
> -   exit(1)
> -
> -   # if portage supports FEATURES=binpkg-multi-instance, then
> -   # cpv_all can return multiple instances per cpv, where
> -   # instances are distinguishable by some extra attributes
> -   # provided by portage's _pkg_str class
> -   bin_dbapi = portage.binarytree(pkgdir=pkgdir, 
> settings=var_dbapi.settings).dbapi
> -   for cpv in bin_dbapi.cpv_all():
> -   mtime = int(bin_dbapi.aux_get(cpv, ['_mtime_'])[0])
> -   if time_limit and mtime >= time_limit:
> -   # time-limit exclusion
> -   continue
> -   # dict is cpv->[pkgs] (supports binpkg-multi-instance)
> -   clean_me.setdefault(cpv, []).append(cpv)
> -
> -   # keep only obsolete ones
> -   if destructive and package_names:
> -   cp_all = dict.fromkeys(var_dbapi.cp_all())
> -   else:
> -   cp_all = {}
> -   for cpv in list(clean_me):
> -   if exclDictMatchCP(exclude,portage.cpv_getkey(cpv)):
> -   # exclusion because of the exclude file
> -   del clean_me[cpv]
> -   continue
> -   if not destructive and port_dbapi.cpv_exists(cpv):
> -   # exclusion because pkg still exists (in porttree)
> -   del clean_me[cpv]
> -   continue
> -   if destructive and var_dbapi.cpv_exists(cpv):
> -   buildtime = var_dbapi.aux_get(cpv, ['BUILD_TIME'])[0]
> -   

[gentoo-portage-dev] [PATCH gentoolkit 2/2] eclean: Add option to delete binpkgs with changed deps

2020-02-20 Thread Matt Turner
Signed-off-by: Matt Turner 
---
 pym/gentoolkit/eclean/cli.py|  7 ++-
 pym/gentoolkit/eclean/search.py | 30 +-
 2 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/pym/gentoolkit/eclean/cli.py b/pym/gentoolkit/eclean/cli.py
index 1a99b3e..39aafd3 100644
--- a/pym/gentoolkit/eclean/cli.py
+++ b/pym/gentoolkit/eclean/cli.py
@@ -147,6 +147,8 @@ def printUsage(_error=None, help=None):
or help in ('all','packages'):
print( "Available", yellow("options"),"for the",
green("packages"),"action:", file=out)
+   print( yellow(" --changed-deps")+
+   "   - delete packages for which ebuild 
dependencies have changed", file=out)
print( yellow(" -i, --ignore-failure")+
" - ignore failure to locate PKGDIR", 
file=out)
print( file=out)
@@ -263,6 +265,8 @@ def parseArgs(options={}):
options['size-limit'] = parseSize(a)
elif o in ("-v", "--verbose") and not options['quiet']:
options['verbose'] = True
+   elif o in ("--changed-deps"):
+   options['changed-deps'] = True
elif o in ("-i", "--ignore-failure"):
options['ignore-failure'] = True
else:
@@ -290,7 +294,7 @@ def parseArgs(options={}):
getopt_options['short']['distfiles'] = "fs:"
getopt_options['long']['distfiles'] = ["fetch-restricted", 
"size-limit="]
getopt_options['short']['packages'] = "i"
-   getopt_options['long']['packages'] = ["ignore-failure"]
+   getopt_options['long']['packages'] = ["ignore-failure", "changed-deps"]
# set default options, except 'nocolor', which is set in main()
options['interactive'] = False
options['pretend'] = False
@@ -303,6 +307,7 @@ def parseArgs(options={}):
options['fetch-restricted'] = False
options['size-limit'] = 0
options['verbose'] = False
+   options['changed-deps'] = False
options['ignore-failure'] = False
# if called by a well-named symlink, set the action accordingly:
action = None
diff --git a/pym/gentoolkit/eclean/search.py b/pym/gentoolkit/eclean/search.py
index 831ba39..da8c286 100644
--- a/pym/gentoolkit/eclean/search.py
+++ b/pym/gentoolkit/eclean/search.py
@@ -13,6 +13,8 @@ import sys
 from functools import partial
 
 import portage
+from portage.dep import Atom, use_reduce
+from portage.dep._slot_operator import strip_slots
 
 import gentoolkit.pprinter as pp
 from gentoolkit.eclean.exclude import (exclDictMatchCP, exclDictExpand,
@@ -488,6 +490,17 @@ class DistfilesSearch(object):
return clean_me, saved
 
 
+def _deps_equal(deps_a, deps_b, eapi, uselist=None):
+"""Compare two dependency lists given a set of USE flags"""
+if deps_a == deps_b: return True
+
+deps_a = use_reduce(deps_a, uselist=uselist, eapi=eapi, token_class=Atom)
+deps_b = use_reduce(deps_b, uselist=uselist, eapi=eapi, token_class=Atom)
+strip_slots(deps_a)
+strip_slots(deps_b)
+return deps_a == deps_b
+
+
 def findPackages(
 options,
 exclude=None,
@@ -564,7 +577,22 @@ def findPackages(
 
 # Exclude if binpkg exists in the porttree and not --deep
 if not destructive and port_dbapi.cpv_exists(cpv):
-continue
+if not options['changed-deps']:
+continue
+
+uselist = bin_dbapi.aux_get(cpv, ['USE'])[0].split()
+all_equal = True
+
+for k in ('RDEPEND', 'PDEPEND'):
+binpkg_deps = bin_dbapi.aux_get(cpv, [k])
+ebuild_deps = port_dbapi.aux_get(cpv, [k])
+
+if not _deps_equal(binpkg_deps, ebuild_deps, cpv.eapi, 
uselist):
+all_equal = False
+break
+
+if all_equal:
+continue
 
 if destructive and var_dbapi.cpv_exists(cpv):
 # Exclude if an instance of the package is installed due to
-- 
2.24.1




[gentoo-portage-dev] [PATCH gentoolkit 1/2] eclean: Rewrite findPackages()

2020-02-20 Thread Matt Turner
I found the original code to be nearly incomprehensible. Instead of
populating a dict of potential binpkgs to remove and then removing from
the to-be-removed list, just selectively add to-be-removed packages.

Signed-off-by: Matt Turner 
---
I switched from tabs to spaces in the process. I can revert back if
desired.

 pym/gentoolkit/eclean/search.py | 189 
 1 file changed, 94 insertions(+), 95 deletions(-)

diff --git a/pym/gentoolkit/eclean/search.py b/pym/gentoolkit/eclean/search.py
index 58bd97e..831ba39 100644
--- a/pym/gentoolkit/eclean/search.py
+++ b/pym/gentoolkit/eclean/search.py
@@ -489,98 +489,97 @@ class DistfilesSearch(object):
 
 
 def findPackages(
-   options,
-   exclude=None,
-   destructive=False,
-   time_limit=0,
-   package_names=False,
-   pkgdir=None,
-   port_dbapi=portage.db[portage.root]["porttree"].dbapi,
-   var_dbapi=portage.db[portage.root]["vartree"].dbapi
-   ):
-   """Find all obsolete binary packages.
-
-   XXX: packages are found only by symlinks.
-   Maybe i should also return .tbz2 files from All/ that have
-   no corresponding symlinks.
-
-   @param options: dict of options determined at runtime
-   @param exclude: an exclusion dict as defined in
-   exclude.parseExcludeFile class.
-   @param destructive: boolean, defaults to False
-   @param time_limit: integer time value as returned by parseTime()
-   @param package_names: boolean, defaults to False.
-   used only if destructive=True
-   @param pkgdir: path to the binary package dir being checked
-   @param port_dbapi: defaults to 
portage.db[portage.root]["porttree"].dbapi
-   can be overridden for tests.
-   @param var_dbapi: defaults to portage.db[portage.root]["vartree"].dbapi
-   can be overridden for tests.
-
-   @rtype: dict
-   @return clean_me i.e. {'cat/pkg-ver.tbz2': [filepath],}
-   """
-   if exclude is None:
-   exclude = {}
-   clean_me = {}
-   # create a full package dictionary
-
-   # now do an access test, os.walk does not error for "no read permission"
-   try:
-   test = os.listdir(pkgdir)
-   del test
-   except EnvironmentError as er:
-   if options['ignore-failure']:
-   exit(0)
-   print( pp.error("Error accessing PKGDIR." ), file=sys.stderr)
-   print( pp.error("(Check your make.conf file and 
environment)."), file=sys.stderr)
-   print( pp.error("Error: %s" %str(er)), file=sys.stderr)
-   exit(1)
-
-   # if portage supports FEATURES=binpkg-multi-instance, then
-   # cpv_all can return multiple instances per cpv, where
-   # instances are distinguishable by some extra attributes
-   # provided by portage's _pkg_str class
-   bin_dbapi = portage.binarytree(pkgdir=pkgdir, 
settings=var_dbapi.settings).dbapi
-   for cpv in bin_dbapi.cpv_all():
-   mtime = int(bin_dbapi.aux_get(cpv, ['_mtime_'])[0])
-   if time_limit and mtime >= time_limit:
-   # time-limit exclusion
-   continue
-   # dict is cpv->[pkgs] (supports binpkg-multi-instance)
-   clean_me.setdefault(cpv, []).append(cpv)
-
-   # keep only obsolete ones
-   if destructive and package_names:
-   cp_all = dict.fromkeys(var_dbapi.cp_all())
-   else:
-   cp_all = {}
-   for cpv in list(clean_me):
-   if exclDictMatchCP(exclude,portage.cpv_getkey(cpv)):
-   # exclusion because of the exclude file
-   del clean_me[cpv]
-   continue
-   if not destructive and port_dbapi.cpv_exists(cpv):
-   # exclusion because pkg still exists (in porttree)
-   del clean_me[cpv]
-   continue
-   if destructive and var_dbapi.cpv_exists(cpv):
-   buildtime = var_dbapi.aux_get(cpv, ['BUILD_TIME'])[0]
-   clean_me[cpv] = [pkg for pkg in clean_me[cpv]
-   # only keep path if BUILD_TIME is identical 
with vartree
-   if bin_dbapi.aux_get(pkg, ['BUILD_TIME'])[0] != 
buildtime]
-   if not clean_me[cpv]:
-   # nothing we can clean for this package
-   del clean_me[cpv]
-   continue
-   if portage.cpv_getkey(cpv) in cp_all and 
port_dbapi.cpv_exists(cpv):
-   # exclusion because of 

Re: [gentoo-portage-dev] [PATCH gentoolkit 2/2] imlate: Dynamically calculate column width in report

2020-01-02 Thread Matt Turner
On Thu, Jan 2, 2020 at 12:41 PM Zac Medico  wrote:
>
> On 1/2/20 10:58 AM, Matt Turner wrote:
> > Also print categories on the line with the package name for much simpler
> > consumption by external tools.
> >
> > Signed-off-by: Matt Turner 
> > ---
> >  pym/gentoolkit/imlate/imlate.py | 20 +---
> >  1 file changed, 13 insertions(+), 7 deletions(-)
>
> Looks good, but you can omit the .keys() calls since direct iteration
> over a dictionary yields the keys.

Thank you. Fixed locally.



[gentoo-portage-dev] [PATCH gentoolkit 2/2] imlate: Dynamically calculate column width in report

2020-01-02 Thread Matt Turner
Also print categories on the line with the package name for much simpler
consumption by external tools.

Signed-off-by: Matt Turner 
---
 pym/gentoolkit/imlate/imlate.py | 20 +---
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/pym/gentoolkit/imlate/imlate.py b/pym/gentoolkit/imlate/imlate.py
index 86d1a7e..a5568d7 100755
--- a/pym/gentoolkit/imlate/imlate.py
+++ b/pym/gentoolkit/imlate/imlate.py
@@ -67,11 +67,18 @@ def _fill( width, line, fill = " " ):
 # create a hopefully pretty result
 def show_result( conf, pkgs ):
# X - len(colX) = space to fill
-   col1 = 40
-   col2 = 20
+   col1 = -1
+   col2 = -1
+   for cat in pkgs.keys():
+   for pkg in pkgs[cat].keys():
+   col1 = max(col1, len(("%s/%s" % (cat, pkg
+   col2 = max(col2, len(pkgs[cat][pkg][1]))
+   col1 += 1
+   col2 += 1
 
_header = "%s candidates for 'gentoo' on '%s'"
-   _helper = "category/package[:SLOT] our version 
best version"
+   _helper = "%s%s%s" % (_fill(col1, "category/package[:SLOT])"),
+ _fill(col2, "our version"), 
"best version")
_cand = ""
header = ""
 
@@ -102,11 +109,10 @@ def show_result( conf, pkgs ):
print(_fill( len( _helper ), "", "-" ), file=out)
 
for cat in sorted( pkgs.keys() ):
-   print("%s/" % cat, file=out)
for pkg in sorted( pkgs[cat].keys() ):
-   print("%s%s%s" % ( _fill( col1, ( "  %s" % pkg ) ),
-   _fill( 
col2, pkgs[cat][pkg][1] ),
-   
pkgs[cat][pkg][0] ), file=out)
+   print("%s%s%s" % (_fill(col1, ("%s/%s" % (cat, pkg))),
+ _fill(col2, 
pkgs[cat][pkg][1]),
+ pkgs[cat][pkg][0] ), 
file=out)
 
if conf["FILE"] != "stdout":
out.close()
-- 
2.24.1




[gentoo-portage-dev] [PATCH gentoolkit 1/2] eclean: Fix typos

2020-01-02 Thread Matt Turner
Signed-off-by: Matt Turner 
---
 pym/gentoolkit/eclean/cli.py| 4 ++--
 pym/gentoolkit/eclean/search.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/pym/gentoolkit/eclean/cli.py b/pym/gentoolkit/eclean/cli.py
index 1d2f52b..1a99b3e 100644
--- a/pym/gentoolkit/eclean/cli.py
+++ b/pym/gentoolkit/eclean/cli.py
@@ -304,7 +304,7 @@ def parseArgs(options={}):
options['size-limit'] = 0
options['verbose'] = False
options['ignore-failure'] = False
-   # if called by a well-named symlink, set the acction accordingly:
+   # if called by a well-named symlink, set the action accordingly:
action = None
# temp print line to ensure it is the svn/branch code running, etc..
#print(  "## svn/branch/gentoolkit_eclean ### ==> ", 
os.path.basename(sys.argv[0]))
@@ -400,7 +400,7 @@ def doAction(action,options,exclude={}, output=None):
)
 
# initialize our cleaner
-   cleaner = CleanUp( output.progress_controller)
+   cleaner = CleanUp(output.progress_controller)
 
# actually clean files if something was found
if clean_me:
diff --git a/pym/gentoolkit/eclean/search.py b/pym/gentoolkit/eclean/search.py
index ce455a3..58bd97e 100644
--- a/pym/gentoolkit/eclean/search.py
+++ b/pym/gentoolkit/eclean/search.py
@@ -574,7 +574,7 @@ def findPackages(
del clean_me[cpv]
continue
if portage.cpv_getkey(cpv) in cp_all and 
port_dbapi.cpv_exists(cpv):
-   # exlusion because of --package-names
+   # exclusion because of --package-names
del clean_me[cpv]
 
# the getname method correctly supports FEATURES=binpkg-multi-instance,
-- 
2.24.1




Re: [gentoo-portage-dev] [PATCH gentoolkit 1/2] eclean: Inline _sort_keys method

2019-12-04 Thread Matt Turner
On Wed, Dec 4, 2019 at 9:17 PM Zac Medico  wrote:
>
> On 12/4/19 6:12 PM, Matt Turner wrote:
> > The boilerplate for calling this method was larger than what it actually
> > contained. Additionally I think this change will allow the loop to run
> > on a generator rather than a full list.
> >
> > Signed-off-by: Matt Turner 
> > ---
> >  pym/gentoolkit/eclean/clean.py | 21 ++---
> >  1 file changed, 6 insertions(+), 15 deletions(-)
> >
> > diff --git a/pym/gentoolkit/eclean/clean.py b/pym/gentoolkit/eclean/clean.py
> > index fd59976..b790de0 100644
> > --- a/pym/gentoolkit/eclean/clean.py
> > +++ b/pym/gentoolkit/eclean/clean.py
> > @@ -36,10 +36,9 @@ class CleanUp(object):
> >   @return: total size that was cleaned
> >   """
> >   file_type = 'file'
> > - clean_keys = self._sort_keys(clean_dict)
> >   clean_size = 0
> > - # clean all entries one by one
> > - for key in clean_keys:
> > + # clean all entries one by one; sorting helps reading
> > + for key in sorted(clean_dict.keys()):
> >   clean_size += self._clean_files(clean_dict[key], key, 
> > file_type)
> >   # return total size of deleted or to delete files
> >   return clean_size
> > @@ -57,10 +56,9 @@ class CleanUp(object):
> >   @return: total size that was cleaned
> >   """
> >   file_type = 'binary package'
> > - clean_keys = self._sort_keys(clean_dict)
> >   clean_size = 0
> > - # clean all entries one by one
> > - for key in clean_keys:
> > + # clean all entries one by one; sorting helps reading
> > + for key in sorted(clean_dict.keys()):
> >   clean_size += self._clean_files(clean_dict[key], key, 
> > file_type)
> >
> >   #  run 'emaint --fix' here
> > @@ -83,10 +81,9 @@ class CleanUp(object):
> >   @return: total size that would be cleaned
> >   """
> >   file_type = 'file'
> > - clean_keys = self._sort_keys(clean_dict)
> >   clean_size = 0
> > - # tally all entries one by one
> > - for key in clean_keys:
> > + # tally all entries one by one; sorting helps reading
> > + for key in sorted(clean_dict.keys()):
> >   key_size = self._get_size(clean_dict[key])
> >   self.controller(key_size, key, clean_dict[key], 
> > file_type)
> >   clean_size += key_size
> > @@ -110,12 +107,6 @@ class CleanUp(object):
> >   print( pp.error("Error: %s" %str(er)), 
> > file=sys.stderr)
> >   return key_size
> >
> > - def _sort_keys(self, clean_dict):
> > - """Returns a list of sorted dictionary keys."""
> > - # sorting helps reading
> > - clean_keys = sorted(clean_dict)
> > - return clean_keys
> > -
> >   def _clean_files(self, files, key, file_type):
> >   """File removal function."""
> >   clean_size = 0
> >
>
> Looks good except you can just use sorted(clean_dict) without calling
> the keys() method.

Thanks Zac! Both fixes you noted (here and in 2/2) are fixed locally.



[gentoo-portage-dev] [PATCH gentoolkit 2/2] eclean: Delete empty directories

2019-12-04 Thread Matt Turner
Closes: https://bugs.gentoo.org/671592
Signed-off-by: Matt Turner 
---
 pym/gentoolkit/eclean/clean.py | 4 
 1 file changed, 4 insertions(+)

diff --git a/pym/gentoolkit/eclean/clean.py b/pym/gentoolkit/eclean/clean.py
index b790de0..89300ce 100644
--- a/pym/gentoolkit/eclean/clean.py
+++ b/pym/gentoolkit/eclean/clean.py
@@ -139,6 +139,10 @@ class CleanUp(object):
# only count size if successfully 
deleted and not a link
if statinfo.st_nlink == 1:
clean_size += statinfo.st_size
+   try:
+   
os.rmdir(os.path.dirname(file_))
+   except OSError as er:
+   pass
except EnvironmentError as er:
print( pp.error("Could not delete 
"+file_), file=sys.stderr)
print( pp.error("Error: %s" %str(er)), 
file=sys.stderr)
-- 
2.23.0




[gentoo-portage-dev] [PATCH gentoolkit 1/2] eclean: Inline _sort_keys method

2019-12-04 Thread Matt Turner
The boilerplate for calling this method was larger than what it actually
contained. Additionally I think this change will allow the loop to run
on a generator rather than a full list.

Signed-off-by: Matt Turner 
---
 pym/gentoolkit/eclean/clean.py | 21 ++---
 1 file changed, 6 insertions(+), 15 deletions(-)

diff --git a/pym/gentoolkit/eclean/clean.py b/pym/gentoolkit/eclean/clean.py
index fd59976..b790de0 100644
--- a/pym/gentoolkit/eclean/clean.py
+++ b/pym/gentoolkit/eclean/clean.py
@@ -36,10 +36,9 @@ class CleanUp(object):
@return: total size that was cleaned
"""
file_type = 'file'
-   clean_keys = self._sort_keys(clean_dict)
clean_size = 0
-   # clean all entries one by one
-   for key in clean_keys:
+   # clean all entries one by one; sorting helps reading
+   for key in sorted(clean_dict.keys()):
clean_size += self._clean_files(clean_dict[key], key, 
file_type)
# return total size of deleted or to delete files
return clean_size
@@ -57,10 +56,9 @@ class CleanUp(object):
@return: total size that was cleaned
"""
file_type = 'binary package'
-   clean_keys = self._sort_keys(clean_dict)
clean_size = 0
-   # clean all entries one by one
-   for key in clean_keys:
+   # clean all entries one by one; sorting helps reading
+   for key in sorted(clean_dict.keys()):
clean_size += self._clean_files(clean_dict[key], key, 
file_type)
 
#  run 'emaint --fix' here
@@ -83,10 +81,9 @@ class CleanUp(object):
@return: total size that would be cleaned
"""
file_type = 'file'
-   clean_keys = self._sort_keys(clean_dict)
clean_size = 0
-   # tally all entries one by one
-   for key in clean_keys:
+   # tally all entries one by one; sorting helps reading
+   for key in sorted(clean_dict.keys()):
key_size = self._get_size(clean_dict[key])
self.controller(key_size, key, clean_dict[key], 
file_type)
clean_size += key_size
@@ -110,12 +107,6 @@ class CleanUp(object):
print( pp.error("Error: %s" %str(er)), 
file=sys.stderr)
return key_size
 
-   def _sort_keys(self, clean_dict):
-   """Returns a list of sorted dictionary keys."""
-   # sorting helps reading
-   clean_keys = sorted(clean_dict)
-   return clean_keys
-
def _clean_files(self, files, key, file_type):
"""File removal function."""
clean_size = 0
-- 
2.23.0