Re: [gentoo-portage-dev] [PATCH] binarytree._read_metadata: return empty strings for undefine values (bug 603826)

2016-12-27 Thread Zac Medico 
On 12/27/2016 03:39 PM, Brian Dolbec wrote:
> On Tue, 27 Dec 2016 13:33:55 -0800
> Zac Medico  wrote:
> 
>> Fix the binarytree._read_metadata method to return empty strings for
>> undefined metadata values, in order to fulfill a contract with the
>> _pkg_str class. This prevents an AttributeError triggered by old
>> binary packages which have undefined repository metadata, as reported
>> in bug 603826.
>>
>> X-Gentoo-bug: 603826
>> X-Gentoo-bug-url: https://bugs.gentoo.org/603826
>> ---
>>  pym/portage/dbapi/bintree.py | 31 +--
>>  pym/portage/versions.py  |  8 +++-
>>  2 files changed, 32 insertions(+), 7 deletions(-)
>>
>> diff --git a/pym/portage/dbapi/bintree.py
>> b/pym/portage/dbapi/bintree.py index f483059..3341889 100644
>> --- a/pym/portage/dbapi/bintree.py
>> +++ b/pym/portage/dbapi/bintree.py
>> @@ -1,4 +1,4 @@
>> -# Copyright 1998-2014 Gentoo Foundation
>> +# Copyright 1998-2016 Gentoo Foundation
>>  # Distributed under the terms of the GNU General Public License v2
>>  
>>  from __future__ import unicode_literals
>> @@ -1041,12 +1041,12 @@ class binarytree(object):
>>  noiselevel=-1)
>>  return
>>  metadata = self._read_metadata(full_path, s)
>> -slot = metadata.get("SLOT")
>> +invalid_depend = False
>>  try:
>>  self._eval_use_flags(cpv, metadata)
>>  except portage.exception.InvalidDependString:
>> -slot = None
>> -if slot is None:
>> +invalid_depend = True
>> +if invalid_depend or not metadata.get("SLOT"):
>>  writemsg(_("!!! Invalid binary package:
>> '%s'\n") % full_path, noiselevel=-1)
>>  return
>> @@ -1126,6 +1126,21 @@ class binarytree(object):
>>  return cpv
>>  
>>  def _read_metadata(self, filename, st, keys=None):
>> +"""
>> +Read metadata from a binary package. The returned
>> metadata
>> +dictionary will contain empty strings for any values
>> that
>> +are undefined (this is important because the
>> _pkg_str class
>> +distinguishes between missing and undefined values).
>> +
>> +@param filename: File path of the binary package
>> +@type filename: string
>> +@param st: stat result for the binary package
>> +@type st: os.stat_result
>> +@param keys: optional list of specific metadata keys
>> to retrieve
>> +@type keys: iterable
>> +@rtype: dict
>> +@return: package metadata
>> +"""
>>  if keys is None:
>>  keys = self.dbapi._aux_cache_keys
>>  metadata = self.dbapi._aux_cache_slot_dict()
>> @@ -1139,10 +1154,14 @@ class binarytree(object):
>>  metadata[k] = _unicode(st.st_size)
>>  else:
>>  v =
>> binary_metadata.get(_unicode_encode(k))
>> -if v is not None:
>> +if v is None:
>> +if k == "EAPI":
>> +metadata[k] = "0"
>> +else:
>> +metadata[k] = ""
>> +else:
>>  v = _unicode_decode(v)
>>  metadata[k] = "
>> ".join(v.split())
>> -metadata.setdefault("EAPI", "0")
>>  return metadata
>>  
>>  def _inject_file(self, pkgindex, cpv, filename):
>> diff --git a/pym/portage/versions.py b/pym/portage/versions.py
>> index a028d93..adfb1c3 100644
>> --- a/pym/portage/versions.py
>> +++ b/pym/portage/versions.py
>> @@ -1,5 +1,5 @@
>>  # versions.py -- core Portage functionality
>> -# Copyright 1998-2014 Gentoo Foundation
>> +# Copyright 1998-2016 Gentoo Foundation
>>  # Distributed under the terms of the GNU General Public License v2
>>  
>>  from __future__ import unicode_literals
>> @@ -359,6 +359,12 @@ class _pkg_str(_unicode):
>>  Instances are typically created in dbapi.cp_list() or the
>> Atom contructor, and propagate from there. Generally, code that
>> pickles these objects will manually convert them to a plain unicode
>> object first. +
>> +Instances of this class will have missing attributes for
>> metadata that
>> +has not been passed into the constructor. The missing
>> attributes are
>> +used to distinguish missing metadata values from undefined
>> metadata values.
>> +For example, the repo attribute will be missing if the
>> 'repository' key
>> +is missing from the metadata dictionary.
>>  """
>>  
>>  def __new__(cls, cpv, metadata=None, settings=None,
>> eapi=None,
> 
> LGTM
> 

Thanks, pushed:

Re: [gentoo-portage-dev] [PATCH] binarytree._read_metadata: return empty strings for undefine values (bug 603826)

2016-12-27 Thread Brian Dolbec 
On Tue, 27 Dec 2016 13:33:55 -0800
Zac Medico  wrote:

> Fix the binarytree._read_metadata method to return empty strings for
> undefined metadata values, in order to fulfill a contract with the
> _pkg_str class. This prevents an AttributeError triggered by old
> binary packages which have undefined repository metadata, as reported
> in bug 603826.
> 
> X-Gentoo-bug: 603826
> X-Gentoo-bug-url: https://bugs.gentoo.org/603826
> ---
>  pym/portage/dbapi/bintree.py | 31 +--
>  pym/portage/versions.py  |  8 +++-
>  2 files changed, 32 insertions(+), 7 deletions(-)
> 
> diff --git a/pym/portage/dbapi/bintree.py
> b/pym/portage/dbapi/bintree.py index f483059..3341889 100644
> --- a/pym/portage/dbapi/bintree.py
> +++ b/pym/portage/dbapi/bintree.py
> @@ -1,4 +1,4 @@
> -# Copyright 1998-2014 Gentoo Foundation
> +# Copyright 1998-2016 Gentoo Foundation
>  # Distributed under the terms of the GNU General Public License v2
>  
>  from __future__ import unicode_literals
> @@ -1041,12 +1041,12 @@ class binarytree(object):
>   noiselevel=-1)
>   return
>   metadata = self._read_metadata(full_path, s)
> - slot = metadata.get("SLOT")
> + invalid_depend = False
>   try:
>   self._eval_use_flags(cpv, metadata)
>   except portage.exception.InvalidDependString:
> - slot = None
> - if slot is None:
> + invalid_depend = True
> + if invalid_depend or not metadata.get("SLOT"):
>   writemsg(_("!!! Invalid binary package:
> '%s'\n") % full_path, noiselevel=-1)
>   return
> @@ -1126,6 +1126,21 @@ class binarytree(object):
>   return cpv
>  
>   def _read_metadata(self, filename, st, keys=None):
> + """
> + Read metadata from a binary package. The returned
> metadata
> + dictionary will contain empty strings for any values
> that
> + are undefined (this is important because the
> _pkg_str class
> + distinguishes between missing and undefined values).
> +
> + @param filename: File path of the binary package
> + @type filename: string
> + @param st: stat result for the binary package
> + @type st: os.stat_result
> + @param keys: optional list of specific metadata keys
> to retrieve
> + @type keys: iterable
> + @rtype: dict
> + @return: package metadata
> + """
>   if keys is None:
>   keys = self.dbapi._aux_cache_keys
>   metadata = self.dbapi._aux_cache_slot_dict()
> @@ -1139,10 +1154,14 @@ class binarytree(object):
>   metadata[k] = _unicode(st.st_size)
>   else:
>   v =
> binary_metadata.get(_unicode_encode(k))
> - if v is not None:
> + if v is None:
> + if k == "EAPI":
> + metadata[k] = "0"
> + else:
> + metadata[k] = ""
> + else:
>   v = _unicode_decode(v)
>   metadata[k] = "
> ".join(v.split())
> - metadata.setdefault("EAPI", "0")
>   return metadata
>  
>   def _inject_file(self, pkgindex, cpv, filename):
> diff --git a/pym/portage/versions.py b/pym/portage/versions.py
> index a028d93..adfb1c3 100644
> --- a/pym/portage/versions.py
> +++ b/pym/portage/versions.py
> @@ -1,5 +1,5 @@
>  # versions.py -- core Portage functionality
> -# Copyright 1998-2014 Gentoo Foundation
> +# Copyright 1998-2016 Gentoo Foundation
>  # Distributed under the terms of the GNU General Public License v2
>  
>  from __future__ import unicode_literals
> @@ -359,6 +359,12 @@ class _pkg_str(_unicode):
>   Instances are typically created in dbapi.cp_list() or the
> Atom contructor, and propagate from there. Generally, code that
> pickles these objects will manually convert them to a plain unicode
> object first. +
> + Instances of this class will have missing attributes for
> metadata that
> + has not been passed into the constructor. The missing
> attributes are
> + used to distinguish missing metadata values from undefined
> metadata values.
> + For example, the repo attribute will be missing if the
> 'repository' key
> + is missing from the metadata dictionary.
>   """
>  
>   def __new__(cls, cpv, metadata=None, settings=None,
> eapi=None,

LGTM

-- 
Brian Dolbec

Re: [gentoo-portage-dev] [PATCH] LinkageMapELF: compute mulilib category for preserved libs (bug 598080)

2016-12-27 Thread Zac Medico 
On 12/27/2016 01:23 PM, Brian Dolbec wrote:
> On Mon, 26 Dec 2016 20:37:39 -0800
> Zac Medico  wrote:
> 
>> When support for parsing ELF headers in order to compute multilib
>> category was added in commit f1c1b8a77eebf7713b32e5f9945690f60f4f46de,
>> the LinkageMapELF class was not updated to do the same for preserved
>> libraries. This has resulted in incorrect preserve-libs handling
>> as reported in bug 598080, for ABIs including x32 where the
>> _approx_multilib_categories mapping is insufficient. This patch fixes
>> LinkageMapELF to compute the multilib category for each preserved
>> library, in the same way as the _post_src_install_soname_symlinks
>> function, so that the LinkageMapELF.findConsumers method will operate
>> correctly with preserved libraries of all ABIs.
>>
>> Fixes: f1c1b8a77eeb ("Generate soname dependency metadata (bug
>> 282639)") X-Gentoo-bug: 598080
>> X-Gentoo-bug-url: https://bugs.gentoo.org/598080
>> ---
>>  pym/portage/util/_dyn_libs/LinkageMapELF.py | 33
>> +++-- 1 file changed, 26 insertions(+), 7
>> deletions(-)
>>
>> diff --git a/pym/portage/util/_dyn_libs/LinkageMapELF.py
>> b/pym/portage/util/_dyn_libs/LinkageMapELF.py index 0b09fe5..a063621
>> 100644 --- a/pym/portage/util/_dyn_libs/LinkageMapELF.py
>> +++ b/pym/portage/util/_dyn_libs/LinkageMapELF.py
>> @@ -4,6 +4,7 @@
>>  import errno
>>  import logging
>>  import subprocess
>> +import sys
>>  
>>  import portage
>>  from portage import _encodings
>> @@ -12,6 +13,7 @@ from portage import _unicode_decode
>>  from portage import _unicode_encode
>>  from portage.cache.mappings import slot_dict_class
>>  from portage.const import EPREFIX
>> +from portage.dep.soname.multilib_category import
>> compute_multilib_category from portage.exception import
>> CommandNotFound, InvalidData from portage.localization import _
>>  from portage.util import getlibpaths
>> @@ -20,6 +22,12 @@ from portage.util import normalize_path
>>  from portage.util import varexpand
>>  from portage.util import writemsg_level
>>  from portage.util._dyn_libs.NeededEntry import NeededEntry
>> +from portage.util.elf.header import ELFHeader
>> +
>> +if sys.hexversion >= 0x300:
>> +_unicode = str
>> +else:
>> +_unicode = unicode
>>  
>>  # Map ELF e_machine values from NEEDED.ELF.2 to approximate multilib
>>  # categories. This approximation will produce incorrect results on
>> x32 @@ -283,15 +291,26 @@ class LinkageMapELF(object):
>>  l = l[3:].rstrip("\n")
>>  if not l:
>>  continue
>> -fields = l.split(";")
>> -if len(fields) < 5:
>> -
>> writemsg_level(_("\nWrong number of fields " \
>> -"returned
>> from scanelf: %s\n\n") % (l,),
>> +try:
>> +entry =
>> NeededEntry.parse("scanelf", l)
>> +except InvalidData as e:
>> +
>> writemsg_level("\n%s\n\n" % (e,), level=logging.ERROR, noiselevel=-1)
>>  continue
>> -fields[1] =
>> fields[1][root_len:]
>> -owner = plibs.pop(fields[1],
>> None)
>> -lines.append((owner,
>> "scanelf", ";".join(fields)))
>> +try:
>> +with
>> open(_unicode_encode(entry.filename,
>> +
>> encoding=_encodings['fs'],
>> +
>> errors='strict'), 'rb') as f:
>> +elf_header =
>> ELFHeader.read(f)
>> +except EnvironmentError as e:
>> +if e.errno !=
>> errno.ENOENT:
>> +raise
>> +# File removed
>> concurrently.
>> +continue
>> +entry.multilib_category =
>> compute_multilib_category(elf_header)
>> +entry.filename =
>> entry.filename[root_len:]
>> +owner =
>> plibs.pop(entry.filename, None)
>> +lines.append((owner,
>> "scanelf", _unicode(entry))) proc.wait()
>>  proc.stdout.close()
>>  
> 
> looks fine
> 

Thanks, pushed:

https://gitweb.gentoo.org/proj/portage.git/commit/?id=cbaee3c3b28f52947c142da8df24d6b0817962f9

-- 
Thanks,
Zac

[gentoo-portage-dev] [PATCH] binarytree._read_metadata: return empty strings for undefine values (bug 603826)

2016-12-27 Thread Zac Medico 
Fix the binarytree._read_metadata method to return empty strings for
undefined metadata values, in order to fulfill a contract with the
_pkg_str class. This prevents an AttributeError triggered by old
binary packages which have undefined repository metadata, as reported
in bug 603826.

X-Gentoo-bug: 603826
X-Gentoo-bug-url: https://bugs.gentoo.org/603826
---
 pym/portage/dbapi/bintree.py | 31 +--
 pym/portage/versions.py  |  8 +++-
 2 files changed, 32 insertions(+), 7 deletions(-)

diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index f483059..3341889 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2014 Gentoo Foundation
+# Copyright 1998-2016 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 from __future__ import unicode_literals
@@ -1041,12 +1041,12 @@ class binarytree(object):
noiselevel=-1)
return
metadata = self._read_metadata(full_path, s)
-   slot = metadata.get("SLOT")
+   invalid_depend = False
try:
self._eval_use_flags(cpv, metadata)
except portage.exception.InvalidDependString:
-   slot = None
-   if slot is None:
+   invalid_depend = True
+   if invalid_depend or not metadata.get("SLOT"):
writemsg(_("!!! Invalid binary package: '%s'\n") % 
full_path,
noiselevel=-1)
return
@@ -1126,6 +1126,21 @@ class binarytree(object):
return cpv
 
def _read_metadata(self, filename, st, keys=None):
+   """
+   Read metadata from a binary package. The returned metadata
+   dictionary will contain empty strings for any values that
+   are undefined (this is important because the _pkg_str class
+   distinguishes between missing and undefined values).
+
+   @param filename: File path of the binary package
+   @type filename: string
+   @param st: stat result for the binary package
+   @type st: os.stat_result
+   @param keys: optional list of specific metadata keys to retrieve
+   @type keys: iterable
+   @rtype: dict
+   @return: package metadata
+   """
if keys is None:
keys = self.dbapi._aux_cache_keys
metadata = self.dbapi._aux_cache_slot_dict()
@@ -1139,10 +1154,14 @@ class binarytree(object):
metadata[k] = _unicode(st.st_size)
else:
v = binary_metadata.get(_unicode_encode(k))
-   if v is not None:
+   if v is None:
+   if k == "EAPI":
+   metadata[k] = "0"
+   else:
+   metadata[k] = ""
+   else:
v = _unicode_decode(v)
metadata[k] = " ".join(v.split())
-   metadata.setdefault("EAPI", "0")
return metadata
 
def _inject_file(self, pkgindex, cpv, filename):
diff --git a/pym/portage/versions.py b/pym/portage/versions.py
index a028d93..adfb1c3 100644
--- a/pym/portage/versions.py
+++ b/pym/portage/versions.py
@@ -1,5 +1,5 @@
 # versions.py -- core Portage functionality
-# Copyright 1998-2014 Gentoo Foundation
+# Copyright 1998-2016 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 from __future__ import unicode_literals
@@ -359,6 +359,12 @@ class _pkg_str(_unicode):
Instances are typically created in dbapi.cp_list() or the Atom 
contructor,
and propagate from there. Generally, code that pickles these objects 
will
manually convert them to a plain unicode object first.
+
+   Instances of this class will have missing attributes for metadata that
+   has not been passed into the constructor. The missing attributes are
+   used to distinguish missing metadata values from undefined metadata 
values.
+   For example, the repo attribute will be missing if the 'repository' key
+   is missing from the metadata dictionary.
"""
 
def __new__(cls, cpv, metadata=None, settings=None, eapi=None,
-- 
2.7.4

Re: [gentoo-portage-dev] [PATCH] LinkageMapELF: compute mulilib category for preserved libs (bug 598080)

2016-12-27 Thread Brian Dolbec 
On Mon, 26 Dec 2016 20:37:39 -0800
Zac Medico  wrote:

> When support for parsing ELF headers in order to compute multilib
> category was added in commit f1c1b8a77eebf7713b32e5f9945690f60f4f46de,
> the LinkageMapELF class was not updated to do the same for preserved
> libraries. This has resulted in incorrect preserve-libs handling
> as reported in bug 598080, for ABIs including x32 where the
> _approx_multilib_categories mapping is insufficient. This patch fixes
> LinkageMapELF to compute the multilib category for each preserved
> library, in the same way as the _post_src_install_soname_symlinks
> function, so that the LinkageMapELF.findConsumers method will operate
> correctly with preserved libraries of all ABIs.
> 
> Fixes: f1c1b8a77eeb ("Generate soname dependency metadata (bug
> 282639)") X-Gentoo-bug: 598080
> X-Gentoo-bug-url: https://bugs.gentoo.org/598080
> ---
>  pym/portage/util/_dyn_libs/LinkageMapELF.py | 33
> +++-- 1 file changed, 26 insertions(+), 7
> deletions(-)
> 
> diff --git a/pym/portage/util/_dyn_libs/LinkageMapELF.py
> b/pym/portage/util/_dyn_libs/LinkageMapELF.py index 0b09fe5..a063621
> 100644 --- a/pym/portage/util/_dyn_libs/LinkageMapELF.py
> +++ b/pym/portage/util/_dyn_libs/LinkageMapELF.py
> @@ -4,6 +4,7 @@
>  import errno
>  import logging
>  import subprocess
> +import sys
>  
>  import portage
>  from portage import _encodings
> @@ -12,6 +13,7 @@ from portage import _unicode_decode
>  from portage import _unicode_encode
>  from portage.cache.mappings import slot_dict_class
>  from portage.const import EPREFIX
> +from portage.dep.soname.multilib_category import
> compute_multilib_category from portage.exception import
> CommandNotFound, InvalidData from portage.localization import _
>  from portage.util import getlibpaths
> @@ -20,6 +22,12 @@ from portage.util import normalize_path
>  from portage.util import varexpand
>  from portage.util import writemsg_level
>  from portage.util._dyn_libs.NeededEntry import NeededEntry
> +from portage.util.elf.header import ELFHeader
> +
> +if sys.hexversion >= 0x300:
> + _unicode = str
> +else:
> + _unicode = unicode
>  
>  # Map ELF e_machine values from NEEDED.ELF.2 to approximate multilib
>  # categories. This approximation will produce incorrect results on
> x32 @@ -283,15 +291,26 @@ class LinkageMapELF(object):
>   l = l[3:].rstrip("\n")
>   if not l:
>   continue
> - fields = l.split(";")
> - if len(fields) < 5:
> -
> writemsg_level(_("\nWrong number of fields " \
> - "returned
> from scanelf: %s\n\n") % (l,),
> + try:
> + entry =
> NeededEntry.parse("scanelf", l)
> + except InvalidData as e:
> +
> writemsg_level("\n%s\n\n" % (e,), level=logging.ERROR, noiselevel=-1)
>   continue
> - fields[1] =
> fields[1][root_len:]
> - owner = plibs.pop(fields[1],
> None)
> - lines.append((owner,
> "scanelf", ";".join(fields)))
> + try:
> + with
> open(_unicode_encode(entry.filename,
> +
> encoding=_encodings['fs'],
> +
> errors='strict'), 'rb') as f:
> + elf_header =
> ELFHeader.read(f)
> + except EnvironmentError as e:
> + if e.errno !=
> errno.ENOENT:
> + raise
> + # File removed
> concurrently.
> + continue
> + entry.multilib_category =
> compute_multilib_category(elf_header)
> + entry.filename =
> entry.filename[root_len:]
> + owner =
> plibs.pop(entry.filename, None)
> + lines.append((owner,
> "scanelf", _unicode(entry))) proc.wait()
>   proc.stdout.close()
>  

looks fine

-- 
Brian Dolbec