Re: [gentoo-portage-dev] [PATCH 2/2] pym/portage/util/locale.py: add a C module to check locale
' : [
>+ 'src/check_locale.c',
>+ ],
>+}
>
> class x_build(build):
> """ Build command with extra build_man call. """
>diff --git a/src/check_locale.c b/src/check_locale.c
>new file mode 100644
>index 000..9762ef2
>--- /dev/null
>+++ b/src/check_locale.c
>@@ -0,0 +1,144 @@
>+/* Copyright 2005-2015 Gentoo Foundation
>+ * Distributed under the terms of the GNU General Public License v2
>+ */
>+
>+#include
>+#include
>+#include
>+
>+static PyObject * portage_c_check_locale(PyObject *, PyObject *);
>+
>+static PyMethodDef CheckLocaleMethods[] = {
>+ {"_c_check_locale", portage_c_check_locale, METH_NOARGS, "Check the
>system locale."},
>+ {NULL, NULL, 0, NULL}
>+};
>+
>+#if PY_MAJOR_VERSION >= 3
>+static struct PyModuleDef moduledef = {
>+ PyModuleDef_HEAD_INIT,
>+ "portage_c_check_locale", /*
>m_name */
>+ "Module for checking the system locale for portage",/*
>m_doc */
>+ -1, /*
>m_size */
>+ CheckLocaleMethods, /*
>m_methods */
>+ NULL, /*
>m_reload */
>+ NULL, /*
>m_traverse */
>+ NULL, /*
>m_clear */
>+ NULL, /*
>m_free */
>+};
>+#endif
>+
>+static PyObject *LocaleError;
>+
>+PyMODINIT_FUNC
>+#if PY_MAJOR_VERSION >= 3
>+PyInit_portage_c_check_locale(void)
>+#else
>+initportage_c_check_locale(void)
>+#endif
>+{
>+ PyObject *m;
>+
>+#if PY_MAJOR_VERSION >= 3
>+ m = PyModule_Create(&moduledef);
>+#else
>+ m = Py_InitModule("portage_c_check_locale", CheckLocaleMethods);
>+#endif
>+
>+ if (m == NULL)
>+#if PY_MAJOR_VERSION >= 3
>+ return NULL;
>+#else
>+ return;
>+#endif
>+
>+ LocaleError = PyErr_NewException("locale.LocaleError", NULL, NULL);
>+ Py_INCREF(LocaleError);
>+ PyModule_AddObject(m, "LocaleError", LocaleError);
>+
>+#if PY_MAJOR_VERSION >= 3
>+ return m;
>+#else
>+ return;
>+#endif
>+}
>+
>+
>+static void
>+error_msg(char msg[], int c[], int rc[], int ac[])
>+{
>+ int i, p;
>+
>+ strcat(msg, " ");
>+ p = strlen(msg);
>+ for (i = 'a'; i <= 'z'; i++)
>+ msg[p++] = c[i-'a'];
>+
>+ strcat(msg, " -> ");
>+ p = strlen(msg);
>+ for (i = 'a'; i <= 'z'; i++)
>+ msg[p++] = rc[i-'a'];
>+ strcat(msg, "\n");
>+
>+ strcat(msg, " expected: ");
>+ p = strlen(msg);
>+ for (i = 'a'; i <= 'z'; i++)
>+ msg[p++] = ac[i-'a'];
>+ strcat(msg, "\n\n");
>+}
>+
>+
>+static PyObject *
>+portage_c_check_locale(PyObject *self, PyObject *args)
>+{
>+ int i, upper = 1, lower = 1;
>+ int lc[26], uc[26], rlc[26], ruc[26];
>+ char msg[1024];
>+
>+ memset(msg, 0, 1024);
>+
>+ for (i = 'a'; i <= 'z'; i++) {
>+ lc[i-'a'] = i;
>+ ruc[i-'a'] = toupper(i);
>+ }
>+
>+ for (i = 'A'; i <= 'Z'; i++) {
>+ uc[i-'A'] = i;
>+ rlc[i-'A'] = tolower(i);
>+ }
>+
>+ for (i = 'a'; i <= 'z'; i++)
>+ if(lc[i-'a'] != rlc[i-'a']) {
>+ lower = 0;
>+ break;
>+ }
>+
>+ for (i = 'A'; i <= 'Z'; i++)
>+ if(uc[i-'A'] != ruc[i-'A']) {
>+ upper = 0;
>+ break;
>+ }
>+
>+ if (lower == 0 || upper == 0) {
>+ strcpy(msg,
>+ "!!! WARNING: The LC_CTYPE variable is set to a locale
>that
>specifies\n"
>+ "!!! transformation between lowercase and uppercase
>ASCII
>characters that\n"
>+ "!!! is different than the one specified by POSIX
>locale. This can
>break\n"
>+ "!!! ebuilds and cause issues in programs that rely on
>the common
>character\n"
>+ "!!! conversion scheme. Please consider enabling
>another locale
>(such as\n"
>+ "!!! en_US.UTF-8) in /etc/locale.gen and setting it as
>LC_CTYPE
>in\n"
>+ "!!! make.conf.\n\n"
>+ );
>+
>+ if (lower == 0)
>+ error_msg(msg, lc, ruc, uc);
>+
>+ if (upper == 0)
>+ error_msg(msg, uc, rlc, lc);
>+ }
>+
>+#if PY_MAJOR_VERSION >= 3
>+ return Py_BuildValue("iy", lower && upper, msg);
>+#else
>+ return Py_BuildValue("is", lower && upper, msg);
>+#endif
>+}
--
Best regards,
Michał Górny (by phone)
Re: [gentoo-portage-dev] [PATCH 2/2] pym/portage/util/locale.py: add a C module to check locale
On Fri, 20 May 2016 06:59:14 -0400
"Anthony G. Basile" wrote:
> On 5/19/16 9:38 AM, Michał Górny wrote:
> > Dnia 19 maja 2016 14:43:38 CEST, "Anthony G. Basile"
> > napisał(a):
> >> From: "Anthony G. Basile"
> >>
> >> The current method to check for the system locale is to use python's
> >> ctypes.util.find_library() to construct a full library path to the
> >> system libc.so which is then passed to ctypes.CDLL(). However,
> >> this gets bogged down in implementation dependant details and
> >> fails with musl.
> >>
> >> We work around this design flaw in ctypes with a small python module
> >> written in C called 'portage_c_check_locale', and only fall back on
> >> the current ctypes-based check when this module is not available.
> >>
> >> This has been tested on glibc, uClibc and musl systems.
> >>
> >> X-Gentoo-bug: 571444
> >> X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=571444
> >> Signed-off-by: Anthony G. Basile
> >
> > To be honest, I don't like this. Do we really want to duplicate all this,
> > including duplicating the complete messages? This really looks messy and
> > unmaintainable.
> >
> > The only reason libc functions were being used was to workaround the hacky
> > built-in case conversion functions. And this is the only part where CDLL
> > was really used.
> >
> > So please change this to provide trivial wrappers around the C library
> > functions, and use them alternatively to ones obtained using CDLL. With a
> > single common code doing all the check logic and messaging.
> >
> >
>
> ctypes is itself a problem and so hacky python built-ins were replaced
> by more hacky python. CDLL shouldn't be used at all. The other problem
> is that non utf-8 encodings cause problems much earlier in the portage
> codebase than the test for a sane environment, which is a bigger problem
> than this small issue. No one checked what happens with python2.7 +
> portage + exotic locale. Since I don't know where this might head in
> the future, I kinda like the standalone potential. The only repeated
> code here is the message. Nonetheless, I can reduce this to just two
> functions, and do something like the following. I assume that's what
> you're suggesting:
>
> try:
> from portage_c_convert_case import _c_toupper, _c_tolower
> libc_toupper = _c_toupper
> libc_lolower = _c_tolower
> except ImportError:
> libc_fn = find_library("c")
> if libc_fn is None:
> return None
> libc = LoadLibrary(libc_fn)
> if libc is None:
> return None
> libc_toupper = libc.toupper
> libc_tolower = libc.tolower
>
>
> Incidentally, another approach, one that I use in bash is as follows. I
> think it requires bash4, and I'm not sure how to pull this into python
> gracefully.
>
> l="abcdefghijklmnopqrstuvwxyz"
> u="ABCDEFGHIJKLMNOPQRSTUVWXYZ"
> ru=${l^^}
> rl=${u,,}
> [[ $l == $rl ]] && echo "lower case okay" || echo "lower case bad"
> [[ $u == $ru ]] && echo "upper case okay" || echo "upper case bad"
Exactly as pointed out above. You have to import tolower()
and toupper() from libc as Python case conversion functions don't give
the same results as 'pure' libc used by bash.
--
Best regards,
Michał Górny
<http://dev.gentoo.org/~mgorny/>
pgpfrBpVYAfyE.pgp
Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH 1/2] portage.package.ebuild.config: Rename iuse_implicit -> iuse_effective
Rename the iuse_implicit variable used in USE_EXPAND handling to iuse_effective, since that is what is actually passed there. Correct naming makes figuring out what the function does much easier. --- pym/portage/package/ebuild/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pym/portage/package/ebuild/config.py b/pym/portage/package/ebuild/config.py index 45b7d08..5f19996 100644 --- a/pym/portage/package/ebuild/config.py +++ b/pym/portage/package/ebuild/config.py @@ -1278,13 +1278,13 @@ class config(object): """ def __init__(self, settings, unfiltered_use, - use, usemask, iuse_implicit, + use, usemask, iuse_effective, use_expand_split, use_expand_dict): self._settings = settings self._unfiltered_use = unfiltered_use self._use = use self._usemask = usemask - self._iuse_implicit = iuse_implicit + self._iuse_effective = iuse_effective self._use_expand_split = use_expand_split self._use_expand_dict = use_expand_dict @@ -1302,7 +1302,7 @@ class config(object): if has_wildcard: var_split = [ x for x in var_split if x != "*" ] has_iuse = set() - for x in self._iuse_implicit: + for x in self._iuse_effective: if x[:prefix_len] == prefix: has_iuse.add(x[prefix_len:]) if has_wildcard: -- 2.8.2
[gentoo-portage-dev] [PATCH 2/2] ebuild.config: Fix filtering all USE_EXPAND variables in EAPI 5+
Ensure that all USE_EXPAND variables are properly filtered and exported
in EAPI 5 and newer, as required by the PMS. This includes exporting
an empty value if no matching flag is provided in IUSE.
Bug: https://bugs.gentoo.org/show_bug.cgi?id=582140
---
pym/portage/eapi.py | 6 +-
pym/portage/package/ebuild/config.py | 11 ---
2 files changed, 13 insertions(+), 4 deletions(-)
diff --git a/pym/portage/eapi.py b/pym/portage/eapi.py
index 1709026..c4fb374 100644
--- a/pym/portage/eapi.py
+++ b/pym/portage/eapi.py
@@ -50,6 +50,9 @@ def eapi_exports_EBUILD_PHASE_FUNC(eapi):
def eapi_exports_REPOSITORY(eapi):
return eapi in ("4-python", "5-progress")
+def eapi_exports_USE_EXPAND_variables(eapi):
+ return eapi not in ("0", "1", "2", "3", "4", "4-python", "4-slot-abi")
+
def eapi_has_pkg_pretend(eapi):
return eapi not in ("0", "1", "2", "3")
@@ -101,7 +104,7 @@ def eapi_has_targetroot(eapi):
_eapi_attrs = collections.namedtuple('_eapi_attrs',
'dots_in_PN dots_in_use_flags exports_EBUILD_PHASE_FUNC '
- 'feature_flag_test feature_flag_targetroot '
+ 'exports_USE_EXPAND_variables feature_flag_test feature_flag_targetroot
'
'hdepend iuse_defaults iuse_effective posixish_locale '
'repo_deps required_use required_use_at_most_one_of slot_operator
slot_deps '
'src_uri_arrows strong_blocks use_deps use_dep_defaults')
@@ -128,6 +131,7 @@ def _get_eapi_attrs(eapi):
dots_in_PN = (eapi is None or eapi_allows_dots_in_PN(eapi)),
dots_in_use_flags = (eapi is None or
eapi_allows_dots_in_use_flags(eapi)),
exports_EBUILD_PHASE_FUNC = (eapi is None or
eapi_exports_EBUILD_PHASE_FUNC(eapi)),
+ exports_USE_EXPAND_variables = (eapi is None or
eapi_exports_USE_EXPAND_variables(eapi)),
feature_flag_test = True,
feature_flag_targetroot = (eapi is not None and
eapi_has_targetroot(eapi)),
hdepend = (eapi is not None and eapi_has_hdepend(eapi)),
diff --git a/pym/portage/package/ebuild/config.py
b/pym/portage/package/ebuild/config.py
index 5f19996..ee1fadb 100644
--- a/pym/portage/package/ebuild/config.py
+++ b/pym/portage/package/ebuild/config.py
@@ -1279,7 +1279,7 @@ class config(object):
def __init__(self, settings, unfiltered_use,
use, usemask, iuse_effective,
- use_expand_split, use_expand_dict):
+ use_expand_split, use_expand_dict,
eapi_exports_USE_EXPAND_variables):
self._settings = settings
self._unfiltered_use = unfiltered_use
self._use = use
@@ -1287,6 +1287,7 @@ class config(object):
self._iuse_effective = iuse_effective
self._use_expand_split = use_expand_split
self._use_expand_dict = use_expand_dict
+ self._eapi_exports_USE_EXPAND_variables =
eapi_exports_USE_EXPAND_variables
def __getitem__(self, key):
prefix = key.lower() + '_'
@@ -1330,7 +1331,7 @@ class config(object):
filtered_var_split.append(x)
var_split = filtered_var_split
- if var_split:
+ if var_split or self._eapi_exports_USE_EXPAND_variables:
value = ' '.join(var_split)
else:
# Don't export empty USE_EXPAND vars unless the
user config
@@ -1725,9 +1726,13 @@ class config(object):
x in self.get('USE_EXPAND', '').split())
lazy_use_expand = self._lazy_use_expand(
self, unfiltered_use, use, self.usemask,
- portage_iuse, use_expand_split, self._use_expand_dict)
+ portage_iuse, use_expand_split, self._use_expand_dict,
+ eapi_attrs.exports_USE_EXPAND_variables)
use_expand_iuses = {}
+ if eapi_attrs.exports_USE_EXPAND_variables:
+ for k in use_expand_split:
+ use_expand_iuses[k] = set()
for x in portage_iuse:
x_split = x.split('_')
if len(x_split) == 1:
--
2.8.2
Re: [gentoo-portage-dev] [PATCH] ebuild: Extend helper-in-global-scope ban to all EAPIs
On Thu, 5 May 2016 23:10:43 +0200 Michał Górny wrote: > Make helper calls in global scope fatal in all supported EAPIs since > this is the behavior required by PMS and all major offenders are fixed > already. > --- > bin/eapi.sh | 4 > bin/ebuild.sh | 30 ++ > 2 files changed, 6 insertions(+), 28 deletions(-) Merged now. -- Best regards, Michał Górny <http://dev.gentoo.org/~mgorny/> pgpfu5F4jC_jU.pgp Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH 2/2] ebuild.config: Fix filtering all USE_EXPAND variables in EAPI 5+
On Sat, 21 May 2016 00:26:40 +0200
Michał Górny wrote:
> Ensure that all USE_EXPAND variables are properly filtered and exported
> in EAPI 5 and newer, as required by the PMS. This includes exporting
> an empty value if no matching flag is provided in IUSE.
>
> Bug: https://bugs.gentoo.org/show_bug.cgi?id=582140
> ---
> pym/portage/eapi.py | 6 +-
> pym/portage/package/ebuild/config.py | 11 ---
> 2 files changed, 13 insertions(+), 4 deletions(-)
>
> diff --git a/pym/portage/eapi.py b/pym/portage/eapi.py
> index 1709026..c4fb374 100644
> --- a/pym/portage/eapi.py
> +++ b/pym/portage/eapi.py
> @@ -50,6 +50,9 @@ def eapi_exports_EBUILD_PHASE_FUNC(eapi):
> def eapi_exports_REPOSITORY(eapi):
> return eapi in ("4-python", "5-progress")
>
> +def eapi_exports_USE_EXPAND_variables(eapi):
> + return eapi not in ("0", "1", "2", "3", "4", "4-python", "4-slot-abi")
> +
> def eapi_has_pkg_pretend(eapi):
> return eapi not in ("0", "1", "2", "3")
>
> @@ -101,7 +104,7 @@ def eapi_has_targetroot(eapi):
>
> _eapi_attrs = collections.namedtuple('_eapi_attrs',
> 'dots_in_PN dots_in_use_flags exports_EBUILD_PHASE_FUNC '
> - 'feature_flag_test feature_flag_targetroot '
> + 'exports_USE_EXPAND_variables feature_flag_test feature_flag_targetroot
> '
> 'hdepend iuse_defaults iuse_effective posixish_locale '
> 'repo_deps required_use required_use_at_most_one_of slot_operator
> slot_deps '
> 'src_uri_arrows strong_blocks use_deps use_dep_defaults')
> @@ -128,6 +131,7 @@ def _get_eapi_attrs(eapi):
> dots_in_PN = (eapi is None or eapi_allows_dots_in_PN(eapi)),
> dots_in_use_flags = (eapi is None or
> eapi_allows_dots_in_use_flags(eapi)),
> exports_EBUILD_PHASE_FUNC = (eapi is None or
> eapi_exports_EBUILD_PHASE_FUNC(eapi)),
> + exports_USE_EXPAND_variables = (eapi is None or
> eapi_exports_USE_EXPAND_variables(eapi)),
> feature_flag_test = True,
> feature_flag_targetroot = (eapi is not None and
> eapi_has_targetroot(eapi)),
> hdepend = (eapi is not None and eapi_has_hdepend(eapi)),
> diff --git a/pym/portage/package/ebuild/config.py
> b/pym/portage/package/ebuild/config.py
> index 5f19996..ee1fadb 100644
> --- a/pym/portage/package/ebuild/config.py
> +++ b/pym/portage/package/ebuild/config.py
> @@ -1279,7 +1279,7 @@ class config(object):
>
> def __init__(self, settings, unfiltered_use,
> use, usemask, iuse_effective,
> - use_expand_split, use_expand_dict):
> + use_expand_split, use_expand_dict,
> eapi_exports_USE_EXPAND_variables):
> self._settings = settings
> self._unfiltered_use = unfiltered_use
> self._use = use
> @@ -1287,6 +1287,7 @@ class config(object):
> self._iuse_effective = iuse_effective
> self._use_expand_split = use_expand_split
> self._use_expand_dict = use_expand_dict
> + self._eapi_exports_USE_EXPAND_variables =
> eapi_exports_USE_EXPAND_variables
>
> def __getitem__(self, key):
> prefix = key.lower() + '_'
> @@ -1330,7 +1331,7 @@ class config(object):
> filtered_var_split.append(x)
> var_split = filtered_var_split
>
> - if var_split:
> + if var_split or self._eapi_exports_USE_EXPAND_variables:
> value = ' '.join(var_split)
> else:
> # Don't export empty USE_EXPAND vars unless the
> user config
> @@ -1725,9 +1726,13 @@ class config(object):
> x in self.get('USE_EXPAND', '').split())
> lazy_use_expand = self._lazy_use_expand(
> self, unfiltered_use, use, self.usemask,
> - portage_iuse, use_expand_split, self._use_expand_dict)
> + portage_iuse, use_expand_split, self._use_expand_dict,
> + eapi_attrs.exports_USE_EXPAND_variables)
>
> use_expand_iuses = {}
> + if eapi_attrs.exports_USE_EXPAND_variables:
> + for k in use_expand_split:
> + use_expand_iuses[k] = set()
> for x in portage_iuse:
> x_split = x.split('_')
> if len(x_split) == 1:
After some thinking, I'll prepare another patch that applies the change
to all EAPIs. The behavior for earlier EAPIs is implementation-defined
by PMS and having it inconsistent will only confuse users.
--
Best regards,
Michał Górny
<http://dev.gentoo.org/~mgorny/>
pgpwxUpygWeHZ.pgp
Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH v2 1/2] portage.package.ebuild.config: Rename iuse_implicit -> iuse_effective
Rename the iuse_implicit variable used in USE_EXPAND handling to iuse_effective, since that is what is actually passed there. Correct naming makes figuring out what the function does much easier. --- pym/portage/package/ebuild/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pym/portage/package/ebuild/config.py b/pym/portage/package/ebuild/config.py index 45b7d08..5f19996 100644 --- a/pym/portage/package/ebuild/config.py +++ b/pym/portage/package/ebuild/config.py @@ -1278,13 +1278,13 @@ class config(object): """ def __init__(self, settings, unfiltered_use, - use, usemask, iuse_implicit, + use, usemask, iuse_effective, use_expand_split, use_expand_dict): self._settings = settings self._unfiltered_use = unfiltered_use self._use = use self._usemask = usemask - self._iuse_implicit = iuse_implicit + self._iuse_effective = iuse_effective self._use_expand_split = use_expand_split self._use_expand_dict = use_expand_dict @@ -1302,7 +1302,7 @@ class config(object): if has_wildcard: var_split = [ x for x in var_split if x != "*" ] has_iuse = set() - for x in self._iuse_implicit: + for x in self._iuse_effective: if x[:prefix_len] == prefix: has_iuse.add(x[prefix_len:]) if has_wildcard: -- 2.8.2
[gentoo-portage-dev] [PATCH v2 2/2] portage.package.ebuild.config: Always export filtered USE_EXPAND vars
Ensure that all USE_EXPAND variables are always exported with filtered
USE flags inside, even if none of those flags are declared in IUSE.
This is the behavior required for EAPI 5+ by the PMS.
Since the behavior for earlier EAPIs is left undefined and having
different behavior would be confusing to users, apply it in earlier
EAPIs as well.
---
bin/ebuild.sh| 6
pym/portage/package/ebuild/config.py | 55 ++--
2 files changed, 3 insertions(+), 58 deletions(-)
diff --git a/bin/ebuild.sh b/bin/ebuild.sh
index 5b3146d..edf885f 100755
--- a/bin/ebuild.sh
+++ b/bin/ebuild.sh
@@ -690,12 +690,6 @@ if ! has "$EBUILD_PHASE" clean cleanrm ; then
fi
fi
-# unset USE_EXPAND variables that contain only the special "*" token
-for x in ${USE_EXPAND} ; do
- [ "${!x}" == "*" ] && unset ${x}
-done
-unset x
-
if has nostrip ${FEATURES} ${RESTRICT} || has strip ${RESTRICT}
then
export DEBUGBUILD=1
diff --git a/pym/portage/package/ebuild/config.py
b/pym/portage/package/ebuild/config.py
index 5f19996..d92f5f6 100644
--- a/pym/portage/package/ebuild/config.py
+++ b/pym/portage/package/ebuild/config.py
@@ -1330,47 +1330,7 @@ class config(object):
filtered_var_split.append(x)
var_split = filtered_var_split
- if var_split:
- value = ' '.join(var_split)
- else:
- # Don't export empty USE_EXPAND vars unless the
user config
- # exports them as empty. This is required for
vars such as
- # LINGUAS, where unset and empty have different
meanings.
- # The special '*' token is understood by
ebuild.sh, which
- # will unset the variable so that things like
LINGUAS work
- # properly (see bug #459350).
- if has_wildcard:
- value = '*'
- else:
- if has_iuse:
- already_set = False
- # Skip the first 'env'
configdict, in order to
- # avoid infinite recursion
here, since that dict's
- # __getitem__ calls the current
__getitem__.
- for d in
self._settings.lookuplist[1:]:
- if key in d:
- already_set =
True
- break
-
- if not already_set:
- for x in
self._unfiltered_use:
- if
x[:prefix_len] == prefix:
-
already_set = True
- break
-
- if already_set:
- value = ''
- else:
- value = '*'
- else:
- # It's not in IUSE, so just
allow the variable content
- # to pass through if it is
defined somewhere. This
- # allows packages that support
LINGUAS but don't
- # declare it in IUSE to use the
variable outside of the
- # USE_EXPAND context.
- value = None
-
- return value
+ return ' '.join(var_split)
def setcpv(self, mycpv, use_cache=None, mydb=None):
"""
@@ -1727,7 +1687,7 @@ class config(object):
self, unfiltered_use, use, self.usemask,
portage_iuse, use_expand_split, self._use_expand_dict)
- use_expand_iuses = {}
+ use_expand_iuses = dict((k, set()) for k in use_expand_split)
for x in portage_iuse:
x_split = x.split('_')
if len(x_split) == 1:
@@ -1735,18 +1695,9 @@ class config(object):
for i in range(len(x_split) - 1):
k = '_'.join(x_split[:i+1])
if k in use_expand_split:
-
[gentoo-portage-dev] [PATCH 3/3] portage.dbapi.vartree: Support exclusions in INSTALL_MASK
Allow INSTALL_MASK patterns to start with '-' to indicate that
a specific match is to be excluded from being masked. In this case,
the last matching pattern determines whether the file is actually
filtered out or kept.
---
pym/portage/dbapi/vartree.py | 10 ++
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 8e5ac43..d02d850 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -3690,19 +3690,21 @@ class dblink(object):
def _is_install_masked(self, relative_path):
ret = False
for pattern in self.settings.install_mask:
+ # if pattern starts with -, possibly exclude this path
+ pat_res = not pattern.startswith('-')
+ if not pat_res:
+ pattern = pattern[1:]
# absolute path pattern
if pattern.startswith('/'):
# match either exact path or one of parent dirs
# the latter is done via matching pattern/*
if (fnmatch.fnmatch(relative_path, pattern[1:])
or
fnmatch.fnmatch(relative_path, pattern[1:] + '/*')):
- ret = True
- break
+ ret = pat_res
# filename
else:
if
fnmatch.fnmatch(os.path.basename(relative_path), pattern):
- ret = True
- break
+ ret = pat_res
return ret
def treewalk(self, srcroot, destroot, inforoot, myebuild, cleanup=0,
--
2.8.3
[gentoo-portage-dev] [PATCH 2/3] Move INSTALL_MASK handling into merging
Introduce a new logic for INSTALL_MASK handling in merging code,
replacing the old code that removed matching files and directories
from imagedir in bash. The new code actually ignores matching files
on-the-fly while testing for file collisions and merging files.
The files are still written to CONTENTS, and output using "###" zing
to indicate being masked, yet are not actually merged to the filesystem.
---
bin/misc-functions.sh| 17 --
pym/portage/dbapi/vartree.py | 102 ++-
pym/portage/package/ebuild/config.py | 2 +-
3 files changed, 65 insertions(+), 56 deletions(-)
diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh
index b42e1d6..4086981 100755
--- a/bin/misc-functions.sh
+++ b/bin/misc-functions.sh
@@ -300,23 +300,6 @@ install_mask() {
set -${shopts}
}
-preinst_mask() {
- if [ -z "${D}" ]; then
-eerror "${FUNCNAME}: D is unset"
-return 1
- fi
-
- if ! ___eapi_has_prefix_variables; then
- local ED=${D}
- fi
-
- # Make sure $PWD is not ${D} so that we don't leave gmon.out files
- # in there in case any tools were built with -pg in CFLAGS.
- cd "${T}"
-
- install_mask "${ED}" "${INSTALL_MASK}"
-}
-
preinst_sfperms() {
if [ -z "${D}" ]; then
eerror "${FUNCNAME}: D is unset"
diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 6209a86..8e5ac43 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -2490,7 +2490,7 @@ class dblink(object):
(statobj.st_dev, statobj.st_ino),
[]).append(relative_path)
- if is_owned:
+ if is_owned and not
self._is_install_masked(relative_path[1:]):
show_unmerge("---",
unmerge_desc["replaced"], file_type, obj)
continue
elif relative_path in cfgfiledict:
@@ -3687,6 +3687,24 @@ class dblink(object):
def _emerge_log(self, msg):
emergelog(False, msg)
+ def _is_install_masked(self, relative_path):
+ ret = False
+ for pattern in self.settings.install_mask:
+ # absolute path pattern
+ if pattern.startswith('/'):
+ # match either exact path or one of parent dirs
+ # the latter is done via matching pattern/*
+ if (fnmatch.fnmatch(relative_path, pattern[1:])
+ or
fnmatch.fnmatch(relative_path, pattern[1:] + '/*')):
+ ret = True
+ break
+ # filename
+ else:
+ if
fnmatch.fnmatch(os.path.basename(relative_path), pattern):
+ ret = True
+ break
+ return ret
+
def treewalk(self, srcroot, destroot, inforoot, myebuild, cleanup=0,
mydbapi=None, prev_mtimes=None, counter=None):
"""
@@ -3846,16 +3864,6 @@ class dblink(object):
max_dblnk = dblnk
self._installed_instance = max_dblnk
- # Apply INSTALL_MASK before collision-protect, since it may
- # be useful to avoid collisions in some scenarios.
- # We cannot detect if this is needed or not here as
INSTALL_MASK can be
- # modified by bashrc files.
- phase = MiscFunctionsProcess(background=False,
- commands=["preinst_mask"], phase="preinst",
- scheduler=self._scheduler, settings=self.settings)
- phase.start()
- phase.wait()
-
# We check for unicode encoding issues after src_install.
However,
# the check must be repeated here for binary packages (it's
# inexpensive since we call os.walk() here anyway).
@@ -3927,6 +3935,10 @@ class dblink(object):
relative_path = fpath[srcroot_len:]
+ # filter on INSTALL_MASK
+ if
self._is_install_masked(relative_path):
+ continue
+
if line_ending_re.search(relative_path)
is not None:
paths_with_newlines.append(relative_path)
@@ -4642,6 +4654,7 @@ class dblink(object):
while mergelist:
[gentoo-portage-dev] [PATCH 1/3] portage.package.ebuild.config: Move FEATURES=no* handling there
Move the code responsible for adding additional paths to INSTALL_MASK
into portage.package.ebuild.config.
---
bin/misc-functions.sh| 13 -
pym/portage/package/ebuild/config.py | 10 ++
2 files changed, 10 insertions(+), 13 deletions(-)
diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh
index 58755a1..b42e1d6 100755
--- a/bin/misc-functions.sh
+++ b/bin/misc-functions.sh
@@ -314,20 +314,7 @@ preinst_mask() {
# in there in case any tools were built with -pg in CFLAGS.
cd "${T}"
- # remove man pages, info pages, docs if requested
- local f
- for f in man info doc; do
- if has no${f} $FEATURES; then
- INSTALL_MASK="${INSTALL_MASK} /usr/share/${f}"
- fi
- done
-
install_mask "${ED}" "${INSTALL_MASK}"
-
- # remove share dir if unnessesary
- if has nodoc $FEATURES || has noman $FEATURES || has noinfo $FEATURES;
then
- rmdir "${ED}usr/share" &> /dev/null
- fi
}
preinst_sfperms() {
diff --git a/pym/portage/package/ebuild/config.py
b/pym/portage/package/ebuild/config.py
index 45b7d08..fcc7ce5 100644
--- a/pym/portage/package/ebuild/config.py
+++ b/pym/portage/package/ebuild/config.py
@@ -1773,6 +1773,16 @@ class config(object):
# setcpv triggers lazy instantiation of things like
_use_manager.
_eapi_cache.clear()
+ # Prepare the final value of INSTALL_MASK
+ install_mask = self["INSTALL_MASK"].split()
+ if 'nodoc' in self.features:
+ install_mask.append("/usr/share/doc")
+ if 'noinfo' in self.features:
+ install_mask.append("/usr/share/info")
+ if 'noman' in self.features:
+ install_mask.append("/usr/share/man")
+ self["INSTALL_MASK"] = ' '.join(install_mask)
+
def _grab_pkg_env(self, penv, container, protected_keys=None):
if protected_keys is None:
protected_keys = ()
--
2.8.3
[gentoo-portage-dev] [PATCH 0/3] INSTALL_MASK redesign, part I
Hello, everyone. As part of new GLEP effort, I'm working on improving INSTALL_MASK support in Portage to make it cleaner and more featureful. Here's the first set of commits resulting from this. The previous implementation of INSTALL_MASK was done purely in bash. Long story short, Portage removed all files matching one of INSTALL_MASK patterns from the installation image. My implementation is done purely as a filter in Python. The ebuild configuration bits now generate final list of INSTALL_MASK. It is afterwards used in vartree bits to filter files while checking for collisions, merging and unmerging appropriately. The major differences/improvements are: * The files are actually left in image directory. Not that it makes any real difference but it is a bit cleaner and more like real mask this way. * The pre-install "removing" output has been replaced by listing files with "###" zing in merging output (which means "not installed due to INSTALL_MASK"). * All masked files are now listed in vdb CONTENTS. Therefore, tools like app-portage/install-mask can now figure out to which packages masks were applied and rebuild them on mask changes appropriately. * Mask exclusions are supported now. Which means you can do e.g.: INSTALL_MASK="/usr/share/locale -/usr/share/locale/foo". * The code is now whitespace-safe. While patterns specified in INSTALL_MASK directly still can not contain spaces, pattern groups will be able to use them. In a few days, I'll try to provide a part II that would implement the actual mask groups. However, I may need some help adding support for the configuration files. I'm not touching PKG_INSTALL_MASK for now since it's harder and outside of the scope of what I'm trying to achieve. Michał Górny (3): portage.package.ebuild.config: Move FEATURES=no* handling there Move INSTALL_MASK handling into merging portage.dbapi.vartree: Support exclusions in INSTALL_MASK bin/misc-functions.sh| 30 -- pym/portage/dbapi/vartree.py | 104 ++- pym/portage/package/ebuild/config.py | 10 3 files changed, 76 insertions(+), 68 deletions(-) -- 2.8.3
Re: [gentoo-portage-dev] [PATCH 2/3] Move INSTALL_MASK handling into merging
On Sun, 22 May 2016 08:56:03 +0200 Michał Górny wrote: > Introduce a new logic for INSTALL_MASK handling in merging code, > replacing the old code that removed matching files and directories > from imagedir in bash. The new code actually ignores matching files > on-the-fly while testing for file collisions and merging files. > The files are still written to CONTENTS, and output using "###" zing > to indicate being masked, yet are not actually merged to the filesystem. > --- > bin/misc-functions.sh| 17 -- > pym/portage/dbapi/vartree.py | 102 > ++- > pym/portage/package/ebuild/config.py | 2 +- > 3 files changed, 65 insertions(+), 56 deletions(-) This actually fails when INSTALL_MASK is not defined. I will be sending another patch shortly. -- Best regards, Michał Górny <http://dev.gentoo.org/~mgorny/> pgpD5HZwPJTkM.pgp Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH v2] Move INSTALL_MASK handling into merging
Introduce a new logic for INSTALL_MASK handling in merging code,
replacing the old code that removed matching files and directories
from imagedir in bash. The new code actually ignores matching files
on-the-fly while testing for file collisions and merging files.
The files are still written to CONTENTS, and output using "###" zing
to indicate being masked, yet are not actually merged to the filesystem.
---
bin/misc-functions.sh| 17 --
pym/portage/dbapi/vartree.py | 102 ++-
pym/portage/package/ebuild/config.py | 4 +-
3 files changed, 66 insertions(+), 57 deletions(-)
diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh
index b42e1d6..4086981 100755
--- a/bin/misc-functions.sh
+++ b/bin/misc-functions.sh
@@ -300,23 +300,6 @@ install_mask() {
set -${shopts}
}
-preinst_mask() {
- if [ -z "${D}" ]; then
-eerror "${FUNCNAME}: D is unset"
-return 1
- fi
-
- if ! ___eapi_has_prefix_variables; then
- local ED=${D}
- fi
-
- # Make sure $PWD is not ${D} so that we don't leave gmon.out files
- # in there in case any tools were built with -pg in CFLAGS.
- cd "${T}"
-
- install_mask "${ED}" "${INSTALL_MASK}"
-}
-
preinst_sfperms() {
if [ -z "${D}" ]; then
eerror "${FUNCNAME}: D is unset"
diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
index 6209a86..8e5ac43 100644
--- a/pym/portage/dbapi/vartree.py
+++ b/pym/portage/dbapi/vartree.py
@@ -2490,7 +2490,7 @@ class dblink(object):
(statobj.st_dev, statobj.st_ino),
[]).append(relative_path)
- if is_owned:
+ if is_owned and not
self._is_install_masked(relative_path[1:]):
show_unmerge("---",
unmerge_desc["replaced"], file_type, obj)
continue
elif relative_path in cfgfiledict:
@@ -3687,6 +3687,24 @@ class dblink(object):
def _emerge_log(self, msg):
emergelog(False, msg)
+ def _is_install_masked(self, relative_path):
+ ret = False
+ for pattern in self.settings.install_mask:
+ # absolute path pattern
+ if pattern.startswith('/'):
+ # match either exact path or one of parent dirs
+ # the latter is done via matching pattern/*
+ if (fnmatch.fnmatch(relative_path, pattern[1:])
+ or
fnmatch.fnmatch(relative_path, pattern[1:] + '/*')):
+ ret = True
+ break
+ # filename
+ else:
+ if
fnmatch.fnmatch(os.path.basename(relative_path), pattern):
+ ret = True
+ break
+ return ret
+
def treewalk(self, srcroot, destroot, inforoot, myebuild, cleanup=0,
mydbapi=None, prev_mtimes=None, counter=None):
"""
@@ -3846,16 +3864,6 @@ class dblink(object):
max_dblnk = dblnk
self._installed_instance = max_dblnk
- # Apply INSTALL_MASK before collision-protect, since it may
- # be useful to avoid collisions in some scenarios.
- # We cannot detect if this is needed or not here as
INSTALL_MASK can be
- # modified by bashrc files.
- phase = MiscFunctionsProcess(background=False,
- commands=["preinst_mask"], phase="preinst",
- scheduler=self._scheduler, settings=self.settings)
- phase.start()
- phase.wait()
-
# We check for unicode encoding issues after src_install.
However,
# the check must be repeated here for binary packages (it's
# inexpensive since we call os.walk() here anyway).
@@ -3927,6 +3935,10 @@ class dblink(object):
relative_path = fpath[srcroot_len:]
+ # filter on INSTALL_MASK
+ if
self._is_install_masked(relative_path):
+ continue
+
if line_ending_re.search(relative_path)
is not None:
paths_with_newlines.append(relative_path)
@@ -4642,6 +4654,7 @@ class dblink(object):
while mergelist:
[gentoo-portage-dev] [PATCH 1/2] portage.util.configparser: Provide common code to handle cp imports
Provide a common code unit to handle portable *ConfigParser imports for all supported Python versions. --- pym/portage/_sets/__init__.py | 10 ++ pym/portage/repository/config.py | 9 + pym/portage/util/_desktop_entry.py | 7 ++- pym/portage/util/configparser.py | 22 ++ 4 files changed, 27 insertions(+), 21 deletions(-) create mode 100644 pym/portage/util/configparser.py diff --git a/pym/portage/_sets/__init__.py b/pym/portage/_sets/__init__.py index 3203521..ec42f7c 100644 --- a/pym/portage/_sets/__init__.py +++ b/pym/portage/_sets/__init__.py @@ -9,14 +9,6 @@ __all__ = ["SETPREFIX", "get_boolean", "SetConfigError", import io import logging import sys -try: - from configparser import NoOptionError, ParsingError - if sys.hexversion >= 0x302: - from configparser import ConfigParser as SafeConfigParser - else: - from configparser import SafeConfigParser -except ImportError: - from ConfigParser import SafeConfigParser, NoOptionError, ParsingError import portage from portage import os from portage import load_mod @@ -29,6 +21,8 @@ from portage.const import _ENABLE_SET_CONFIG from portage.exception import PackageSetNotFound from portage.localization import _ from portage.util import writemsg_level +from portage.util.configparser import (SafeConfigParser, + NoOptionError, ParsingError) SETPREFIX = "@" diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py index 00319fe..9039886 100644 --- a/pym/portage/repository/config.py +++ b/pym/portage/repository/config.py @@ -9,14 +9,6 @@ import warnings import sys import re -try: - from configparser import Error as ConfigParserError - if sys.hexversion >= 0x302: - from configparser import ConfigParser as SafeConfigParser - else: - from configparser import SafeConfigParser -except ImportError: - from ConfigParser import SafeConfigParser, Error as ConfigParserError import portage from portage import eclass_cache, os from portage.const import (MANIFEST2_HASH_FUNCTIONS, MANIFEST2_REQUIRED_HASH, @@ -25,6 +17,7 @@ from portage.eapi import eapi_allows_directories_on_profile_level_and_repository from portage.env.loaders import KeyValuePairFileLoader from portage.util import (normalize_path, read_corresponding_eapi_file, shlex_split, stack_lists, writemsg, writemsg_level, _recursive_file_list) +from portage.util.configparser import SafeConfigParser, ConfigParserError from portage.util._path import isdir_raise_eaccess from portage.util.path import first_existing from portage.localization import _ diff --git a/pym/portage/util/_desktop_entry.py b/pym/portage/util/_desktop_entry.py index 0b49547..95a015e 100644 --- a/pym/portage/util/_desktop_entry.py +++ b/pym/portage/util/_desktop_entry.py @@ -6,14 +6,11 @@ import re import subprocess import sys -try: - from configparser import Error as ConfigParserError, RawConfigParser -except ImportError: - from ConfigParser import Error as ConfigParserError, RawConfigParser - import portage from portage import _encodings, _unicode_encode, _unicode_decode from portage.util import writemsg +from portage.util.configparser import ConfigParserError, RawConfigParser + def parse_desktop_entry(path): """ diff --git a/pym/portage/util/configparser.py b/pym/portage/util/configparser.py new file mode 100644 index 000..d305052 --- /dev/null +++ b/pym/portage/util/configparser.py @@ -0,0 +1,22 @@ +# Copyright 2016 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +__all__ = ['ConfigParserError', 'NoOptionError', 'ParsingError', + 'RawConfigParser', 'SafeConfigParser'] + +# the following scary compatibility thing provides two classes: +# - SafeConfigParser that provides safe interpolation for values, +# - RawConfigParser that provides no interpolation for values. + +import sys + +try: + from configparser import (Error as ConfigParserError, + NoOptionError, ParsingError, RawConfigParser) + if sys.hexversion >= 0x302: + from configparser import ConfigParser as SafeConfigParser + else: + from configparser import SafeConfigParser +except ImportError: + from ConfigParser import (Error as ConfigParserError, + NoOptionError, ParsingError, RawConfigParser, SafeConfigParser) -- 2.8.3
[gentoo-portage-dev] [PATCH 2/2] portage.util.configparser: Commonize portable config file reading routine
---
pym/portage/_sets/__init__.py | 29 ++-
pym/portage/repository/config.py | 38 ++---
pym/portage/util/_desktop_entry.py | 20 ++---
pym/portage/util/configparser.py | 57 +-
4 files changed, 64 insertions(+), 80 deletions(-)
diff --git a/pym/portage/_sets/__init__.py b/pym/portage/_sets/__init__.py
index ec42f7c..6d69bda 100644
--- a/pym/portage/_sets/__init__.py
+++ b/pym/portage/_sets/__init__.py
@@ -22,7 +22,7 @@ from portage.exception import PackageSetNotFound
from portage.localization import _
from portage.util import writemsg_level
from portage.util.configparser import (SafeConfigParser,
- NoOptionError, ParsingError)
+ NoOptionError, ParsingError, read_configs)
SETPREFIX = "@"
@@ -50,32 +50,7 @@ class SetConfig(object):
})
if _ENABLE_SET_CONFIG:
- # use read_file/readfp in order to control decoding of
unicode
- try:
- # Python >=3.2
- read_file = self._parser.read_file
- except AttributeError:
- read_file = self._parser.readfp
-
- for p in paths:
- f = None
- try:
- f = io.open(_unicode_encode(p,
- encoding=_encodings['fs'],
errors='strict'),
- mode='r',
encoding=_encodings['repo.content'],
- errors='replace')
- except EnvironmentError:
- pass
- else:
- try:
- read_file(f)
- except ParsingError as e:
- writemsg_level(_unicode_decode(
- _("!!! Error while
reading sets config file: %s\n")
- ) % e,
level=logging.ERROR, noiselevel=-1)
- finally:
- if f is not None:
- f.close()
+ read_configs(self._parser, paths)
else:
self._create_default_config()
diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py
index 9039886..a23f4bd 100644
--- a/pym/portage/repository/config.py
+++ b/pym/portage/repository/config.py
@@ -17,7 +17,8 @@ from portage.eapi import
eapi_allows_directories_on_profile_level_and_repository
from portage.env.loaders import KeyValuePairFileLoader
from portage.util import (normalize_path, read_corresponding_eapi_file,
shlex_split,
stack_lists, writemsg, writemsg_level, _recursive_file_list)
-from portage.util.configparser import SafeConfigParser, ConfigParserError
+from portage.util.configparser import (SafeConfigParser, ConfigParserError,
+ read_configs)
from portage.util._path import isdir_raise_eaccess
from portage.util.path import first_existing
from portage.localization import _
@@ -542,15 +543,6 @@ class RepoConfigLoader(object):
"""Parse files in paths to load config"""
parser = SafeConfigParser(defaults=default_opts)
- # use read_file/readfp in order to control decoding of unicode
- try:
- # Python >=3.2
- read_file = parser.read_file
- source_kwarg = 'source'
- except AttributeError:
- read_file = parser.readfp
- source_kwarg = 'filename'
-
recursive_paths = []
for p in paths:
if isinstance(p, basestring):
@@ -558,31 +550,7 @@ class RepoConfigLoader(object):
else:
recursive_paths.append(p)
- for p in recursive_paths:
- if isinstance(p, basestring):
- f = None
- try:
- f = io.open(_unicode_encode(p,
- encoding=_encodings['fs'],
errors='strict'),
- mode='r',
encoding=_encodings['repo.content'],
- errors='replace')
- except EnvironmentError:
- pass
- else:
- # The 'source' keyword argument is
needed since otherwise
-
Re: [gentoo-portage-dev] [PATCH 2/2] pym/portage/util/locale.py: add a C module to help check locale
pers = {}
> +x_c_helpers = {
> + 'portage_c_convert_case' : [
> + 'src/portage_c_convert_case.c',
> + ],
> +}
>
> class x_build(build):
> """ Build command with extra build_man call. """
> diff --git a/src/portage_c_convert_case.c b/src/portage_c_convert_case.c
> new file mode 100644
> index 000..f60b0c2
> --- /dev/null
> +++ b/src/portage_c_convert_case.c
> @@ -0,0 +1,94 @@
> +/* Copyright 2005-2016 Gentoo Foundation
> + * Distributed under the terms of the GNU General Public License v2
> + */
> +
> +#include
> +#include
> +
> +static PyObject * portage_c_tolower(PyObject *, PyObject *);
> +static PyObject * portage_c_toupper(PyObject *, PyObject *);
> +
> +static PyMethodDef ConvertCaseMethods[] = {
> + {"_c_tolower", portage_c_tolower, METH_VARARGS, "Convert to lower case
> using system locale."},
> + {"_c_toupper", portage_c_toupper, METH_VARARGS, "Convert to upper case
> using system locale."},
> + {NULL, NULL, 0, NULL}
You should include stdlib.h or stdio.h for NULL.
> +};
> +
> +#if PY_MAJOR_VERSION >= 3
> +static struct PyModuleDef moduledef = {
> + PyModuleDef_HEAD_INIT,
> + "portage_c_convert_case", /*
> m_name */
> + "Module for converting case using the system locale", /*
> m_doc */
> + -1, /*
> m_size */
> + ConvertCaseMethods, /*
> m_methods */
> + NULL, /*
> m_reload */
> + NULL, /*
> m_traverse */
> + NULL, /*
> m_clear */
> + NULL, /*
> m_free */
> +};
> +#endif
> +
> +static PyObject *ConvertCaseError;
> +
> +PyMODINIT_FUNC
> +#if PY_MAJOR_VERSION >= 3
> +PyInit_portage_c_convert_case(void)
> +#else
> +initportage_c_convert_case(void)
> +#endif
> +{
> + PyObject *m;
> +
> +#if PY_MAJOR_VERSION >= 3
> + m = PyModule_Create(&moduledef);
> +#else
> + m = Py_InitModule("portage_c_convert_case", ConvertCaseMethods);
> +#endif
> +
> + if (m == NULL)
> +#if PY_MAJOR_VERSION >= 3
> + return NULL;
> +#else
> + return;
> +#endif
> +
> + ConvertCaseError =
> PyErr_NewException("portage_c_convert_case.ConvertCaseError", NULL, NULL);
> + Py_INCREF(ConvertCaseError);
> + PyModule_AddObject(m, "ConvertCaseError", ConvertCaseError);
> +
> +#if PY_MAJOR_VERSION >= 3
> + return m;
> +#else
> + return;
> +#endif
> +}
To be honest, I think we'd be happier having two big #ifdefs for init
funcs rather than one function with a lot of #ifdefs, and the common
ConvertCaseError part in a separate static function.
> +
> +
> +static PyObject *
> +portage_c_tolower(PyObject *self, PyObject *args)
> +{
> + int c;
> +
> + if (!PyArg_ParseTuple(args, "i", &c))
> + {
> + PyErr_SetString(ConvertCaseError, "_c_tolower: PyArg_ParseTuple
> failed");
From PyArg_ParseTuple() [1]:
| on failure, it returns false and raises the appropriate exception.
So I don't think you need or should use a custom exception here.
[1]:https://docs.python.org/2/c-api/arg.html#c.PyArg_ParseTuple
> + return NULL;
> + }
> +
> + return Py_BuildValue("i", tolower(c));
> +}
> +
> +
> +static PyObject *
> +portage_c_toupper(PyObject *self, PyObject *args)
> +{
> + int c;
> +
> + if (!PyArg_ParseTuple(args, "i", &c))
> + {
> + PyErr_SetString(ConvertCaseError, "_c_toupper: PyArg_ParseTuple
> failed");
> + return NULL;
> + }
> +
> + return Py_BuildValue("i", toupper(c));
> +}
Thanks a lot for your effort. This is really getting in shape.
--
Best regards,
Michał Górny
<http://dev.gentoo.org/~mgorny/>
pgp9y04POWsEG.pgp
Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH 2/2] pym/portage/util/locale.py: add a C module to help check locale
On Mon, 23 May 2016 08:08:18 -0400
"Anthony G. Basile" wrote:
> On 5/23/16 2:44 AM, Michał Górny wrote:
> > On Sun, 22 May 2016 13:04:40 -0400
> > "Anthony G. Basile" wrote:
> >
> >> From: "Anthony G. Basile"
> >>
> >> The current method to check for a sane system locale is to use python's
> >> ctypes.util.find_library() to construct a full library path to the
> >> system libc.so and pass that path to ctypes.CDLL() so we can call
> >> toupper() and tolower() directly. However, this gets bogged down in
> >> implementation details and fails with musl.
> >>
> >> We work around this design flaw in ctypes with a small python module
> >> written in C which provides thin wrappers to toupper() and tolower(),
> >> and only fall back on the current ctypes-based check when this module
> >> is not available.
> >>
> >> This has been tested on glibc, uClibc and musl systems.
> >>
> >> X-Gentoo-bug: 571444
> >> X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=571444
> >>
> >> Signed-off-by: Anthony G. Basile
> >> ---
> >> pym/portage/util/locale.py | 32 ++-
> >> setup.py | 6 ++-
> >> src/portage_c_convert_case.c | 94
> >>
> >> 3 files changed, 121 insertions(+), 11 deletions(-)
> >> create mode 100644 src/portage_c_convert_case.c
> >>
> >> diff --git a/pym/portage/util/locale.py b/pym/portage/util/locale.py
> >> index 2a15ea1..85ddd2b 100644
> >> --- a/pym/portage/util/locale.py
> >> +++ b/pym/portage/util/locale.py
> >> @@ -11,6 +11,7 @@ from __future__ import absolute_import, unicode_literals
> >> import locale
> >> import logging
> >> import os
> >> +import sys
> >> import textwrap
> >> import traceback
> >>
> >> @@ -34,18 +35,26 @@ def _check_locale(silent):
> >>"""
> >>The inner locale check function.
> >>"""
> >> -
> >> - libc_fn = find_library("c")
> >> - if libc_fn is None:
> >> - return None
> >> - libc = LoadLibrary(libc_fn)
> >> - if libc is None:
> >> - return None
> >> + try:
> >> + from portage_c_convert_case import _c_toupper, _c_tolower
> >> + libc_tolower = _c_tolower
> >> + libc_toupper = _c_toupper
> >
> > Now I'm being picky... but if you named the functions toupper()
> > and tolower(), you could actually import the whole module as 'libc'
> > and have less code!
>
> I see what you're saying, and its tempting because its elegant, but I'm
> afraid of a clash of names. I've got a bad feeling this will get us
> into trouble later.
>
> Let me play with this and see what happens.
I don't think this will be problematic since things like this happen
in Python all the time ;-). And after all, C function names can be
different than Python function names.
> > Also it would be nice to actually make the module more generic. There
> > are more places where we use CDLL, and all of them could eventually be
> > supported by the module (unshare() would be much better done in C, for
> > example).
>
> Yeah I get your point here. Let me convince myself first.
I've got a killer argument: right now we hardcode constants from Linux
headers in the Python code!
Not that I'm asking you to actually add code for that as well. Just
rename the module to something more generic like portage.util.libc ;-).
> >> + except ImportError:
> >> + writemsg_level("!!! Unable to import
> >> portage_c_convert_case\n!!!\n",
> >> + level=logging.WARNING, noiselevel=-1)
> >
> > Do we really want to warn verbosely about this? I think it'd be
> > a pretty common case for people running the git checkout.
>
> This should stay. Its good to know that the module is not being
> imported and silently falling back on the ctypes stuff.
>
> 1) its only going to happen in the rare occasion that you're using
> something like a turkish locale and can't import the module.
Wrong. This happens before the check is done, so it will be output
every time Portage is started, also with good locale.
> 2) people who do a git checkout should add
> PYTHONPATH=build/lib.linux-x86_64-3.4 to their env to test the module.
> I can add something to testpath. Users
Re: [gentoo-portage-dev] [PATCH 2/2] portage.util.configparser: Commonize portable config file reading routine
On Sun, 22 May 2016 11:04:51 -0700 Zac Medico wrote: > On 05/22/2016 01:41 AM, Michał Górny wrote: > > --- > > pym/portage/_sets/__init__.py | 29 ++- > > pym/portage/repository/config.py | 38 ++--- > > pym/portage/util/_desktop_entry.py | 20 ++--- > > pym/portage/util/configparser.py | 57 > > +- > > 4 files changed, 64 insertions(+), 80 deletions(-) > > These patches look good. Note that the _native_kwargs function is not > really needed anymore, because it was just a workaround for this issue > which only affected python 2.6.4 and earlier: > >http://bugs.python.org/issue4978 Pushed now, thanks. I'll do a global _native_kwargs cleanup in a separate patch. -- Best regards, Michał Górny <http://dev.gentoo.org/~mgorny/> pgpXQJxBE7cZ3.pgp Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH] Remove unneeded native_kwargs wrapper
Remove the native_kwargs wrapper and all uses of it. As pointed out by
Zac Medico, this was only needed for Python < 2.6.5, and we no longer
support those versions for quite some time.
---
bin/portageq | 2 +-
pym/_emerge/MiscFunctionsProcess.py | 3 +--
pym/_emerge/actions.py| 2 +-
pym/_emerge/depgraph.py | 12
pym/_emerge/resolver/output.py| 3 +--
pym/portage/__init__.py | 9 -
pym/portage/dbapi/porttree.py | 4 ++--
pym/portage/dbapi/vartree.py | 3 +--
pym/portage/news.py | 3 +--
pym/portage/package/ebuild/doebuild.py| 6 +++---
pym/portage/repository/config.py | 2 +-
pym/portage/sync/modules/cvs/cvs.py | 4 ++--
pym/portage/sync/modules/git/git.py | 4 ++--
pym/portage/sync/modules/rsync/rsync.py | 4 ++--
pym/portage/sync/modules/svn/svn.py | 6 +++---
pym/portage/sync/modules/webrsync/webrsync.py | 2 +-
pym/portage/util/__init__.py | 2 +-
pym/portage/util/configparser.py | 5 ++---
repoman/pym/repoman/modules/scan/ebuild/checks.py | 2 +-
19 files changed, 30 insertions(+), 48 deletions(-)
diff --git a/bin/portageq b/bin/portageq
index 832d004..d645635 100755
--- a/bin/portageq
+++ b/bin/portageq
@@ -1281,7 +1281,7 @@ def add_pquery_arguments(parser):
kwargs["help"] = opt_info["help"]
except KeyError:
pass
- arg_group.add_argument(*pargs,
**portage._native_kwargs(kwargs))
+ arg_group.add_argument(*pargs, **kwargs)
def usage(argv):
diff --git a/pym/_emerge/MiscFunctionsProcess.py
b/pym/_emerge/MiscFunctionsProcess.py
index b7f5892..99cf598 100644
--- a/pym/_emerge/MiscFunctionsProcess.py
+++ b/pym/_emerge/MiscFunctionsProcess.py
@@ -40,8 +40,7 @@ class MiscFunctionsProcess(AbstractEbuildProcess):
# think this is a real phase.
phase_backup = self.settings.pop("EBUILD_PHASE", None)
try:
- return spawn(" ".join(args), self.settings,
- **portage._native_kwargs(kwargs))
+ return spawn(" ".join(args), self.settings, **kwargs)
finally:
if phase_backup is not None:
self.settings["EBUILD_PHASE"] = phase_backup
diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py
index 59626ad..2ca7902 100644
--- a/pym/_emerge/actions.py
+++ b/pym/_emerge/actions.py
@@ -2378,7 +2378,7 @@ def load_emerge_config(emerge_config=None, **kargs):
if v and v.strip():
kwargs[k] = v
emerge_config.trees = portage.create_trees(trees=emerge_config.trees,
- **portage._native_kwargs(kwargs))
+ **kwargs)
for root_trees in emerge_config.trees.values():
settings = root_trees["vartree"].settings
diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
index cfaafa3..f78f08d 100644
--- a/pym/_emerge/depgraph.py
+++ b/pym/_emerge/depgraph.py
@@ -4175,8 +4175,7 @@ class depgraph(object):
pargs, kwargs = item
kwargs = kwargs.copy()
kwargs['collect_use_changes'] = True
- if not self._show_unsatisfied_dep(*pargs,
- **portage._native_kwargs(kwargs)):
+ if not self._show_unsatisfied_dep(*pargs,
**kwargs):
remaining_items.append(item)
if len(remaining_items) !=
len(self._dynamic_config._unsatisfied_deps_for_display):
self._dynamic_config._unsatisfied_deps_for_display = remaining_items
@@ -4327,8 +4326,7 @@ class depgraph(object):
not been scheduled for replacement.
"""
kwargs["trees"] = self._dynamic_config._graph_trees
- return self._select_atoms_highest_available(*pargs,
- **portage._native_kwargs(kwargs))
+ return self._select_atoms_highest_available(*pargs, **kwargs)
def _select_atoms_highest_available(self, root, depstring,
myuse=None, parent=None, strict=True, trees=None,
priority=None):
@@ -8511,8 +8509,7 @@ class depgraph(object):
writemsg("\n", noiselevel=-1)
for pargs, kwargs in
self._dynamic_config._unsatisfied_deps_for_display:
- self._show_unsatisfied_dep(*pargs,
- **portage._native
Re: [gentoo-portage-dev] [PATCH 3/3] pym/portage/util/locale.py: add a C module to help check locale
> + PyObject *m;
> + m = PyModule_Create(&moduledef);
> + return m;
> +}
> +#else
> +initlibc(void)
> +{
> + Py_InitModule("libc", LibcMethods);
> +}
> +#endif
> +
> +
> +static PyObject *
> +_libc_tolower(PyObject *self, PyObject *args)
> +{
> + int c;
> +
> + if (!PyArg_ParseTuple(args, "i", &c))
> + return NULL;
> +
> + return Py_BuildValue("i", tolower(c));
> +}
> +
> +
> +static PyObject *
> +_libc_toupper(PyObject *self, PyObject *args)
> +{
> + int c;
> +
> + if (!PyArg_ParseTuple(args, "i", &c))
> + return NULL;
> +
> + return Py_BuildValue("i", toupper(c));
> +}
Aside from that, it look nice and shiny now. Thanks a lot!
--
Best regards,
Michał Górny
<http://dev.gentoo.org/~mgorny/>
pgpMDaEA9fng0.pgp
Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH 3/3] pym/portage/util/locale.py: add a C module to help check locale
On Sun, 29 May 2016 02:37:50 -0400
"Anthony G. Basile" wrote:
> On 5/29/16 2:30 AM, Michał Górny wrote:
> > On Fri, 27 May 2016 10:26:44 -0400
> > "Anthony G. Basile" wrote:
> >
> >> From: "Anthony G. Basile"
> >>
> >> The current method to check for a sane system locale is to use python's
> >> ctypes.util.find_library() to construct a full library path to the
> >> system libc.so and pass that path to ctypes.CDLL() so we can call
> >> toupper() and tolower() directly. However, this gets bogged down in
> >> implementation details and fails with musl.
> >>
> >> We work around this design flaw in ctypes with a small python module
> >> written in C which provides thin wrappers to toupper() and tolower(),
> >> and only fall back on the current ctypes-based check when this module
> >> is not available.
> >>
> >> This has been tested on glibc, uClibc and musl systems.
> >>
> >> X-Gentoo-bug: 571444
> >> X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=571444
> >>
> >> Signed-off-by: Anthony G. Basile
> >> ---
> >> pym/portage/util/locale.py | 16 ++-
> >> setup.py | 6 +++-
> >> src/portage_util_libc.c| 70
> >> ++
> >> 3 files changed, 84 insertions(+), 8 deletions(-)
> >> create mode 100644 src/portage_util_libc.c
> >>
> >> diff --git a/pym/portage/util/locale.py b/pym/portage/util/locale.py
> >> index 093eb86..5b09945 100644
> >> --- a/pym/portage/util/locale.py
> >> +++ b/pym/portage/util/locale.py
> >> @@ -34,13 +34,15 @@ def _check_locale(silent):
> >>"""
> >>The inner locale check function.
> >>"""
> >> -
> >> - libc_fn = find_library("c")
> >> - if libc_fn is None:
> >> - return None
> >> - libc = LoadLibrary(libc_fn)
> >> - if libc is None:
> >> - return None
> >> + try:
> >> + from portage.util import libc
> >> + except ImportError:
> >> + libc_fn = find_library("c")
> >> + if libc_fn is None:
> >> + return None
> >> + libc = LoadLibrary(libc_fn)
> >> + if libc is None:
> >> + return None
> >>
> >>lc = list(range(ord('a'), ord('z')+1))
> >>uc = list(range(ord('A'), ord('Z')+1))
> >> diff --git a/setup.py b/setup.py
> >> index 25429bc..5ca8156 100755
> >> --- a/setup.py
> >> +++ b/setup.py
> >> @@ -47,7 +47,11 @@ x_scripts = {
> >> # Dictionary custom modules written in C/C++ here. The structure is
> >> # key = module name
> >> # value = list of C/C++ source code, path relative to top source
> >> directory
> >> -x_c_helpers = {}
> >> +x_c_helpers = {
> >> + 'portage.util.libc' : [
> >> + 'src/portage_util_libc.c',
> >> + ],
> >> +}
> >>
> >> class x_build(build):
> >>""" Build command with extra build_man call. """
> >> diff --git a/src/portage_util_libc.c b/src/portage_util_libc.c
> >> new file mode 100644
> >> index 000..00b09c2
> >> --- /dev/null
> >> +++ b/src/portage_util_libc.c
> >> @@ -0,0 +1,70 @@
> >> +/* Copyright 2005-2016 Gentoo Foundation
> >> + * Distributed under the terms of the GNU General Public License v2
> >> + */
> >> +
> >> +#include
> >> +#include
> >> +#include
> >> +
> >> +static PyObject * _libc_tolower(PyObject *, PyObject *);
> >> +static PyObject * _libc_toupper(PyObject *, PyObject *);
> >> +
> >> +static PyMethodDef LibcMethods[] = {
> >> + {"tolower", _libc_tolower, METH_VARARGS, "Convert to lower case using
> >> system locale."},
> >> + {"toupper", _libc_toupper, METH_VARARGS, "Convert to upper case using
> >> system locale."},
> >> + {NULL, NULL, 0, NULL}
> >> +};
> >> +
> >> +#if PY_MAJOR_VERSION >= 3
> >> +static struct PyModuleDef moduledef = {
> >> + PyModuleDef_HEAD_INIT,
> >> + "libc", /*
> >> m_name */
&
Re: [gentoo-portage-dev] [PATCH] Remove unneeded native_kwargs wrapper
On Tue, 24 May 2016 11:08:44 +0200 Alexander Berntsen wrote: > -BEGIN PGP SIGNED MESSAGE- > Hash: SHA512 > > Welp that took forever to read through... I didn't see anything wrong > with it. So if tests pass and everything looks OK to you, go ahead and > push it. Merged now. -- Best regards, Michał Górny <http://dev.gentoo.org/~mgorny/> pgpWLd3EwKFeo.pgp Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH 4/3] portage.package.ebuild.config: Support path groups from install-mask.conf
---
.../package/ebuild/_config/InstallMaskManager.py | 59 ++
pym/portage/package/ebuild/config.py | 34 -
pym/portage/util/configparser.py | 19 ++-
3 files changed, 110 insertions(+), 2 deletions(-)
create mode 100644 pym/portage/package/ebuild/_config/InstallMaskManager.py
diff --git a/pym/portage/package/ebuild/_config/InstallMaskManager.py
b/pym/portage/package/ebuild/_config/InstallMaskManager.py
new file mode 100644
index 000..96cb539
--- /dev/null
+++ b/pym/portage/package/ebuild/_config/InstallMaskManager.py
@@ -0,0 +1,59 @@
+# Copyright 2010-2016 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+__all__ = (
+ 'InstallMaskManager',
+)
+
+import sys
+
+from portage import os
+from portage.localization import _
+from portage.util import writemsg
+from portage.util.configparser import (SafeConfigParser, ConfigParserError,
+ MultiValueConfigParserDict, read_configs)
+
+
+class InstallMaskManager(object):
+ def __init__(self, repositories, abs_user_config, user_config=True):
+ self._groups = {}
+
+ # read repository defined groups
+ self._read_config_from_repositories(repositories)
+
+ if user_config:
+ self._read_config(os.path.join(abs_user_config,
'install-mask.conf'), True)
+
+ def _read_config_from_repositories(self, repositories):
+ for r in repositories.repos_with_profiles():
+ self._read_config(os.path.join(r.location, 'metadata',
'install-mask.conf'))
+
+ def _read_config(self, path, is_user_config=False):
+ # use separate parsers to detect collisions properly
+ cfp_kwargs = {}
+ if sys.hexversion >= 0x0302:
+ cfp_kwargs['strict'] = False
+ parser = SafeConfigParser(dict_type=MultiValueConfigParserDict,
+ **cfp_kwargs)
+ try:
+ read_configs(parser, [path])
+ except ConfigParserError as e:
+ writemsg(
+ _("!!! Error while reading %s: %s\n") % (path,
e),
+ noiselevel=-1)
+ return
+
+ for sname in parser.sections():
+ if not is_user_config and sname in self._groups:
+ writemsg(
+ _("!!! Error while reading %s:
duplicate group %s found\n") % (path, sname),
+ noiselevel=-1)
+ continue
+ if not parser.has_option(sname, 'path'):
+ continue
+
+ paths = parser.get(sname, 'path').split('\n')
+ self._groups[sname] = paths
+
+ def expand_group(self, gname):
+ return self._groups[gname]
diff --git a/pym/portage/package/ebuild/config.py
b/pym/portage/package/ebuild/config.py
index 9d13703..dfbd7f2 100644
--- a/pym/portage/package/ebuild/config.py
+++ b/pym/portage/package/ebuild/config.py
@@ -60,6 +60,7 @@ from portage.package.ebuild._config.features_set import
features_set
from portage.package.ebuild._config.KeywordsManager import KeywordsManager
from portage.package.ebuild._config.LicenseManager import LicenseManager
from portage.package.ebuild._config.UseManager import UseManager
+from portage.package.ebuild._config.InstallMaskManager import
InstallMaskManager
from portage.package.ebuild._config.LocationsManager import LocationsManager
from portage.package.ebuild._config.MaskManager import MaskManager
from portage.package.ebuild._config.VirtualsManager import VirtualsManager
@@ -277,6 +278,7 @@ class config(object):
# force instantiation of lazy immutable objects when
cloning, so
# that they're not instantiated more than once
self._keywords_manager_obj = clone._keywords_manager
+ self._install_mask_manager_obj =
clone._install_mask_manager
self._mask_manager_obj = clone._mask_manager
# shared mutable attributes
@@ -329,6 +331,7 @@ class config(object):
else:
# lazily instantiated objects
self._keywords_manager_obj = None
+ self._install_mask_manager_obj = None
self._mask_manager_obj = None
self._virtuals_manager_obj = None
@@ -1032,6 +1035,15 @@ class config(object):
return self._keywords_manager_obj
@property
+ def _install_mask_manager(self):
+ if self._install_mask_manager_obj is None:
+ self._install_mask_manager_obj = InstallMaskManager(
+
Re: [gentoo-portage-dev] New meeting
Dnia 3 czerwca 2016 19:06:38 CEST, Alexander Berntsen napisał(a): >-BEGIN PGP SIGNED MESSAGE- >Hash: SHA512 > >It seems the three of us who've replied are all available 2016-06-07 >at 2pm UTC. That's 4pm for me. Check when it is in your timezone[0], >and I'll see you then! And remember to update the agenda[1] with your >items. Could you add the topic of PMS compliance patches for me? Thanks to our very helpful www team, wiki is completely inaccessible to people who are not around a PC. I'll be in travel during the meeting, and I'll hopefully join via phone unless I fall asleep too hard :-). In case the latter happens, short note the releases topic: I'd really appreciate if we could: 1. Get at least patches 1-3 (preferably all 4) of my INSTALL_MASK improvements merged, 2. Do a release with them and the global-score use* ban (already in master), 3. Delay USE_EXPAND patches post release. The global-scope use* uses are already fixed, so urgent release with the change will prevent people from reintroducing them. The INSTALL_MASK enhancements would give people more tools for lingua filtering. Therefore, it would be good to have it early, and to mention them in the news item before LINGUAS are adjusted to adhere to the PMS. > > >[0] ><http://whenisgood.net/ResultsPopup?event=portage&code=y8rwaw9&slot=146530800> >[1] <https://wiki.gentoo.org/wiki/Project:Portage/Meetings#Agenda> >- -- >Alexander >berna...@gentoo.org >https://secure.plaimi.net/~alexander >-BEGIN PGP SIGNATURE- >Version: GnuPG v2 > >iQIcBAEBCgAGBQJXUbkeAAoJENQqWdRUGk8BtC0QAJ9DBSBqya9vfWGUT7AVXYMV >tAjdNXhs8jJ/ygCEX6MkRvYFqjRuHDIKlGKNpNR96HA7M9j6oxyjySN4ccxxtG7R >T4dBSKPBrU7SEyOeTjtacgXqvbXBJQKrAsAEdJwyr8f5HPmys+Y1GsWqtOJT3N2a >q8X6AZzKRft3XatXi0UahTUq4R2/Ln/RfelE0QQGjMEFCzxdj5nZBbyJdfroSlBK >g9KkHX9QLfoCEbvI8Fe0dksWVmFV8ESwnDvkoNEb+A5ODSTBRvqkaOc3T3hLGEQM >zGS/FT4de38A29ScGtbVWaM/VeDAfUj9hmu+tzSxzN1sNPHpRPn/dKDco71aFaIe >gNLMhLmFyiJftThMV7bnGmuc6EvtLl72oM/Ca/4WI8ZDO4XqkuWIth1vdZKyq3wm >3xI1Cwsow2Fjg0kN8e2nkKgZHw/3ulEZnvaC9QijkCjJpjxD013kHtd3xhndBmAi >gJMtQf4MOtoluiq7wKVQMUQb6yty8jyWjiHrO9YUwwhXFodfZ10/aXwx3aK20e2U >BgyVxpG+gxwkBJbuEQWIQbgvQHxyuwOlRvaE7KyRvsTLmrbMQjaf8ya7VX/Tzl6H >EJxQpEIwqARgVMY6WluPRSAg6vqB1f/ozLntpjqCwInyHYO1y2AS2++VPLusoZkC >Mtr9L1PTgtVfGWf2edy8 >=PUFC >-END PGP SIGNATURE- -- Best regards, Michał Górny (by phone)
Re: [gentoo-portage-dev] [PATCH 4/3] portage.package.ebuild.config: Support path groups from install-mask.conf
On Tue, 31 May 2016 17:58:34 +0200 Michał Górny wrote: > --- > .../package/ebuild/_config/InstallMaskManager.py | 59 > ++ > pym/portage/package/ebuild/config.py | 34 - > pym/portage/util/configparser.py | 19 ++- > 3 files changed, 110 insertions(+), 2 deletions(-) > create mode 100644 pym/portage/package/ebuild/_config/InstallMaskManager.py Please disregard this one. After discussion with ulm, the spec is changing and I will be providing an updated patch later. However, the remaining three still apply. -- Best regards, Michał Górny <http://dev.gentoo.org/~mgorny/> pgpbdoNQzqiWc.pgp Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH v2] Move INSTALL_MASK handling into merging
Dnia 12 czerwca 2016 11:10:55 CEST, Zac Medico napisał(a): >On 05/22/2016 01:21 AM, Michał Górny wrote: >> Introduce a new logic for INSTALL_MASK handling in merging code, >> replacing the old code that removed matching files and directories >> from imagedir in bash. The new code actually ignores matching files >> on-the-fly while testing for file collisions and merging files. >> The files are still written to CONTENTS, and output using "###" zing >> to indicate being masked, yet are not actually merged to the >filesystem. > >Since collision-protect relies on existing files in its collision test, >install-masked files are no longer going to trigger collisions. Then, >since the install-masked files are still written to CONTENTS, it's >possible for the unmerge of one package to unmerge colliding files that >belong to another package! > >There are a number of ways to solve this problem. For example, we could >have the unmerge code ignore any files in CONTENTS that match the >INSTALL_MASK value that was used at merge time. Hmm, thinking about this more widely (i.e. actually thinking rather than mimicking the old behavior), I think it would be better to actually use the original file set for collision-protect. This will make it possible to detect collisions that would otherwise be hidden via INSTALL_MASK. -- Best regards, Michał Górny (by phone)
Re: [gentoo-portage-dev] [PATCH 3/3] portage.dbapi.vartree: Support exclusions in INSTALL_MASK
Dnia 12 czerwca 2016 11:20:36 CEST, Zac Medico napisał(a):
>On 05/21/2016 11:56 PM, Michał Górny wrote:
>> Allow INSTALL_MASK patterns to start with '-' to indicate that
>> a specific match is to be excluded from being masked. In this case,
>> the last matching pattern determines whether the file is actually
>> filtered out or kept.
>> ---
>> pym/portage/dbapi/vartree.py | 10 ++
>> 1 file changed, 6 insertions(+), 4 deletions(-)
>>
>> diff --git a/pym/portage/dbapi/vartree.py
>b/pym/portage/dbapi/vartree.py
>> index 8e5ac43..d02d850 100644
>> --- a/pym/portage/dbapi/vartree.py
>> +++ b/pym/portage/dbapi/vartree.py
>> @@ -3690,19 +3690,21 @@ class dblink(object):
>> def _is_install_masked(self, relative_path):
>> ret = False
>> for pattern in self.settings.install_mask:
>> +# if pattern starts with -, possibly exclude this path
>> +pat_res = not pattern.startswith('-')
>> +if not pat_res:
>> +pattern = pattern[1:]
>> # absolute path pattern
>> if pattern.startswith('/'):
>> # match either exact path or one of parent dirs
>> # the latter is done via matching pattern/*
>> if (fnmatch.fnmatch(relative_path, pattern[1:])
>> or
>> fnmatch.fnmatch(relative_path, pattern[1:] + '/*')):
>> -ret = True
>> -break
>> +ret = pat_res
>> # filename
>> else:
>> if
>> fnmatch.fnmatch(os.path.basename(relative_path), pattern):
>> -ret = True
>> -break
>> +ret = pat_res
>> return ret
>>
>> def treewalk(self, srcroot, destroot, inforoot, myebuild,
>cleanup=0,
>>
>
>In order to implement this with pre-compiled patterns, we can use a
>separate regular expression to represent all of the exclusions.
That won't work since exclusive and inclusive patterns can be stacked, and for
this to work they are applied in order.
E.g. /usr/foo -/usr/foo/bar /usr/foo/bar/baz
Should remove all foo subdirectories except for bar, and also baz inside bar.
--
Best regards,
Michał Górny (by phone)
Re: [gentoo-portage-dev] [PATCH 1/3] portage.package.ebuild.config: Move FEATURES=no* handling there
Dnia 12 czerwca 2016 09:28:14 CEST, Zac Medico napisał(a):
>On 06/12/2016 12:19 AM, Zac Medico wrote:
>> On 05/21/2016 11:56 PM, Michał Górny wrote:
>>> diff --git a/pym/portage/package/ebuild/config.py
>b/pym/portage/package/ebuild/config.py
>>> index 45b7d08..fcc7ce5 100644
>>> --- a/pym/portage/package/ebuild/config.py
>>> +++ b/pym/portage/package/ebuild/config.py
>>> @@ -1773,6 +1773,16 @@ class config(object):
>>> # setcpv triggers lazy instantiation of things like
>>> _use_manager.
>>> _eapi_cache.clear()
>>>
>>> + # Prepare the final value of INSTALL_MASK
>>> + install_mask = self["INSTALL_MASK"].split()
>>> + if 'nodoc' in self.features:
>>> + install_mask.append("/usr/share/doc")
>>> + if 'noinfo' in self.features:
>>> + install_mask.append("/usr/share/info")
>>> + if 'noman' in self.features:
>>> + install_mask.append("/usr/share/man")
>>> + self["INSTALL_MASK"] = ' '.join(install_mask)
>>> +
>>> def _grab_pkg_env(self, penv, container, protected_keys=None):
>>> if protected_keys is None:
>>> protected_keys = ()
>>>
>>
>> I'm concerned that these values can be appended more than once,
>causing
>> the variable to contain duplicate values, because setcpv only calls
>> reset when the has_changed variable is True. So, we only want to call
>> this code when has_changed is True. In fact, this code can go
>> immediately after the reset call here:
>>
>> if has_changed:
>> self.reset(keeping_pkg=1)
>>
>
>Actually, it's more tricky than that, because we need to account for
>both global FEATURES settings and package.env FEATURES settings, and my
>above statements do not account for the global settings.
>
>Also need to consider the case where these features are enabled
>globally, and then *disabled* via package.env!
Then maybe we should just ban them and tell people to use INSTALL_MASK directly
instead. Their behavior is quite unclear anyway, considering later possible
exclusions.
--
Best regards,
Michał Górny (by phone)
Re: [gentoo-portage-dev] [PATCH 3/3] portage.dbapi.vartree: Support exclusions in INSTALL_MASK
Dnia 12 czerwca 2016 11:43:35 CEST, Zac Medico napisał(a):
>On 06/12/2016 02:31 AM, Michał Górny wrote:
>> Dnia 12 czerwca 2016 11:20:36 CEST, Zac Medico
>napisał(a):
>>> On 05/21/2016 11:56 PM, Michał Górny wrote:
>>>> Allow INSTALL_MASK patterns to start with '-' to indicate that
>>>> a specific match is to be excluded from being masked. In this case,
>>>> the last matching pattern determines whether the file is actually
>>>> filtered out or kept.
>>>> ---
>>>> pym/portage/dbapi/vartree.py | 10 ++
>>>> 1 file changed, 6 insertions(+), 4 deletions(-)
>>>>
>>>> diff --git a/pym/portage/dbapi/vartree.py
>>> b/pym/portage/dbapi/vartree.py
>>>> index 8e5ac43..d02d850 100644
>>>> --- a/pym/portage/dbapi/vartree.py
>>>> +++ b/pym/portage/dbapi/vartree.py
>>>> @@ -3690,19 +3690,21 @@ class dblink(object):
>>>>def _is_install_masked(self, relative_path):
>>>>ret = False
>>>>for pattern in self.settings.install_mask:
>>>> + # if pattern starts with -, possibly exclude this path
>>>> + pat_res = not pattern.startswith('-')
>>>> + if not pat_res:
>>>> + pattern = pattern[1:]
>>>># absolute path pattern
>>>>if pattern.startswith('/'):
>>>># match either exact path or one of parent dirs
>>>># the latter is done via matching pattern/*
>>>>if (fnmatch.fnmatch(relative_path, pattern[1:])
>>>>or
>>>> fnmatch.fnmatch(relative_path, pattern[1:] + '/*')):
>>>> - ret = True
>>>> - break
>>>> + ret = pat_res
>>>># filename
>>>>else:
>>>>if
>>>> fnmatch.fnmatch(os.path.basename(relative_path), pattern):
>>>> - ret = True
>>>> - break
>>>> + ret = pat_res
>>>>return ret
>>>>
>>>>def treewalk(self, srcroot, destroot, inforoot, myebuild,
>>> cleanup=0,
>>>>
>>>
>>> In order to implement this with pre-compiled patterns, we can use a
>>> separate regular expression to represent all of the exclusions.
>>
>> That won't work since exclusive and inclusive patterns can be
>stacked, and for this to work they are applied in order.
>>
>> E.g. /usr/foo -/usr/foo/bar /usr/foo/bar/baz
>>
>> Should remove all foo subdirectories except for bar, and also baz
>inside bar.
>
>Hmm, I suppose there could be an optimized implementation to handle
>INSTALL_MASK settings where there are no such ordering constraints.
Is this really worth the effort? I'm using the patch for a while and I/O's the
bottleneck, not CPU. If you really insist on optimizing this, i guess we could
precompile all patterns separately.
--
Best regards,
Michał Górny (by phone)
Re: [gentoo-portage-dev] [PATCH v2] Move INSTALL_MASK handling into merging
Dnia 12 czerwca 2016 11:49:26 CEST, Zac Medico napisał(a): >On 06/12/2016 02:28 AM, Michał Górny wrote: >> Dnia 12 czerwca 2016 11:10:55 CEST, Zac Medico >napisał(a): >>> On 05/22/2016 01:21 AM, Michał Górny wrote: >>>> Introduce a new logic for INSTALL_MASK handling in merging code, >>>> replacing the old code that removed matching files and directories >>>> from imagedir in bash. The new code actually ignores matching files >>>> on-the-fly while testing for file collisions and merging files. >>>> The files are still written to CONTENTS, and output using "###" >zing >>>> to indicate being masked, yet are not actually merged to the >>> filesystem. >>> >>> Since collision-protect relies on existing files in its collision >test, >>> install-masked files are no longer going to trigger collisions. >Then, >>> since the install-masked files are still written to CONTENTS, it's >>> possible for the unmerge of one package to unmerge colliding files >that >>> belong to another package! >>> >>> There are a number of ways to solve this problem. For example, we >could >>> have the unmerge code ignore any files in CONTENTS that match the >>> INSTALL_MASK value that was used at merge time. >> >> Hmm, thinking about this more widely (i.e. actually thinking rather >than mimicking the old behavior), I think it would be better to >actually use the original file set for collision-protect. This will >make it possible to detect collisions that would otherwise be hidden >via INSTALL_MASK. > >Even then, we have to carefully consider how the absence of installed >files affects the collision test. It's safest for the unmerge code to >be >aware of the merge time INSTALL_MASK setting, and not try to unmerge >the >install-masked files. But then it wouldn't unmerge the newly masked files as well. Getting this right will require a lot of effort, and we're less likely to screw something up if we keep it simple. -- Best regards, Michał Górny (by phone)
Re: [gentoo-portage-dev] [PATCH] ebuild: Extend helper-in-global-scope ban to all EAPIs
On Sun, 26 Jun 2016 17:07:09 -0700 Zac Medico wrote: > On 05/20/2016 03:27 PM, Michał Górny wrote: > > On Thu, 5 May 2016 23:10:43 +0200 > > Michał Górny wrote: > > > >> Make helper calls in global scope fatal in all supported EAPIs since > >> this is the behavior required by PMS and all major offenders are fixed > >> already. > >> --- > >> bin/eapi.sh | 4 > >> bin/ebuild.sh | 30 ++ > >> 2 files changed, 6 insertions(+), 28 deletions(-) > > > > Merged now. > > > > Seems we might need to revert this patch: > > https://forums.gentoo.org/viewtopic-t-1047248.html If we revert it, people will stop caring for another year... -- Best regards, Michał Górny <http://dev.gentoo.org/~mgorny/> pgpjRRa6t3u7q.pgp Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH] repoman: Make LIVEVCS.* checks fatal
Make LIVEVCS.* checks fatal to prevent people from committing ebuilds using live eclasses instead of package.masking them afterwards by QA. --- repoman/pym/repoman/qa_data.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/repoman/pym/repoman/qa_data.py b/repoman/pym/repoman/qa_data.py index 48ab389..738368e 100644 --- a/repoman/pym/repoman/qa_data.py +++ b/repoman/pym/repoman/qa_data.py @@ -270,8 +270,6 @@ qawarnings = set(( "repo.eapi.deprecated", "usage.obsolete", "upstream.workaround", - "LIVEVCS.stable", - "LIVEVCS.unmasked", "IUSE.rubydeprecated", )) -- 2.9.0
Re: [gentoo-portage-dev] [PATCH] repoman: Make LIVEVCS.* checks fatal
On Tue, 28 Jun 2016 12:40:07 -0700 Zac Medico wrote: > On 06/28/2016 11:30 AM, Michał Górny wrote: > > Make LIVEVCS.* checks fatal to prevent people from committing ebuilds > > using live eclasses instead of package.masking them afterwards by QA. > > --- > > repoman/pym/repoman/qa_data.py | 2 -- > > 1 file changed, 2 deletions(-) > > > > diff --git a/repoman/pym/repoman/qa_data.py b/repoman/pym/repoman/qa_data.py > > index 48ab389..738368e 100644 > > --- a/repoman/pym/repoman/qa_data.py > > +++ b/repoman/pym/repoman/qa_data.py > > @@ -270,8 +270,6 @@ qawarnings = set(( > > "repo.eapi.deprecated", > > "usage.obsolete", > > "upstream.workaround", > > - "LIVEVCS.stable", > > - "LIVEVCS.unmasked", > > "IUSE.rubydeprecated", > > )) > > > > > > Looks good. Signed and pushed! -- Best regards, Michał Górny <http://dev.gentoo.org/~mgorny/> pgpr98ZWnpyED.pgp Description: OpenPGP digital signature
Re: [gentoo-portage-dev] Leadership election
On Tue, 11 Oct 2016 01:59:38 -0700 Zac Medico wrote: > On 10/11/2016 12:06 AM, Alexander Berntsen wrote: > > Right, voting. I vote for... Brian? Yeah. Brian! > > > > I vote for Brian too. Brian! :) ++Brian. -- Best regards, Michał Górny <http://dev.gentoo.org/~mgorny/> pgpoSx5652awM.pgp Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH] [sync] Increase the default git sync-depth to 10
Increate the default git sync-depth from 1 to 10, to ensure that gentoo-mirror git clones retain at least a single original signed commit. --- pym/portage/sync/modules/git/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pym/portage/sync/modules/git/__init__.py b/pym/portage/sync/modules/git/__init__.py index d5eb5c6..54070fc 100644 --- a/pym/portage/sync/modules/git/__init__.py +++ b/pym/portage/sync/modules/git/__init__.py @@ -18,7 +18,7 @@ class CheckGitConfig(CheckSyncConfig): def check_depth(self): d = self.repo.sync_depth # default - self.repo.sync_depth = 1 + self.repo.sync_depth = 10 if d is not None: try: -- 2.10.2
[gentoo-portage-dev] [PATCH] [sync] Run `git update-index --refresh` when doing shallow pulls
Run `git update-index --refresh` to force proper index recheck before
running `git reset --merge` on a shallow pull. This fixes syncing on
some filesystem configurations including overlayfs on squashfs.
---
pym/portage/sync/modules/git/git.py | 21 -
1 file changed, 16 insertions(+), 5 deletions(-)
diff --git a/pym/portage/sync/modules/git/git.py
b/pym/portage/sync/modules/git/git.py
index 3734b80..dc94ec9 100644
--- a/pym/portage/sync/modules/git/git.py
+++ b/pym/portage/sync/modules/git/git.py
@@ -89,7 +89,7 @@ class GitSync(NewBase):
else:
# Since the default merge strategy typically fails when
# the depth is not unlimited, use `git fetch` followed
by
- # `git reset --merge`.
+ # `git update-index --refresh`, then `git reset
--merge`.
try:
remote_branch = portage._unicode_decode(
subprocess.check_output([self.bin_command, 'rev-parse',
@@ -116,12 +116,23 @@ class GitSync(NewBase):
**self.spawn_kwargs)
if exitcode == os.EX_OK and self.repo.sync_depth is not None:
- reset_cmd = [self.bin_command, 'reset', '--merge',
remote_branch]
- if quiet:
- reset_cmd.append('--quiet')
- exitcode = subprocess.call(reset_cmd,
+ # update-index --refresh is needed on some filesystems
+ # (e.g. with overlayfs on squashfs)
+ update_index_cmd = [self.bin_command, 'update-index']
+ if quiet: # -q needs to go first
+ update_index_cmd.append('-q')
+ update_index_cmd.append('--refresh')
+
+ exitcode = subprocess.call(update_index_cmd,
cwd=portage._unicode_encode(self.repo.location))
+ if exitcode == os.EX_OK:
+ reset_cmd = [self.bin_command, 'reset',
'--merge', remote_branch]
+ if quiet:
+ reset_cmd.append('--quiet')
+ exitcode = subprocess.call(reset_cmd,
+
cwd=portage._unicode_encode(self.repo.location))
+
if exitcode != os.EX_OK:
msg = "!!! git pull error in %s" % self.repo.location
self.logger(self.xterm_titles, msg)
--
2.10.2
Re: [gentoo-portage-dev] [PATCH] [sync] Increase the default git sync-depth to 10
On Sun, 30 Oct 2016 13:22:51 -0700 Zac Medico wrote: > On Sun, Oct 30, 2016 at 1:19 PM, Zac Medico wrote: > > On 10/30/2016 12:08 PM, Michał Górny wrote: > >> Increate the default git sync-depth from 1 to 10, to ensure that > >> gentoo-mirror git clones retain at least a single original signed > >> commit. > > > > Why wouldn't the last commit always be signed? > > Oh, it could be a merge commit. Couldn't the original signed commit be > much more than 10 commit behind though? No, gentoo-mirror is only doing a single-level merge commit to combine signed changes with cache. I'm using 10 as round safe default that shouldn't take up too much space. -- Best regards, Michał Górny <http://dev.gentoo.org/~mgorny/> pgp91imwmLMoU.pgp Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH] [sync] Run `git update-index --refresh` when doing shallow pulls
On Sun, 30 Oct 2016 13:39:53 -0700 Zac Medico wrote: > On 10/30/2016 12:23 PM, Michał Górny wrote: > > Run `git update-index --refresh` to force proper index recheck before > > running `git reset --merge` on a shallow pull. This fixes syncing on > > some filesystem configurations including overlayfs on squashfs. > > Maybe we should use --really-refresh in case the --assume-unchanged flag > gets toggled on somehow (maybe via .git/config)? I don't know. I think we can revisit it if it really becomes a problem for anyone. -- Best regards, Michał Górny <http://dev.gentoo.org/~mgorny/> pgpVmnmuWjxmS.pgp Description: OpenPGP digital signature
[gentoo-portage-dev] [PATCH] [sync] Increase the default git sync-depth to 10
Increate the default git sync-depth from 1 to 10, to ensure that gentoo-mirror git clones retain at least a single original signed commit. --- man/portage.5| 10 -- pym/portage/sync/modules/git/__init__.py | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/man/portage.5 b/man/portage.5 index 963f49d..c1e4d97 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -965,8 +965,14 @@ Specifies priority of given repository. Specifies CVS repository. .TP .B sync\-depth -Specifies clone depth to use for DVCS repositories. Defaults to 1 (only -the newest commit). If set to 0, the depth is unlimited. +Specifies clone depth to use for DVCS repositories. Defaults to 10 +(commits deep). If set to 0, the depth is unlimited. +.br +The default depth of 10 was chosen as a compromise between space +and bandwidth savings, and maintaining a history of recent commits. +It is especially important for gentoo-mirror repositories where the most +recent commits are automated and unsigned, and it is necessary to +rewind the history to the newest signed commit for OpenPGP verification. .TP .B sync\-git\-clone\-extra\-opts Extra options to give to git when cloning repository (git clone). diff --git a/pym/portage/sync/modules/git/__init__.py b/pym/portage/sync/modules/git/__init__.py index d5eb5c6..54070fc 100644 --- a/pym/portage/sync/modules/git/__init__.py +++ b/pym/portage/sync/modules/git/__init__.py @@ -18,7 +18,7 @@ class CheckGitConfig(CheckSyncConfig): def check_depth(self): d = self.repo.sync_depth # default - self.repo.sync_depth = 1 + self.repo.sync_depth = 10 if d is not None: try: -- 2.10.2
Re: [gentoo-portage-dev] [PATCH] [sync] Run `git update-index --refresh` when doing shallow pulls
On Sun, 30 Oct 2016 13:53:05 -0700 Zac Medico wrote: > On 10/30/2016 01:44 PM, Michał Górny wrote: > > On Sun, 30 Oct 2016 13:39:53 -0700 > > Zac Medico wrote: > > > >> On 10/30/2016 12:23 PM, Michał Górny wrote: > >>> Run `git update-index --refresh` to force proper index recheck before > >>> running `git reset --merge` on a shallow pull. This fixes syncing on > >>> some filesystem configurations including overlayfs on squashfs. > >> > >> Maybe we should use --really-refresh in case the --assume-unchanged flag > >> gets toggled on somehow (maybe via .git/config)? > > > > I don't know. I think we can revisit it if it really becomes a problem > > for anyone. > > OK, sounds good. The patch LGTM. Pushed. -- Best regards, Michał Górny <http://dev.gentoo.org/~mgorny/> pgp_3opTsGQ3Z.pgp Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH] [sync] Increase the default git sync-depth to 10
On Sun, 30 Oct 2016 14:44:26 -0700 Zac Medico wrote: > On 10/30/2016 02:34 PM, Michał Górny wrote: > > Increate the default git sync-depth from 1 to 10, to ensure that > > s/Increate/Increase/ Fixed. > > gentoo-mirror git clones retain at least a single original signed > > commit. > > --- > > man/portage.5| 10 -- > > pym/portage/sync/modules/git/__init__.py | 2 +- > > 2 files changed, 9 insertions(+), 3 deletions(-) > > > > diff --git a/man/portage.5 b/man/portage.5 > > index 963f49d..c1e4d97 100644 > > --- a/man/portage.5 > > +++ b/man/portage.5 > > @@ -965,8 +965,14 @@ Specifies priority of given repository. > > Specifies CVS repository. > > .TP > > .B sync\-depth > > -Specifies clone depth to use for DVCS repositories. Defaults to 1 (only > > -the newest commit). If set to 0, the depth is unlimited. > > +Specifies clone depth to use for DVCS repositories. Defaults to 10 > > +(commits deep). If set to 0, the depth is unlimited. > > +.br > > +The default depth of 10 was chosen as a compromise between space > > +and bandwidth savings, and maintaining a history of recent commits. > > +It is especially important for gentoo-mirror repositories where the most > > +recent commits are automated and unsigned, and it is necessary to > > +rewind the history to the newest signed commit for OpenPGP verification. > > Shouldn't people feel uneasy about the last commit being unverifiable? I > would think that that last commit should be signed with an > infrastructure key. I've even written a blog post [1] about that. Long story short, trusting some random key used by automated process running on remote server with no real security is insane. I've made a script that verifies underlying repo commit instead, and diffs for metadata changes. [1]:https://blogs.gentoo.org/mgorny/2016/04/15/why-automated-gentoo-mirror-commits-are-not-signed-and-how-to-verify-them-2/ -- Best regards, Michał Górny <http://dev.gentoo.org/~mgorny/> pgpgqKE6ZVgvK.pgp Description: OpenPGP digital signature
Re: [gentoo-portage-dev] Re: [PATCH] [sync] Run `git update-index --refresh` when doing shallow pulls
On Mon, 31 Oct 2016 07:16:24 + (UTC)
Martin Vaeth wrote:
> Michał Górny wrote:
> > + if quiet: # -q needs to go first
> > + update_index_cmd.append('-q')
>
> The options -q --unmerged (which are implicitly passed by "git status")
> are not only to suppress verbosity:
> The git-update-index man page has to say for these options:
>
> : the default behavior is to error out. This option makes
> : git update-index continue anyway.
>
> and if I understood the git source code correctly, without
> these options git will indeed break some loops early in the
> "error" case, i.e. it will perhaps not do a full update of
> the index in the "error" case.
It's a 'best effort' attempt at making git not error out
in the following merge. If it doesn't work, the merge will print
the error.
--
Best regards,
Michał Górny
<http://dev.gentoo.org/~mgorny/>
pgpWdkbJQrA03.pgp
Description: OpenPGP digital signature
Re: [gentoo-portage-dev] portage-2.3.2 stable request?
On Fri, 4 Nov 2016 13:19:39 -0700 Zac Medico wrote: > On 11/04/2016 01:14 PM, Brian Dolbec wrote: > > On Thu, 3 Nov 2016 15:55:23 -0700 > > Zac Medico wrote: > > > >> In about a week, portage-2.3.2 will be eligible for a stable request. > >> > >> The only potential problem that I've noticed is the complaint about > >> changes from bug 552814 causing issues for people using git sync with > >> overlay filesystems, but setting sync-depth = 0 gives those users a > >> workaround. There's also bug 597838, about the sync-depth setting > >> being ineffective, but I only know of a couple of people that have > >> been able to reproduce that. > >> > >> So, do we want to do a stable request portage-2.3.2 when the time > >> comes? > > > > I'm not sure. Do we -r1 it adding a patch or two and ask it be stabled? > > > > There are just 4 commits since 2.3.2, and they all look good. Maybe we > should just cut a 2.3.3 release and wait another 30 days (we also need > to stabilize app-crypt/gkeys since it's needed by emerge-webrsync now). Wouldn't it be better to have a really working version of gkeys before it's stabilized? Like one that could be used without having to create custom configuration files and/or run it as root? -- Best regards, Michał Górny <http://dev.gentoo.org/~mgorny/> pgpVR3redBmlG.pgp Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH] sync: call git prune before shallow fetch (bug 599008)
On Sat, 5 Nov 2016 13:43:15 -0700
Zac Medico wrote:
> This is necessary in order to avoid "There are too many unreachable
> loose objects" warnings from automatic git gc calls.
>
> X-Gentoo-Bug: 599008
> X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=599008
> ---
> pym/portage/sync/modules/git/git.py | 6 ++
> 1 file changed, 6 insertions(+)
>
> diff --git a/pym/portage/sync/modules/git/git.py
> b/pym/portage/sync/modules/git/git.py
> index f288733..c90cf88 100644
> --- a/pym/portage/sync/modules/git/git.py
> +++ b/pym/portage/sync/modules/git/git.py
> @@ -101,6 +101,12 @@ class GitSync(NewBase):
> writemsg_level(msg + "\n", level=logging.ERROR,
> noiselevel=-1)
> return (e.returncode, False)
>
> + # For shallow fetch, unreachable objects must be pruned
> + # manually, since otherwise automatic git gc calls will
> + # eventually warn about them (see bug 599008).
> + subprocess.call(['git', 'prune'],
> + cwd=portage._unicode_encode(self.repo.location))
> +
> git_cmd_opts += " --depth %d" % self.repo.sync_depth
> git_cmd = "%s fetch %s%s" % (self.bin_command,
> remote_branch.partition('/')[0], git_cmd_opts)
Does it have a performance impact?
--
Best regards,
Michał Górny
pgpxPy9bIdGpf.pgp
Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH] sync: call git prune before shallow fetch (bug 599008)
On Sat, 5 Nov 2016 15:11:10 -0700
Zac Medico wrote:
> On 11/05/2016 02:50 PM, Michał Górny wrote:
> > On Sat, 5 Nov 2016 13:43:15 -0700
> > Zac Medico wrote:
> >
> >> This is necessary in order to avoid "There are too many unreachable
> >> loose objects" warnings from automatic git gc calls.
> >>
> >> X-Gentoo-Bug: 599008
> >> X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=599008
> >> ---
> >> pym/portage/sync/modules/git/git.py | 6 ++
> >> 1 file changed, 6 insertions(+)
> >>
> >> diff --git a/pym/portage/sync/modules/git/git.py
> >> b/pym/portage/sync/modules/git/git.py
> >> index f288733..c90cf88 100644
> >> --- a/pym/portage/sync/modules/git/git.py
> >> +++ b/pym/portage/sync/modules/git/git.py
> >> @@ -101,6 +101,12 @@ class GitSync(NewBase):
> >>writemsg_level(msg + "\n", level=logging.ERROR,
> >> noiselevel=-1)
> >>return (e.returncode, False)
> >>
> >> + # For shallow fetch, unreachable objects must be pruned
> >> + # manually, since otherwise automatic git gc calls will
> >> + # eventually warn about them (see bug 599008).
> >> + subprocess.call(['git', 'prune'],
> >> + cwd=portage._unicode_encode(self.repo.location))
> >> +
> >>git_cmd_opts += " --depth %d" % self.repo.sync_depth
> >>git_cmd = "%s fetch %s%s" % (self.bin_command,
> >>remote_branch.partition('/')[0], git_cmd_opts)
> >
> > Does it have a performance impact?
>
> Yes, it takes about 20 seconds on my laptop. I suppose we could make
> this an optional thing, so that those people can do it manually if they
> want.
So we have improvement from at most few seconds for normal 'git pull'
to around a minute for shallow pull?
--
Best regards,
Michał Górny
pgpLtvazW4Jsz.pgp
Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH] sync: call git prune before shallow fetch (bug 599008)
On Sat, 5 Nov 2016 15:56:20 -0700
Zac Medico wrote:
> On 11/05/2016 03:22 PM, Michał Górny wrote:
> > On Sat, 5 Nov 2016 15:11:10 -0700
> > Zac Medico wrote:
> >
> >> On 11/05/2016 02:50 PM, Michał Górny wrote:
> >>> On Sat, 5 Nov 2016 13:43:15 -0700
> >>> Zac Medico wrote:
> >>>
> >>>> This is necessary in order to avoid "There are too many unreachable
> >>>> loose objects" warnings from automatic git gc calls.
> >>>>
> >>>> X-Gentoo-Bug: 599008
> >>>> X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=599008
> >>>> ---
> >>>> pym/portage/sync/modules/git/git.py | 6 ++
> >>>> 1 file changed, 6 insertions(+)
> >>>>
> >>>> diff --git a/pym/portage/sync/modules/git/git.py
> >>>> b/pym/portage/sync/modules/git/git.py
> >>>> index f288733..c90cf88 100644
> >>>> --- a/pym/portage/sync/modules/git/git.py
> >>>> +++ b/pym/portage/sync/modules/git/git.py
> >>>> @@ -101,6 +101,12 @@ class GitSync(NewBase):
> >>>> writemsg_level(msg + "\n",
> >>>> level=logging.ERROR, noiselevel=-1)
> >>>> return (e.returncode, False)
> >>>>
> >>>> +# For shallow fetch, unreachable objects must
> >>>> be pruned
> >>>> +# manually, since otherwise automatic git gc
> >>>> calls will
> >>>> +# eventually warn about them (see bug 599008).
> >>>> +subprocess.call(['git', 'prune'],
> >>>> +
> >>>> cwd=portage._unicode_encode(self.repo.location))
> >>>> +
> >>>> git_cmd_opts += " --depth %d" %
> >>>> self.repo.sync_depth
> >>>> git_cmd = "%s fetch %s%s" % (self.bin_command,
> >>>> remote_branch.partition('/')[0],
> >>>> git_cmd_opts)
> >>>
> >>> Does it have a performance impact?
> >>
> >> Yes, it takes about 20 seconds on my laptop. I suppose we could make
> >> this an optional thing, so that those people can do it manually if they
> >> want.
> >
> > So we have improvement from at most few seconds for normal 'git pull'
> > to around a minute for shallow pull?
>
> Well we've got a least 3 resources to consider:
>
> 1) network bandwidth
> 2) disk usage
> 3) sync time
>
> For me, sync time doesn't really matter that much, but I suppose it
> might for some people.
For a common user, network bandwidth is not a problem with git (except
maybe for the huge initial clone). Especially when syncing frequently,
the gain from subsequent --depth=1 is negligible. When syncing rarely,
you probably prefer snapshots anyway.
I doubt this could be of benefit even to dial-up users; that is,
that more time would be saved on fetching than lost on all the ops
needed to make things continue to work. The additional data won't
affect the data plan users much probably either.
Especially that Gentoo is all about fetching distfiles that are huge
compared to the git updates for the repository.
As for the disk usage, again, the difference should be negligible.
The major difference is done on initial fetch. Of course, regularly
pruning the repository will reduce its size. But then, pruning it will
non-shallow fetches would probably achieve a similar effect thanks to
delta compression.
That leaves the sync time. Which is becoming worse than rsync.
--
Best regards,
Michał Górny
<http://dev.gentoo.org/~mgorny/>
pgpxVXUiCYwdc.pgp
Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH v2] parse_metadata_use: prefer first language found (bug 599060)
On Sun, 6 Nov 2016 12:37:22 -0800
Zac Medico wrote:
> Descriptions may exist for multiple languages, so ignore all except
> the first description found for a particular value of restrict, so
> that use.local.desc content is consistent.
>
> X-Gentoo-Bug: 599060
> X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=599060
> ---
> pym/portage/xml/metadata.py | 10 ++
> 1 file changed, 10 insertions(+)
>
> [PATCH v2] fix to handle multiple values of restrict
>
> diff --git a/pym/portage/xml/metadata.py b/pym/portage/xml/metadata.py
> index 4940bfb..1bc1c78 100644
> --- a/pym/portage/xml/metadata.py
> +++ b/pym/portage/xml/metadata.py
> @@ -455,6 +455,16 @@ def parse_metadata_use(xml_tree):
> if pkg_flag is not None:
> flag_restrict = flag.get("restrict")
>
> + # Descriptions may exist for multiple
> languages, so
> + # ignore all except the first description found
> for a
> + # particular value of restrict (see bug 599060).
> + try:
> + uselist[pkg_flag][flag_restrict]
> + except KeyError:
> + pass
> + else:
> + continue
> +
> # emulate the Element.itertext() method from
> python-2.7
> inner_text = []
> stack = []
This really doesn't fix the bug at hand. We should always prefer
English descriptions, either through lang="en" or lang="".
--
Best regards,
Michał Górny
<http://dev.gentoo.org/~mgorny/>
pgplpQgZxt5pz.pgp
Description: OpenPGP digital signature
Re: [gentoo-portage-dev] Vote to revert all changes related to shallow git fetch from bug 552814
On Sun, 6 Nov 2016 11:26:06 -0800 Zac Medico wrote: > Given the performance issues introduced by `git update-index` and `git > prune`, shallow fetch doesn't seem to be a practical default at this time. > > In order to prepare a release that will be eligible for stabilization, I > propose that we revert all of the changes related to bug 552814: > > https://gitweb.gentoo.org/proj/portage.git/commit/?id=84413bb1dd9df322568ce25efc5b7854a43d03c7 > https://gitweb.gentoo.org/proj/portage.git/commit/?id=55aef9bf297ef8cbf29921acb454449d01313818 > https://gitweb.gentoo.org/proj/portage.git/commit/?id=f5d258656de3db54af06fbca9b8da5217d3802f4 > https://gitweb.gentoo.org/proj/portage.git/commit/?id=f77fcd6b0b4ebb49ca62f5767cd5c931127c3dbb > https://gitweb.gentoo.org/proj/portage.git/commit/?id=d075422a8902617833ec945d94beb0bb334d44c9 Please do that. The sooner it is done, the less likely it is that users have too many loose objects to not be able to clean up the mess automatically during periodic 'git gc'. -- Best regards, Michał Górny <http://dev.gentoo.org/~mgorny/> pgpqTwH99fREO.pgp Description: OpenPGP digital signature
Re: [gentoo-portage-dev] [PATCH] Run distcc-pump server throughout src_configure() to src_install()
Dnia 2014-09-07, nie o godzinie 12:16 +0200, Michał Górny pisze: > Start distcc-pump server in all phases throughout src_configure() to > src_install() rather than in src_compile() alone. Since each of those > phases may involve some compilations, we should try to take full > advantage of distcc in all of them. Moreover, this silences distcc > warnings about being unable to connect to the server when compiling. Pushed. -- Best regards, Michał Górny signature.asc Description: This is a digitally signed message part
Re: [gentoo-portage-dev] [PATCH] sys-apps/portage: add native-extensions USE flag (bug 571444)
W dniu 01.02.2017, śro o godzinie 09∶06 -0800, użytkownik Zac Medico
napisał:
> The native-extensions USE flag will enable building of the
> libc bindings. This is not enabled by default because it does
> not support cross compilation.
>
> X-Gentoo-bug: 571444
> X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=571444
> ---
> sys-apps/portage/metadata.xml| 1 +
> sys-apps/portage/portage-.ebuild | 7 ++-
> 2 files changed, 7 insertions(+), 1 deletion(-)
>
> diff --git a/sys-apps/portage/metadata.xml b/sys-
> apps/portage/metadata.xml
> index e032ea5..882d3ba 100644
> --- a/sys-apps/portage/metadata.xml
> +++ b/sys-apps/portage/metadata.xml
> @@ -12,6 +12,7 @@
>
> Build html API documentation with
> epydoc.
> Use inter-process communication between portage
> and running ebuilds.
> +Build native extensions. Cross-
> compilation is not supported.
> Preserve extended attributes (filesystem-
> stored metadata) when installing files. Usually only required for
> hardened systems.
>
>
> diff --git a/sys-apps/portage/portage-.ebuild b/sys-
> apps/portage/portage-.ebuild
> index 6a6f515..981db26 100644
> --- a/sys-apps/portage/portage-.ebuild
> +++ b/sys-apps/portage/portage-.ebuild
> @@ -19,7 +19,7 @@ HOMEPAGE="https://wiki.gentoo.org/wiki/Project:Porta
> ge"
> LICENSE="GPL-2"
> KEYWORDS=""
> SLOT="0"
> -IUSE="build doc epydoc +ipc linguas_ru selinux xattr"
> +IUSE="build doc epydoc +ipc linguas_ru native-extensions selinux
> xattr"
Wouldn't it be better to enable it by default?
>
> DEPEND="!build? ( $(python_gen_impl_dep 'ssl(+)') )
> >=app-arch/tar-1.27
> @@ -85,6 +85,11 @@ pkg_setup() {
> python_prepare_all() {
> distutils-r1_python_prepare_all
>
> + if use native-extensions; then
> + printf "[build_ext]\nportage-ext-modules=true" >> \
> + setup.cfg || die
> + fi
> +
> if ! use ipc ; then
> einfo "Disabling ipc..."
> sed -e "s:_enable_ipc_daemon =
> True:_enable_ipc_daemon = False:" \
--
Best regards,
Michał Górny
signature.asc
Description: This is a digitally signed message part
Re: [gentoo-portage-dev] [PATCH] sys-apps/portage: add native-extensions USE flag (bug 571444)
W dniu 01.02.2017, śro o godzinie 20∶46 -0500, użytkownik Michael Orlitzky napisał: > On 02/01/2017 04:03 PM, Michał Górny wrote: > > > SLOT="0" > > > -IUSE="build doc epydoc +ipc linguas_ru selinux xattr" > > > +IUSE="build doc epydoc +ipc linguas_ru native-extensions selinux > > > xattr" > > > > Wouldn't it be better to enable it by default? > > > > Please don't enshrine personal preferences into IUSE defaults unless > they're critical to the package. You're saying nonsense. There is no reason for most of Gentoo users to default to crippled Portage. -- Best regards, Michał Górny signature.asc Description: This is a digitally signed message part
Re: [gentoo-portage-dev] [PATCH 08/18] Add tentative support for EAPI6 eapply_user function
W dniu 09.02.2017, czw o godzinie 14∶39 -0800, użytkownik Zac Medico
napisał:
> On 12/01/2014 01:28 PM, Michał Górny wrote:
> > Add support for the user patch applying function.
> > ---
> > bin/eapi.sh | 4
> > bin/phase-helpers.sh | 22 ++
> > 2 files changed, 26 insertions(+)
> >
> > diff --git a/bin/eapi.sh b/bin/eapi.sh
> > index 8bb..6e78750 100644
> > --- a/bin/eapi.sh
> > +++ b/bin/eapi.sh
> > @@ -76,6 +76,10 @@ ___eapi_has_eapply() {
> > [[ ! ${1-${EAPI}} =~
> > ^(0|1|2|3|4|4-python|4-slot-abi|5|5-hdepend|5-progress)$ ]]
> > }
> >
> > +___eapi_has_eapply_user() {
> > + [[ ! ${1-${EAPI}} =~
> > ^(0|1|2|3|4|4-python|4-slot-abi|5|5-hdepend|5-progress)$ ]]
> > +}
> > +
> > ___eapi_has_master_repositories() {
> > [[ ${1-${EAPI}} =~ ^(5-progress)$ ]]
> > }
> > diff --git a/bin/phase-helpers.sh b/bin/phase-helpers.sh
> > index e9fbbb4..f4b64ee 100644
> > --- a/bin/phase-helpers.sh
> > +++ b/bin/phase-helpers.sh
> > @@ -986,6 +986,28 @@ if ___eapi_has_eapply; then
> > }
> > fi
> >
> > +if ___eapi_has_eapply_user; then
> > + eapply_user() {
> > + local basedir=${PORTAGE_CONFIGROOT%/}/etc/portage/patches
> > +
> > + local d applied
> > + # possibilities:
> > + # 1. ${CATEGORY}/${P}-${PR} (note: -r0 desired to avoid applying
> > + #${P} twice)
> > + # 2. ${CATEGORY}/${P}
> > + # 3. ${CATEGORY}/${PN}
> > + # all of the above may be optionally followed by a slot
> > + for d in
> > "${basedir}"/${CATEGORY}/{${P}-${PR},${P},${PN}}{,:${SLOT%/*}}; do
> > + if [[ -d ${d} ]]; then
> > + eapply "${d}"
> > + applied=1
>
> I think it should break out of the loop here, like epatch_user does.
As the comment above suggests, it was intentional that all directories
are used.
> It doesn't make sense to apply more-specific patches before
> less-specific patches, does it?
Maybe. It would probably be most reasonable to sort them all by
filename, and apply in that order. Also allowing patch with the same
filename to override/skip patch from less specific directory.
> Maybe we can just treat this as a bug fix? Is anyone relying on the
> multiple directory usage?
That sounds like a major behavior change for a 'fix'. I'm using multiple
directories though it's all pretty much a workaround solution, so I
guess it doesn't matter if we keep it stable.
>
> > + fi
> > + done
> > +
> > + [[ -n ${applied} ]] && ewarn "User patches applied."
> > + }
> > +fi
> > +
> > if ___eapi_has_master_repositories; then
> > master_repositories() {
> > local output repository=$1 retval
> >
>
>
--
Best regards,
Michał Górny
signature.asc
Description: This is a digitally signed message part
[gentoo-portage-dev] [PATCH 1/3] repoman: Sort deprecated eclasses
--- repoman/pym/repoman/modules/scan/ebuild/checks.py | 10 +- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/repoman/pym/repoman/modules/scan/ebuild/checks.py b/repoman/pym/repoman/modules/scan/ebuild/checks.py index 15e225156..286021524 100644 --- a/repoman/pym/repoman/modules/scan/ebuild/checks.py +++ b/repoman/pym/repoman/modules/scan/ebuild/checks.py @@ -416,19 +416,19 @@ class InheritDeprecated(LineCheck): "base": False, "bash-completion": "bash-completion-r1", "boost-utils": False, + "clutter": "gnome2", "distutils": "distutils-r1", "games": False, "gems": "ruby-fakegem", - "mono": "mono-env", - "python": "python-r1 / python-single-r1 / python-any-r1", - "ruby": "ruby-ng", - "x-modular": "xorg-2", "gst-plugins-bad": "gstreamer", "gst-plugins-base": "gstreamer", "gst-plugins-good": "gstreamer", "gst-plugins-ugly": "gstreamer", "gst-plugins10": "gstreamer", - "clutter": "gnome2", + "mono": "mono-env", + "python": "python-r1 / python-single-r1 / python-any-r1", + "ruby": "ruby-ng", + "x-modular": "xorg-2", } _inherit_re = re.compile(r'^\s*inherit\s(.*)$') -- 2.11.1
[gentoo-portage-dev] [PATCH 3/3] repoman: Deprecate confutils.eclass
--- repoman/pym/repoman/modules/scan/ebuild/checks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/repoman/pym/repoman/modules/scan/ebuild/checks.py b/repoman/pym/repoman/modules/scan/ebuild/checks.py index d091a92b7..06afad5f7 100644 --- a/repoman/pym/repoman/modules/scan/ebuild/checks.py +++ b/repoman/pym/repoman/modules/scan/ebuild/checks.py @@ -417,6 +417,7 @@ class InheritDeprecated(LineCheck): "bash-completion": "bash-completion-r1", "boost-utils": False, "clutter": "gnome2", + "confutils": False, "distutils": "distutils-r1", "games": False, "gems": "ruby-fakegem", -- 2.11.1
[gentoo-portage-dev] [PATCH 2/3] repoman: Deprecate gpe.eclass
--- repoman/pym/repoman/modules/scan/ebuild/checks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/repoman/pym/repoman/modules/scan/ebuild/checks.py b/repoman/pym/repoman/modules/scan/ebuild/checks.py index 286021524..d091a92b7 100644 --- a/repoman/pym/repoman/modules/scan/ebuild/checks.py +++ b/repoman/pym/repoman/modules/scan/ebuild/checks.py @@ -420,6 +420,7 @@ class InheritDeprecated(LineCheck): "distutils": "distutils-r1", "games": False, "gems": "ruby-fakegem", + "gpe": False, "gst-plugins-bad": "gstreamer", "gst-plugins-base": "gstreamer", "gst-plugins-good": "gstreamer", -- 2.11.1
Re: [gentoo-portage-dev] [PATCH 3/3] repoman: Deprecate confutils.eclass
W dniu 22.02.2017, śro o godzinie 01∶15 -0800, użytkownik Zac Medico napisał: > This whole series looks good. Pushed, thanks. -- Best regards, Michał Górny signature.asc Description: This is a digitally signed message part
[gentoo-portage-dev] [PATCH 2/3] checksum: Remove redundant internal fallbacks
Remove the internal digest fallbacks since they have no real use
nowadays. The hashlib versions are preferred later in the logic anyway,
and they are available since Python 2.5.
---
pym/portage/checksum.py | 19 ++-
1 file changed, 2 insertions(+), 17 deletions(-)
diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py
index 319252315..8b4d96e30 100644
--- a/pym/portage/checksum.py
+++ b/pym/portage/checksum.py
@@ -18,8 +18,8 @@ import tempfile
# most preferred first. Please keep this in sync with logic below.
#
#
-# MD5: python-fchksum, hashlib, mhash, hashlib/md5
-# SHA1: hashlib, mhash, hashlib/sha1
+# MD5: python-fchksum, hashlib, mhash
+# SHA1: hashlib, mhash
# SHA256: hashlib, pycrypto, mhash
# SHA512: hashlib, mhash
# RMD160: hashlib, pycrypto, mhash
@@ -77,21 +77,6 @@ class _generate_hash_function(object):
# Define hash functions, try to use the best module available. Later
definitions
# override earlier ones
-# Use the internal modules as last fallback
-try:
- from hashlib import md5 as _new_md5
-except ImportError:
- from md5 import new as _new_md5
-
-md5hash = _generate_hash_function("MD5", _new_md5, origin="internal")
-
-try:
- from hashlib import sha1 as _new_sha1
-except ImportError:
- from sha import new as _new_sha1
-
-sha1hash = _generate_hash_function("SHA1", _new_sha1, origin="internal")
-
# Try to use mhash if available
# mhash causes GIL presently, so it gets less priority than hashlib and
# pycrypto. However, it might be the only accelerated implementation of
--
2.12.0
[gentoo-portage-dev] [PATCH 1/3] checksum: Add summary on top to help following the logic
Add a summary of all supported digest algorithms on top of the file
along with the supported implementations and their order of preference.
This will help people follow the crazy logic below.
---
pym/portage/checksum.py | 15 ++-
1 file changed, 14 insertions(+), 1 deletion(-)
diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py
index cdf467003..319252315 100644
--- a/pym/portage/checksum.py
+++ b/pym/portage/checksum.py
@@ -1,5 +1,5 @@
# checksum.py -- core Portage functionality
-# Copyright 1998-2014 Gentoo Foundation
+# Copyright 1998-2017 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
import portage
@@ -13,6 +13,19 @@ import stat
import subprocess
import tempfile
+
+# Summary of all available hashes and their implementations,
+# most preferred first. Please keep this in sync with logic below.
+#
+#
+# MD5: python-fchksum, hashlib, mhash, hashlib/md5
+# SHA1: hashlib, mhash, hashlib/sha1
+# SHA256: hashlib, pycrypto, mhash
+# SHA512: hashlib, mhash
+# RMD160: hashlib, pycrypto, mhash
+# WHIRLPOOL: hashlib, mhash, bundled
+
+
#dict of all available hash functions
hashfunc_map = {}
hashorigin_map = {}
--
2.12.0
[gentoo-portage-dev] [PATCHES] Little cleanup of checksum + BLAKE2*/SHA3
Hi, Here's a quick set of patches to the checksum module: 1) adds a summary table on top to ease the reading of the horrible code below, 2) removes dead fallbacks for md5/sha1 that apply only to py<2.5, 3) adds BLAKE2b/s and SHA3 hashes available in Python 3.6+. Sadly, I wasn't able to find any other (fallback) Python provider of the latter at the moment, so it will take a while before we can start using it anywhere. -- Best regards, Michał Górny
[gentoo-portage-dev] [PATCH 3/3] checksum: Add blake2* and sha3 hashes from hashlib 3.6+
Add initial support for using the new SHA3_256 and SHA3_512, as well
as competetive BLAKE2b and BLAKE2s hashes that are now provided
in hashlib module of Python 3.6.
---
pym/portage/checksum.py | 14 +-
pym/portage/const.py| 6 ++
2 files changed, 15 insertions(+), 5 deletions(-)
diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py
index 8b4d96e30..a46b820af 100644
--- a/pym/portage/checksum.py
+++ b/pym/portage/checksum.py
@@ -24,6 +24,10 @@ import tempfile
# SHA512: hashlib, mhash
# RMD160: hashlib, pycrypto, mhash
# WHIRLPOOL: hashlib, mhash, bundled
+# BLAKE2B (512): hashlib (3.6+)
+# BLAKE2S (512): hashlib (3.6+)
+# SHA3_256: hashlib (3.6+)
+# SHA3_512: hashlib (3.6+)
#dict of all available hash functions
@@ -121,7 +125,15 @@ try:
sha1hash = _generate_hash_function("SHA1", hashlib.sha1,
origin="hashlib")
sha256hash = _generate_hash_function("SHA256", hashlib.sha256,
origin="hashlib")
sha512hash = _generate_hash_function("SHA512", hashlib.sha512,
origin="hashlib")
- for local_name, hash_name in (("rmd160", "ripemd160"), ("whirlpool",
"whirlpool")):
+ for local_name, hash_name in (
+ ("rmd160", "ripemd160"),
+ ("whirlpool", "whirlpool"),
+ # available since Python 3.6
+ ("BLAKE2B", "blake2b"),
+ ("BLAKE2S", "blake2s"),
+ ("SHA3_256", "sha3_256"),
+ ("SHA3_512", "sha3_512"),
+ ):
try:
hashlib.new(hash_name)
except ValueError:
diff --git a/pym/portage/const.py b/pym/portage/const.py
index 179efce98..0cef2e8ae 100644
--- a/pym/portage/const.py
+++ b/pym/portage/const.py
@@ -224,9 +224,6 @@ MANIFEST1_REQUIRED_HASH = "MD5"
# - Set manifest-hashes in gentoo-x86/metadata/layout.conf as follows:
# manifest-hashes = SHA512 WHIRLPOOL
#
-# After SHA-3 is approved:
-# - Add new hashes to MANIFEST2_HASH_*.
-#
# After SHA-3 is supported in stable portage:
# - Set manifest-hashes in gentoo-x86/metadata/layout.conf as follows:
# manifest-hashes = SHA3 SHA512 WHIRLPOOL
@@ -234,7 +231,8 @@ MANIFEST1_REQUIRED_HASH = "MD5"
# After layout.conf settings correspond to defaults in stable portage:
# - Remove redundant settings from gentoo-x86/metadata/layout.conf.
-MANIFEST2_HASH_FUNCTIONS = ("SHA256", "SHA512", "WHIRLPOOL")
+MANIFEST2_HASH_FUNCTIONS = ("SHA256", "SHA512", "WHIRLPOOL",
+ "BLAKE2B", "BLAKE2S", "SHA3_256", "SHA3_512")
MANIFEST2_HASH_DEFAULTS = frozenset(["SHA256", "SHA512", "WHIRLPOOL"])
MANIFEST2_REQUIRED_HASH = "SHA256"
--
2.12.0
Re: [gentoo-portage-dev] [PATCH 2/3] checksum: Remove redundant internal fallbacks
W dniu 28.02.2017, wto o godzinie 10∶01 +0100, użytkownik Michał Górny
napisał:
> Remove the internal digest fallbacks since they have no real use
> nowadays. The hashlib versions are preferred later in the logic anyway,
> and they are available since Python 2.5.
> ---
> pym/portage/checksum.py | 19 ++-
> 1 file changed, 2 insertions(+), 17 deletions(-)
>
> diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py
> index 319252315..8b4d96e30 100644
> --- a/pym/portage/checksum.py
> +++ b/pym/portage/checksum.py
> @@ -18,8 +18,8 @@ import tempfile
> # most preferred first. Please keep this in sync with logic below.
> #
> #
> -# MD5: python-fchksum, hashlib, mhash, hashlib/md5
> -# SHA1: hashlib, mhash, hashlib/sha1
> +# MD5: python-fchksum, hashlib, mhash
> +# SHA1: hashlib, mhash
> # SHA256: hashlib, pycrypto, mhash
> # SHA512: hashlib, mhash
> # RMD160: hashlib, pycrypto, mhash
> @@ -77,21 +77,6 @@ class _generate_hash_function(object):
> # Define hash functions, try to use the best module available. Later
> definitions
> # override earlier ones
>
> -# Use the internal modules as last fallback
> -try:
> - from hashlib import md5 as _new_md5
> -except ImportError:
> - from md5 import new as _new_md5
Ok, I see it now that there's still some dead code in Portage
referencing the _new_md5 thingy directly. I'll look into fixing that.
> -
> -md5hash = _generate_hash_function("MD5", _new_md5, origin="internal")
> -
> -try:
> - from hashlib import sha1 as _new_sha1
> -except ImportError:
> - from sha import new as _new_sha1
> -
> -sha1hash = _generate_hash_function("SHA1", _new_sha1, origin="internal")
> -
> # Try to use mhash if available
> # mhash causes GIL presently, so it gets less priority than hashlib and
> # pycrypto. However, it might be the only accelerated implementation of
--
Best regards,
Michał Górny
signature.asc
Description: This is a digitally signed message part
[gentoo-portage-dev] [PATCHES] Fix md5 refs in vartree
Hi, Here's a prequel to my other patch set. It cleans up the use of MD5 in vartree. Currently, MD5 is loaded from three different locations -- most of merging code uses portage.checksum high-level functions, one bit uses implementation detail of portage.checksum and one bit imports hashlib directly. I've replaced the use of portage.checksum implementation detail with another use of hashlib, and removed the compatibility for Python < 2.5. I think it's a reasonable temporary measure until someone on friend terms with the code reworks it not to use MD5. -- Best regards, Michał Górny
[gentoo-portage-dev] [PATCH 1/2] vartree: Use hashlib lazy-import, kill py<2.5 compat
--- pym/portage/dbapi/vartree.py | 9 +++-- 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 5053801e6..2a612d162 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -1,4 +1,4 @@ -# Copyright 1998-2014 Gentoo Foundation +# Copyright 1998-2017 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import division, unicode_literals @@ -9,6 +9,7 @@ __all__ = [ import portage portage.proxy.lazyimport.lazyimport(globals(), + 'hashlib:md5', 'portage.checksum:_perform_md5_merge@perform_md5', 'portage.data:portage_gid,portage_uid,secpass', 'portage.dbapi.dep_expand:dep_expand', @@ -1118,11 +1119,7 @@ class vardbapi(dbapi): packages that own it. This is used to optimize owner lookups by narrowing the search down to a smaller number of packages. """ - try: - from hashlib import md5 as _new_hash - except ImportError: - from md5 import new as _new_hash - + _new_hash = md5 _hash_bits = 16 _hex_chars = _hash_bits // 4 -- 2.12.0
[gentoo-portage-dev] [PATCH 2/2] vartree: Replace uses of internal portage.checksum._new_md5 var
Use the public hashlib.md5 method that is required for other code in the module already. --- pym/portage/dbapi/vartree.py | 6 ++ 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 2a612d162..c421dc50b 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -4690,8 +4690,7 @@ class dblink(object): os.unlink(mysrc) os.symlink(myto, mysrc) - mymd5 = portage.checksum._new_md5( - _unicode_encode(myto)).hexdigest() + mymd5 = md5(_unicode_encode(myto)).hexdigest() protected = False if stat.S_ISLNK(mymode) or stat.S_ISREG(mymode): @@ -4725,8 +4724,7 @@ class dblink(object): # of the link target path string is used # for cfgfiledict (symlinks are # protected since bug #485598). - destmd5 = portage.checksum._new_md5( - _unicode_encode(mydest_link)).hexdigest() + destmd5 = md5(_unicode_encode(mydest_link)).hexdigest() elif stat.S_ISREG(mydmode): destmd5 = perform_md5(mydest, -- 2.12.0
[gentoo-portage-dev] [PATCH 1/2] checksum: Fix overriding fallbacks on broken pycrypto
The pycrypto override used the same variables as actual hash functions
before determining whether its functions are useful. As a result, if
pycrypto had a broken module and no hash function was generated,
the possible previous implementation was replaced by None.
---
pym/portage/checksum.py | 12 ++--
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py
index a46b820af..fc38417a7 100644
--- a/pym/portage/checksum.py
+++ b/pym/portage/checksum.py
@@ -105,14 +105,14 @@ except ImportError:
# is broken somehow.
try:
from Crypto.Hash import SHA256, RIPEMD
- sha256hash = getattr(SHA256, 'new', None)
- if sha256hash is not None:
+ sha256hash_ = getattr(SHA256, 'new', None)
+ if sha256hash_ is not None:
sha256hash = _generate_hash_function("SHA256",
- sha256hash, origin="pycrypto")
- rmd160hash = getattr(RIPEMD, 'new', None)
- if rmd160hash is not None:
+ sha256hash_, origin="pycrypto")
+ rmd160hash_ = getattr(RIPEMD, 'new', None)
+ if rmd160hash_ is not None:
rmd160hash = _generate_hash_function("RMD160",
- rmd160hash, origin="pycrypto")
+ rmd160hash_, origin="pycrypto")
except ImportError:
pass
--
2.12.0
[gentoo-portage-dev] [PATCH 2/2] checksum: Add pycryptodome fallbacks for SHA3 and BLAKE2
---
pym/portage/checksum.py | 22 ++
1 file changed, 22 insertions(+)
diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py
index fc38417a7..042a0a745 100644
--- a/pym/portage/checksum.py
+++ b/pym/portage/checksum.py
@@ -116,6 +116,28 @@ try:
except ImportError:
pass
+try:
+ # added in pycryptodome
+ from Crypto.Hash import BLAKE2b, BLAKE2s, SHA3_256, SHA3_512
+ blake2bhash_ = getattr(BLAKE2b, 'new', None)
+ if blake2bhash_ is not None:
+ blake2bhash = _generate_hash_function("BLAKE2B",
+ blake2bhash_, origin="pycrypto")
+ blake2shash_ = getattr(BLAKE2s, 'new', None)
+ if blake2shash_ is not None:
+ blake2shash = _generate_hash_function("BLAKE2S",
+ blake2shash_, origin="pycrypto")
+ sha3_256hash_ = getattr(SHA3_256, 'new', None)
+ if sha3_256hash_ is not None:
+ sha3_256hash = _generate_hash_function("SHA3_256",
+ sha3_256hash_, origin="pycrypto")
+ sha3_512hash_ = getattr(SHA3_512, 'new', None)
+ if sha3_512hash_ is not None:
+ sha3_512hash = _generate_hash_function("SHA3_512",
+ sha3_512hash_, origin="pycrypto")
+except ImportError:
+ pass
+
# Use hashlib from python-2.5 if available and prefer it over pycrypto and
internal fallbacks.
# Need special handling for RMD160/WHIRLPOOL as they may not always be
provided by hashlib.
try:
--
2.12.0
Re: [gentoo-portage-dev] [PATCHES] Fix md5 refs in vartree
W dniu 28.02.2017, wto o godzinie 11∶03 -0800, użytkownik Zac Medico napisał: > On 02/28/2017 02:12 AM, Michał Górny wrote: > > Hi, > > > > Here's a prequel to my other patch set. It cleans up the use of MD5 > > in vartree. > > > > Currently, MD5 is loaded from three different locations -- most of > > merging code uses portage.checksum high-level functions, one bit uses > > implementation detail of portage.checksum and one bit imports hashlib > > directly. > > > > I've replaced the use of portage.checksum implementation detail with > > another use of hashlib, and removed the compatibility for Python < 2.5. > > I think it's a reasonable temporary measure until someone on friend > > terms with the code reworks it not to use MD5. > > > > -- > > Best regards, > > Michał Górny > > > > > > Both series look good to me. Thanks. Merged both yesterday. -- Best regards, Michał Górny signature.asc Description: This is a digitally signed message part
Re: [gentoo-portage-dev] [PATCH 1/2] checksum: Fix overriding fallbacks on broken pycrypto
W dniu 28.02.2017, wto o godzinie 23∶57 -0800, użytkownik Zac Medico
napisał:
> On 02/28/2017 11:34 PM, Michał Górny wrote:
> > The pycrypto override used the same variables as actual hash functions
> > before determining whether its functions are useful. As a result, if
> > pycrypto had a broken module and no hash function was generated,
> > the possible previous implementation was replaced by None.
> > ---
> > pym/portage/checksum.py | 12 ++--
> > 1 file changed, 6 insertions(+), 6 deletions(-)
> >
> > diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py
> > index a46b820af..fc38417a7 100644
> > --- a/pym/portage/checksum.py
> > +++ b/pym/portage/checksum.py
> > @@ -105,14 +105,14 @@ except ImportError:
> > # is broken somehow.
> > try:
> > from Crypto.Hash import SHA256, RIPEMD
> > - sha256hash = getattr(SHA256, 'new', None)
> > - if sha256hash is not None:
> > + sha256hash_ = getattr(SHA256, 'new', None)
> > + if sha256hash_ is not None:
> > sha256hash = _generate_hash_function("SHA256",
> > - sha256hash, origin="pycrypto")
> > - rmd160hash = getattr(RIPEMD, 'new', None)
> > - if rmd160hash is not None:
> > + sha256hash_, origin="pycrypto")
> > + rmd160hash_ = getattr(RIPEMD, 'new', None)
> > + if rmd160hash_ is not None:
> > rmd160hash = _generate_hash_function("RMD160",
> > - rmd160hash, origin="pycrypto")
> > + rmd160hash_, origin="pycrypto")
> > except ImportError:
> > pass
> >
> >
>
> Looks good.
Thanks. Merged both patches.
--
Best regards,
Michał Górny
signature.asc
Description: This is a digitally signed message part
Re: [gentoo-portage-dev] [PATCH] movefile: support in-kernel file copying on Linux (bug 607868)
copying. There is an implicit hole at the
> + * end of the file.
> + */
> +offset_hole = lseek(fd_in, offset_data, SEEK_HOLE);
> +if (offset_hole < 0) {
> +error = 1;
> +break;
> +}
> +
> +/* Revert SEEK_HOLE offset change, since we're going
> + * to copy the data that comes before the hole.
> + */
> +if (lseek(fd_in, offset_out, SEEK_SET) < 0) {
> +error = 1;
> +break;
> +}
> +
> +copyfunc_ret = copyfunc(fd_out,
> +fd_in,
> +&offset_out,
> +offset_hole - offset_data);
> +
> +if (copyfunc_ret < 0) {
> +if ((errno == EXDEV || errno == ENOSYS) &&
> +copyfunc == cfr_wrapper) {
> +/* Use sendfile instead of copy_file_range for
> + * cross-device copies, or when the copy_file_range
> + * syscall is not available (less than Linux 4.5).
> + */
> +copyfunc = sendfile;
> +copyfunc_ret = copyfunc(fd_out,
> +fd_in,
> +&offset_out,
> +offset_hole - offset_data);
> +
> +if (copyfunc_ret < 0) {
I think you still should have a proper fallback for sendfile() refusing
to do its job.
> +error = 1;
> +break;
> +}
> +}
> +else {
> +error = 1;
> +break;
> + }
> +}
> +}
> +#else
> +/* Less than Linux 3.1 does not support SEEK_DATA or copy_file_range,
> + * so just use sendfile for in-kernel copy. This will fail for Linux
> + * versions from 2.6.0 to 2.6.32, because sendfile does not support
> + * writing to regular files. The caller should perform a runtime
> + * test to verify that this function works with the running kernel.
> + */
> +struct stat sb;
> +int stat_acquired;
> +
> +if (!PyArg_ParseTuple(args, "ii", &fd_in, &fd_out))
> +return NULL;
> +
> +eintr_retry = 1;
> +offset_out = 0;
> +stat_acquired = 0;
> +
> +while (eintr_retry) {
> +
> +error = 0;
> +
> +Py_BEGIN_ALLOW_THREADS
> +
> +if (!stat_acquired && fstat(fd_in, &sb) < 0) {
> +error = 1;
> +}
> +else {
> +stat_acquired = 1;
> +while (offset_out < sb.st_size) {
> +copyfunc_ret = sendfile(fd_out,
> +fd_in,
> +&offset_out,
> +sb.st_size - offset_out);
> +
> +if (copyfunc_ret < 0) {
Likewise, especially that old kernels may refuse file-to-file copy here.
> +error = 1;
> +break;
> +}
> +}
> +}
> +#endif
> +Py_END_ALLOW_THREADS
> +
> +if (!(error && errno == EINTR && PyErr_CheckSignals() == 0))
> +eintr_retry = 0;
> +}
> +
> +if (error)
> +return PyErr_SetFromErrno(PyExc_OSError);
> +
> +return Py_BuildValue("i", offset_out);
> +}
--
Best regards,
Michał Górny
signature.asc
Description: This is a digitally signed message part
Re: [gentoo-portage-dev] [PATCH v2] movefile: support in-kernel file copying on Linux (bug 607868)
+if (errno == EINVAL && !offset_out) {
> +lseek_works = 0;
> +}
> +error = 1;
> +break;
> +}
> +
> +copyfunc_ret = copyfunc(fd_out,
> +fd_in,
> +&offset_out,
> +len);
> +
> +if (copyfunc_ret < 0) {
> +if ((errno == EXDEV || errno == ENOSYS) &&
> +copyfunc == cfr_wrapper) {
> +/* Use sendfile instead of copy_file_range for
> + * cross-device copies, or when the copy_file_range
> + * syscall is not available (less than Linux 4.5).
> + */
> +copyfunc = sendfile;
> +copyfunc_ret = copyfunc(fd_out,
> +fd_in,
> +&offset_out,
> +len);
> +
> +if (copyfunc_ret < 0) {
> +/* On Linux, if lseek succeeded above, then
> + * sendfile should have worked here too, so
> + * don't bother to fallback for EINVAL here.
> + */
> +error = 1;
> +break;
> +}
> +}
> +else {
> +error = 1;
> +break;
> +}
> +}
> +}
> +}
> +
> +/* Less than Linux 3.1 does not support SEEK_DATA or copy_file_range,
> + * so just use sendfile for in-kernel copy. This will fail for Linux
> + * versions from 2.6.0 to 2.6.32, because sendfile does not support
> + * writing to regular files.
> + */
> +if (sendfile_works && !lseek_works) {
> +error = 0;
> +
> +if (!stat_in_acquired && fstat(fd_in, &stat_in) < 0) {
> +error = 1;
> +}
> +else {
> +stat_in_acquired = 1;
> +while (offset_out < stat_in.st_size) {
> +copyfunc_ret = sendfile(fd_out,
> +fd_in,
> +&offset_out,
> +stat_in.st_size - offset_out);
> +
> +if (copyfunc_ret < 0) {
> +if (errno == EINVAL && !offset_out) {
> +sendfile_works = 0;
> +}
> +error = 1;
> +break;
> +}
> +}
> +}
> +}
> +
> +/* This implementation will work on any kernel. */
> +if (!sendfile_works) {
> +error = 0;
> +
> +if (!stat_out_acquired && fstat(fd_in, &stat_out) < 0) {
> +error = 1;
> +}
> +else {
> +stat_out_acquired = 1;
> +if (buf == NULL)
> +buf = malloc(stat_out.st_blksize);
> +if (buf == NULL) {
> +error = 1;
> +}
> +else {
> +while (1) {
> +/* Some bytes may still be buffered from the
> + * previous iteration of the outer loop.
> + */
> +if (!buf_bytes) {
> +buf_offset = 0;
> +buf_bytes = read(fd_in, buf,
> stat_out.st_blksize);
> +
> +if (!buf_bytes) {
> +/* EOF */
> +break;
> +}
> +else if (buf_bytes < 0) {
> +error = 1;
> +break;
> +}
> +}
> +
> +copyfunc_ret = write(fd_out, buf + buf_offset,
> buf_bytes);
> +if (copyfunc_ret < 0) {
> +error = 1;
> +break;
> +}
> +
> +buf_bytes -= copyfunc_ret;
> +buf_offset += copyfunc_ret;
> +}
> +}
> +}
> +}
> +
> +Py_END_ALLOW_THREADS
> +
> +if (!(error && errno == EINTR && PyErr_CheckSignals() == 0))
> +eintr_retry = 0;
To be honest, I can't do it but it'd be really a good idea if someone
could analyze all the code paths where the code above could give 'error
= 1' and ensure that:
a. there is no case when we error=1 and leave errno unmodified/unset
(i.e. can accidentally leave earlier EINTR there),
b. we can correctly recover from any EINTR, i.e. if EINTR causes some
function to be interrupted in the middle, our next iteration starts
safely.
> +}
> +
> +free(buf);
if (buf != NULL) ?
> +
> +if (error)
> +return PyErr_SetFromErrno(PyExc_OSError);
> +
> +return Py_BuildValue("i", offset_out);
> +}
--
Best regards,
Michał Górny
signature.asc
Description: This is a digitally signed message part
Re: [gentoo-portage-dev] [PATCH] emerge: auto-enable --with-bdeps if --usepkg is not enabled (bug 598444)
W dniu 04.03.2017, sob o godzinie 01∶34 -0800, użytkownik Zac Medico napisał: > It's useful to enable --with-bdeps by default so that @world updates > will update all packages that are not eligible for removal by > emerge --depclean. However, many users of binary packages do not want > unnecessary build time dependencies installed, therefore do not > auto-enable --with-bdeps for installation actions when the --usepkg > option is enabled. > > X-Gentoo-bug: 598444 > X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=598444 > --- > man/emerge.1 | 18 ++- > pym/_emerge/create_depgraph_params.py| 4 + > pym/_emerge/depgraph.py | 4 +- > pym/portage/tests/resolver/ResolverPlayground.py | 5 + > pym/portage/tests/resolver/test_bdeps.py | 197 > +++ > 5 files changed, 223 insertions(+), 5 deletions(-) > create mode 100644 pym/portage/tests/resolver/test_bdeps.py Just to be clear, this will break stuff for people who have more than one slot of LLVM enabled since different slots build-depend on colliding packages. -- Best regards, Michał Górny signature.asc Description: This is a digitally signed message part
Re: [gentoo-portage-dev] [PATCH] emerge: auto-enable --with-bdeps if --usepkg is not enabled (bug 598444)
W dniu 04.03.2017, sob o godzinie 02∶30 -0800, użytkownik Zac Medico napisał: > On 03/04/2017 01:59 AM, Michał Górny wrote: > > W dniu 04.03.2017, sob o godzinie 01∶34 -0800, użytkownik Zac Medico > > napisał: > > > It's useful to enable --with-bdeps by default so that @world updates > > > will update all packages that are not eligible for removal by > > > emerge --depclean. However, many users of binary packages do not want > > > unnecessary build time dependencies installed, therefore do not > > > auto-enable --with-bdeps for installation actions when the --usepkg > > > option is enabled. > > > > > > X-Gentoo-bug: 598444 > > > X-Gentoo-bug-url: https://bugs.gentoo.org/show_bug.cgi?id=598444 > > > --- > > > man/emerge.1 | 18 ++- > > > pym/_emerge/create_depgraph_params.py| 4 + > > > pym/_emerge/depgraph.py | 4 +- > > > pym/portage/tests/resolver/ResolverPlayground.py | 5 + > > > pym/portage/tests/resolver/test_bdeps.py | 197 > > > +++ > > > 5 files changed, 223 insertions(+), 5 deletions(-) > > > create mode 100644 pym/portage/tests/resolver/test_bdeps.py > > > > Just to be clear, this will break stuff for people who have more than > > one slot of LLVM enabled since different slots build-depend on colliding > > packages. > > Hmm, that's unfortunate, because putting --with-bdeps=n in > EMERGE_DEFAULT_OPTS is not good for --depclean. I'll have to think about > a better way to handle this case. We've already got a bug open for this > sort of situation here: > > https://bugs.gentoo.org/show_bug.cgi?id=427938 Eventually I want to try to accept more lit versions for LLVM. However, it kinda fails when LLVM upgrades gtest and lit is bound to a specific output. -- Best regards, Michał Górny signature.asc Description: This is a digitally signed message part
[gentoo-portage-dev] [PATCHES] Checksums pt. 3 -- one more fallback for SHA3
Hi, Just a quick patchset fixing fallback doc and adding pysha3 support. Next time, I'll probably replace the whole horrible fallback logic there with something cleaner (and lazy). -- Best regards, Michał Górny
[gentoo-portage-dev] [PATCH 2/2] portage.checksum: Support pysha3 fallback for SHA3
pysha3 provides a stand-alone FIPS-compliant SHA3 implementation that
can be used as a fallback for Python < 3.6.
---
pym/portage/checksum.py | 14 --
1 file changed, 12 insertions(+), 2 deletions(-)
diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py
index ac11d3f4b..3e61acdec 100644
--- a/pym/portage/checksum.py
+++ b/pym/portage/checksum.py
@@ -10,6 +10,7 @@ from portage import _encodings
from portage import _unicode_decode, _unicode_encode
import errno
import stat
+import sys
import subprocess
import tempfile
@@ -26,8 +27,8 @@ import tempfile
# WHIRLPOOL: hashlib, mhash, bundled
# BLAKE2B (512): hashlib (3.6+), pycrypto
# BLAKE2S (512): hashlib (3.6+), pycrypto
-# SHA3_256: hashlib (3.6+), pycrypto
-# SHA3_512: hashlib (3.6+), pycrypto
+# SHA3_256: hashlib (3.6+), pysha3, pycrypto
+# SHA3_512: hashlib (3.6+), pysha3, pycrypto
#dict of all available hash functions
@@ -138,6 +139,15 @@ try:
except ImportError:
pass
+# Support using pysha3 as fallback for python<3.6
+try:
+ import sha3
+
+ sha3_256hash = _generate_hash_function("SHA3_256", sha3.sha3_256,
origin="pysha3")
+ sha3_512hash = _generate_hash_function("SHA3_512", sha3.sha3_512,
origin="pysha3")
+except ImportError:
+ pass
+
# Use hashlib from python-2.5 if available and prefer it over pycrypto and
internal fallbacks.
# Need special handling for RMD160/WHIRLPOOL as they may not always be
provided by hashlib.
try:
--
2.12.0
[gentoo-portage-dev] [PATCH 1/2] portage.checksum: Update fallback doc for SHA3/BLAKE2*
--- pym/portage/checksum.py | 8 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py index 042a0a745..ac11d3f4b 100644 --- a/pym/portage/checksum.py +++ b/pym/portage/checksum.py @@ -24,10 +24,10 @@ import tempfile # SHA512: hashlib, mhash # RMD160: hashlib, pycrypto, mhash # WHIRLPOOL: hashlib, mhash, bundled -# BLAKE2B (512): hashlib (3.6+) -# BLAKE2S (512): hashlib (3.6+) -# SHA3_256: hashlib (3.6+) -# SHA3_512: hashlib (3.6+) +# BLAKE2B (512): hashlib (3.6+), pycrypto +# BLAKE2S (512): hashlib (3.6+), pycrypto +# SHA3_256: hashlib (3.6+), pycrypto +# SHA3_512: hashlib (3.6+), pycrypto #dict of all available hash functions -- 2.12.0
[gentoo-portage-dev] [PATCH 04/14] portage.checksum: Support getting byte string checksums
Add a checksum_str() method to Portage hashes and a matching function to make it possible to compute checksums of arbitrary bytestrings rather than just files. --- pym/portage/checksum.py | 29 + 1 file changed, 29 insertions(+) diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py index 67d6a544f..9ba251f29 100644 --- a/pym/portage/checksum.py +++ b/pym/portage/checksum.py @@ -59,6 +59,18 @@ class _generate_hash_function(object): hashfunc_map[hashtype] = self hashorigin_map[hashtype] = origin + def checksum_str(self, data): + """ + Obtain a checksum of a byte-string. + + @param data: Data to hash + @type data: bytes + @return: The hash of the data (hex-digest) + """ + checksum = self._hashobject() + checksum.update(data) + return checksum.hexdigest() + def checksum_file(self, filename): """ Run a checksum against a file. @@ -461,3 +473,20 @@ def perform_multiple_checksums(filename, hashes=["MD5"], calc_prelink=0): raise portage.exception.DigestException(x+" hash function not available (needs dev-python/pycrypto or >=dev-lang/python-2.5)") rVal[x] = perform_checksum(filename, x, calc_prelink)[0] return rVal + + +def checksum_str(data, hashname="MD5"): + """ + Run a specific checksum against a byte string. + + @param filename: Data to checksum + @type filename: Bytes + @param hashname: The type of hash function to run + @type hashname: String + @rtype: String + @return: The hash (hex-digest) of the data + """ + if hashname not in hashfunc_map: + raise portage.exception.DigestException(hashname + \ + " hash function not available (needs dev-python/pycrypto)") + return hashfunc_map[hashname].checksum_str(data) -- 2.12.0
[gentoo-portage-dev] [PATCHES] portage.checksum hacking, pt. 4
Hi, Here's a huge batch of patches for portage.checksum and relevant stuff. It's not a complete rewrite as I have originally planned, and it's still not the code I'd like to see but functionally it has been improved a lot. Notable changes: 1. Other modules no longer use hashfunc_map, now they use the public functions instead. 2. A method to checksum byte strings (i.e. stuff in-memory) has been added. For better compatibility, practically unused python-fchksum support has been removed. [TODO: possibly port other uses of hashlib in Portage?] 3. Tests for all hash functions have been added. This makes it possible to verify that different implementations are working correctly. It is especially important because some of the Keccak (SHA-3) implementations are using pre-FIPS Keccak version. [TODO: make it possible to test all available fallbacks, not only the used function] 4. Fallbacks have been reworked so that the relevant modules are loaded only if a better implementation is not available already. Fallbacks for functions that are guaranteed to be available in hashlib have been removed. 5. Support using pygcrypt as provider of optimized hash function fallbacks (for ripemd160, Whirlpool and SHA3). 6. Support for Streebog (GOST R 34.11-2012) hash algorithm, provided either by pygcrypt or pure Python pygost module. [TODO: find a better pure Python provider] -- Best regards, Michał Górny
[gentoo-portage-dev] [PATCH 02/14] portage.checksum: Remove python-fchksum support
Remove the support for MD5 implementation from python-fchksum package.
The package is rarely installed, supports Python 2.7 only and the code
handles checksumming a whole file only.
---
pym/portage/checksum.py | 11 +--
1 file changed, 1 insertion(+), 10 deletions(-)
diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py
index 3e61acdec..9f88f7e65 100644
--- a/pym/portage/checksum.py
+++ b/pym/portage/checksum.py
@@ -19,7 +19,7 @@ import tempfile
# most preferred first. Please keep this in sync with logic below.
#
#
-# MD5: python-fchksum, hashlib, mhash
+# MD5: hashlib, mhash
# SHA1: hashlib, mhash
# SHA256: hashlib, pycrypto, mhash
# SHA512: hashlib, mhash
@@ -186,15 +186,6 @@ if "WHIRLPOOL" not in hashfunc_map:
from portage.util.whirlpool import new as _new_whirlpool
whirlpoolhash = _generate_hash_function("WHIRLPOOL", _new_whirlpool,
origin="bundled")
-# Use python-fchksum if available, prefer it over all other MD5 implementations
-try:
- from fchksum import fmd5t as md5hash
- hashfunc_map["MD5"] = md5hash
- hashorigin_map["MD5"] = "python-fchksum"
-
-except ImportError:
- pass
-
# There is only one implementation for size
def getsize(filename):
size = os.stat(filename).st_size
--
2.12.0
[gentoo-portage-dev] [PATCH 01/14] Use public API: hashfunc_map -> get_valid_checksum_keys()
Use get_valid_checksum_keys() function instead of accessing hashfunc_map
directly throughout the Portage.
---
pym/portage/_emirrordist/FetchTask.py | 2 +-
pym/portage/dbapi/bintree.py | 4 ++--
pym/portage/eclass_cache.py | 2 +-
pym/portage/manifest.py | 4 ++--
pym/portage/package/ebuild/fetch.py | 10 +-
5 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/pym/portage/_emirrordist/FetchTask.py
b/pym/portage/_emirrordist/FetchTask.py
index 64de67582..203b8c213 100644
--- a/pym/portage/_emirrordist/FetchTask.py
+++ b/pym/portage/_emirrordist/FetchTask.py
@@ -574,7 +574,7 @@ class FetchTask(CompositeTask):
else:
for hash_name in self.digests:
if hash_name != "size" and \
- hash_name in
portage.checksum.hashfunc_map:
+ hash_name in
portage.checksum.get_valid_checksum_keys():
return hash_name
return None
diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index ae9e7caa2..12c3d3e51 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -7,7 +7,7 @@ __all__ = ["bindbapi", "binarytree"]
import portage
portage.proxy.lazyimport.lazyimport(globals(),
- 'portage.checksum:hashfunc_map,perform_multiple_checksums,' + \
+ 'portage.checksum:get_valid_checksum_keys,perform_multiple_checksums,'
+ \
'verify_all,_apply_hash_filter,_hash_filter',
'portage.dbapi.dep_expand:dep_expand',
'portage.dep:dep_getkey,isjustname,isvalidatom,match_from_list',
@@ -1546,7 +1546,7 @@ class binarytree(object):
if metadata is None:
return digests
- for k in hashfunc_map:
+ for k in get_valid_checksum_keys():
v = metadata.get(k)
if not v:
continue
diff --git a/pym/portage/eclass_cache.py b/pym/portage/eclass_cache.py
index 2988d25d6..d2d9e2710 100644
--- a/pym/portage/eclass_cache.py
+++ b/pym/portage/eclass_cache.py
@@ -48,7 +48,7 @@ class hashed_path(object):
# we don't care to allow .mD5 as an alias for .md5
raise AttributeError(attr)
hashname = attr.upper()
- if hashname not in checksum.hashfunc_map:
+ if hashname not in checksum.get_valid_checksum_keys():
raise AttributeError(attr)
val = checksum.perform_checksum(self.location, hashname)[0]
setattr(self, attr, val)
diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py
index 7278e21c1..28a77ba87 100644
--- a/pym/portage/manifest.py
+++ b/pym/portage/manifest.py
@@ -13,7 +13,7 @@ import warnings
import portage
portage.proxy.lazyimport.lazyimport(globals(),
- 'portage.checksum:hashfunc_map,perform_multiple_checksums,' + \
+ 'portage.checksum:get_valid_checksum_keys,perform_multiple_checksums,'
+ \
'verify_all,_apply_hash_filter,_filter_unaccelarated_hashes',
'portage.repository.config:_find_invalid_path_char',
'portage.util:write_atomic,writemsg_level',
@@ -154,7 +154,7 @@ class Manifest(object):
self.hashes.update(hashes.intersection(MANIFEST2_HASH_FUNCTIONS))
self.hashes.difference_update(hashname for hashname in \
- list(self.hashes) if hashname not in hashfunc_map)
+ list(self.hashes) if hashname not in
get_valid_checksum_keys())
self.hashes.add("size")
self.hashes.add(MANIFEST2_REQUIRED_HASH)
for t in MANIFEST2_IDENTIFIERS:
diff --git a/pym/portage/package/ebuild/fetch.py
b/pym/portage/package/ebuild/fetch.py
index 8755b7522..5c4b74446 100644
--- a/pym/portage/package/ebuild/fetch.py
+++ b/pym/portage/package/ebuild/fetch.py
@@ -29,7 +29,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
from portage import OrderedDict, os, selinux, shutil, _encodings, \
_shell_quote, _unicode_encode
-from portage.checksum import (hashfunc_map, perform_md5, verify_all,
+from portage.checksum import (get_valid_checksum_keys, perform_md5, verify_all,
_filter_unaccelarated_hashes, _hash_filter, _apply_hash_filter)
from portage.const import BASH_BINARY, CUSTOM_MIRRORS_FILE, \
GLOBAL_CONFIG_PATH
@@ -551,6 +551,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
distdir_writable = can_fetch and not fetch_to_ro
failed_files = set()
restrict_fetch_msg = False
+ valid_hashes = frozenset(get_valid_checksum_keys())
+ valid_hashes.discard("size")
for myfile in filedict:
"""
@@ -564,11 +566,9 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
orig_digests = my
[gentoo-portage-dev] [PATCH 06/14] portage.checksum: Fix BLAKE2* fallbacks from pycryptodome
Fix BLAKE2* fallback functions to explicitly provide digest length as
required by pycryptodome.
---
pym/portage/checksum.py | 6 --
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py
index 9ba251f29..82e8bec00 100644
--- a/pym/portage/checksum.py
+++ b/pym/portage/checksum.py
@@ -132,14 +132,16 @@ except ImportError:
try:
# added in pycryptodome
from Crypto.Hash import BLAKE2b, BLAKE2s, SHA3_256, SHA3_512
+ import functools
+
blake2bhash_ = getattr(BLAKE2b, 'new', None)
if blake2bhash_ is not None:
blake2bhash = _generate_hash_function("BLAKE2B",
- blake2bhash_, origin="pycrypto")
+ functools.partial(blake2bhash_, digest_bytes=64),
origin="pycrypto")
blake2shash_ = getattr(BLAKE2s, 'new', None)
if blake2shash_ is not None:
blake2shash = _generate_hash_function("BLAKE2S",
- blake2shash_, origin="pycrypto")
+ functools.partial(blake2shash_, digest_bytes=32),
origin="pycrypto")
sha3_256hash_ = getattr(SHA3_256, 'new', None)
if sha3_256hash_ is not None:
sha3_256hash = _generate_hash_function("SHA3_256",
--
2.12.0
[gentoo-portage-dev] [PATCH 03/14] portage.checksum: create explicit checksum_file() method
Make the file checksum generation code use an explicit checksum_file()
method rather than implicit __call__. This should be more readable,
and make it cleanly possible to add more methods.
---
pym/portage/checksum.py | 15 +--
1 file changed, 9 insertions(+), 6 deletions(-)
diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py
index 9f88f7e65..67d6a544f 100644
--- a/pym/portage/checksum.py
+++ b/pym/portage/checksum.py
@@ -59,7 +59,7 @@ class _generate_hash_function(object):
hashfunc_map[hashtype] = self
hashorigin_map[hashtype] = origin
- def __call__(self, filename):
+ def checksum_file(self, filename):
"""
Run a checksum against a file.
@@ -186,11 +186,14 @@ if "WHIRLPOOL" not in hashfunc_map:
from portage.util.whirlpool import new as _new_whirlpool
whirlpoolhash = _generate_hash_function("WHIRLPOOL", _new_whirlpool,
origin="bundled")
+
# There is only one implementation for size
-def getsize(filename):
- size = os.stat(filename).st_size
- return (size, size)
-hashfunc_map["size"] = getsize
+class SizeHash(object):
+ def checksum_file(self, filename):
+ size = os.stat(filename).st_size
+ return (size, size)
+
+hashfunc_map["size"] = SizeHash()
# end actual hash functions
@@ -420,7 +423,7 @@ def perform_checksum(filename, hashname="MD5",
calc_prelink=0):
if hashname not in hashfunc_map:
raise
portage.exception.DigestException(hashname + \
" hash function not available (needs
dev-python/pycrypto)")
- myhash, mysize = hashfunc_map[hashname](myfilename)
+ myhash, mysize =
hashfunc_map[hashname].checksum_file(myfilename)
except (OSError, IOError) as e:
if e.errno in (errno.ENOENT, errno.ESTALE):
raise portage.exception.FileNotFound(myfilename)
--
2.12.0
[gentoo-portage-dev] [PATCH 07/14] portage.checksum: Remove fallbacks for algorithms guaranteed since py2.7
The MD5, SHA1 and SHA2 algorithms are guaranteed to be available in
hashlib for Python 2.7 and newer, making the fallbacks to other
implementations meaningless. Remove them to simplify the code.
---
pym/portage/checksum.py | 18 +-
1 file changed, 5 insertions(+), 13 deletions(-)
diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py
index 82e8bec00..7812791ad 100644
--- a/pym/portage/checksum.py
+++ b/pym/portage/checksum.py
@@ -19,10 +19,10 @@ import tempfile
# most preferred first. Please keep this in sync with logic below.
#
#
-# MD5: hashlib, mhash
-# SHA1: hashlib, mhash
-# SHA256: hashlib, pycrypto, mhash
-# SHA512: hashlib, mhash
+# MD5: hashlib
+# SHA1: hashlib
+# SHA256: hashlib
+# SHA512: hashlib
# RMD160: hashlib, pycrypto, mhash
# WHIRLPOOL: hashlib, mhash, bundled
# BLAKE2B (512): hashlib (3.6+), pycrypto
@@ -100,10 +100,6 @@ class _generate_hash_function(object):
# WHIRLPOOL available.
try:
import mhash, functools
- md5hash = _generate_hash_function("MD5", functools.partial(mhash.MHASH,
mhash.MHASH_MD5), origin="mhash")
- sha1hash = _generate_hash_function("SHA1",
functools.partial(mhash.MHASH, mhash.MHASH_SHA1), origin="mhash")
- sha256hash = _generate_hash_function("SHA256",
functools.partial(mhash.MHASH, mhash.MHASH_SHA256), origin="mhash")
- sha512hash = _generate_hash_function("SHA512",
functools.partial(mhash.MHASH, mhash.MHASH_SHA512), origin="mhash")
for local_name, hash_name in (("rmd160", "ripemd160"), ("whirlpool",
"whirlpool")):
if hasattr(mhash, 'MHASH_%s' % local_name.upper()):
globals()['%shash' % local_name] = \
@@ -117,11 +113,7 @@ except ImportError:
# Check for 'new' attributes, since they can be missing if the module
# is broken somehow.
try:
- from Crypto.Hash import SHA256, RIPEMD
- sha256hash_ = getattr(SHA256, 'new', None)
- if sha256hash_ is not None:
- sha256hash = _generate_hash_function("SHA256",
- sha256hash_, origin="pycrypto")
+ from Crypto.Hash import RIPEMD
rmd160hash_ = getattr(RIPEMD, 'new', None)
if rmd160hash_ is not None:
rmd160hash = _generate_hash_function("RMD160",
--
2.12.0
[gentoo-portage-dev] [PATCH 08/14] portage.checksum: Remove exception handling for missing hashlib
Remove the try-except block for potential ImportError of hashlib.
The hashlib module should be available in all supported Python versions,
and we do not really test or support the case when it is not available.
---
pym/portage/checksum.py | 55 ++---
1 file changed, 25 insertions(+), 30 deletions(-)
diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py
index 7812791ad..68ed44fa9 100644
--- a/pym/portage/checksum.py
+++ b/pym/portage/checksum.py
@@ -9,6 +9,8 @@ from portage import os
from portage import _encodings
from portage import _unicode_decode, _unicode_encode
import errno
+import functools
+import hashlib
import stat
import sys
import subprocess
@@ -99,7 +101,7 @@ class _generate_hash_function(object):
# pycrypto. However, it might be the only accelerated implementation of
# WHIRLPOOL available.
try:
- import mhash, functools
+ import mhash
for local_name, hash_name in (("rmd160", "ripemd160"), ("whirlpool",
"whirlpool")):
if hasattr(mhash, 'MHASH_%s' % local_name.upper()):
globals()['%shash' % local_name] = \
@@ -124,7 +126,6 @@ except ImportError:
try:
# added in pycryptodome
from Crypto.Hash import BLAKE2b, BLAKE2s, SHA3_256, SHA3_512
- import functools
blake2bhash_ = getattr(BLAKE2b, 'new', None)
if blake2bhash_ is not None:
@@ -156,34 +157,28 @@ except ImportError:
# Use hashlib from python-2.5 if available and prefer it over pycrypto and
internal fallbacks.
# Need special handling for RMD160/WHIRLPOOL as they may not always be
provided by hashlib.
-try:
- import hashlib, functools
-
- md5hash = _generate_hash_function("MD5", hashlib.md5, origin="hashlib")
- sha1hash = _generate_hash_function("SHA1", hashlib.sha1,
origin="hashlib")
- sha256hash = _generate_hash_function("SHA256", hashlib.sha256,
origin="hashlib")
- sha512hash = _generate_hash_function("SHA512", hashlib.sha512,
origin="hashlib")
- for local_name, hash_name in (
- ("rmd160", "ripemd160"),
- ("whirlpool", "whirlpool"),
- # available since Python 3.6
- ("BLAKE2B", "blake2b"),
- ("BLAKE2S", "blake2s"),
- ("SHA3_256", "sha3_256"),
- ("SHA3_512", "sha3_512"),
- ):
- try:
- hashlib.new(hash_name)
- except ValueError:
- pass
- else:
- globals()['%shash' % local_name] = \
- _generate_hash_function(local_name.upper(), \
- functools.partial(hashlib.new, hash_name), \
- origin='hashlib')
-
-except ImportError:
- pass
+md5hash = _generate_hash_function("MD5", hashlib.md5, origin="hashlib")
+sha1hash = _generate_hash_function("SHA1", hashlib.sha1, origin="hashlib")
+sha256hash = _generate_hash_function("SHA256", hashlib.sha256,
origin="hashlib")
+sha512hash = _generate_hash_function("SHA512", hashlib.sha512,
origin="hashlib")
+for local_name, hash_name in (
+ ("rmd160", "ripemd160"),
+ ("whirlpool", "whirlpool"),
+ # available since Python 3.6
+ ("BLAKE2B", "blake2b"),
+ ("BLAKE2S", "blake2s"),
+ ("SHA3_256", "sha3_256"),
+ ("SHA3_512", "sha3_512"),
+ ):
+ try:
+ hashlib.new(hash_name)
+ except ValueError:
+ pass
+ else:
+ globals()['%shash' % local_name] = \
+ _generate_hash_function(local_name.upper(), \
+ functools.partial(hashlib.new, hash_name), \
+ origin='hashlib')
_whirlpool_unaccelerated = False
if "WHIRLPOOL" not in hashfunc_map:
--
2.12.0
[gentoo-portage-dev] [PATCH 05/14] tests: Add minimal safety checks for checksums
---
pym/portage/tests/util/test_checksum.py | 88 +
1 file changed, 88 insertions(+)
create mode 100644 pym/portage/tests/util/test_checksum.py
diff --git a/pym/portage/tests/util/test_checksum.py
b/pym/portage/tests/util/test_checksum.py
new file mode 100644
index 0..72b8cef3f
--- /dev/null
+++ b/pym/portage/tests/util/test_checksum.py
@@ -0,0 +1,88 @@
+# Copyright 2011-2017 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from portage.tests import TestCase
+
+from portage.checksum import checksum_str
+from portage.exception import DigestException
+
+class ChecksumTestCase(TestCase):
+ text = b'Some test string used to check if the hash works'
+
+ def test_md5(self):
+ self.assertEqual(checksum_str(b'', 'MD5'),
+ 'd41d8cd98f00b204e9800998ecf8427e')
+ self.assertEqual(checksum_str(self.text, 'MD5'),
+ '094c3bf4732f59b39d577e9726f1e934')
+
+ def test_sha1(self):
+ self.assertEqual(checksum_str(b'', 'SHA1'),
+ 'da39a3ee5e6b4b0d3255bfef95601890afd80709')
+ self.assertEqual(checksum_str(self.text, 'SHA1'),
+ '5c572017d4e4d49e4aa03a2eda12dbb54a1e2e4f')
+
+ def test_sha256(self):
+ self.assertEqual(checksum_str(b'', 'SHA256'),
+
'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855')
+ self.assertEqual(checksum_str(self.text, 'SHA256'),
+
'e3d4a1135181fe156d61455615bb6296198e8ca5b2f20ddeb85cb4cd27f62320')
+
+ def test_sha512(self):
+ self.assertEqual(checksum_str(b'', 'SHA512'),
+
'cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e')
+ self.assertEqual(checksum_str(self.text, 'SHA512'),
+
'c8eaa902d48a2c82c2185a92f1c8bab8115c63c8d7a9966a8e8e81b07abcb9762f4707a6b27075e9d720277ba9fec072a59840d6355dd2ee64681d8f39a50856')
+
+ def test_rmd160(self):
+ try:
+ self.assertEqual(checksum_str(b'', 'RMD160'),
+
'9c1185a5c5e9fc54612808977ee8f548b2258d31')
+ self.assertEqual(checksum_str(self.text, 'RMD160'),
+
'fc453174f63fc011d6f64abd2c45fb6a53c8239b')
+ except DigestException:
+ self.skipTest('RMD160 implementation not available')
+
+ def test_whirlpool(self):
+ try:
+ self.assertEqual(checksum_str(b'', 'WHIRLPOOL'),
+
'19fa61d75522a4669b44e39c1d2e1726c530232130d407f89afee0964997f7a73e83be698b288febcf88e3e03c4f0757ea8964e59b63d93708b138cc42a66eb3')
+ self.assertEqual(checksum_str(self.text, 'WHIRLPOOL'),
+
'8f556a079b87057f19e0880eed6d833e40c916f4b133196f6842281a2517873074d399832470c11ee251696b4844a10197714a069ba3e3415c8a4eced8f91b48')
+ except DigestException:
+ self.skipTest('WHIRLPOOL implementation not available')
+
+ def test_blake2b(self):
+ try:
+ self.assertEqual(checksum_str(b'', 'BLAKE2B'),
+
'786a02f742015903c6c6fd852552d272912f4740e15847618a86e217f71f5419d25e1031afee585313896444934eb04b903a685b1448b755d56f701afe9be2ce')
+ self.assertEqual(checksum_str(self.text, 'BLAKE2B'),
+
'84cb3c88838c7147bc9797c6525f812adcdcb40137f9c075963e3a3ed1fe06aaeeb4d2bb5589bad286864dc1aa834cfc4d66b8d7e4d4a246d91d45ce3a6eee43')
+ except DigestException:
+ self.skipTest('BLAKE2B implementation not available')
+
+ def test_blake2s(self):
+ try:
+ self.assertEqual(checksum_str(b'', 'BLAKE2S'),
+
'69217a3079908094e11121d042354a7c1f55b6482ca1a51e1b250dfd1ed0eef9')
+ self.assertEqual(checksum_str(self.text, 'BLAKE2S'),
+
'823ab2429f27690450efe888b0404d092fe2ee72a9bd63d5342c251b4dbb373d')
+ except DigestException:
+ self.skipTest('BLAKE2S implementation not available')
+
+ def test_sha3_256(self):
+ try:
+ self.assertEqual(checksum_str(b'', 'SHA3_256'),
+
'a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a')
+ self.assertEqual(checksum_str(self.text, 'SHA3_256'),
+
'932fc0498ebb865406f9b6606280939283aa8a148562e39fd095a5d22bdec5c6')
+
[gentoo-portage-dev] [PATCH 09/14] portage.checksum: Stop exposing global hash variables
Stop exposing global variables such as 'md5hash'. Those are not used
anywhere anymore, exposing them makes the code more complex and makes it
easy to accidentally fail to set them properly (this happened already
for SHA3). Instead, rely on them being inserted into hashfunc_map.
---
pym/portage/checksum.py | 42 --
1 file changed, 20 insertions(+), 22 deletions(-)
diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py
index 68ed44fa9..659012cdc 100644
--- a/pym/portage/checksum.py
+++ b/pym/portage/checksum.py
@@ -102,11 +102,10 @@ class _generate_hash_function(object):
# WHIRLPOOL available.
try:
import mhash
- for local_name, hash_name in (("rmd160", "ripemd160"), ("whirlpool",
"whirlpool")):
- if hasattr(mhash, 'MHASH_%s' % local_name.upper()):
- globals()['%shash' % local_name] = \
- _generate_hash_function(local_name.upper(), \
- functools.partial(mhash.MHASH, getattr(mhash,
'MHASH_%s' % hash_name.upper())), \
+ for local_name, hash_name in (("RMD160", "RIPEMD160"), ("WHIRLPOOL",
"WHIRLPOOL")):
+ if hasattr(mhash, 'MHASH_%s' % hash_name):
+ _generate_hash_function(local_name,
+ functools.partial(mhash.MHASH, getattr(mhash,
'MHASH_%s' % hash_name)),
origin='mhash')
except ImportError:
pass
@@ -118,7 +117,7 @@ try:
from Crypto.Hash import RIPEMD
rmd160hash_ = getattr(RIPEMD, 'new', None)
if rmd160hash_ is not None:
- rmd160hash = _generate_hash_function("RMD160",
+ _generate_hash_function("RMD160",
rmd160hash_, origin="pycrypto")
except ImportError:
pass
@@ -129,19 +128,19 @@ try:
blake2bhash_ = getattr(BLAKE2b, 'new', None)
if blake2bhash_ is not None:
- blake2bhash = _generate_hash_function("BLAKE2B",
+ _generate_hash_function("BLAKE2B",
functools.partial(blake2bhash_, digest_bytes=64),
origin="pycrypto")
blake2shash_ = getattr(BLAKE2s, 'new', None)
if blake2shash_ is not None:
- blake2shash = _generate_hash_function("BLAKE2S",
+ _generate_hash_function("BLAKE2S",
functools.partial(blake2shash_, digest_bytes=32),
origin="pycrypto")
sha3_256hash_ = getattr(SHA3_256, 'new', None)
if sha3_256hash_ is not None:
- sha3_256hash = _generate_hash_function("SHA3_256",
+ _generate_hash_function("SHA3_256",
sha3_256hash_, origin="pycrypto")
sha3_512hash_ = getattr(SHA3_512, 'new', None)
if sha3_512hash_ is not None:
- sha3_512hash = _generate_hash_function("SHA3_512",
+ _generate_hash_function("SHA3_512",
sha3_512hash_, origin="pycrypto")
except ImportError:
pass
@@ -150,20 +149,20 @@ except ImportError:
try:
import sha3
- sha3_256hash = _generate_hash_function("SHA3_256", sha3.sha3_256,
origin="pysha3")
- sha3_512hash = _generate_hash_function("SHA3_512", sha3.sha3_512,
origin="pysha3")
+ _generate_hash_function("SHA3_256", sha3.sha3_256, origin="pysha3")
+ _generate_hash_function("SHA3_512", sha3.sha3_512, origin="pysha3")
except ImportError:
pass
# Use hashlib from python-2.5 if available and prefer it over pycrypto and
internal fallbacks.
# Need special handling for RMD160/WHIRLPOOL as they may not always be
provided by hashlib.
-md5hash = _generate_hash_function("MD5", hashlib.md5, origin="hashlib")
-sha1hash = _generate_hash_function("SHA1", hashlib.sha1, origin="hashlib")
-sha256hash = _generate_hash_function("SHA256", hashlib.sha256,
origin="hashlib")
-sha512hash = _generate_hash_function("SHA512", hashlib.sha512,
origin="hashlib")
+_generate_hash_function("MD5", hashlib.md5, origin="hashlib")
+_generate_hash_function("SHA1", hashlib.sha1, origin="hashlib")
+_generate_hash_function("SHA256", hashlib.sha256, origin="hashlib")
+_generate_hash_function("SHA512", hashlib.sha512, origin="hashlib")
for local_name, hash_name in (
- ("rmd160", "ripemd160"),
- ("whirlpool", "whirlpool"),
+ ("RMD160", "ripemd160"),
+ ("WHIRLPOOL", "whirlpool"),
# available since Python 3.6
("BLAKE2B", "blake2b"),
("BLAKE2S", "blake2s"),
@@ -175,9 +174,8 @@ for local_name, hash_name in (
except ValueError:
pass
else:
- globals()['%shash' % local_name] = \
- _generate_hash_function(local_name.upper(), \
- functools.partial(hashlib.new, hash_name), \
+ _generate_hash_function(local_name,
+ functools.partial(hashlib.new
[gentoo-portage-dev] [PATCH 11/14] portage.checksum: Support pygcrypt as optimized fallback
pygcrypt uses libgcrypt which provides support for ripemd160, whirlpool,
SHA3, plus some algorithms not provided by any other library.
---
pym/portage/checksum.py | 38 ++
1 file changed, 38 insertions(+)
diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py
index 2c482f5e7..92b41b133 100644
--- a/pym/portage/checksum.py
+++ b/pym/portage/checksum.py
@@ -135,6 +135,44 @@ if "SHA3_256" not in hashfunc_map or "SHA3_512" not in
hashfunc_map:
pass
+# Support pygcrypt as fallback using optimized routines from libgcrypt
+# (GnuPG).
+gcrypt_algos = frozenset(('RMD160', 'WHIRLPOOL', 'SHA3_256', 'SHA3_512'))
+if gcrypt_algos.difference(hashfunc_map):
+ try:
+ import binascii
+ import pygcrypt.hashcontext
+
+ class GCryptHashWrapper(object):
+ def __init__(self, algo):
+ self._obj =
pygcrypt.hashcontext.HashContext(algo=algo)
+
+ def update(self, data):
+ self._obj.write(data)
+
+ def hexdigest(self):
+ return
binascii.b2a_hex(self._obj.read()).decode()
+
+ name_mapping = {
+ 'RMD160': 'ripemd160',
+ 'WHIRLPOOL': 'whirlpool',
+ 'SHA3_256': 'sha3-256',
+ 'SHA3_512': 'sha3-512',
+ }
+
+ for local_name, gcry_name in name_mapping.items():
+ try:
+ pygcrypt.hashcontext.HashContext(algo=gcry_name)
+ except Exception: # yes, it throws Exception...
+ pass
+ else:
+ _generate_hash_function(local_name,
+
functools.partial(GCryptHashWrapper, gcry_name),
+ origin="pygcrypt")
+ except ImportError:
+ pass
+
+
# Use pycrypto when available, prefer it over the internal fallbacks
# Check for 'new' attributes, since they can be missing if the module
# is broken somehow.
--
2.12.0
[gentoo-portage-dev] [PATCH 12/14] Support STREEBOG{256,512} hash function (from pygcrypt), #597736
---
pym/portage/checksum.py | 5 -
pym/portage/const.py| 3 ++-
pym/portage/tests/util/test_checksum.py | 18 ++
3 files changed, 24 insertions(+), 2 deletions(-)
diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py
index 92b41b133..3ee100c3f 100644
--- a/pym/portage/checksum.py
+++ b/pym/portage/checksum.py
@@ -137,7 +137,8 @@ if "SHA3_256" not in hashfunc_map or "SHA3_512" not in
hashfunc_map:
# Support pygcrypt as fallback using optimized routines from libgcrypt
# (GnuPG).
-gcrypt_algos = frozenset(('RMD160', 'WHIRLPOOL', 'SHA3_256', 'SHA3_512'))
+gcrypt_algos = frozenset(('RMD160', 'WHIRLPOOL', 'SHA3_256', 'SHA3_512',
+ 'STREEBOG256', 'STREEBOG512'))
if gcrypt_algos.difference(hashfunc_map):
try:
import binascii
@@ -158,6 +159,8 @@ if gcrypt_algos.difference(hashfunc_map):
'WHIRLPOOL': 'whirlpool',
'SHA3_256': 'sha3-256',
'SHA3_512': 'sha3-512',
+ 'STREEBOG256': 'stribog256',
+ 'STREEBOG512': 'stribog512',
}
for local_name, gcry_name in name_mapping.items():
diff --git a/pym/portage/const.py b/pym/portage/const.py
index 0cef2e8ae..7e415ba9c 100644
--- a/pym/portage/const.py
+++ b/pym/portage/const.py
@@ -232,7 +232,8 @@ MANIFEST1_REQUIRED_HASH = "MD5"
# - Remove redundant settings from gentoo-x86/metadata/layout.conf.
MANIFEST2_HASH_FUNCTIONS = ("SHA256", "SHA512", "WHIRLPOOL",
- "BLAKE2B", "BLAKE2S", "SHA3_256", "SHA3_512")
+ "BLAKE2B", "BLAKE2S", "SHA3_256", "SHA3_512",
+ "STREEBOG256", "STREEBOG512")
MANIFEST2_HASH_DEFAULTS = frozenset(["SHA256", "SHA512", "WHIRLPOOL"])
MANIFEST2_REQUIRED_HASH = "SHA256"
diff --git a/pym/portage/tests/util/test_checksum.py
b/pym/portage/tests/util/test_checksum.py
index 72b8cef3f..01ac8f9d0 100644
--- a/pym/portage/tests/util/test_checksum.py
+++ b/pym/portage/tests/util/test_checksum.py
@@ -86,3 +86,21 @@ class ChecksumTestCase(TestCase):
'6634c004dc31822fa65c2f1e2e3bbf0cfa35085653cca1ca9ca42f8f3f13c908405e0b665918146181c9fc9a9d793fc05429d669c35a55517820dfaa071425ca')
except DigestException:
self.skipTest('SHA3_512 implementation not available')
+
+ def test_streebog256(self):
+ try:
+ self.assertEqual(checksum_str(b'', 'STREEBOG256'),
+
'3f539a213e97c802cc229d474c6aa32a825a360b2a933a949fd925208d9ce1bb')
+ self.assertEqual(checksum_str(self.text, 'STREEBOG256'),
+
'4992f1239c46f15b89e7b83ded4d83fb5966da3692788a4a1a6d118f78c08444')
+ except DigestException:
+ self.skipTest('STREEBOG256 implementation not
available')
+
+ def test_streebog512(self):
+ try:
+ self.assertEqual(checksum_str(b'', 'STREEBOG512'),
+
'8e945da209aa869f0455928529bcae4679e9873ab707b55315f56ceb98bef0a7362f715528356ee83cda5f2aac4c6ad2ba3a715c1bcd81cb8e9f90bf4c1c1a8a')
+ self.assertEqual(checksum_str(self.text, 'STREEBOG512'),
+
'330f5c26437f4e22c0163c72b12e93b8c27202f0750627355bdee43a0e0b253c90fbf0a27adbe5414019ff01ed84b7b240a1da1cbe10fae3adffc39c2d87a51f')
+ except DigestException:
+ self.skipTest('STREEBOG512 implementation not
available')
--
2.12.0
[gentoo-portage-dev] [PATCH 14/14] travis: Install additional (pure Python) crypto providers if necessary
Install pysha3 to provide the SHA3 routines on Python < 3.6, and install
pygost to provide a Streebog implementation.
---
.travis.yml | 9 -
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/.travis.yml b/.travis.yml
index 7fe3c06aa..196e3520a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,7 +8,14 @@ python:
- pypy
# command to install dependencies
-install: "pip install lxml"
+install:
+- pip install lxml
+# python3.6+ has sha3 built-in, for older versions install pysha3
+# (except for pypy where pysha3 is broken)
+- "[[ ${TRAVIS_PYTHON_VERSION} == 3.[6789] || ${TRAVIS_PYTHON_VERSION} ==
pypy ]] || pip install pysha3"
+# always install pygost for Streebog
+# (except for py3.3 which it does not support)
+- "[[ ${TRAVIS_PYTHON_VERSION} == 3.3 ]] || pip install pygost"
script:
- printf "[build_ext]\nportage-ext-modules=true" >> setup.cfg
--
2.12.0
[gentoo-portage-dev] [PATCH 10/14] portage.checksum: Reorder to avoid loading redundant impls
Reorder the checksum implementations to start with the most preferred
implementation, and try fallbacks only if the most preferred
implementation is not available. Most importantly, this means that
Portage will no longer attempt to load all hash libraries in the system,
especially when it can satisfy its requirements with hashlib.
---
pym/portage/checksum.py | 145 +---
1 file changed, 87 insertions(+), 58 deletions(-)
diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py
index 659012cdc..2c482f5e7 100644
--- a/pym/portage/checksum.py
+++ b/pym/portage/checksum.py
@@ -93,66 +93,11 @@ class _generate_hash_function(object):
return (checksum.hexdigest(), size)
-# Define hash functions, try to use the best module available. Later
definitions
-# override earlier ones
-# Try to use mhash if available
-# mhash causes GIL presently, so it gets less priority than hashlib and
-# pycrypto. However, it might be the only accelerated implementation of
-# WHIRLPOOL available.
-try:
- import mhash
- for local_name, hash_name in (("RMD160", "RIPEMD160"), ("WHIRLPOOL",
"WHIRLPOOL")):
- if hasattr(mhash, 'MHASH_%s' % hash_name):
- _generate_hash_function(local_name,
- functools.partial(mhash.MHASH, getattr(mhash,
'MHASH_%s' % hash_name)),
- origin='mhash')
-except ImportError:
- pass
-
-# Use pycrypto when available, prefer it over the internal fallbacks
-# Check for 'new' attributes, since they can be missing if the module
-# is broken somehow.
-try:
- from Crypto.Hash import RIPEMD
- rmd160hash_ = getattr(RIPEMD, 'new', None)
- if rmd160hash_ is not None:
- _generate_hash_function("RMD160",
- rmd160hash_, origin="pycrypto")
-except ImportError:
- pass
-
-try:
- # added in pycryptodome
- from Crypto.Hash import BLAKE2b, BLAKE2s, SHA3_256, SHA3_512
-
- blake2bhash_ = getattr(BLAKE2b, 'new', None)
- if blake2bhash_ is not None:
- _generate_hash_function("BLAKE2B",
- functools.partial(blake2bhash_, digest_bytes=64),
origin="pycrypto")
- blake2shash_ = getattr(BLAKE2s, 'new', None)
- if blake2shash_ is not None:
- _generate_hash_function("BLAKE2S",
- functools.partial(blake2shash_, digest_bytes=32),
origin="pycrypto")
- sha3_256hash_ = getattr(SHA3_256, 'new', None)
- if sha3_256hash_ is not None:
- _generate_hash_function("SHA3_256",
- sha3_256hash_, origin="pycrypto")
- sha3_512hash_ = getattr(SHA3_512, 'new', None)
- if sha3_512hash_ is not None:
- _generate_hash_function("SHA3_512",
- sha3_512hash_, origin="pycrypto")
-except ImportError:
- pass
-
-# Support using pysha3 as fallback for python<3.6
-try:
- import sha3
+# Define hash functions, try to use the best module available. Preferred
+# modules should go first, latter ones should check if the hashes aren't
+# already defined.
- _generate_hash_function("SHA3_256", sha3.sha3_256, origin="pysha3")
- _generate_hash_function("SHA3_512", sha3.sha3_512, origin="pysha3")
-except ImportError:
- pass
# Use hashlib from python-2.5 if available and prefer it over pycrypto and
internal fallbacks.
# Need special handling for RMD160/WHIRLPOOL as they may not always be
provided by hashlib.
@@ -178,6 +123,89 @@ for local_name, hash_name in (
functools.partial(hashlib.new, hash_name),
origin='hashlib')
+
+# Support using pysha3 as fallback for python<3.6
+if "SHA3_256" not in hashfunc_map or "SHA3_512" not in hashfunc_map:
+ try:
+ import sha3
+
+ _generate_hash_function("SHA3_256", sha3.sha3_256,
origin="pysha3")
+ _generate_hash_function("SHA3_512", sha3.sha3_512,
origin="pysha3")
+ except ImportError:
+ pass
+
+
+# Use pycrypto when available, prefer it over the internal fallbacks
+# Check for 'new' attributes, since they can be missing if the module
+# is broken somehow.
+if 'RMD160' not in hashfunc_map:
+ try:
+ from Crypto.Hash import RIPEMD
+ rmd160hash_ = getattr(RIPEMD, 'new', None)
+ if rmd160hash_ is not None:
+ _generate_hash_function("RMD160",
+ rmd160hash_, origin="pycrypto")
+ except ImportError:
+ pass
+
+# The following hashes were added in pycryptodome (pycrypto fork)
+if 'BLAKE2B' not in hashfunc_map:
+ try:
+ from Crypto.Hash import BLAKE2b
+ blake2bhash_ = getattr(BLAKE2b, 'new', None)
+ if blake2bhash_ is not None:
+ _generate_hash_function("BLAKE2B",
+
[gentoo-portage-dev] [PATCH 13/14] portage.checksum: Support pygost as fallback Streebog provider
Support the pure Python implementation of Streebog in pygost as
a fallback algorithm. The code is horrible (it stores all the data
in memory before hashing), so it is really intended as last fallback.
---
pym/portage/checksum.py | 15 +++
1 file changed, 15 insertions(+)
diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py
index 3ee100c3f..df896533f 100644
--- a/pym/portage/checksum.py
+++ b/pym/portage/checksum.py
@@ -247,6 +247,21 @@ if 'RMD160' not in hashfunc_map or 'WHIRLPOOL' not in
hashfunc_map:
pass
+# Support pygost as fallback streebog provider
+# It's mostly provided as a reference implementation; it's pure Python,
+# slow and reads all data to memory (i.e. doesn't hash on update()...)
+if 'STREEBOG256' not in hashfunc_map or 'STREEBOG512' not in hashfunc_map:
+ try:
+ import pygost.gost34112012
+
+ _generate_hash_function("STREEBOG256",
+ functools.partial(pygost.gost34112012.GOST34112012,
digest_size=32), origin="pygost")
+ _generate_hash_function("STREEBOG512",
+ functools.partial(pygost.gost34112012.GOST34112012,
digest_size=64), origin="pygost")
+ except ImportError:
+ pass
+
+
_whirlpool_unaccelerated = False
if "WHIRLPOOL" not in hashfunc_map:
# Bundled WHIRLPOOL implementation
--
2.12.0
Re: [gentoo-portage-dev] [PATCH 01/14] Use public API: hashfunc_map -> get_valid_checksum_keys()
W dniu 12.03.2017, nie o godzinie 13∶36 -0700, użytkownik Zac Medico napisał: > On Sun, Mar 12, 2017 at 11:59 AM, Michał Górny wrote: > > Use get_valid_checksum_keys() function instead of accessing hashfunc_map > > directly throughout the Portage. > > --- > > pym/portage/_emirrordist/FetchTask.py | 2 +- > > pym/portage/dbapi/bintree.py | 4 ++-- > > pym/portage/eclass_cache.py | 2 +- > > pym/portage/manifest.py | 4 ++-- > > pym/portage/package/ebuild/fetch.py | 10 +- > > 5 files changed, 11 insertions(+), 11 deletions(-) > > > > diff --git a/pym/portage/_emirrordist/FetchTask.py > > b/pym/portage/_emirrordist/FetchTask.py > > index 64de67582..203b8c213 100644 > > --- a/pym/portage/_emirrordist/FetchTask.py > > +++ b/pym/portage/_emirrordist/FetchTask.py > > @@ -574,7 +574,7 @@ class FetchTask(CompositeTask): > > else: > > for hash_name in self.digests: > > if hash_name != "size" and \ > > - hash_name in > > portage.checksum.hashfunc_map: > > + hash_name in > > portage.checksum.get_valid_checksum_keys(): > > The get_valid_checksum_keys function creates a list for each call, so > this is a lot less efficient that dict hash table lookup that it was > before. Does it really matter though? AFAICS this is just a matter of optimizing the function, e.g. putting the keys in a frozenset. -- Best regards, Michał Górny signature.asc Description: This is a digitally signed message part
[gentoo-portage-dev] [PATCH] portage.checksum: Store supported hash types in a frozenset
Copy the list of supported hash types (hashfunc_map dict) into
a frozenset to support efficient access in the public API.
---
pym/portage/checksum.py | 25 ++---
1 file changed, 14 insertions(+), 11 deletions(-)
diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py
index df896533f..ff132751b 100644
--- a/pym/portage/checksum.py
+++ b/pym/portage/checksum.py
@@ -278,6 +278,9 @@ class SizeHash(object):
hashfunc_map["size"] = SizeHash()
+# cache all supported hash methods in a frozenset
+hashfunc_keys = frozenset(hashfunc_map)
+
# end actual hash functions
@@ -312,15 +315,15 @@ def _perform_md5_merge(x, **kwargs):
def perform_all(x, calc_prelink=0):
mydict = {}
- for k in hashfunc_map:
+ for k in hashfunc_keys:
mydict[k] = perform_checksum(x, k, calc_prelink)[0]
return mydict
def get_valid_checksum_keys():
- return list(hashfunc_map)
+ return hashfunc_keys
def get_hash_origin(hashtype):
- if hashtype not in hashfunc_map:
+ if hashtype not in hashfunc_keys:
raise KeyError(hashtype)
return hashorigin_map.get(hashtype, "unknown")
@@ -334,7 +337,7 @@ def _filter_unaccelarated_hashes(digests):
due to minimization of dependencies.
"""
if _whirlpool_unaccelerated and "WHIRLPOOL" in digests:
- verifiable_hash_types = set(digests).intersection(hashfunc_map)
+ verifiable_hash_types = set(digests).intersection(hashfunc_keys)
verifiable_hash_types.discard("size")
if len(verifiable_hash_types) > 1:
digests = dict(digests)
@@ -383,7 +386,7 @@ def _apply_hash_filter(digests, hash_filter):
@type hash_filter: callable
"""
- verifiable_hash_types = set(digests).intersection(hashfunc_map)
+ verifiable_hash_types = set(digests).intersection(hashfunc_keys)
verifiable_hash_types.discard("size")
modified = False
if len(verifiable_hash_types) > 1:
@@ -430,10 +433,10 @@ def verify_all(filename, mydict, calc_prelink=0,
strict=0):
raise portage.exception.FileNotFound(filename)
return False, (str(e), None, None)
- verifiable_hash_types = set(mydict).intersection(hashfunc_map)
+ verifiable_hash_types = set(mydict).intersection(hashfunc_keys)
verifiable_hash_types.discard("size")
if not verifiable_hash_types:
- expected = set(hashfunc_map)
+ expected = set(hashfunc_keys)
expected.discard("size")
expected = list(expected)
expected.sort()
@@ -448,7 +451,7 @@ def verify_all(filename, mydict, calc_prelink=0, strict=0):
for x in sorted(mydict):
if x == "size":
continue
- elif x in hashfunc_map:
+ elif x in hashfunc_keys:
myhash = perform_checksum(filename, x,
calc_prelink=calc_prelink)[0]
if mydict[x] != myhash:
if strict:
@@ -504,7 +507,7 @@ def perform_checksum(filename, hashname="MD5",
calc_prelink=0):
# This happens during uninstallation of prelink.
prelink_capable = False
try:
- if hashname not in hashfunc_map:
+ if hashname not in hashfunc_keys:
raise
portage.exception.DigestException(hashname + \
" hash function not available (needs
dev-python/pycrypto)")
myhash, mysize =
hashfunc_map[hashname].checksum_file(myfilename)
@@ -541,7 +544,7 @@ def perform_multiple_checksums(filename, hashes=["MD5"],
calc_prelink=0):
"""
rVal = {}
for x in hashes:
- if x not in hashfunc_map:
+ if x not in hashfunc_keys:
raise portage.exception.DigestException(x+" hash
function not available (needs dev-python/pycrypto or >=dev-lang/python-2.5)")
rVal[x] = perform_checksum(filename, x, calc_prelink)[0]
return rVal
@@ -558,7 +561,7 @@ def checksum_str(data, hashname="MD5"):
@rtype: String
@return: The hash (hex-digest) of the data
"""
- if hashname not in hashfunc_map:
+ if hashname not in hashfunc_keys:
raise portage.exception.DigestException(hashname + \
" hash function not available (needs
dev-python/pycrypto)")
return hashfunc_map[hashname].checksum_str(data)
--
2.12.0
Re: [gentoo-portage-dev] [PATCHES] portage.checksum hacking, pt. 4
W dniu 12.03.2017, nie o godzinie 21∶31 -0700, użytkownik Zac Medico napisał: > On Sun, Mar 12, 2017 at 11:59 AM, Michał Górny wrote: > > Hi, > > > > Here's a huge batch of patches for portage.checksum and relevant stuff. > > It's not a complete rewrite as I have originally planned, and it's still > > not the code I'd like to see but functionally it has been improved a lot. > > The series looks good to me, except we should change the > get_valid_checksum_keys function to return a frozenset. Thanks a lot, all merged now. -- Best regards, Michał Górny signature.asc Description: This is a digitally signed message part
[gentoo-portage-dev] [PATCH] portage.package.ebuild: Use a fake FILESDIR to catch invalid accesses
Use a model of fake FILESDIR path to ensure that invalid accesses to
FILESDIR will result in failures rather than being silently allowed by
Portage. This mostly involves accesses in the global scope and pkg_*
phases, although the current model does not cover the latter completely
(i.e. does not guarantee that the directory is removed post src_*).
This model aims to follow PMS wording quite precisely. The value of
FILESDIR is meant to be stable throughout the build process, and it is
reliably set to a temporary directory path. However, since the path is
not guaranteed to be present outside src_*, the directory symlink is not
actually created before src_* phases.
== Review notes ==
This is the final version that actually works ;-). I did a full egencache
run with it and the only packages that fail are eblit users (all of them
have bugs open with deadline 1 week from now). There might be a few more
silly failures, so I'll test it for a while, and ask Toralf to possibly
do some tinderboxing.
---
man/ebuild.5 | 6 +++---
pym/_emerge/EbuildPhase.py | 4 +++-
pym/portage/package/ebuild/config.py | 3 ---
pym/portage/package/ebuild/doebuild.py | 6 +-
pym/portage/package/ebuild/prepare_build_dirs.py | 13 +
5 files changed, 24 insertions(+), 8 deletions(-)
diff --git a/man/ebuild.5 b/man/ebuild.5
index 72b8b6905..e4c866cd2 100644
--- a/man/ebuild.5
+++ b/man/ebuild.5
@@ -411,9 +411,9 @@ not be defined. It is autogenerated from the \fBSRC_URI\fR
variable.
.B WORKDIR\fR = \fI"${PORTAGE_TMPDIR}/portage/${CATEGORY}/${PF}/work"
Contains the path to the package build root. Do not modify this variable.
.TP
-.B FILESDIR\fR = \fI"${repository_location}/${CATEGORY}/${PN}/files"
-Contains the path to the 'files' subdirectory in the package specific
-location in given repository. Do not modify this variable.
+.B FILESDIR\fR = \fI"${PORTAGE_TMPDIR}/${CATEGORY}/${PF}/files"
+Contains the path to the directory in which package-specific auxiliary
+files are located. Do not modify this variable.
.TP
.B EBUILD_PHASE
Contains the abreviated name of the phase function that is
diff --git a/pym/_emerge/EbuildPhase.py b/pym/_emerge/EbuildPhase.py
index fc185fcfd..504c812ea 100644
--- a/pym/_emerge/EbuildPhase.py
+++ b/pym/_emerge/EbuildPhase.py
@@ -11,7 +11,8 @@ from _emerge.BinpkgEnvExtractor import BinpkgEnvExtractor
from _emerge.MiscFunctionsProcess import MiscFunctionsProcess
from _emerge.EbuildProcess import EbuildProcess
from _emerge.CompositeTask import CompositeTask
-from portage.package.ebuild.prepare_build_dirs import _prepare_workdir
+from portage.package.ebuild.prepare_build_dirs import (_prepare_workdir,
+ _prepare_fake_filesdir)
from portage.util import writemsg
try:
@@ -229,6 +230,7 @@ class EbuildPhase(CompositeTask):
# ownership since tar can change that too.
os.utime(settings["WORKDIR"], None)
_prepare_workdir(settings)
+ _prepare_fake_filesdir(settings)
elif self.phase == "install":
out = io.StringIO()
_post_src_install_write_metadata(settings)
diff --git a/pym/portage/package/ebuild/config.py
b/pym/portage/package/ebuild/config.py
index ef29afeab..f8043dbf5 100644
--- a/pym/portage/package/ebuild/config.py
+++ b/pym/portage/package/ebuild/config.py
@@ -2784,9 +2784,6 @@ class config(object):
mydict.pop("EPREFIX", None)
mydict.pop("EROOT", None)
- if phase == 'depend':
- mydict.pop('FILESDIR', None)
-
if phase not in ("pretend", "setup", "preinst", "postinst") or \
not eapi_exports_replace_vars(eapi):
mydict.pop("REPLACING_VERSIONS", None)
diff --git a/pym/portage/package/ebuild/doebuild.py
b/pym/portage/package/ebuild/doebuild.py
index 4baae17b1..2f80a28ee 100644
--- a/pym/portage/package/ebuild/doebuild.py
+++ b/pym/portage/package/ebuild/doebuild.py
@@ -31,6 +31,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.package.ebuild.digestcheck:digestcheck',
'portage.package.ebuild.digestgen:digestgen',
'portage.package.ebuild.fetch:fetch',
+ 'portage.package.ebuild.prepare_build_dirs:_prepare_fake_filesdir',
'portage.package.ebuild._ipc.QueryCommand:QueryCommand',
'portage.dep._slot_operator:evaluate_slot_operator_equal_deps',
'portage.package.ebuild._spawn_nofetch:spawn_nofetch',
@@ -164,6 +165,9 @@ def _doebuild_spawn(phase, settings, actionmap=None,
**kwargs):
os.path.basename(EBUILD_SH_BINARY))),
ebuild_sh_arg)
+ if phase == 'unpack':
+ _prepare_fake_filesdir(settings)
+
settings['EBUILD_PHASE'] = phase
try:
return
Re: [gentoo-portage-dev] [PATCH] portage.package.ebuild: Use a fake FILESDIR to catch invalid accesses
On czw, 2017-03-16 at 21:29 +0100, Ulrich Mueller wrote: > > > > > > On Thu, 16 Mar 2017, Michał Górny wrote: > > + mysettings["FILESDIR"] = os.path.join(settings["PORTAGE_BUILDDIR"], > > "files") > > I believe that this contradicts current PMS 11.1, which defines > FILESDIR as follows: "The full path to the package's files directory, > used for small support files or patches. See section 4.3." > > So maybe you should first propose a clarification of the PMS wording? I think that 'full' in this context means 'absolute'. I don't think this really requires us to use the 'original' path. But sure, I'll submit a patch in a few minutes. -- Best regards, Michał Górny signature.asc Description: This is a digitally signed message part
[gentoo-portage-dev] [PATCH v2] portage.package.ebuild: Use a fake FILESDIR to catch invalid accesses
Use a model of fake FILESDIR path to ensure that invalid accesses to
FILESDIR will result in failures rather than being silently allowed by
Portage. This mostly involves accesses in the global scope and pkg_*
phases, although the current model does not cover the latter completely
(i.e. does not guarantee that the directory is removed post src_*).
This model aims to follow PMS wording quite precisely. The value of
FILESDIR is meant to be stable throughout the build process, and it is
reliably set to a temporary directory path. However, since the path is
not guaranteed to be present outside src_*, the directory symlink is not
actually created before src_* phases.
---
man/ebuild.5 | 6 +++---
pym/_emerge/EbuildPhase.py | 6 +-
pym/portage/package/ebuild/config.py | 3 ---
pym/portage/package/ebuild/doebuild.py | 2 +-
pym/portage/package/ebuild/prepare_build_dirs.py | 13 +
5 files changed, 22 insertions(+), 8 deletions(-)
// CHANGED IN v2:
// - fixed FILESDIR to be created before unpack, not after
// - removed dupe logic
diff --git a/man/ebuild.5 b/man/ebuild.5
index 72b8b6905..e4c866cd2 100644
--- a/man/ebuild.5
+++ b/man/ebuild.5
@@ -411,9 +411,9 @@ not be defined. It is autogenerated from the \fBSRC_URI\fR
variable.
.B WORKDIR\fR = \fI"${PORTAGE_TMPDIR}/portage/${CATEGORY}/${PF}/work"
Contains the path to the package build root. Do not modify this variable.
.TP
-.B FILESDIR\fR = \fI"${repository_location}/${CATEGORY}/${PN}/files"
-Contains the path to the 'files' subdirectory in the package specific
-location in given repository. Do not modify this variable.
+.B FILESDIR\fR = \fI"${PORTAGE_TMPDIR}/${CATEGORY}/${PF}/files"
+Contains the path to the directory in which package-specific auxiliary
+files are located. Do not modify this variable.
.TP
.B EBUILD_PHASE
Contains the abreviated name of the phase function that is
diff --git a/pym/_emerge/EbuildPhase.py b/pym/_emerge/EbuildPhase.py
index fc185fcfd..4b5cbf8fa 100644
--- a/pym/_emerge/EbuildPhase.py
+++ b/pym/_emerge/EbuildPhase.py
@@ -11,7 +11,8 @@ from _emerge.BinpkgEnvExtractor import BinpkgEnvExtractor
from _emerge.MiscFunctionsProcess import MiscFunctionsProcess
from _emerge.EbuildProcess import EbuildProcess
from _emerge.CompositeTask import CompositeTask
-from portage.package.ebuild.prepare_build_dirs import _prepare_workdir
+from portage.package.ebuild.prepare_build_dirs import (_prepare_workdir,
+ _prepare_fake_filesdir)
from portage.util import writemsg
try:
@@ -129,6 +130,9 @@ class EbuildPhase(CompositeTask):
# If the environment.bz2 doesn't exist, then ebuild.sh
will
# source the ebuild as a fallback.
+ if self.phase == "unpack":
+ _prepare_fake_filesdir(self.settings)
+
self._start_lock()
def _env_extractor_exit(self, env_extractor):
diff --git a/pym/portage/package/ebuild/config.py
b/pym/portage/package/ebuild/config.py
index ef29afeab..f8043dbf5 100644
--- a/pym/portage/package/ebuild/config.py
+++ b/pym/portage/package/ebuild/config.py
@@ -2784,9 +2784,6 @@ class config(object):
mydict.pop("EPREFIX", None)
mydict.pop("EROOT", None)
- if phase == 'depend':
- mydict.pop('FILESDIR', None)
-
if phase not in ("pretend", "setup", "preinst", "postinst") or \
not eapi_exports_replace_vars(eapi):
mydict.pop("REPLACING_VERSIONS", None)
diff --git a/pym/portage/package/ebuild/doebuild.py
b/pym/portage/package/ebuild/doebuild.py
index 4baae17b1..e7db54bcf 100644
--- a/pym/portage/package/ebuild/doebuild.py
+++ b/pym/portage/package/ebuild/doebuild.py
@@ -337,7 +337,6 @@ def doebuild_environment(myebuild, mydo, myroot=None,
settings=None,
mysettings["EBUILD"] = ebuild_path
mysettings["O"]= pkg_dir
mysettings.configdict["pkg"]["CATEGORY"] = cat
- mysettings["FILESDIR"] = pkg_dir+"/files"
mysettings["PF"] = mypv
if hasattr(mydbapi, 'repositories'):
@@ -390,6 +389,7 @@ def doebuild_environment(myebuild, mydo, myroot=None,
settings=None,
mysettings["WORKDIR"] = os.path.join(mysettings["PORTAGE_BUILDDIR"],
"work")
mysettings["D"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], "image")
+ os.sep
mysettings["T"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], "temp")
+ mysettings["FILESDIR"] = os.path.join(settings["PORTAGE_BUILDDIR"],
"files")
# Prefix forward compatability
eprefix_lstrip = mysettings["EPREFIX"].lstrip(os.sep)
diff --git a/pym/portage/package/ebuild/prepare_build_dirs.py
b/pym/portage/package/ebuild/prepare_build_dirs.py
index 7e5249b66..e3ae318bd 100644
--- a/pym/portage/package/ebuild/prepare_build_dirs.py
+++ b/pym/portage/packag
[gentoo-portage-dev] [PATCHES] Restrict DISTDIR to src_unpack and further
Hi, Here's a not-so-quick batch of patches that adjust DISTDIR shadow dir handling to be alike the new handling of FILESDIR (and needs to be applied on top of that). The idea is that DISTDIR has a consistent value throughout all the phases (as required by the PMS) and references the shadow directory. The directory itself is not created until src_unpack, so mistaken accesses in earlier phases are caught early. The accesses in pkg_pre/post phases are not caught yet. As usual, this patch can be applied after reasonable set of blockers in ::gentoo is resolved. -- Best regards, Michał Górny
[gentoo-portage-dev] [PATCH 4/4] _emerge.Ebuild*: delay creating DISTDIR shadow until src_unpack
---
pym/_emerge/EbuildExecuter.py | 4
pym/_emerge/EbuildPhase.py| 4 +++-
2 files changed, 3 insertions(+), 5 deletions(-)
diff --git a/pym/_emerge/EbuildExecuter.py b/pym/_emerge/EbuildExecuter.py
index 7a454c3e6..81257aa9e 100644
--- a/pym/_emerge/EbuildExecuter.py
+++ b/pym/_emerge/EbuildExecuter.py
@@ -8,7 +8,6 @@ import portage
from portage import os
from portage.eapi import eapi_has_src_prepare_and_src_configure, \
eapi_exports_replace_vars
-from portage.package.ebuild.prepare_build_dirs import _prepare_fake_distdir
class EbuildExecuter(CompositeTask):
@@ -25,9 +24,6 @@ class EbuildExecuter(CompositeTask):
cleanup = 0
portage.prepare_build_dirs(pkg.root, settings, cleanup)
- alist = settings.configdict["pkg"].get("A", "").split()
- _prepare_fake_distdir(settings, alist)
-
if eapi_exports_replace_vars(settings['EAPI']):
vardb = pkg.root_config.trees['vartree'].dbapi
settings["REPLACING_VERSIONS"] = " ".join(
diff --git a/pym/_emerge/EbuildPhase.py b/pym/_emerge/EbuildPhase.py
index 4b5cbf8fa..535b64a9c 100644
--- a/pym/_emerge/EbuildPhase.py
+++ b/pym/_emerge/EbuildPhase.py
@@ -12,7 +12,7 @@ from _emerge.MiscFunctionsProcess import MiscFunctionsProcess
from _emerge.EbuildProcess import EbuildProcess
from _emerge.CompositeTask import CompositeTask
from portage.package.ebuild.prepare_build_dirs import (_prepare_workdir,
- _prepare_fake_filesdir)
+ _prepare_fake_distdir, _prepare_fake_filesdir)
from portage.util import writemsg
try:
@@ -131,6 +131,8 @@ class EbuildPhase(CompositeTask):
# source the ebuild as a fallback.
if self.phase == "unpack":
+ alist = self.settings.configdict["pkg"].get("A",
"").split()
+ _prepare_fake_distdir(self.settings, alist)
_prepare_fake_filesdir(self.settings)
self._start_lock()
--
2.12.0
[gentoo-portage-dev] [PATCH 3/4] portage.package.ebuild.doebuild: Override DISTDIR unconditionally, #612972
Ensure that DISTDIR is always defined to the path to the shadow
directory. This ensures that PMS rules for consistent value are
followed, and that no global scope calls should be able to access
the distfile directory. This also ensures that global-scope assignments
(e.g. in PATCHES) do not work around the shadow directory.
---
pym/portage/package/ebuild/doebuild.py | 9 +
pym/portage/package/ebuild/prepare_build_dirs.py | 6 ++
2 files changed, 7 insertions(+), 8 deletions(-)
diff --git a/pym/portage/package/ebuild/doebuild.py
b/pym/portage/package/ebuild/doebuild.py
index 15e4abb48..8efc08334 100644
--- a/pym/portage/package/ebuild/doebuild.py
+++ b/pym/portage/package/ebuild/doebuild.py
@@ -348,7 +348,8 @@ def doebuild_environment(myebuild, mydo, myroot=None,
settings=None,
mysettings["PORTDIR"] = os.path.realpath(mysettings["PORTDIR"])
mysettings.pop("PORTDIR_OVERLAY", None)
- mysettings["DISTDIR"] = os.path.realpath(mysettings["DISTDIR"])
+ if "PORTAGE_ACTUAL_DISTDIR" not in mysettings:
+ mysettings["PORTAGE_ACTUAL_DISTDIR"] =
os.path.realpath(mysettings["DISTDIR"])
mysettings["RPMDIR"] = os.path.realpath(mysettings["RPMDIR"])
mysettings["ECLASSDIR"] = mysettings["PORTDIR"]+"/eclass"
@@ -390,6 +391,7 @@ def doebuild_environment(myebuild, mydo, myroot=None,
settings=None,
mysettings["WORKDIR"] = os.path.join(mysettings["PORTAGE_BUILDDIR"],
"work")
mysettings["D"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], "image")
+ os.sep
mysettings["T"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], "temp")
+ mysettings["DISTDIR"] = os.path.join(settings["PORTAGE_BUILDDIR"],
"distdir")
mysettings["FILESDIR"] = os.path.join(settings["PORTAGE_BUILDDIR"],
"files")
# Prefix forward compatability
@@ -1207,9 +1209,8 @@ def doebuild(myebuild, mydo, _unused=DeprecationWarning,
settings=None, debug=0,
mysettings.pop("REPLACING_VERSIONS", None)
# Make sure that DISTDIR is restored to it's normal value
before we return!
- if "PORTAGE_ACTUAL_DISTDIR" in mysettings:
- mysettings["DISTDIR"] =
mysettings["PORTAGE_ACTUAL_DISTDIR"]
- del mysettings["PORTAGE_ACTUAL_DISTDIR"]
+ mysettings["DISTDIR"] = mysettings["PORTAGE_ACTUAL_DISTDIR"]
+ del mysettings["PORTAGE_ACTUAL_DISTDIR"]
if logfile and not returnpid:
try:
diff --git a/pym/portage/package/ebuild/prepare_build_dirs.py
b/pym/portage/package/ebuild/prepare_build_dirs.py
index 0fa43ba59..6528d550e 100644
--- a/pym/portage/package/ebuild/prepare_build_dirs.py
+++ b/pym/portage/package/ebuild/prepare_build_dirs.py
@@ -411,10 +411,8 @@ def _prepare_fake_filesdir(settings):
os.symlink(real_filesdir, symlink_path)
def _prepare_fake_distdir(settings, alist):
- orig_distdir = settings["DISTDIR"]
- settings["PORTAGE_ACTUAL_DISTDIR"] = orig_distdir
- edpath = settings["DISTDIR"] = \
- os.path.join(settings["PORTAGE_BUILDDIR"], "distdir")
+ orig_distdir = settings["PORTAGE_ACTUAL_DISTDIR"]
+ edpath = settings["DISTDIR"]
portage.util.ensure_dirs(edpath, gid=portage_gid, mode=0o755)
# Remove any unexpected files or directories.
--
2.12.0
