Re: [gentoo-portage-dev] [PATCH] repos.conf: rename sync-depth option to clone-depth

[Brian Dolbec]

On Fri, 17 Feb 2017 18:53:09 -0800
Zac Medico  wrote:

> Since sync-depth actually controls clone depth, rename it
> to clone-depth, and show a warning message when the sync-depth
> option has been specified:
> 
> UserWarning: repos.conf: sync-depth is deprecated, use clone-depth
> instead
> 
> This makes it feasible to change the meaning of sync-depth in
> the future (it could be used to control git pull depth).
> 
> X-Gentoo-Bug: 552814
> X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=552814
> ---
>  man/portage.5|  7 +--
>  pym/portage/repository/config.py | 16 
>  pym/portage/sync/modules/git/__init__.py | 14 +-
>  pym/portage/sync/modules/git/git.py  |  4 +++-
>  4 files changed, 29 insertions(+), 12 deletions(-)
> 
> diff --git a/man/portage.5 b/man/portage.5
> index 415817a..82e979e 100644
> --- a/man/portage.5
> +++ b/man/portage.5
> @@ -925,6 +925,10 @@ Valid values: yes, no, true, false.
>  If unset, the repo will be treated as set
>  yes, true.
>  .TP
> +.B clone\-depth
> +Specifies clone depth to use for DVCS repositories. Defaults to 1
> (only +the newest commit). If set to 0, the depth is unlimited.
> +.TP
>  .B eclass\-overrides
>  Makes given repository inherit eclasses from specified repositories.
>  .br
> @@ -972,8 +976,7 @@ Valid values: true, false.
>  Specifies CVS repository.
>  .TP
>  .B sync\-depth
> -Specifies clone depth to use for DVCS repositories. Defaults to 1
> (only -the newest commit). If set to 0, the depth is unlimited.
> +This is a deprecated alias for the \fBclone\-depth\fR option.
>  .TP
>  .B sync\-git\-clone\-extra\-opts
>  Extra options to give to git when cloning repository (git clone).
> diff --git a/pym/portage/repository/config.py
> b/pym/portage/repository/config.py index 67c717d..b65ed97 100644
> --- a/pym/portage/repository/config.py
> +++ b/pym/portage/repository/config.py
> @@ -75,7 +75,8 @@ class RepoConfig(object):
>   """Stores config of one repository"""
>  
>   __slots__ = ('aliases', 'allow_missing_manifest',
> 'allow_provide_virtual',
> - 'auto_sync', 'cache_formats', 'create_manifest',
> 'disable_manifest',
> + 'auto_sync', 'cache_formats', 'clone_depth',
> + 'create_manifest', 'disable_manifest',
>   'eapi', 'eclass_db', 'eclass_locations',
> 'eclass_overrides', 'find_invalid_path_char', 'force', 'format',
> 'local_config', 'location', 'main_repo', 'manifest_hashes',
> 'masters', 'missing_repo_name', @@ -168,7 +169,13 @@ class
> RepoConfig(object): auto_sync = auto_sync.strip().lower()
>   self.auto_sync = auto_sync
>  
> + self.clone_depth = repo_opts.get('clone-depth')
>   self.sync_depth = repo_opts.get('sync-depth')
> +
> + if self.sync_depth is not None:
> + warnings.warn(_("repos.conf: sync-depth is
> deprecated,"
> + " use clone-depth instead"))
> +
>   self.sync_hooks_only_on_change = repo_opts.get(
>   'sync-hooks-only-on-change',
> 'false').lower() == 'true' 
> @@ -505,7 +512,8 @@ class RepoConfigLoader(object):
>   if repos_conf_opts is not
> None: # Selectively copy only the attributes which
>   # repos.conf is
> allowed to override.
> - for k in ('aliases',
> 'auto_sync', 'eclass_overrides',
> + for k in ('aliases',
> 'auto_sync',
> + 'clone_depth',
> 'eclass_overrides', 'force', 'masters', 'priority',
> 'strict_misc_digests', 'sync_depth', 'sync_hooks_only_on_change',
>   'sync_type',
> 'sync_umask', 'sync_uri', 'sync_user', @@ -929,8 +937,8 @@ class
> RepoConfigLoader(object): 
>   def config_string(self):
>   bool_keys = ("strict_misc_digests",)
> - str_or_int_keys = ("auto_sync", "format", "location",
> - "main_repo", "priority",
> + str_or_int_keys = ("auto_sync", "clone_depth",
> "format", "location",
> + "main_repo", "priority", "sync_depth",
>   "sync_type", "sync_umask", "sync_uri",
> 'sync_user') str_tuple_keys = ("aliases", "eclass_overrides", "force")
>   repo_config_tuple_keys = ("masters",)
> diff --git a/pym/portage/sync/modules/git/__init__.py
> b/pym/portage/sync/modules/git/__init__.py index d5eb5c6..2df60e3
> 100644 --- a/pym/portage/sync/modules/git/__init__.py
> +++ b/pym/portage/sync/modules/git/__init__.py
> @@ -16,22 +16,26 @@ class CheckGitConfig(CheckSyncConfig):
>   self.checks.append('check_depth')
>  
>   def check_depth(self):
> - d = self.repo.sync_depth
> + for attr in ('clone_depth', 'sync_depth'):
> +

Re: [gentoo-portage-dev] [PATCH] repoman: use regular expression to detect line continuations

[Brian Dolbec]

On Tue, 21 Feb 2017 16:31:56 -0800
Zac Medico  wrote:

> Use a regular expression to detect line continuations, instead
> of the unicode_escape codec, since the unicode_escape codec is
> not really intended to be used this way.
> 
> This solves an issue with python3.6, where a DeprecationWarning
> is triggered by ebuilds containing escape sequences, like this
> warning triggered by a sed expression in the dev-db/sqlite
> ebuilds:
> 
> DeprecationWarning: invalid escape sequence '\['
> ---
>  repoman/pym/repoman/modules/scan/ebuild/checks.py | 28
> +++ 1 file changed, 8 insertions(+), 20
> deletions(-)
> 
> diff --git a/repoman/pym/repoman/modules/scan/ebuild/checks.py
> b/repoman/pym/repoman/modules/scan/ebuild/checks.py index
> 15e2251..d21bf0c 100644 ---
> a/repoman/pym/repoman/modules/scan/ebuild/checks.py +++
> b/repoman/pym/repoman/modules/scan/ebuild/checks.py @@ -8,8 +8,8 @@
> and correctness of an ebuild.""" 
>  from __future__ import unicode_literals
>  
> -import codecs
>  from itertools import chain
> +import operator
>  import re
>  import time
>  
> @@ -923,11 +923,10 @@ def checks_init(experimental_inherit=False):
>  
>  _here_doc_re = re.compile(r'.*<<[-]?(\w+)\s*(>\s*\S+\s*)?$')
>  _ignore_comment_re = re.compile(r'^\s*#')
> +_continuation_re = re.compile(r'(\\)*$')
>  
>  
>  def run_checks(contents, pkg):
> - unicode_escape_codec = codecs.lookup('unicode_escape')
> - unicode_escape = lambda x: unicode_escape_codec.decode(x)[0]
>   if _constant_checks is None:
>   checks_init()
>   checks = _constant_checks
> @@ -957,32 +956,21 @@ def run_checks(contents, pkg):
>   #   cow
>   # This will merge these lines like so:
>   #   inherit foo bar moo cow
> - try:
> - # A normal line will end in the two bytes:
> <\> <\n>.  So decoding
> - # that will result in python thinking the
> <\n> is being escaped
> - # and eat the single <\> which makes it hard
> for us to detect.
> - # Instead, strip the newline (which we know
> all lines have), and
> - # append a <0>.  Then when python escapes
> it, if the line ended
> - # in a <\>, we'll end up with a <\0> marker
> to key off of.  This
> - # shouldn't be a problem with any valid
> ebuild ...
> - line_escaped =
> unicode_escape(line.rstrip('\n') + '0')
> - except SystemExit:
> - raise
> - except:
> - # Who knows what kind of crazy crap an
> ebuild will have
> - # in it -- don't allow it to kill us.
> - line_escaped = line
> + # A line ending with an even number of backslashes
> does not count,
> + # because the last backslash is escaped. Therefore,
> search for an
> + # odd number of backslashes.
> + line_escaped =
> operator.sub(*_continuation_re.search(line).span()) % 2 == 1 if
> multiline: # Chop off the \ and \n bytes from the previous line.
>   multiline = multiline[:-2] + line
> - if not line_escaped.endswith('\0'):
> + if not line_escaped:
>   line = multiline
>   num = multinum
>   multiline = None
>   else:
>   continue
>   else:
> - if line_escaped.endswith('\0'):
> + if line_escaped:
>   multinum = num
>   multiline = line
>   continue

Code seems fine to me, I trust you ;)

-- 
Brian Dolbec 

[gentoo-portage-dev] [PATCH] repoman: use regular expression to detect line continuations

[Zac Medico]

Use a regular expression to detect line continuations, instead
of the unicode_escape codec, since the unicode_escape codec is
not really intended to be used this way.

This solves an issue with python3.6, where a DeprecationWarning
is triggered by ebuilds containing escape sequences, like this
warning triggered by a sed expression in the dev-db/sqlite
ebuilds:

DeprecationWarning: invalid escape sequence '\['
---
 repoman/pym/repoman/modules/scan/ebuild/checks.py | 28 +++
 1 file changed, 8 insertions(+), 20 deletions(-)

diff --git a/repoman/pym/repoman/modules/scan/ebuild/checks.py 
b/repoman/pym/repoman/modules/scan/ebuild/checks.py
index 15e2251..d21bf0c 100644
--- a/repoman/pym/repoman/modules/scan/ebuild/checks.py
+++ b/repoman/pym/repoman/modules/scan/ebuild/checks.py
@@ -8,8 +8,8 @@ and correctness of an ebuild."""
 
 from __future__ import unicode_literals
 
-import codecs
 from itertools import chain
+import operator
 import re
 import time
 
@@ -923,11 +923,10 @@ def checks_init(experimental_inherit=False):
 
 _here_doc_re = re.compile(r'.*<<[-]?(\w+)\s*(>\s*\S+\s*)?$')
 _ignore_comment_re = re.compile(r'^\s*#')
+_continuation_re = re.compile(r'(\\)*$')
 
 
 def run_checks(contents, pkg):
-   unicode_escape_codec = codecs.lookup('unicode_escape')
-   unicode_escape = lambda x: unicode_escape_codec.decode(x)[0]
if _constant_checks is None:
checks_init()
checks = _constant_checks
@@ -957,32 +956,21 @@ def run_checks(contents, pkg):
#   cow
# This will merge these lines like so:
#   inherit foo bar moo cow
-   try:
-   # A normal line will end in the two bytes: <\> <\n>.  
So decoding
-   # that will result in python thinking the <\n> is being 
escaped
-   # and eat the single <\> which makes it hard for us to 
detect.
-   # Instead, strip the newline (which we know all lines 
have), and
-   # append a <0>.  Then when python escapes it, if the 
line ended
-   # in a <\>, we'll end up with a <\0> marker to key off 
of.  This
-   # shouldn't be a problem with any valid ebuild ...
-   line_escaped = unicode_escape(line.rstrip('\n') + '0')
-   except SystemExit:
-   raise
-   except:
-   # Who knows what kind of crazy crap an ebuild will have
-   # in it -- don't allow it to kill us.
-   line_escaped = line
+   # A line ending with an even number of backslashes does not 
count,
+   # because the last backslash is escaped. Therefore, search for 
an
+   # odd number of backslashes.
+   line_escaped = 
operator.sub(*_continuation_re.search(line).span()) % 2 == 1
if multiline:
# Chop off the \ and \n bytes from the previous line.
multiline = multiline[:-2] + line
-   if not line_escaped.endswith('\0'):
+   if not line_escaped:
line = multiline
num = multinum
multiline = None
else:
continue
else:
-   if line_escaped.endswith('\0'):
+   if line_escaped:
multinum = num
multiline = line
continue
-- 
2.10.2