On Thu, Feb 15, 2018 at 12:49 AM, Zac Medico <zmed...@gentoo.org> wrote:
> Use sys.stdin.buffer instead of sys.stdin.buffer.raw, for buffered input. > Also use raw bytes instead of unicode strings, in order to avoid making > assumptions about character encodings, and also to avoid overhead from > unicode decoding/encoding. > > Since the % operator does not support bytes operands in python3.4, use > the + operator to format strings of bytes. > > Bug: https://bugs.gentoo.org/647654 > --- > [PATCH v2] changes: > * don't use % operator with bytes operands, for python3.4 compat > * add unit test that compares expected output to actual output > Sorry to be unclear; if there is existing test coverage via emerge I think that is fine; I was just worried there were none. I need to learn to check travis-ci for these pullreqs. > > bin/filter-bash-environment.py | 47 +++++------ > pym/portage/tests/bin/test_filter_bash_env.py | 113 > ++++++++++++++++++++++++++ > 2 files changed, 135 insertions(+), 25 deletions(-) > create mode 100644 pym/portage/tests/bin/test_filter_bash_env.py > > diff --git a/bin/filter-bash-environment.py b/bin/filter-bash-environment. > py > index a4cdc5429..668aa7452 100755 > --- a/bin/filter-bash-environment.py > +++ b/bin/filter-bash-environment.py > @@ -2,21 +2,19 @@ > # Copyright 1999-2014 Gentoo Foundation > # Distributed under the terms of the GNU General Public License v2 > > -import codecs > -import io > import os > import re > import sys > > -here_doc_re = re.compile(r'.*\s<<[-]?(\w+)$') > -func_start_re = re.compile(r'^[-\w]+\s*\(\)\s*$') > -func_end_re = re.compile(r'^\}$') > +here_doc_re = re.compile(br'.*\s<<[-]?(\w+)$') > +func_start_re = re.compile(br'^[-\w]+\s*\(\)\s*$') > +func_end_re = re.compile(br'^\}$') > > -var_assign_re = re.compile(r'(^|^declare\s+-\ > S+\s+|^declare\s+|^export\s+)([^=\s]+)=("|\')?.*$') > -close_quote_re = re.compile(r'(\\"|"|\')\s*$') > -readonly_re = re.compile(r'^declare\s+-(\S*)r(\S*)\s+') > +var_assign_re = re.compile(br'(^|^declare\s+-\ > S+\s+|^declare\s+|^export\s+)([^=\s]+)=("|\')?.*$') > +close_quote_re = re.compile(br'(\\"|"|\')\s*$') > +readonly_re = re.compile(br'^declare\s+-(\S*)r(\S*)\s+') > # declare without assignment > -var_declare_re = re.compile(r'^declare(\s+-\S+)?\s+([^=\s]+)\s*$') > +var_declare_re = re.compile(br'^declare(\s+-\S+)?\s+([^=\s]+)\s*$') > > def have_end_quote(quote, line): > """ > @@ -32,16 +30,16 @@ def have_end_quote(quote, line): > def filter_declare_readonly_opt(line): > readonly_match = readonly_re.match(line) > if readonly_match is not None: > - declare_opts = '' > + declare_opts = b'' > for i in (1, 2): > group = readonly_match.group(i) > if group is not None: > declare_opts += group > if declare_opts: > - line = 'declare -%s %s' % \ > - (declare_opts, line[readonly_match.end():]) > + line = b'declare -' + declare_opts + \ > + b' ' + line[readonly_match.end():] > else: > - line = 'declare ' + line[readonly_match.end():] > + line = b'declare ' + line[readonly_match.end():] > return line > > def filter_bash_environment(pattern, file_in, file_out): > @@ -57,7 +55,7 @@ def filter_bash_environment(pattern, file_in, file_out): > for line in file_in: > if multi_line_quote is not None: > if not multi_line_quote_filter: > - file_out.write(line.replace("\1", "")) > + file_out.write(line.replace(b"\1", b"")) > if have_end_quote(multi_line_quote, line): > multi_line_quote = None > multi_line_quote_filter = None > @@ -78,7 +76,7 @@ def filter_bash_environment(pattern, file_in, file_out): > multi_line_quote_filter = > filter_this > if not filter_this: > line = filter_declare_readonly_opt( > line) > - file_out.write(line.replace("\1", > "")) > + file_out.write(line.replace(b"\1", > b"")) > continue > else: > declare_match = var_declare_re.match(line) > @@ -98,7 +96,7 @@ def filter_bash_environment(pattern, file_in, file_out): > continue > here_doc = here_doc_re.match(line) > if here_doc is not None: > - here_doc_delim = re.compile("^%s$" % > here_doc.group(1)) > + here_doc_delim = re.compile(b'^%s' + > here_doc.group(1) + b'$') > file_out.write(line) > continue > # Note: here-documents are handled before functions since > otherwise > @@ -141,18 +139,17 @@ if __name__ == "__main__": > file_in = sys.stdin > file_out = sys.stdout > if sys.hexversion >= 0x3000000: > - file_in = codecs.iterdecode(sys.stdin.buffer.raw, > - 'utf_8', errors='replace') > - file_out = io.TextIOWrapper(sys.stdout.buffer, > - 'utf_8', errors='backslashreplace') > - > - var_pattern = args[0].split() > + file_in = sys.stdin.buffer > + file_out = sys.stdout.buffer > + var_pattern = os.fsencode(args[0]).split() > + else: > + var_pattern = args[0].split() > > # Filter invalid variable names that are not supported by bash. > - var_pattern.append(r'\d.*') > - var_pattern.append(r'.*\W.*') > + var_pattern.append(br'\d.*') > + var_pattern.append(br'.*\W.*') > > - var_pattern = "^(%s)$" % "|".join(var_pattern) > + var_pattern = b'^(' + b'|'.join(var_pattern) + b')$' > filter_bash_environment( > re.compile(var_pattern), file_in, file_out) > file_out.flush() > diff --git a/pym/portage/tests/bin/test_filter_bash_env.py > b/pym/portage/tests/bin/test_filter_bash_env.py > new file mode 100644 > index 000000000..8cf643768 > --- /dev/null > +++ b/pym/portage/tests/bin/test_filter_bash_env.py > @@ -0,0 +1,113 @@ > +# Copyright 2018 Gentoo Foundation > +# Distributed under the terms of the GNU General Public License v2 > + > +import difflib > +import os > +import subprocess > + > +from portage.const import PORTAGE_BIN_PATH > +from portage.tests import TestCase > + > + > +class TestFilterBashEnv(TestCase): > + def testTestFilterBashEnv(self): > + > + test_cases = ( > + ( > + 'RDEPEND BASH.* _EPATCH_ECLASS', > + b'''declare -ir BASHPID="28997" > +declare -rx A="portage-2.3.24.tar.bz2" > +declare -- DESKTOP_DATABASE_DIR="/usr/share/applications" > +declare PDEPEND=" > + !build? ( > + >=net-misc/rsync-2.6.4 > + userland_GNU? ( >=sys-apps/coreutils-6.4 ) > + ) " > +declare RDEPEND=" > + >=app-arch/tar-1.27 > + dev-lang/python-exec:2" > +declare -x PF="portage-2.3.24" > +declare -a PYTHON_COMPAT=([0]="pypy" [1]="python3_4" [2]="python3_5" > [3]="python3_6" [4]="python2_7") > +declare -- _EPATCH_ECLASS="1" > +declare -- _EUTILS_ECLASS="1" > +declare -- f > +get_libdir () > +{ > + local CONF_LIBDIR; > + if [ -n "${CONF_LIBDIR_OVERRIDE}" ]; then > + echo ${CONF_LIBDIR_OVERRIDE}; > + else > + get_abi_LIBDIR; > + fi > +} > +make_wrapper () > +{ > + cat <<-EOF > +export ${var}="\${${var}}:${EPREFIX}${libdir}" > +EOF > +} > +use_if_iuse () > +{ > + in_iuse $1 || return 1; > + use $1 > +} > +''', > + b'''declare -x A="portage-2.3.24.tar.bz2" > +declare -- DESKTOP_DATABASE_DIR="/usr/share/applications" > +declare PDEPEND=" > + !build? ( > + >=net-misc/rsync-2.6.4 > + userland_GNU? ( >=sys-apps/coreutils-6.4 ) > + ) " > +declare -x PF="portage-2.3.24" > +declare -a PYTHON_COMPAT=([0]="pypy" [1]="python3_4" [2]="python3_5" > [3]="python3_6" [4]="python2_7") > +declare -- _EUTILS_ECLASS="1" > +declare -- f > +get_libdir () > +{ > + local CONF_LIBDIR; > + if [ -n "${CONF_LIBDIR_OVERRIDE}" ]; then > + echo ${CONF_LIBDIR_OVERRIDE}; > + else > + get_abi_LIBDIR; > + fi > +} > +make_wrapper () > +{ > + cat <<-EOF > +export ${var}="\${${var}}:${EPREFIX}${libdir}" > +EOF > +} > +use_if_iuse () > +{ > + in_iuse $1 || return 1; > + use $1 > +} > +'''), > + ) > + > + for filter_vars, env_in, env_out in test_cases: > + proc = None > + try: > + proc = subprocess.Popen( > + [ > + > os.path.join(PORTAGE_BIN_PATH, 'filter-bash-environment.py'), > + filter_vars, > + ], > + stdin=subprocess.PIPE, > + stdout=subprocess.PIPE, > + ) > + proc.stdin.write(env_in) > + proc.stdin.close() > + result = proc.stdout.read() > + finally: > + if proc is not None: > + proc.stdin.close() > + proc.wait() + proc.stdout.close() > + > + diff = list(difflib.unified_diff( > + env_out.decode().splitlines(), > + result.decode().splitlines())) > + > + self.assertEqual(diff, []) > -- > 2.13.6 > > >
