Use sys.stdin.buffer instead of sys.stdin.buffer.raw, for buffered input. Also use raw bytes instead of unicode strings, in order to avoid making assumptions about character encodings, and also to avoid overhead from unicode decoding/encoding.
Since the % operator does not support bytes operands in python3.4, use the + operator to format strings of bytes. Bug: https://bugs.gentoo.org/647654 --- [PATCH v2] changes: * don't use % operator with bytes operands, for python3.4 compat * add unit test that compares expected output to actual output bin/filter-bash-environment.py | 47 +++++------ pym/portage/tests/bin/test_filter_bash_env.py | 113 ++++++++++++++++++++++++++ 2 files changed, 135 insertions(+), 25 deletions(-) create mode 100644 pym/portage/tests/bin/test_filter_bash_env.py diff --git a/bin/filter-bash-environment.py b/bin/filter-bash-environment.py index a4cdc5429..668aa7452 100755 --- a/bin/filter-bash-environment.py +++ b/bin/filter-bash-environment.py @@ -2,21 +2,19 @@ # Copyright 1999-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -import codecs -import io import os import re import sys -here_doc_re = re.compile(r'.*\s<<[-]?(\w+)$') -func_start_re = re.compile(r'^[-\w]+\s*\(\)\s*$') -func_end_re = re.compile(r'^\}$') +here_doc_re = re.compile(br'.*\s<<[-]?(\w+)$') +func_start_re = re.compile(br'^[-\w]+\s*\(\)\s*$') +func_end_re = re.compile(br'^\}$') -var_assign_re = re.compile(r'(^|^declare\s+-\S+\s+|^declare\s+|^export\s+)([^=\s]+)=("|\')?.*$') -close_quote_re = re.compile(r'(\\"|"|\')\s*$') -readonly_re = re.compile(r'^declare\s+-(\S*)r(\S*)\s+') +var_assign_re = re.compile(br'(^|^declare\s+-\S+\s+|^declare\s+|^export\s+)([^=\s]+)=("|\')?.*$') +close_quote_re = re.compile(br'(\\"|"|\')\s*$') +readonly_re = re.compile(br'^declare\s+-(\S*)r(\S*)\s+') # declare without assignment -var_declare_re = re.compile(r'^declare(\s+-\S+)?\s+([^=\s]+)\s*$') +var_declare_re = re.compile(br'^declare(\s+-\S+)?\s+([^=\s]+)\s*$') def have_end_quote(quote, line): """ @@ -32,16 +30,16 @@ def have_end_quote(quote, line): def filter_declare_readonly_opt(line): readonly_match = readonly_re.match(line) if readonly_match is not None: - declare_opts = '' + declare_opts = b'' for i in (1, 2): group = readonly_match.group(i) if group is not None: declare_opts += group if declare_opts: - line = 'declare -%s %s' % \ - (declare_opts, line[readonly_match.end():]) + line = b'declare -' + declare_opts + \ + b' ' + line[readonly_match.end():] else: - line = 'declare ' + line[readonly_match.end():] + line = b'declare ' + line[readonly_match.end():] return line def filter_bash_environment(pattern, file_in, file_out): @@ -57,7 +55,7 @@ def filter_bash_environment(pattern, file_in, file_out): for line in file_in: if multi_line_quote is not None: if not multi_line_quote_filter: - file_out.write(line.replace("\1", "")) + file_out.write(line.replace(b"\1", b"")) if have_end_quote(multi_line_quote, line): multi_line_quote = None multi_line_quote_filter = None @@ -78,7 +76,7 @@ def filter_bash_environment(pattern, file_in, file_out): multi_line_quote_filter = filter_this if not filter_this: line = filter_declare_readonly_opt(line) - file_out.write(line.replace("\1", "")) + file_out.write(line.replace(b"\1", b"")) continue else: declare_match = var_declare_re.match(line) @@ -98,7 +96,7 @@ def filter_bash_environment(pattern, file_in, file_out): continue here_doc = here_doc_re.match(line) if here_doc is not None: - here_doc_delim = re.compile("^%s$" % here_doc.group(1)) + here_doc_delim = re.compile(b'^%s' + here_doc.group(1) + b'$') file_out.write(line) continue # Note: here-documents are handled before functions since otherwise @@ -141,18 +139,17 @@ if __name__ == "__main__": file_in = sys.stdin file_out = sys.stdout if sys.hexversion >= 0x3000000: - file_in = codecs.iterdecode(sys.stdin.buffer.raw, - 'utf_8', errors='replace') - file_out = io.TextIOWrapper(sys.stdout.buffer, - 'utf_8', errors='backslashreplace') - - var_pattern = args[0].split() + file_in = sys.stdin.buffer + file_out = sys.stdout.buffer + var_pattern = os.fsencode(args[0]).split() + else: + var_pattern = args[0].split() # Filter invalid variable names that are not supported by bash. - var_pattern.append(r'\d.*') - var_pattern.append(r'.*\W.*') + var_pattern.append(br'\d.*') + var_pattern.append(br'.*\W.*') - var_pattern = "^(%s)$" % "|".join(var_pattern) + var_pattern = b'^(' + b'|'.join(var_pattern) + b')$' filter_bash_environment( re.compile(var_pattern), file_in, file_out) file_out.flush() diff --git a/pym/portage/tests/bin/test_filter_bash_env.py b/pym/portage/tests/bin/test_filter_bash_env.py new file mode 100644 index 000000000..8cf643768 --- /dev/null +++ b/pym/portage/tests/bin/test_filter_bash_env.py @@ -0,0 +1,113 @@ +# Copyright 2018 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import difflib +import os +import subprocess + +from portage.const import PORTAGE_BIN_PATH +from portage.tests import TestCase + + +class TestFilterBashEnv(TestCase): + def testTestFilterBashEnv(self): + + test_cases = ( + ( + 'RDEPEND BASH.* _EPATCH_ECLASS', + b'''declare -ir BASHPID="28997" +declare -rx A="portage-2.3.24.tar.bz2" +declare -- DESKTOP_DATABASE_DIR="/usr/share/applications" +declare PDEPEND=" + !build? ( + >=net-misc/rsync-2.6.4 + userland_GNU? ( >=sys-apps/coreutils-6.4 ) + ) " +declare RDEPEND=" + >=app-arch/tar-1.27 + dev-lang/python-exec:2" +declare -x PF="portage-2.3.24" +declare -a PYTHON_COMPAT=([0]="pypy" [1]="python3_4" [2]="python3_5" [3]="python3_6" [4]="python2_7") +declare -- _EPATCH_ECLASS="1" +declare -- _EUTILS_ECLASS="1" +declare -- f +get_libdir () +{ + local CONF_LIBDIR; + if [ -n "${CONF_LIBDIR_OVERRIDE}" ]; then + echo ${CONF_LIBDIR_OVERRIDE}; + else + get_abi_LIBDIR; + fi +} +make_wrapper () +{ + cat <<-EOF +export ${var}="\${${var}}:${EPREFIX}${libdir}" +EOF +} +use_if_iuse () +{ + in_iuse $1 || return 1; + use $1 +} +''', + b'''declare -x A="portage-2.3.24.tar.bz2" +declare -- DESKTOP_DATABASE_DIR="/usr/share/applications" +declare PDEPEND=" + !build? ( + >=net-misc/rsync-2.6.4 + userland_GNU? ( >=sys-apps/coreutils-6.4 ) + ) " +declare -x PF="portage-2.3.24" +declare -a PYTHON_COMPAT=([0]="pypy" [1]="python3_4" [2]="python3_5" [3]="python3_6" [4]="python2_7") +declare -- _EUTILS_ECLASS="1" +declare -- f +get_libdir () +{ + local CONF_LIBDIR; + if [ -n "${CONF_LIBDIR_OVERRIDE}" ]; then + echo ${CONF_LIBDIR_OVERRIDE}; + else + get_abi_LIBDIR; + fi +} +make_wrapper () +{ + cat <<-EOF +export ${var}="\${${var}}:${EPREFIX}${libdir}" +EOF +} +use_if_iuse () +{ + in_iuse $1 || return 1; + use $1 +} +'''), + ) + + for filter_vars, env_in, env_out in test_cases: + proc = None + try: + proc = subprocess.Popen( + [ + os.path.join(PORTAGE_BIN_PATH, 'filter-bash-environment.py'), + filter_vars, + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + proc.stdin.write(env_in) + proc.stdin.close() + result = proc.stdout.read() + finally: + if proc is not None: + proc.stdin.close() + proc.wait() + proc.stdout.close() + + diff = list(difflib.unified_diff( + env_out.decode().splitlines(), + result.decode().splitlines())) + + self.assertEqual(diff, []) -- 2.13.6
