3 new commits in tox: https://bitbucket.org/hpk42/tox/commits/f8af400bcecc/ Changeset: f8af400bcecc User: hpk42 Date: 2013-10-22 10:59:09 Summary: fix issue129: tox now uses Popen(..., universal_newlines=True) to force creation of unicode stdout/stderr streams. fixes a problem on specific platform configs when creating virtualenvs with Python3.3. Thanks Jorgen Schäfer or investigation and solution sketch. Affected #: 4 files
diff -r 88a503e7e5aefe509be8f6c3108a84baa20c3c26 -r f8af400bceccb5b95a6e5ef407f973d21b466c51 CHANGELOG --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,11 @@ 1.6.2.dev --------- +- fix issue129: tox now uses Popen(..., universal_newlines=True) to force + creation of unicode stdout/stderr streams. fixes a problem on specific + platform configs when creating virtualenvs with Python3.3. Thanks Jorgen Schäfer + or investigation and solution sketch. + - fix issue128: enable full substitution in install_command, thanks for the PR to Ronald Evers diff -r 88a503e7e5aefe509be8f6c3108a84baa20c3c26 -r f8af400bceccb5b95a6e5ef407f973d21b466c51 tox/_cmdline.py --- a/tox/_cmdline.py +++ b/tox/_cmdline.py @@ -93,7 +93,8 @@ if cwd is None: # XXX cwd = self.session.config.cwd cwd = py.path.local() - popen = self._popen(args, cwd, env=env, stdout=f, stderr=STDOUT) + popen = self._popen(args, cwd, env=env, + stdout=f, stderr=STDOUT) popen.outpath = outpath popen.args = [str(x) for x in args] popen.cwd = cwd @@ -150,6 +151,7 @@ if env is None: env = os.environ.copy() return self.session.popen(args, shell=False, cwd=str(cwd), + universal_newlines=True, stdout=stdout, stderr=stderr, env=env) diff -r 88a503e7e5aefe509be8f6c3108a84baa20c3c26 -r f8af400bceccb5b95a6e5ef407f973d21b466c51 tox/_pytestplugin.py --- a/tox/_pytestplugin.py +++ b/tox/_pytestplugin.py @@ -130,6 +130,7 @@ def make_emptydir(self, path): pass def popen(self, args, cwd, shell=None, + universal_newlines=False, stdout=None, stderr=None, env=None): pm = pcallMock(args, cwd, env, stdout, stderr, shell) self._pcalls.append(pm) diff -r 88a503e7e5aefe509be8f6c3108a84baa20c3c26 -r f8af400bceccb5b95a6e5ef407f973d21b466c51 tox/_venv.py --- a/tox/_venv.py +++ b/tox/_venv.py @@ -228,7 +228,7 @@ args = [self.envconfig.envpython, str(setup_py), '--name'] output = action.popen(args, cwd=setupdir, redirect=False, returnout=True) - name = output.strip().decode('utf-8') + name = output.strip() egg_info = setupdir.join('.'.join((name, 'egg-info'))) for conf_file in (setup_py, setup_cfg): if (not egg_info.check() or (conf_file.check() https://bitbucket.org/hpk42/tox/commits/f15199a9ec78/ Changeset: f15199a9ec78 User: hpk42 Date: 2013-10-22 11:04:11 Summary: more windows parsing fixes Affected #: 5 files diff -r f8af400bceccb5b95a6e5ef407f973d21b466c51 -r f15199a9ec78e052b9f9513a34e0e5768e8affdd CHANGELOG --- a/CHANGELOG +++ b/CHANGELOG @@ -9,7 +9,8 @@ - fix issue128: enable full substitution in install_command, thanks for the PR to Ronald Evers -- fix windows parsing/escaping +- rework and simplify "commands" parsing and in particular posargs + substitutions to avoid various win32/posix related quoting issues. 1.6.1 ----- diff -r f8af400bceccb5b95a6e5ef407f973d21b466c51 -r f15199a9ec78e052b9f9513a34e0e5768e8affdd setup.py --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ description='virtualenv-based automation of test activities', long_description=open("README.rst").read(), url='http://tox.testrun.org/', - version='1.6.2.dev1', + version='1.6.2.dev2', license='http://opensource.org/licenses/MIT', platforms=['unix', 'linux', 'osx', 'cygwin', 'win32'], author='holger krekel', diff -r f8af400bceccb5b95a6e5ef407f973d21b466c51 -r f15199a9ec78e052b9f9513a34e0e5768e8affdd tests/test_config.py --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1088,14 +1088,14 @@ parsed = list(p.words()) assert parsed == ['nosetests', ' ', '-v', ' ', '-a', ' ', '!deferred', ' ', '--with-doctest', ' ', '[]'] -def test_argv_unquote_single_args(): - argv = ["hello", '"hello2"', "'hello3'"] - newargv = unquote_single_args(argv) - assert newargv == ["hello", "hello2", "hello3"] -def test_argv_roundrobin(): - argv = ["hello", "this\\that"] - assert string2argv(argv2string(argv)) == argv - argv = ["hello world"] - assert string2argv(argv2string(argv)) == argv + @pytest.mark.skipif("sys.platform != 'win32'") + def test_commands_with_backslash(self, newconfig): + config = newconfig([r"hello\world"], """ + [testenv:py26] + commands = some {posargs} + """) + envconfig = config.envconfigs["py26"] + assert envconfig.commands[0] == ["some", r"hello\world"] + diff -r f8af400bceccb5b95a6e5ef407f973d21b466c51 -r f15199a9ec78e052b9f9513a34e0e5768e8affdd tox/__init__.py --- a/tox/__init__.py +++ b/tox/__init__.py @@ -1,5 +1,5 @@ # -__version__ = '1.6.2.dev1' +__version__ = '1.6.2.dev2' class exception: class Error(Exception): diff -r f8af400bceccb5b95a6e5ef407f973d21b466c51 -r f15199a9ec78e052b9f9513a34e0e5768e8affdd tox/_config.py --- a/tox/_config.py +++ b/tox/_config.py @@ -413,7 +413,7 @@ def addsubstitutions(self, _posargs=None, **kw): self._subs.update(kw) if _posargs: - self._subs['_posargs'] = _posargs + self.posargs = _posargs def getpath(self, section, name, defaultpath): toxinidir = self._subs['toxinidir'] @@ -468,27 +468,38 @@ return commandlist def _processcommand(self, command): - posargs = self._subs.get('_posargs', None) - words = list(CommandParser(command).words()) - new_command = '' - for word in words: - if word == '[]': + posargs = getattr(self, "posargs", None) + + # special treat posargs which might contain multiple arguments + # in their defaults + newcommand = "" + for word in CommandParser(command).words(): + if word.startswith("{posargs:") and word.endswith("}"): if posargs: - new_command += ' '.join(posargs) + word = "{posargs}" + else: + word = word[9:-1] + newcommand += word + + # now we can properly parse the command + argv = [] + for arg in shlex.split(newcommand): + if arg in ('[]', "{posargs}"): + if posargs: + argv.extend(posargs) continue - - new_word = self._replace(word, quote=True) - # two passes; we might have substitutions in the result - new_word = self._replace(new_word, quote=True) - new_command += new_word - - return shlex.split(new_command.strip()) + new_arg = "" + for word in CommandParser(arg).words(): + new_word = self._replace(word) + new_word = self._replace(new_word) + new_arg += new_word + argv.append(new_arg) + return argv def getargv(self, section, name, default=None, replace=True): command = self.getdefault( - section, name, default=default, replace=replace) - - return string2argv(command.strip()) + section, name, default=default, replace=False) + return self._processcommand(command.strip()) def getbool(self, section, name, default=None): s = self.getdefault(section, name, default) @@ -527,22 +538,7 @@ #print "getdefault", section, name, "returned", repr(x) return x - def _replace_posargs(self, match, quote): - return self._do_replace_posargs(lambda: match.group('substitution_value')) - - def _do_replace_posargs(self, value_func): - posargs = self._subs.get('_posargs', None) - - if posargs: - return argv2string(posargs) - - value = value_func() - if value: - return value - - return '' - - def _replace_env(self, match, quote): + def _replace_env(self, match): envkey = match.group('substitution_value') if not envkey: raise tox.exception.ConfigError( @@ -555,7 +551,7 @@ return os.environ[envkey] - def _substitute_from_other_section(self, key, quote): + def _substitute_from_other_section(self, key): if key.startswith("[") and "]" in key: i = key.find("]") section, item = key[1:i], key[i+1:] @@ -566,37 +562,25 @@ x = str(self._cfg[section][item]) self._subststack.append((section, item)) try: - return self._replace(x, quote=quote) + return self._replace(x) finally: self._subststack.pop() raise tox.exception.ConfigError( "substitution key %r not found" % key) - def _replace_substitution(self, match, quote): + def _replace_substitution(self, match): sub_key = match.group('substitution_value') val = self._subs.get(sub_key, None) if val is None: - val = self._substitute_from_other_section(sub_key, quote) + val = self._substitute_from_other_section(sub_key) if py.builtin.callable(val): val = val() - if quote: - return '"%s"' % str(val).replace('"', r'\"') - else: - return str(val) + return str(val) - def _is_bare_posargs(self, groupdict): - return groupdict.get('substitution_value', None) == 'posargs' \ - and not groupdict.get('sub_type') - - def _replace_match(self, match, quote): + def _replace_match(self, match): g = match.groupdict() - # special case: posargs. If there is a 'posargs' substitution value - # and no type, handle it as empty posargs - if self._is_bare_posargs(g): - return self._do_replace_posargs(lambda: '') - # special case: opts and packages. Leave {opts} and # {packages} intact, they are replaced manually in # _venv.VirtualEnv.run_install_command. @@ -605,7 +589,6 @@ return '{%s}' % sub_value handlers = { - 'posargs' : self._replace_posargs, 'env' : self._replace_env, None : self._replace_substitution, } @@ -619,22 +602,11 @@ except KeyError: raise tox.exception.ConfigError("No support for the %s substitution type" % sub_type) - # quoting is done in handlers, as at least posargs handling is special: - # all of its arguments are inserted as separate parameters - return handler(match, quote) + return handler(match) - def _replace_match_quote(self, match): - return self._replace_match(match, quote=True) - def _replace_match_no_quote(self, match): - return self._replace_match(match, quote=False) - - def _replace(self, x, quote=False): + def _replace(self, x): if '{' in x: - if quote: - replace_func = self._replace_match_quote - else: - replace_func = self._replace_match_no_quote - return RE_ITEM_REF.sub(replace_func, x) + return RE_ITEM_REF.sub(self._replace_match, x) return x def _parse_command(self, command): @@ -706,18 +678,3 @@ return 'jenkins' return None - -def unquote_single_args(argv): - newargv = [] - for arg in argv: - if len(arg) >=2 and arg[0] == arg[-1]: - if arg[0] in ("'", '"'): - arg = arg[1:-1] - newargv.append(arg) - return newargv - -def string2argv(cmd): - return unquote_single_args(shlex.split(cmd, posix=False)) - -def argv2string(argv): - return subprocess.list2cmdline(argv) https://bitbucket.org/hpk42/tox/commits/ca1c1d8f9586/ Changeset: ca1c1d8f9586 User: hpk42 Date: 2013-10-22 11:04:11 Summary: a command line specified --installpkg trumps any develop option Affected #: 3 files diff -r f15199a9ec78e052b9f9513a34e0e5768e8affdd -r ca1c1d8f95869a3ca6e4af9fb3f84779e84438b8 CHANGELOG --- a/CHANGELOG +++ b/CHANGELOG @@ -12,6 +12,9 @@ - rework and simplify "commands" parsing and in particular posargs substitutions to avoid various win32/posix related quoting issues. +- make sure that the --installpkg option trumps any usedevelop settings + in tox.ini or + 1.6.1 ----- diff -r f15199a9ec78e052b9f9513a34e0e5768e8affdd -r ca1c1d8f95869a3ca6e4af9fb3f84779e84438b8 tests/test_config.py --- a/tests/test_config.py +++ b/tests/test_config.py @@ -308,7 +308,8 @@ assert argvlist[0] == ["cmd1"] assert argvlist[1] == ["cmd2", "value2", "other"] - def test_positional_arguments_are_only_replaced_when_standing_alone(self, tmpdir, newconfig): + def test_positional_arguments_are_only_replaced_when_standing_alone(self, + tmpdir, newconfig): config = newconfig(""" [section] key= @@ -406,6 +407,13 @@ assert envconfig.envlogdir == envconfig.envdir.join("log") assert envconfig.setenv is None + def test_installpkg_tops_develop(self, newconfig): + config = newconfig(["--installpkg=abc"], """ + [testenv] + usedevelop = True + """) + assert not config.envconfigs["python"].develop + def test_specific_command_overrides(self, tmpdir, newconfig): config = newconfig(""" [testenv] diff -r f15199a9ec78e052b9f9513a34e0e5768e8affdd -r ca1c1d8f95869a3ca6e4af9fb3f84779e84438b8 tox/_config.py --- a/tox/_config.py +++ b/tox/_config.py @@ -272,7 +272,8 @@ vc.config = config reader = IniReader(self._cfg, fallbacksections=["testenv"]) reader.addsubstitutions(**subs) - vc.develop = reader.getbool(section, "usedevelop", config.option.develop) + vc.develop = not config.option.installpkg and \ + reader.getbool(section, "usedevelop", config.option.develop) vc.envdir = reader.getpath(section, "envdir", "{toxworkdir}/%s" % name) vc.args_are_paths = reader.getbool(section, "args_are_paths", True) if reader.getdefault(section, "python", None): Repository URL: https://bitbucket.org/hpk42/tox/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email. _______________________________________________ pytest-commit mailing list pytest-commit@python.org https://mail.python.org/mailman/listinfo/pytest-commit