4 new commits in tox: https://bitbucket.org/hpk42/tox/commits/128a5ad7a9a6/ Changeset: 128a5ad7a9a6 Branch: issue285 User: hpk42 Date: 2015-12-07 11:38:55+00:00 Summary: refactor setenv processing into its own class so that we can cleanly implement lazyness and get rid of all kinds of ordering problems. Affected #: 4 files
diff -r 0f1846de3af1d21c3d33099c47ef7d961620f362 -r 128a5ad7a9a68825dc837b25e7594e2d92809c40 CHANGELOG --- a/CHANGELOG +++ b/CHANGELOG @@ -1,7 +1,10 @@ 2.3.0 (unreleased) ----- -- fix issue285 (WIP) setenv processing with self-references +- fix issue285 make setenv processing fully lazy to fix regressions + of tox-2.2.X and so that we can now have testenv attributes like + "basepython" depend on environment variables that are set in + a setenv section. - allow "#" in commands. This is slightly incompatible with commands sections that used a comment after a "\" line continuation. diff -r 0f1846de3af1d21c3d33099c47ef7d961620f362 -r 128a5ad7a9a68825dc837b25e7594e2d92809c40 tests/test_config.py --- a/tests/test_config.py +++ b/tests/test_config.py @@ -644,7 +644,7 @@ assert envconfig.usedevelop is False assert envconfig.ignore_errors is False assert envconfig.envlogdir == envconfig.envdir.join("log") - assert list(envconfig.setenv.keys()) == ['PYTHONHASHSEED'] + assert list(envconfig.setenv.definitions.keys()) == ['PYTHONHASHSEED'] hashseed = envconfig.setenv['PYTHONHASHSEED'] assert isinstance(hashseed, str) # The following line checks that hashseed parses to an integer. @@ -1516,7 +1516,7 @@ return envconfigs["python"] def _check_hashseed(self, envconfig, expected): - assert envconfig.setenv == {'PYTHONHASHSEED': expected} + assert envconfig.setenv['PYTHONHASHSEED'] == expected def _check_testenv(self, newconfig, expected, args=None, tox_ini=None): envconfig = self._get_envconfig(newconfig, args=args, tox_ini=tox_ini) @@ -1565,7 +1565,7 @@ def test_noset(self, tmpdir, newconfig): args = ['--hashseed', 'noset'] envconfig = self._get_envconfig(newconfig, args=args) - assert envconfig.setenv == {} + assert not envconfig.setenv.definitions def test_noset_with_setenv(self, tmpdir, newconfig): tox_ini = """ @@ -1610,18 +1610,32 @@ class TestSetenv: - def test_getdict_lazy(self, tmpdir, newconfig): + def test_getdict_lazy(self, tmpdir, newconfig, monkeypatch): + monkeypatch.setenv("X", "2") config = newconfig(""" [testenv:X] key0 = key1 = {env:X} - key2 = {env:X:1} + key2 = {env:Y:1} """) envconfig = config.envconfigs["X"] - val = envconfig._reader.getdict_lazy("key0") - assert val == {"key1": "{env:X}", - "key2": "{env:X:1}"} + val = envconfig._reader.getdict_setenv("key0") + assert val["key1"] == "2" + assert val["key2"] == "1" + def test_getdict_lazy_update(self, tmpdir, newconfig, monkeypatch): + monkeypatch.setenv("X", "2") + config = newconfig(""" + [testenv:X] + key0 = + key1 = {env:X} + key2 = {env:Y:1} + """) + envconfig = config.envconfigs["X"] + val = envconfig._reader.getdict_setenv("key0") + d = {} + d.update(val) + assert d == {"key1": "2", "key2": "1"} def test_setenv_uses_os_environ(self, tmpdir, newconfig, monkeypatch): monkeypatch.setenv("X", "1") diff -r 0f1846de3af1d21c3d33099c47ef7d961620f362 -r 128a5ad7a9a68825dc837b25e7594e2d92809c40 tox.ini --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ commands=echo {posargs} [testenv] -commands= py.test --timeout=180 {posargs} +commands= py.test --timeout=180 {posargs:tests} deps=pytest>=2.3.5 pytest-timeout diff -r 0f1846de3af1d21c3d33099c47ef7d961620f362 -r 128a5ad7a9a68825dc837b25e7594e2d92809c40 tox/config.py --- a/tox/config.py +++ b/tox/config.py @@ -26,6 +26,8 @@ hookimpl = pluggy.HookimplMarker("tox") +_dummy = object() + def get_plugin_manager(): # initialize plugin manager @@ -253,6 +255,47 @@ setattr(namespace, self.dest, 0) +class SetenvDict: + def __init__(self, dict, reader): + self.reader = reader + self.definitions = dict + self.resolved = {} + self._lookupstack = [] + + def __contains__(self, name): + return name in self.definitions + + def get(self, name, default=None): + try: + return self.resolved[name] + except KeyError: + try: + if name in self._lookupstack: + raise KeyError("recursion") + val = self.definitions[name] + except KeyError: + return os.environ.get(name, default) + self._lookupstack.append(name) + try: + self.resolved[name] = res = self.reader._replace(val) + finally: + self._lookupstack.pop() + return res + + def __getitem__(self, name): + x = self.get(name, _dummy) + if x is _dummy: + raise KeyError(name) + return x + + def keys(self): + return self.definitions.keys() + + def __setitem__(self, name, value): + self.definitions[name] = value + self.resolved[name] = value + + @hookimpl def tox_addoption(parser): # formatter_class=argparse.ArgumentDefaultsHelpFormatter) @@ -330,32 +373,15 @@ # add various core venv interpreter attributes def setenv(testenv_config, value): setenv = value - reader = testenv_config._reader - - # we need to resolve environment variable substitution - - replacing = [] # for detecting direct recursion - def setenv_reader(name): - if name in setenv and name not in replacing: - return setenv[name] - return os.environ.get(name) - reader.set_envreader(setenv_reader) - - for name, value in setenv.items(): - replacing.append(name) - setenv[name] = reader._replace(value) - replacing.pop() - config = testenv_config.config if "PYTHONHASHSEED" not in setenv and config.hashseed is not None: setenv['PYTHONHASHSEED'] = config.hashseed return setenv parser.add_testenv_attribute( - name="setenv", type="dict_lazy", postprocess=setenv, + name="setenv", type="dict_setenv", postprocess=setenv, help="list of X=Y lines with environment variable settings") - def basepython_default(testenv_config, value): if value is None: for f in testenv_config.factors: @@ -532,21 +558,33 @@ self.factors = factors self._reader = reader - @property - def envbindir(self): + def get_envbindir(self): """ path to directory where scripts/binaries reside. """ - if sys.platform == "win32": + if (sys.platform == "win32" + and "jython" not in self.basepython + and "pypy" not in self.basepython): return self.envdir.join("Scripts") else: return self.envdir.join("bin") @property + def envbindir(self): + return self.get_envbindir() + + @property def envpython(self): """ path to python executable. """ - return self.envbindir.join(self.basepython) + return self.get_envpython() - # no @property to avoid early calling (see callable(subst[key]) checks) - def envsitepackagesdir(self): + def get_envpython(self): + """ path to python/jython executable. """ + if "jython" in str(self.basepython): + name = "jython" + else: + name = "python" + return self.envbindir.join(name) + + def get_envsitepackagesdir(self): """ return sitepackagesdir of the virtualenv environment. (only available during execution, not parsing) """ @@ -707,10 +745,13 @@ vc = TestenvConfig(config=config, envname=name, factors=factors, reader=reader) reader.addsubstitutions(**subs) reader.addsubstitutions(envname=name) + reader.addsubstitutions(envbindir=vc.get_envbindir, + envsitepackagesdir=vc.get_envsitepackagesdir, + envpython=vc.get_envpython) for env_attr in config._testenv_attr: atype = env_attr.type - if atype in ("bool", "path", "string", "dict", "dict_lazy", "argv", "argvlist"): + if atype in ("bool", "path", "string", "dict", "dict_setenv", "argv", "argvlist"): meth = getattr(reader, "get" + atype) res = meth(env_attr.name, env_attr.default) elif atype == "space-separated-list": @@ -727,9 +768,6 @@ if atype == "path": reader.addsubstitutions(**{env_attr.name: res}) - if env_attr.name == "basepython": - reader.addsubstitutions(envbindir=vc.envbindir, envpython=vc.envpython, - envsitepackagesdir=vc.envsitepackagesdir) return vc def _getenvdata(self, reader): @@ -818,13 +856,12 @@ self.factors = factors self._subs = {} self._subststack = [] - self._envreader = os.environ.get - - def set_envreader(self, envreader): - self._envreader = envreader + self._setenv = None def get_environ_value(self, name): - return self._envreader(name) + if self._setenv is None: + return os.environ.get(name) + return self._setenv.get(name) def addsubstitutions(self, _posargs=None, **kw): self._subs.update(kw) @@ -847,9 +884,11 @@ value = self.getstring(name, None) return self._getdict(value, default=default, sep=sep) - def getdict_lazy(self, name, default=None, sep="\n"): - value = self.getstring(name, None, replace="noenv") - return self._getdict(value, default=default, sep=sep) + def getdict_setenv(self, name, default=None, sep="\n"): + value = self.getstring(name, None, replace=False) + definitions = self._getdict(value, default=default, sep=sep) + self._setenv = SetenvDict(definitions, reader=self) + return self._setenv def _getdict(self, value, default, sep): if value is None: @@ -903,7 +942,7 @@ x = self._apply_factors(x) if replace and x and hasattr(x, 'replace'): - x = self._replace(x, name=name, opt_replace_env=(replace!="noenv")) + x = self._replace(x, name=name) # print "getstring", self.section_name, name, "returned", repr(x) return x @@ -920,14 +959,14 @@ lines = s.strip().splitlines() return '\n'.join(filter(None, map(factor_line, lines))) - def _replace(self, value, name=None, section_name=None, opt_replace_env=True): + def _replace(self, value, name=None, section_name=None): if '{' not in value: return value section_name = section_name if section_name else self.section_name self._subststack.append((section_name, name)) try: - return Replacer(self, opt_replace_env=opt_replace_env).do_replace(value) + return Replacer(self).do_replace(value) finally: assert self._subststack.pop() == (section_name, name) @@ -942,10 +981,8 @@ ''', re.VERBOSE) - - def __init__(self, reader, opt_replace_env): + def __init__(self, reader): self.reader = reader - self.opt_replace_env = opt_replace_env def do_replace(self, x): return self.RE_ITEM_REF.sub(self._replace_match, x) @@ -967,11 +1004,10 @@ "Malformed substitution; no substitution type provided") if sub_type == "env": - if self.opt_replace_env: - return self._replace_env(match) - return "{env:%s}" %(g["substitution_value"]) - if sub_type != None: - raise tox.exception.ConfigError("No support for the %s substitution type" % sub_type) + return self._replace_env(match) + if sub_type is not None: + raise tox.exception.ConfigError( + "No support for the %s substitution type" % sub_type) return self._replace_substitution(match) def _replace_env(self, match): @@ -1007,8 +1043,7 @@ raise ValueError('%s already in %s' % ( (section, item), self.reader._subststack)) x = str(cfg[section][item]) - return self.reader._replace(x, name=item, section_name=section, - opt_replace_env=self.opt_replace_env) + return self.reader._replace(x, name=item, section_name=section) raise tox.exception.ConfigError( "substitution key %r not found" % key) @@ -1023,7 +1058,6 @@ return str(val) - class _ArgvlistReader: @classmethod def getargvlist(cls, reader, value): https://bitbucket.org/hpk42/tox/commits/1bf39138fe84/ Changeset: 1bf39138fe84 Branch: issue285 User: hpk42 Date: 2015-12-07 11:39:34+00:00 Summary: reshuffle tests related to setenv processing and integrate nelfin's cross-section test but mark it as xfailing because i am not sure we need to go to the trouble Affected #: 5 files diff -r 128a5ad7a9a68825dc837b25e7594e2d92809c40 -r 1bf39138fe845f4a95092fc69dbfee4a7d854176 CHANGELOG --- a/CHANGELOG +++ b/CHANGELOG @@ -1,10 +1,11 @@ 2.3.0 (unreleased) ----- -- fix issue285 make setenv processing fully lazy to fix regressions +- fix issue285: make setenv processing fully lazy to fix regressions of tox-2.2.X and so that we can now have testenv attributes like "basepython" depend on environment variables that are set in - a setenv section. + a setenv section. Thanks Nelfin for some tests and initial + work on a PR. - allow "#" in commands. This is slightly incompatible with commands sections that used a comment after a "\" line continuation. diff -r 128a5ad7a9a68825dc837b25e7594e2d92809c40 -r 1bf39138fe845f4a95092fc69dbfee4a7d854176 setup.py --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ description='virtualenv-based automation of test activities', long_description=open("README.rst").read(), url='http://tox.testrun.org/', - version='2.3.0.dev1', + version='2.3.0.dev2', license='http://opensource.org/licenses/MIT', platforms=['unix', 'linux', 'osx', 'cygwin', 'win32'], author='holger krekel', diff -r 128a5ad7a9a68825dc837b25e7594e2d92809c40 -r 1bf39138fe845f4a95092fc69dbfee4a7d854176 tests/test_config.py --- a/tests/test_config.py +++ b/tests/test_config.py @@ -735,46 +735,6 @@ if bp == "jython": assert envconfig.envpython == envconfig.envbindir.join(bp) - def test_setenv_overrides(self, tmpdir, newconfig): - config = newconfig(""" - [testenv] - setenv = - PYTHONPATH = something - ANOTHER_VAL=else - """) - assert len(config.envconfigs) == 1 - envconfig = config.envconfigs['python'] - assert 'PYTHONPATH' in envconfig.setenv - assert 'ANOTHER_VAL' in envconfig.setenv - assert envconfig.setenv['PYTHONPATH'] == 'something' - assert envconfig.setenv['ANOTHER_VAL'] == 'else' - - def test_setenv_with_envdir_and_basepython(self, tmpdir, newconfig): - config = newconfig(""" - [testenv] - setenv = - VAL = {envdir} - basepython = {env:VAL} - """) - assert len(config.envconfigs) == 1 - envconfig = config.envconfigs['python'] - assert 'VAL' in envconfig.setenv - assert envconfig.setenv['VAL'] == envconfig.envdir - assert envconfig.basepython == envconfig.envdir - - def test_setenv_ordering_1(self, tmpdir, newconfig): - config = newconfig(""" - [testenv] - setenv= - VAL={envdir} - commands=echo {env:VAL} - """) - assert len(config.envconfigs) == 1 - envconfig = config.envconfigs['python'] - assert 'VAL' in envconfig.setenv - assert envconfig.setenv['VAL'] == envconfig.envdir - assert str(envconfig.envdir) in envconfig.commands[0] - @pytest.mark.parametrize("plat", ["win32", "linux2"]) def test_passenv_as_multiline_list(self, tmpdir, newconfig, monkeypatch, plat): monkeypatch.setattr(sys, "platform", plat) @@ -1672,6 +1632,61 @@ """) assert config.envconfigs["env1"].setenv["X"] == "3" + def test_setenv_overrides(self, tmpdir, newconfig): + config = newconfig(""" + [testenv] + setenv = + PYTHONPATH = something + ANOTHER_VAL=else + """) + assert len(config.envconfigs) == 1 + envconfig = config.envconfigs['python'] + assert 'PYTHONPATH' in envconfig.setenv + assert 'ANOTHER_VAL' in envconfig.setenv + assert envconfig.setenv['PYTHONPATH'] == 'something' + assert envconfig.setenv['ANOTHER_VAL'] == 'else' + + def test_setenv_with_envdir_and_basepython(self, tmpdir, newconfig): + config = newconfig(""" + [testenv] + setenv = + VAL = {envdir} + basepython = {env:VAL} + """) + assert len(config.envconfigs) == 1 + envconfig = config.envconfigs['python'] + assert 'VAL' in envconfig.setenv + assert envconfig.setenv['VAL'] == envconfig.envdir + assert envconfig.basepython == envconfig.envdir + + def test_setenv_ordering_1(self, tmpdir, newconfig): + config = newconfig(""" + [testenv] + setenv= + VAL={envdir} + commands=echo {env:VAL} + """) + assert len(config.envconfigs) == 1 + envconfig = config.envconfigs['python'] + assert 'VAL' in envconfig.setenv + assert envconfig.setenv['VAL'] == envconfig.envdir + assert str(envconfig.envdir) in envconfig.commands[0] + + @pytest.mark.xfail(reason="we don't implement cross-section substitution for setenv") + def test_setenv_cross_section_subst(self, monkeypatch, newconfig): + """test that we can do cross-section substitution with setenv""" + monkeypatch.delenv('TEST', raising=False) + config = newconfig(""" + [section] + x = + NOT_TEST={env:TEST:defaultvalue} + + [testenv] + setenv = {[section]x} + """) + envconfig = config.envconfigs["python"] + assert envconfig.setenv["NOT_TEST"] == "defaultvalue" + class TestIndexServer: def test_indexserver(self, tmpdir, newconfig): diff -r 128a5ad7a9a68825dc837b25e7594e2d92809c40 -r 1bf39138fe845f4a95092fc69dbfee4a7d854176 tox/__init__.py --- a/tox/__init__.py +++ b/tox/__init__.py @@ -1,5 +1,5 @@ # -__version__ = '2.3.0.dev1' +__version__ = '2.3.0.dev2' from .hookspecs import hookspec, hookimpl # noqa diff -r 128a5ad7a9a68825dc837b25e7594e2d92809c40 -r 1bf39138fe845f4a95092fc69dbfee4a7d854176 tox/config.py --- a/tox/config.py +++ b/tox/config.py @@ -271,7 +271,7 @@ except KeyError: try: if name in self._lookupstack: - raise KeyError("recursion") + raise KeyError(name) val = self.definitions[name] except KeyError: return os.environ.get(name, default) @@ -1028,7 +1028,8 @@ if envvalue is None: if default is None: raise tox.exception.ConfigError( - "substitution env:%r: unknown environment variable %r" % + "substitution env:%r: unknown environment variable %r " + " or recursive definition." % (envkey, envkey)) return default return envvalue https://bitbucket.org/hpk42/tox/commits/2565db0f6c44/ Changeset: 2565db0f6c44 Branch: issue285 User: hpk42 Date: 2015-12-07 11:41:10+00:00 Summary: merge default Affected #: 6 files diff -r 1bf39138fe845f4a95092fc69dbfee4a7d854176 -r 2565db0f6c44a138226ee990b295c4cc8f53003d CHANGELOG --- a/CHANGELOG +++ b/CHANGELOG @@ -13,6 +13,9 @@ - fix issue289: fix build_sphinx target, thanks Barry Warsaw. +- fix issue252: allow environment names with special characters. + Thanks Julien Castets for initial PR and patience. + 2.2.1 ----- diff -r 1bf39138fe845f4a95092fc69dbfee4a7d854176 -r 2565db0f6c44a138226ee990b295c4cc8f53003d doc/config-v2.txt --- a/doc/config-v2.txt +++ b/doc/config-v2.txt @@ -143,7 +143,7 @@ A testenv can define a new ``platform`` setting. If its value is not contained in the string obtained from calling -``platform.platform()`` the environment will be skipped. +``sys.platform`` the environment will be skipped. Expanding the ``envlist`` setting ---------------------------------------------------------- diff -r 1bf39138fe845f4a95092fc69dbfee4a7d854176 -r 2565db0f6c44a138226ee990b295c4cc8f53003d tests/test_z_cmdline.py --- a/tests/test_z_cmdline.py +++ b/tests/test_z_cmdline.py @@ -301,6 +301,23 @@ ]) +def test_venv_special_chars_issue252(cmd, initproj): + initproj("pkg123-0.7", filedefs={ + 'tests': {'test_hello.py': "def test_hello(): pass"}, + 'tox.ini': ''' + [tox] + envlist = special&&1 + [testenv:special&&1] + changedir=tests + ''' + }) + result = cmd.run("tox", ) + assert result.ret == 0 + result.stdout.fnmatch_lines([ + "*installed*pkg123*" + ]) + + def test_unknown_environment(cmd, initproj): initproj("env123-0.7", filedefs={ 'tox.ini': '' diff -r 1bf39138fe845f4a95092fc69dbfee4a7d854176 -r 2565db0f6c44a138226ee990b295c4cc8f53003d tox/session.py --- a/tox/session.py +++ b/tox/session.py @@ -533,11 +533,11 @@ action = self.newaction(venv, "envreport") with action: pip = venv.getcommandpath("pip") - # we can't really call internal helpers here easily :/ - # output = venv._pcall([str(pip), "freeze"], - # cwd=self.config.toxinidir, - # action=action) - output = py.process.cmdexec("%s freeze" % (pip)) + output = venv._pcall([str(pip), "freeze"], + cwd=self.config.toxinidir, + action=action) + # the output contains a mime-header, skip it + output = output.split("\n\n")[-1] packages = output.strip().split("\n") action.setactivity("installed", ",".join(packages)) envlog = self.resultlog.get_envlog(venv.name) https://bitbucket.org/hpk42/tox/commits/bcddfde43e80/ Changeset: bcddfde43e80 Branch: issue285 User: hpk42 Date: 2015-12-07 11:49:18+00:00 Summary: add py35 to default test envs and automize versions generated in doc Affected #: 4 files diff -r 2565db0f6c44a138226ee990b295c4cc8f53003d -r bcddfde43e804ea8284cfa2e29a47dcb31a8bc62 doc/Makefile --- a/doc/Makefile +++ b/doc/Makefile @@ -12,6 +12,8 @@ PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +SITETARGET=$(shell ./_getdoctarget.py) + .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest help: @@ -36,8 +38,10 @@ clean: -rm -rf $(BUILDDIR)/* + install: clean html @rsync -avz $(BUILDDIR)/html/ testrun.org:/www/testrun.org/tox/latest + @rsync -avz $(BUILDDIR)/html/ testrun.org:/www/testrun.org/tox/$(SITETARGET) #dev #latexpdf #@scp $(BUILDDIR)/latex/*.pdf testrun.org:www-tox/latest diff -r 2565db0f6c44a138226ee990b295c4cc8f53003d -r bcddfde43e804ea8284cfa2e29a47dcb31a8bc62 doc/_getdoctarget.py --- /dev/null +++ b/doc/_getdoctarget.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python + +import py + +def get_version_string(): + fn = py.path.local(__file__).join("..", "..", + "tox", "__init__.py") + for line in fn.readlines(): + if "version" in line and not line.strip().startswith('#'): + return eval(line.split("=")[-1]) + +def get_minor_version_string(): + return ".".join(get_version_string().split(".")[:2]) + +if __name__ == "__main__": + print (get_minor_version_string()) diff -r 2565db0f6c44a138226ee990b295c4cc8f53003d -r bcddfde43e804ea8284cfa2e29a47dcb31a8bc62 doc/conf.py --- a/doc/conf.py +++ b/doc/conf.py @@ -13,6 +13,13 @@ import sys, os +# The short X.Y version. +sys.path.insert(0, os.path.dirname(__file__)) +import _getdoctarget + +version = _getdoctarget.get_minor_version_string() +release = _getdoctarget.get_version_string() + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. @@ -47,9 +54,6 @@ # |version| and |release|, also used in various other places throughout the # built documents. # -# The short X.Y version. -release = "2.2" -version = "2.2.0" # The full version, including alpha/beta/rc tags. # The language for content autogenerated by Sphinx. Refer to documentation diff -r 2565db0f6c44a138226ee990b295c4cc8f53003d -r bcddfde43e804ea8284cfa2e29a47dcb31a8bc62 tox.ini --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist=py27,py26,py34,py33,pypy,flakes,py26-bare +envlist=py27,py26,py34,py33,py35,pypy,flakes,py26-bare [testenv:X] commands=echo {posargs} Repository URL: https://bitbucket.org/hpk42/tox/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email. _______________________________________________ pytest-commit mailing list pytest-commit@python.org https://mail.python.org/mailman/listinfo/pytest-commit