6 new commits in tox: https://bitbucket.org/hpk42/tox/commits/1de999d6ecf9/ Changeset: 1de999d6ecf9 Branch: pluggy User: hpk42 Date: 2015-05-11 09:26:07+00:00 Summary: add changelog entry for pluggy Affected #: 1 file
diff -r 5bf717febabda1de4f8db338e983832d54c25e59 -r 1de999d6ecf98715ce179636a522b86f5dd9b8dc CHANGELOG --- a/CHANGELOG +++ b/CHANGELOG @@ -32,6 +32,9 @@ - fix issue240: allow to specify empty argument list without it being rewritten to ".". Thanks Daniel Hahler. +- introduce experimental (not much documented yet) plugin system + based on pytest's externalized "pluggy" system. + See tox/hookspecs.py for the current hooks. 1.9.2 ----------- https://bitbucket.org/hpk42/tox/commits/af0751f4403a/ Changeset: af0751f4403a Branch: abort_by_default_when_a_command_fails User: hpk42 Date: 2015-05-11 10:11:57+00:00 Summary: close branch Affected #: 0 files https://bitbucket.org/hpk42/tox/commits/7f1bcc90f673/ Changeset: 7f1bcc90f673 Branch: pluggy User: hpk42 Date: 2015-05-11 10:06:39+00:00 Summary: refactor testenv section parser to work by registering ini attributes at tox_addoption() time. Introduce new "--help-ini" or "--hi" option to show all testenv variables. Affected #: 7 files diff -r 1de999d6ecf98715ce179636a522b86f5dd9b8dc -r 7f1bcc90f673122444ce1efac01206da661c84ff CHANGELOG --- a/CHANGELOG +++ b/CHANGELOG @@ -36,6 +36,10 @@ based on pytest's externalized "pluggy" system. See tox/hookspecs.py for the current hooks. +- introduce parser.add_testenv_attribute() to register an ini-variable + for testenv sections. Can be used from plugins through the + tox_add_option hook. + 1.9.2 ----------- diff -r 1de999d6ecf98715ce179636a522b86f5dd9b8dc -r 7f1bcc90f673122444ce1efac01206da661c84ff tests/test_config.py --- a/tests/test_config.py +++ b/tests/test_config.py @@ -18,7 +18,7 @@ assert config.toxworkdir.realpath() == tmpdir.join(".tox").realpath() assert config.envconfigs['py1'].basepython == sys.executable assert config.envconfigs['py1'].deps == [] - assert not config.envconfigs['py1'].platform + assert config.envconfigs['py1'].platform == ".*" def test_config_parsing_multienv(self, tmpdir, newconfig): config = newconfig([], """ @@ -92,12 +92,12 @@ """ Ensure correct parseini._is_same_dep is working with a few samples. """ - assert parseini._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3') - assert parseini._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3>=2.0') - assert parseini._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3>2.0') - assert parseini._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3<2.0') - assert parseini._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3<=2.0') - assert not parseini._is_same_dep('pkg_hello-world3==1.0', 'otherpkg>=2.0') + assert DepOption._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3') + assert DepOption._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3>=2.0') + assert DepOption._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3>2.0') + assert DepOption._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3<2.0') + assert DepOption._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3<=2.0') + assert not DepOption._is_same_dep('pkg_hello-world3==1.0', 'otherpkg>=2.0') class TestConfigPlatform: @@ -219,8 +219,8 @@ commands = echo {[section]key} """) - reader = IniReader(config._cfg) - x = reader.getargvlist("testenv", "commands") + reader = SectionReader("testenv", config._cfg) + x = reader.getargvlist("commands") assert x == [["echo", "whatever"]] def test_command_substitution_from_other_section_multiline(self, newconfig): @@ -244,8 +244,8 @@ # comment is omitted echo {[base]commands} """) - reader = IniReader(config._cfg) - x = reader.getargvlist("testenv", "commands") + reader = SectionReader("testenv", config._cfg) + x = reader.getargvlist("commands") assert x == [ "cmd1 param11 param12".split(), "cmd2 param21 param22".split(), @@ -256,16 +256,16 @@ class TestIniParser: - def test_getdefault_single(self, tmpdir, newconfig): + def test_getstring_single(self, tmpdir, newconfig): config = newconfig(""" [section] key=value """) - reader = IniReader(config._cfg) - x = reader.getdefault("section", "key") + reader = SectionReader("section", config._cfg) + x = reader.getstring("key") assert x == "value" - assert not reader.getdefault("section", "hello") - x = reader.getdefault("section", "hello", "world") + assert not reader.getstring("hello") + x = reader.getstring("hello", "world") assert x == "world" def test_missing_substitution(self, tmpdir, newconfig): @@ -273,40 +273,40 @@ [mydefault] key2={xyz} """) - reader = IniReader(config._cfg, fallbacksections=['mydefault']) + reader = SectionReader("mydefault", config._cfg, fallbacksections=['mydefault']) assert reader is not None - py.test.raises(tox.exception.ConfigError, - 'reader.getdefault("mydefault", "key2")') + with py.test.raises(tox.exception.ConfigError): + reader.getstring("key2") - def test_getdefault_fallback_sections(self, tmpdir, newconfig): + def test_getstring_fallback_sections(self, tmpdir, newconfig): config = newconfig(""" [mydefault] key2=value2 [section] key=value """) - reader = IniReader(config._cfg, fallbacksections=['mydefault']) - x = reader.getdefault("section", "key2") + reader = SectionReader("section", config._cfg, fallbacksections=['mydefault']) + x = reader.getstring("key2") assert x == "value2" - x = reader.getdefault("section", "key3") + x = reader.getstring("key3") assert not x - x = reader.getdefault("section", "key3", "world") + x = reader.getstring("key3", "world") assert x == "world" - def test_getdefault_substitution(self, tmpdir, newconfig): + def test_getstring_substitution(self, tmpdir, newconfig): config = newconfig(""" [mydefault] key2={value2} [section] key={value} """) - reader = IniReader(config._cfg, fallbacksections=['mydefault']) + reader = SectionReader("section", config._cfg, fallbacksections=['mydefault']) reader.addsubstitutions(value="newvalue", value2="newvalue2") - x = reader.getdefault("section", "key2") + x = reader.getstring("key2") assert x == "newvalue2" - x = reader.getdefault("section", "key3") + x = reader.getstring("key3") assert not x - x = reader.getdefault("section", "key3", "{value2}") + x = reader.getstring("key3", "{value2}") assert x == "newvalue2" def test_getlist(self, tmpdir, newconfig): @@ -316,9 +316,9 @@ item1 {item2} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions(item1="not", item2="grr") - x = reader.getlist("section", "key2") + x = reader.getlist("key2") assert x == ['item1', 'grr'] def test_getdict(self, tmpdir, newconfig): @@ -328,28 +328,31 @@ key1=item1 key2={item2} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions(item1="not", item2="grr") - x = reader.getdict("section", "key2") + x = reader.getdict("key2") assert 'key1' in x assert 'key2' in x assert x['key1'] == 'item1' assert x['key2'] == 'grr' - def test_getdefault_environment_substitution(self, monkeypatch, newconfig): + x = reader.getdict("key3", {1: 2}) + assert x == {1: 2} + + def test_getstring_environment_substitution(self, monkeypatch, newconfig): monkeypatch.setenv("KEY1", "hello") config = newconfig(""" [section] key1={env:KEY1} key2={env:KEY2} """) - reader = IniReader(config._cfg) - x = reader.getdefault("section", "key1") + reader = SectionReader("section", config._cfg) + x = reader.getstring("key1") assert x == "hello" - py.test.raises(tox.exception.ConfigError, - 'reader.getdefault("section", "key2")') + with py.test.raises(tox.exception.ConfigError): + reader.getstring("key2") - def test_getdefault_environment_substitution_with_default(self, monkeypatch, newconfig): + def test_getstring_environment_substitution_with_default(self, monkeypatch, newconfig): monkeypatch.setenv("KEY1", "hello") config = newconfig(""" [section] @@ -357,12 +360,12 @@ key2={env:KEY2:DEFAULT_VALUE} key3={env:KEY3:} """) - reader = IniReader(config._cfg) - x = reader.getdefault("section", "key1") + reader = SectionReader("section", config._cfg) + x = reader.getstring("key1") assert x == "hello" - x = reader.getdefault("section", "key2") + x = reader.getstring("key2") assert x == "DEFAULT_VALUE" - x = reader.getdefault("section", "key3") + x = reader.getstring("key3") assert x == "" def test_value_matches_section_substituion(self): @@ -373,15 +376,15 @@ assert is_section_substitution("{[setup]}") is None assert is_section_substitution("{[setup] commands}") is None - def test_getdefault_other_section_substitution(self, newconfig): + def test_getstring_other_section_substitution(self, newconfig): config = newconfig(""" [section] key = rue [testenv] key = t{[section]key} """) - reader = IniReader(config._cfg) - x = reader.getdefault("testenv", "key") + reader = SectionReader("testenv", config._cfg) + x = reader.getstring("key") assert x == "true" def test_argvlist(self, tmpdir, newconfig): @@ -391,12 +394,12 @@ cmd1 {item1} {item2} cmd2 {item2} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions(item1="with space", item2="grr") # py.test.raises(tox.exception.ConfigError, - # "reader.getargvlist('section', 'key1')") - assert reader.getargvlist('section', 'key1') == [] - x = reader.getargvlist("section", "key2") + # "reader.getargvlist('key1')") + assert reader.getargvlist('key1') == [] + x = reader.getargvlist("key2") assert x == [["cmd1", "with", "space", "grr"], ["cmd2", "grr"]] @@ -405,9 +408,9 @@ [section] comm = py.test {posargs} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions([r"hello\this"]) - argv = reader.getargv("section", "comm") + argv = reader.getargv("comm") assert argv == ["py.test", "hello\\this"] def test_argvlist_multiline(self, tmpdir, newconfig): @@ -417,12 +420,12 @@ cmd1 {item1} \ # a comment {item2} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions(item1="with space", item2="grr") # py.test.raises(tox.exception.ConfigError, - # "reader.getargvlist('section', 'key1')") - assert reader.getargvlist('section', 'key1') == [] - x = reader.getargvlist("section", "key2") + # "reader.getargvlist('key1')") + assert reader.getargvlist('key1') == [] + x = reader.getargvlist("key2") assert x == [["cmd1", "with", "space", "grr"]] def test_argvlist_quoting_in_command(self, tmpdir, newconfig): @@ -432,8 +435,8 @@ cmd1 'with space' \ # a comment 'after the comment' """) - reader = IniReader(config._cfg) - x = reader.getargvlist("section", "key1") + reader = SectionReader("section", config._cfg) + x = reader.getargvlist("key1") assert x == [["cmd1", "with space", "after the comment"]] def test_argvlist_positional_substitution(self, tmpdir, newconfig): @@ -444,22 +447,22 @@ cmd2 {posargs:{item2} \ other} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) posargs = ['hello', 'world'] reader.addsubstitutions(posargs, item2="value2") # py.test.raises(tox.exception.ConfigError, - # "reader.getargvlist('section', 'key1')") - assert reader.getargvlist('section', 'key1') == [] - argvlist = reader.getargvlist("section", "key2") + # "reader.getargvlist('key1')") + assert reader.getargvlist('key1') == [] + argvlist = reader.getargvlist("key2") assert argvlist[0] == ["cmd1"] + posargs assert argvlist[1] == ["cmd2"] + posargs - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions([], item2="value2") # py.test.raises(tox.exception.ConfigError, - # "reader.getargvlist('section', 'key1')") - assert reader.getargvlist('section', 'key1') == [] - argvlist = reader.getargvlist("section", "key2") + # "reader.getargvlist('key1')") + assert reader.getargvlist('key1') == [] + argvlist = reader.getargvlist("key2") assert argvlist[0] == ["cmd1"] assert argvlist[1] == ["cmd2", "value2", "other"] @@ -471,10 +474,10 @@ cmd2 -f '{posargs}' cmd3 -f {posargs} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions(["foo", "bar"]) - assert reader.getargvlist('section', 'key1') == [] - x = reader.getargvlist("section", "key2") + assert reader.getargvlist('key1') == [] + x = reader.getargvlist("key2") assert x == [["cmd1", "--foo-args=foo bar"], ["cmd2", "-f", "foo bar"], ["cmd3", "-f", "foo", "bar"]] @@ -485,10 +488,10 @@ key2= cmd1 -f {posargs} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions(["foo", "'bar", "baz'"]) - assert reader.getargvlist('section', 'key1') == [] - x = reader.getargvlist("section", "key2") + assert reader.getargvlist('key1') == [] + x = reader.getargvlist("key2") assert x == [["cmd1", "-f", "foo", "bar baz"]] def test_positional_arguments_are_only_replaced_when_standing_alone(self, tmpdir, newconfig): @@ -500,11 +503,11 @@ cmd2 -m '\'something\'' [] cmd3 something[]else """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) posargs = ['hello', 'world'] reader.addsubstitutions(posargs) - argvlist = reader.getargvlist('section', 'key') + argvlist = reader.getargvlist('key') assert argvlist[0] == ['cmd0'] + posargs assert argvlist[1] == ['cmd1', '-m', '[abc]'] assert argvlist[2] == ['cmd2', '-m', "something"] + posargs @@ -516,32 +519,32 @@ key = py.test -n5 --junitxml={envlogdir}/junit-{envname}.xml [] """ config = newconfig(inisource) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) posargs = ['hello', 'world'] reader.addsubstitutions(posargs, envlogdir='ENV_LOG_DIR', envname='ENV_NAME') expected = [ 'py.test', '-n5', '--junitxml=ENV_LOG_DIR/junit-ENV_NAME.xml', 'hello', 'world' ] - assert reader.getargvlist('section', 'key')[0] == expected + assert reader.getargvlist('key')[0] == expected def test_getargv(self, newconfig): config = newconfig(""" [section] key=some command "with quoting" """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) expected = ['some', 'command', 'with quoting'] - assert reader.getargv('section', 'key') == expected + assert reader.getargv('key') == expected def test_getpath(self, tmpdir, newconfig): config = newconfig(""" [section] path1={HELLO} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions(toxinidir=tmpdir, HELLO="mypath") - x = reader.getpath("section", "path1", tmpdir) + x = reader.getpath("path1", tmpdir) assert x == tmpdir.join("mypath") def test_getbool(self, tmpdir, newconfig): @@ -553,13 +556,13 @@ key2a=falsE key5=yes """) - reader = IniReader(config._cfg) - assert reader.getbool("section", "key1") is True - assert reader.getbool("section", "key1a") is True - assert reader.getbool("section", "key2") is False - assert reader.getbool("section", "key2a") is False - py.test.raises(KeyError, 'reader.getbool("section", "key3")') - py.test.raises(tox.exception.ConfigError, 'reader.getbool("section", "key5")') + reader = SectionReader("section", config._cfg) + assert reader.getbool("key1") is True + assert reader.getbool("key1a") is True + assert reader.getbool("key2") is False + assert reader.getbool("key2a") is False + py.test.raises(KeyError, 'reader.getbool("key3")') + py.test.raises(tox.exception.ConfigError, 'reader.getbool("key5")') class TestConfigTestEnv: @@ -585,7 +588,7 @@ assert envconfig.commands == [["xyz", "--abc"]] assert envconfig.changedir == config.setupdir assert envconfig.sitepackages is False - assert envconfig.develop is False + assert envconfig.usedevelop is False assert envconfig.envlogdir == envconfig.envdir.join("log") assert list(envconfig.setenv.keys()) == ['PYTHONHASHSEED'] hashseed = envconfig.setenv['PYTHONHASHSEED'] @@ -605,7 +608,7 @@ [testenv] usedevelop = True """) - assert not config.envconfigs["python"].develop + assert not config.envconfigs["python"].usedevelop def test_specific_command_overrides(self, tmpdir, newconfig): config = newconfig(""" @@ -1371,7 +1374,7 @@ def test_noset(self, tmpdir, newconfig): args = ['--hashseed', 'noset'] envconfig = self._get_envconfig(newconfig, args=args) - assert envconfig.setenv is None + assert envconfig.setenv == {} def test_noset_with_setenv(self, tmpdir, newconfig): tox_ini = """ diff -r 1de999d6ecf98715ce179636a522b86f5dd9b8dc -r 7f1bcc90f673122444ce1efac01206da661c84ff tests/test_venv.py --- a/tests/test_venv.py +++ b/tests/test_venv.py @@ -464,7 +464,7 @@ venv = VirtualEnv(envconfig, session=mocksession) venv.update() cconfig = venv._getliveconfig() - cconfig.develop = True + cconfig.usedevelop = True cconfig.writeconfig(venv.path_config) mocksession._clearmocks() venv.update() diff -r 1de999d6ecf98715ce179636a522b86f5dd9b8dc -r 7f1bcc90f673122444ce1efac01206da661c84ff tox.ini --- a/tox.ini +++ b/tox.ini @@ -5,8 +5,10 @@ commands=echo {posargs} [testenv] -commands=py.test --junitxml={envlogdir}/junit-{envname}.xml {posargs} +commands= py.test --timeout=60 {posargs} + deps=pytest>=2.3.5 + pytest-timeout [testenv:docs] basepython=python @@ -14,11 +16,10 @@ deps=sphinx {[testenv]deps} commands= - py.test -v \ - --junitxml={envlogdir}/junit-{envname}.xml \ - check_sphinx.py {posargs} + py.test -v check_sphinx.py {posargs} [testenv:flakes] +qwe = 123 deps = pytest-flakes>=0.2 pytest-pep8 diff -r 1de999d6ecf98715ce179636a522b86f5dd9b8dc -r 7f1bcc90f673122444ce1efac01206da661c84ff tox/_cmdline.py --- a/tox/_cmdline.py +++ b/tox/_cmdline.py @@ -25,12 +25,34 @@ def main(args=None): try: config = parseconfig(args) + if config.option.help: + show_help(config) + raise SystemExit(0) + elif config.option.helpini: + show_help_ini(config) + raise SystemExit(0) retcode = Session(config).runcommand() raise SystemExit(retcode) except KeyboardInterrupt: raise SystemExit(2) +def show_help(config): + tw = py.io.TerminalWriter() + tw.write(config._parser.format_help()) + tw.line() + + +def show_help_ini(config): + tw = py.io.TerminalWriter() + tw.sep("-", "per-testenv attributes") + for env_attr in config._testenv_attr: + tw.line("%-15s %-8s default: %s" % + (env_attr.name, "<" + env_attr.type + ">", env_attr.default), bold=True) + tw.line(env_attr.help) + tw.line() + + class Action(object): def __init__(self, session, venv, msg, args): self.venv = venv @@ -487,7 +509,7 @@ venv.status = "platform mismatch" continue # we simply omit non-matching platforms if self.setupenv(venv): - if venv.envconfig.develop: + if venv.envconfig.usedevelop: self.developpkg(venv, self.config.setupdir) elif self.config.skipsdist or venv.envconfig.skip_install: self.finishvenv(venv) @@ -566,7 +588,7 @@ self.report.line(" deps=%s" % envconfig.deps) self.report.line(" envdir= %s" % envconfig.envdir) self.report.line(" downloadcache=%s" % envconfig.downloadcache) - self.report.line(" usedevelop=%s" % envconfig.develop) + self.report.line(" usedevelop=%s" % envconfig.usedevelop) def showenvs(self): for env in self.config.envlist: diff -r 1de999d6ecf98715ce179636a522b86f5dd9b8dc -r 7f1bcc90f673122444ce1efac01206da661c84ff tox/_config.py --- a/tox/_config.py +++ b/tox/_config.py @@ -38,6 +38,122 @@ return pm +class MyParser: + def __init__(self): + self.argparser = argparse.ArgumentParser( + description="tox options", add_help=False) + self._testenv_attr = [] + + def add_argument(self, *args, **kwargs): + return self.argparser.add_argument(*args, **kwargs) + + def add_testenv_attribute(self, name, type, help, default=None, postprocess=None): + self._testenv_attr.append(VenvAttribute(name, type, default, help, postprocess)) + + def add_testenv_attribute_obj(self, obj): + assert hasattr(obj, "name") + assert hasattr(obj, "type") + assert hasattr(obj, "help") + assert hasattr(obj, "postprocess") + self._testenv_attr.append(obj) + + def parse_args(self, args): + return self.argparser.parse_args(args) + + def format_help(self): + return self.argparser.format_help() + + +class VenvAttribute: + def __init__(self, name, type, default, help, postprocess): + self.name = name + self.type = type + self.default = default + self.help = help + self.postprocess = postprocess + + +class DepOption: + name = "deps" + type = "line-list" + help = "each line specifies a dependency in pip/setuptools format." + default = () + + def postprocess(self, config, reader, section_val): + deps = [] + for depline in section_val: + m = re.match(r":(\w+):\s*(\S+)", depline) + if m: + iname, name = m.groups() + ixserver = config.indexserver[iname] + else: + name = depline.strip() + ixserver = None + name = self._replace_forced_dep(name, config) + deps.append(DepConfig(name, ixserver)) + return deps + + def _replace_forced_dep(self, name, config): + """ + Override the given dependency config name taking --force-dep-version + option into account. + + :param name: dep config, for example ["pkg==1.0", "other==2.0"]. + :param config: Config instance + :return: the new dependency that should be used for virtual environments + """ + if not config.option.force_dep: + return name + for forced_dep in config.option.force_dep: + if self._is_same_dep(forced_dep, name): + return forced_dep + return name + + @classmethod + def _is_same_dep(cls, dep1, dep2): + """ + Returns True if both dependency definitions refer to the + same package, even if versions differ. + """ + dep1_name = pkg_resources.Requirement.parse(dep1).project_name + dep2_name = pkg_resources.Requirement.parse(dep2).project_name + return dep1_name == dep2_name + + +class PosargsOption: + name = "args_are_paths" + type = "bool" + default = True + help = "treat positional args in commands as paths" + + def postprocess(self, config, reader, section_val): + args = config.option.args + if args: + if section_val: + args = [] + for arg in config.option.args: + if arg: + origpath = config.invocationcwd.join(arg, abs=True) + if origpath.check(): + arg = reader.getpath("changedir", ".").bestrelpath(origpath) + args.append(arg) + reader.addsubstitutions(args) + return section_val + + +class InstallcmdOption: + name = "install_command" + type = "argv" + default = "pip install {opts} {packages}" + help = "install command for dependencies and package under test." + + def postprocess(self, config, reader, section_val): + if '{packages}' not in section_val: + raise tox.exception.ConfigError( + "'install_command' must contain '{packages}' substitution") + return section_val + + def parseconfig(args=None): """ :param list[str] args: Optional list of arguments. @@ -52,13 +168,15 @@ args = sys.argv[1:] # prepare command line options - parser = argparse.ArgumentParser(description=__doc__) + parser = MyParser() pm.hook.tox_addoption(parser=parser) # parse command line options option = parser.parse_args(args) interpreters = tox.interpreters.Interpreters(hook=pm.hook) config = Config(pluginmanager=pm, option=option, interpreters=interpreters) + config._parser = parser + config._testenv_attr = parser._testenv_attr # parse ini file basename = config.option.configfile @@ -111,6 +229,10 @@ parser.add_argument("--version", nargs=0, action=VersionAction, dest="version", help="report version information to stdout.") + parser.add_argument("-h", "--help", action="store_true", dest="help", + help="show help about options") + parser.add_argument("--help-ini", "--hi", action="store_true", dest="helpini", + help="show help about ini-names") parser.add_argument("-v", nargs=0, action=CountAction, default=0, dest="verbosity", help="increase verbosity of reporting output.") @@ -156,6 +278,7 @@ "all commands and results involved. This will turn off " "pass-through output from running test commands which is " "instead captured into the json result file.") + # We choose 1 to 4294967295 because it is the range of PYTHONHASHSEED. parser.add_argument("--hashseed", action="store", metavar="SEED", default=None, @@ -175,7 +298,130 @@ parser.add_argument("args", nargs="*", help="additional arguments available to command positional substitution") - return parser + + # add various core venv interpreter attributes + + parser.add_testenv_attribute( + name="envdir", type="path", default="{toxworkdir}/{envname}", + help="venv directory") + + parser.add_testenv_attribute( + name="envtmpdir", type="path", default="{envdir}/tmp", + help="venv temporary directory") + + parser.add_testenv_attribute( + name="envlogdir", type="path", default="{envdir}/log", + help="venv log directory") + + def downloadcache(config, reader, section_val): + if section_val: + # env var, if present, takes precedence + downloadcache = os.environ.get("PIP_DOWNLOAD_CACHE", section_val) + return py.path.local(downloadcache) + + parser.add_testenv_attribute( + name="downloadcache", type="string", default=None, postprocess=downloadcache, + help="(deprecated) set PIP_DOWNLOAD_CACHE.") + + parser.add_testenv_attribute( + name="changedir", type="path", default="{toxinidir}", + help="directory to change to when running commands") + + parser.add_testenv_attribute_obj(PosargsOption()) + + parser.add_testenv_attribute( + name="skip_install", type="bool", default=False, + help="Do not install the current package. This can be used when " + "you need the virtualenv management but do not want to install " + "the current package") + + def recreate(config, reader, section_val): + if config.option.recreate: + return True + return section_val + + parser.add_testenv_attribute( + name="recreate", type="bool", default=False, postprocess=recreate, + help="always recreate this test environment.") + + def setenv(config, reader, section_val): + setenv = section_val + if "PYTHONHASHSEED" not in setenv and config.hashseed is not None: + setenv['PYTHONHASHSEED'] = config.hashseed + return setenv + + parser.add_testenv_attribute( + name="setenv", type="dict", postprocess=setenv, + help="list of X=Y lines with environment variable settings") + + def passenv(config, reader, section_val): + passenv = set(["PATH"]) + if sys.platform == "win32": + passenv.add("SYSTEMROOT") # needed for python's crypto module + passenv.add("PATHEXT") # needed for discovering executables + for spec in section_val: + for name in os.environ: + if fnmatchcase(name.upper(), spec.upper()): + passenv.add(name) + return passenv + + parser.add_testenv_attribute( + name="passenv", type="space-separated-list", postprocess=passenv, + help="environment variables names which shall be passed " + "from tox invocation to test environment when executing commands.") + + parser.add_testenv_attribute( + name="whitelist_externals", type="line-list", + help="each lines specifies a path or basename for which tox will not warn " + "about it coming from outside the test environment.") + + parser.add_testenv_attribute( + name="platform", type="string", default=".*", + help="regular expression which must match against ``sys.platform``. " + "otherwise testenv will be skipped.") + + def sitepackages(config, reader, section_val): + return config.option.sitepackages or section_val + + parser.add_testenv_attribute( + name="sitepackages", type="bool", default=False, postprocess=sitepackages, + help="Set to ``True`` if you want to create virtual environments that also " + "have access to globally installed packages.") + + def pip_pre(config, reader, section_val): + return config.option.pre or section_val + + parser.add_testenv_attribute( + name="pip_pre", type="bool", default=False, postprocess=pip_pre, + help="If ``True``, adds ``--pre`` to the ``opts`` passed to " + "the install command. ") + + def develop(config, reader, section_val): + return not config.option.installpkg and (section_val or config.option.develop) + + parser.add_testenv_attribute( + name="usedevelop", type="bool", postprocess=develop, default=False, + help="install package in develop/editable mode") + + def basepython_default(config, reader, section_val): + if section_val is None: + for f in reader.factors: + if f in default_factors: + return default_factors[f] + return sys.executable + return str(section_val) + + parser.add_testenv_attribute( + name="basepython", type="string", default=None, postprocess=basepython_default, + help="executable name or path of interpreter used to create a " + "virtual test environment.") + + parser.add_testenv_attribute_obj(InstallcmdOption()) + parser.add_testenv_attribute_obj(DepOption()) + + parser.add_testenv_attribute( + name="commands", type="argvlist", default="", + help="each line specifies a test command and can use substitution.") class Config(object): @@ -272,12 +518,10 @@ self.config = config ctxname = getcontextname() if ctxname == "jenkins": - reader = IniReader(self._cfg, fallbacksections=['tox']) - toxsection = "tox:%s" % ctxname + reader = SectionReader("tox:jenkins", self._cfg, fallbacksections=['tox']) distshare_default = "{toxworkdir}/distshare" elif not ctxname: - reader = IniReader(self._cfg) - toxsection = "tox" + reader = SectionReader("tox", self._cfg) distshare_default = "{homedir}/.tox/distshare" else: raise ValueError("invalid context") @@ -292,18 +536,17 @@ reader.addsubstitutions(toxinidir=config.toxinidir, homedir=config.homedir) - config.toxworkdir = reader.getpath(toxsection, "toxworkdir", - "{toxinidir}/.tox") - config.minversion = reader.getdefault(toxsection, "minversion", None) + config.toxworkdir = reader.getpath("toxworkdir", "{toxinidir}/.tox") + config.minversion = reader.getstring("minversion", None) if not config.option.skip_missing_interpreters: config.option.skip_missing_interpreters = \ - reader.getbool(toxsection, "skip_missing_interpreters", False) + reader.getbool("skip_missing_interpreters", False) # determine indexserver dictionary config.indexserver = {'default': IndexServerConfig('default')} prefix = "indexserver" - for line in reader.getlist(toxsection, prefix): + for line in reader.getlist(prefix): name, url = map(lambda x: x.strip(), line.split("=", 1)) config.indexserver[name] = IndexServerConfig(name, url) @@ -328,16 +571,15 @@ config.indexserver[name] = IndexServerConfig(name, override) reader.addsubstitutions(toxworkdir=config.toxworkdir) - config.distdir = reader.getpath(toxsection, "distdir", "{toxworkdir}/dist") + config.distdir = reader.getpath("distdir", "{toxworkdir}/dist") reader.addsubstitutions(distdir=config.distdir) - config.distshare = reader.getpath(toxsection, "distshare", - distshare_default) + config.distshare = reader.getpath("distshare", distshare_default) reader.addsubstitutions(distshare=config.distshare) - config.sdistsrc = reader.getpath(toxsection, "sdistsrc", None) - config.setupdir = reader.getpath(toxsection, "setupdir", "{toxinidir}") + config.sdistsrc = reader.getpath("sdistsrc", None) + config.setupdir = reader.getpath("setupdir", "{toxinidir}") config.logdir = config.toxworkdir.join("log") - config.envlist, all_envs = self._getenvdata(reader, toxsection) + config.envlist, all_envs = self._getenvdata(reader) # factors used in config or predefined known_factors = self._list_section_factors("testenv") @@ -345,7 +587,7 @@ known_factors.add("python") # factors stated in config envlist - stated_envlist = reader.getdefault(toxsection, "envlist", replace=False) + stated_envlist = reader.getstring("envlist", replace=False) if stated_envlist: for env in _split_env(stated_envlist): known_factors.update(env.split('-')) @@ -356,13 +598,13 @@ factors = set(name.split('-')) if section in self._cfg or factors <= known_factors: config.envconfigs[name] = \ - self._makeenvconfig(name, section, reader._subs, config) + self.make_envconfig(name, section, reader._subs, config) all_develop = all(name in config.envconfigs - and config.envconfigs[name].develop + and config.envconfigs[name].usedevelop for name in config.envlist) - config.skipsdist = reader.getbool(toxsection, "skipsdist", all_develop) + config.skipsdist = reader.getbool("skipsdist", all_develop) def _list_section_factors(self, section): factors = set() @@ -372,115 +614,51 @@ factors.update(*mapcat(_split_factor_expr, exprs)) return factors - def _makeenvconfig(self, name, section, subs, config): + def make_envconfig(self, name, section, subs, config): vc = VenvConfig(config=config, envname=name) factors = set(name.split('-')) - reader = IniReader(self._cfg, fallbacksections=["testenv"], factors=factors) + reader = SectionReader(section, self._cfg, fallbacksections=["testenv"], + factors=factors) reader.addsubstitutions(**subs) - vc.develop = ( - not config.option.installpkg - and reader.getbool(section, "usedevelop", config.option.develop)) - vc.envdir = reader.getpath(section, "envdir", "{toxworkdir}/%s" % name) - vc.args_are_paths = reader.getbool(section, "args_are_paths", True) - if reader.getdefault(section, "python", None): - raise tox.exception.ConfigError( - "'python=' key was renamed to 'basepython='") - bp = next((default_factors[f] for f in factors if f in default_factors), - sys.executable) - vc.basepython = reader.getdefault(section, "basepython", bp) + reader.addsubstitutions(envname=name) - reader.addsubstitutions(envdir=vc.envdir, envname=vc.envname, - envbindir=vc.envbindir, envpython=vc.envpython, - envsitepackagesdir=vc.envsitepackagesdir) - vc.envtmpdir = reader.getpath(section, "tmpdir", "{envdir}/tmp") - vc.envlogdir = reader.getpath(section, "envlogdir", "{envdir}/log") - reader.addsubstitutions(envlogdir=vc.envlogdir, envtmpdir=vc.envtmpdir) - vc.changedir = reader.getpath(section, "changedir", "{toxinidir}") - if config.option.recreate: - vc.recreate = True - else: - vc.recreate = reader.getbool(section, "recreate", False) - args = config.option.args - if args: - if vc.args_are_paths: - args = [] - for arg in config.option.args: - if arg: - origpath = config.invocationcwd.join(arg, abs=True) - if origpath.check(): - arg = vc.changedir.bestrelpath(origpath) - args.append(arg) - reader.addsubstitutions(args) - setenv = {} - if config.hashseed is not None: - setenv['PYTHONHASHSEED'] = config.hashseed - setenv.update(reader.getdict(section, 'setenv')) + for env_attr in config._testenv_attr: + atype = env_attr.type + if atype in ("bool", "path", "string", "dict", "argv", "argvlist"): + meth = getattr(reader, "get" + atype) + res = meth(env_attr.name, env_attr.default) + elif atype == "space-separated-list": + res = reader.getlist(env_attr.name, sep=" ") + elif atype == "line-list": + res = reader.getlist(env_attr.name, sep="\n") + else: + raise ValueError("unknown type %r" % (atype,)) - # read passenv - vc.passenv = set(["PATH"]) - if sys.platform == "win32": - vc.passenv.add("SYSTEMROOT") # needed for python's crypto module - vc.passenv.add("PATHEXT") # needed for discovering executables - for spec in reader.getlist(section, "passenv", sep=" "): - for name in os.environ: - if fnmatchcase(name.lower(), spec.lower()): - vc.passenv.add(name) + if env_attr.postprocess: + res = env_attr.postprocess(config, reader, res) + setattr(vc, env_attr.name, res) - vc.setenv = setenv - if not vc.setenv: - vc.setenv = None + if atype == "path": + reader.addsubstitutions(**{env_attr.name: res}) - vc.commands = reader.getargvlist(section, "commands") - vc.whitelist_externals = reader.getlist(section, - "whitelist_externals") - vc.deps = [] - for depline in reader.getlist(section, "deps"): - m = re.match(r":(\w+):\s*(\S+)", depline) - if m: - iname, name = m.groups() - ixserver = config.indexserver[iname] - else: - name = depline.strip() - ixserver = None - name = self._replace_forced_dep(name, config) - vc.deps.append(DepConfig(name, ixserver)) + if env_attr.name == "install_command": + reader.addsubstitutions(envbindir=vc.envbindir, envpython=vc.envpython, + envsitepackagesdir=vc.envsitepackagesdir) - platform = "" - for platform in reader.getlist(section, "platform"): - if platform.strip(): - break - vc.platform = platform - - vc.sitepackages = ( - self.config.option.sitepackages - or reader.getbool(section, "sitepackages", False)) - - vc.downloadcache = None - downloadcache = reader.getdefault(section, "downloadcache") - if downloadcache: - # env var, if present, takes precedence - downloadcache = os.environ.get("PIP_DOWNLOAD_CACHE", downloadcache) - vc.downloadcache = py.path.local(downloadcache) - - vc.install_command = reader.getargv( - section, - "install_command", - "pip install {opts} {packages}", - ) - if '{packages}' not in vc.install_command: - raise tox.exception.ConfigError( - "'install_command' must contain '{packages}' substitution") - vc.pip_pre = config.option.pre or reader.getbool( - section, "pip_pre", False) - - vc.skip_install = reader.getbool(section, "skip_install", False) - + # XXX introduce some testenv verification like this: + # try: + # sec = self._cfg[section] + # except KeyError: + # sec = self._cfg["testenv"] + # for name in sec: + # if name not in names: + # print ("unknown testenv attribute: %r" % (name,)) return vc - def _getenvdata(self, reader, toxsection): + def _getenvdata(self, reader): envstr = self.config.option.env \ or os.environ.get("TOXENV") \ - or reader.getdefault(toxsection, "envlist", replace=False) \ + or reader.getstring("envlist", replace=False) \ or [] envlist = _split_env(envstr) @@ -497,32 +675,6 @@ return envlist, all_envs - def _replace_forced_dep(self, name, config): - """ - Override the given dependency config name taking --force-dep-version - option into account. - - :param name: dep config, for example ["pkg==1.0", "other==2.0"]. - :param config: Config instance - :return: the new dependency that should be used for virtual environments - """ - if not config.option.force_dep: - return name - for forced_dep in config.option.force_dep: - if self._is_same_dep(forced_dep, name): - return forced_dep - return name - - @classmethod - def _is_same_dep(cls, dep1, dep2): - """ - Returns True if both dependency definitions refer to the - same package, even if versions differ. - """ - dep1_name = pkg_resources.Requirement.parse(dep1).project_name - dep2_name = pkg_resources.Requirement.parse(dep2).project_name - return dep1_name == dep2_name - def _split_env(env): """if handed a list, action="append" was used for -e """ @@ -589,8 +741,9 @@ re.VERBOSE) -class IniReader: - def __init__(self, cfgparser, fallbacksections=None, factors=()): +class SectionReader: + def __init__(self, section_name, cfgparser, fallbacksections=None, factors=()): + self.section_name = section_name self._cfg = cfgparser self.fallbacksections = fallbacksections or [] self.factors = factors @@ -602,129 +755,39 @@ if _posargs: self.posargs = _posargs - def getpath(self, section, name, defaultpath): + def getpath(self, name, defaultpath): toxinidir = self._subs['toxinidir'] - path = self.getdefault(section, name, defaultpath) + path = self.getstring(name, defaultpath) if path is None: return path return toxinidir.join(path, abs=True) - def getlist(self, section, name, sep="\n"): - s = self.getdefault(section, name, None) + def getlist(self, name, sep="\n"): + s = self.getstring(name, None) if s is None: return [] return [x.strip() for x in s.split(sep) if x.strip()] - def getdict(self, section, name, sep="\n"): - s = self.getdefault(section, name, None) + def getdict(self, name, default=None, sep="\n"): + s = self.getstring(name, None) if s is None: - return {} + return default or {} value = {} for line in s.split(sep): - if not line.strip(): - continue - name, rest = line.split('=', 1) - value[name.strip()] = rest.strip() + if line.strip(): + name, rest = line.split('=', 1) + value[name.strip()] = rest.strip() return value - def getargvlist(self, section, name): - """Get arguments for every parsed command. - - :param str section: Section name in the configuration. - :param str name: Key name in a section. - :rtype: list[list[str]] - :raise :class:`tox.exception.ConfigError`: - line-continuation ends nowhere while resolving for specified section - """ - content = self.getdefault(section, name, '', replace=False) - return self._parse_commands(section, name, content) - - def _parse_commands(self, section, name, content): - """Parse commands from key content in specified section. - - :param str section: Section name in the configuration. - :param str name: Key name in a section. - :param str content: Content stored by key. - - :rtype: list[list[str]] - :raise :class:`tox.exception.ConfigError`: - line-continuation ends nowhere while resolving for specified section - """ - commands = [] - current_command = "" - for line in content.splitlines(): - line = line.rstrip() - i = line.find("#") - if i != -1: - line = line[:i].rstrip() - if not line: - continue - if line.endswith("\\"): - current_command += " " + line[:-1] - continue - current_command += line - - if is_section_substitution(current_command): - replaced = self._replace(current_command) - commands.extend(self._parse_commands(section, name, replaced)) - else: - commands.append(self._processcommand(current_command)) - current_command = "" - else: - if current_command: - raise tox.exception.ConfigError( - "line-continuation ends nowhere while resolving for [%s] %s" % - (section, name)) - return commands - - def _processcommand(self, command): - posargs = getattr(self, "posargs", None) - - # Iterate through each word of the command substituting as - # appropriate to construct the new command string. This - # string is then broken up into exec argv components using - # shlex. - newcommand = "" - for word in CommandParser(command).words(): - if word == "{posargs}" or word == "[]": - if posargs: - newcommand += " ".join(posargs) - continue - elif word.startswith("{posargs:") and word.endswith("}"): - if posargs: - newcommand += " ".join(posargs) - continue - else: - word = word[9:-1] - new_arg = "" - new_word = self._replace(word) - new_word = self._replace(new_word) - new_arg += new_word - newcommand += new_arg - - # Construct shlex object that will not escape any values, - # use all values as is in argv. - shlexer = shlex.shlex(newcommand, posix=True) - shlexer.whitespace_split = True - shlexer.escape = '' - shlexer.commenters = '' - argv = list(shlexer) - return argv - - def getargv(self, section, name, default=None, replace=True): - command = self.getdefault( - section, name, default=default, replace=False) - return self._processcommand(command.strip()) - - def getbool(self, section, name, default=None): - s = self.getdefault(section, name, default) + def getbool(self, name, default=None): + s = self.getstring(name, default) if not s: s = default if s is None: raise KeyError("no config value [%s] %s found" % ( - section, name)) + self.section_name, name)) if not isinstance(s, bool): if s.lower() == "true": @@ -736,9 +799,16 @@ "boolean value %r needs to be 'True' or 'False'") return s - def getdefault(self, section, name, default=None, replace=True): + def getargvlist(self, name, default=""): + s = self.getstring(name, default, replace=False) + return _ArgvlistReader.getargvlist(self, s) + + def getargv(self, name, default=""): + return self.getargvlist(name, default)[0] + + def getstring(self, name, default=None, replace=True): x = None - for s in [section] + self.fallbacksections: + for s in [self.section_name] + self.fallbacksections: try: x = self._cfg[s][name] break @@ -751,12 +821,12 @@ x = self._apply_factors(x) if replace and x and hasattr(x, 'replace'): - self._subststack.append((section, name)) + self._subststack.append((self.section_name, name)) try: x = self._replace(x) finally: - assert self._subststack.pop() == (section, name) - # print "getdefault", section, name, "returned", repr(x) + assert self._subststack.pop() == (self.section_name, name) + # print "getstring", self.section_name, name, "returned", repr(x) return x def _apply_factors(self, s): @@ -852,8 +922,80 @@ return RE_ITEM_REF.sub(self._replace_match, x) return x - def _parse_command(self, command): - pass + +class _ArgvlistReader: + @classmethod + def getargvlist(cls, reader, section_val): + """Parse ``commands`` argvlist multiline string. + + :param str name: Key name in a section. + :param str section_val: Content stored by key. + + :rtype: list[list[str]] + :raise :class:`tox.exception.ConfigError`: + line-continuation ends nowhere while resolving for specified section + """ + commands = [] + current_command = "" + for line in section_val.splitlines(): + line = line.rstrip() + i = line.find("#") + if i != -1: + line = line[:i].rstrip() + if not line: + continue + if line.endswith("\\"): + current_command += " " + line[:-1] + continue + current_command += line + + if is_section_substitution(current_command): + replaced = reader._replace(current_command) + commands.extend(cls.getargvlist(reader, replaced)) + else: + commands.append(cls.processcommand(reader, current_command)) + current_command = "" + else: + if current_command: + raise tox.exception.ConfigError( + "line-continuation ends nowhere while resolving for [%s] %s" % + (reader.section_name, "commands")) + return commands + + @classmethod + def processcommand(cls, reader, command): + posargs = getattr(reader, "posargs", None) + + # Iterate through each word of the command substituting as + # appropriate to construct the new command string. This + # string is then broken up into exec argv components using + # shlex. + newcommand = "" + for word in CommandParser(command).words(): + if word == "{posargs}" or word == "[]": + if posargs: + newcommand += " ".join(posargs) + continue + elif word.startswith("{posargs:") and word.endswith("}"): + if posargs: + newcommand += " ".join(posargs) + continue + else: + word = word[9:-1] + new_arg = "" + new_word = reader._replace(word) + new_word = reader._replace(new_word) + new_arg += new_word + newcommand += new_arg + + # Construct shlex object that will not escape any values, + # use all values as is in argv. + shlexer = shlex.shlex(newcommand, posix=True) + shlexer.whitespace_split = True + shlexer.escape = '' + shlexer.commenters = '' + argv = list(shlexer) + return argv class CommandParser(object): diff -r 1de999d6ecf98715ce179636a522b86f5dd9b8dc -r 7f1bcc90f673122444ce1efac01206da661c84ff tox/_venv.py --- a/tox/_venv.py +++ b/tox/_venv.py @@ -10,17 +10,17 @@ class CreationConfig: def __init__(self, md5, python, version, sitepackages, - develop, deps): + usedevelop, deps): self.md5 = md5 self.python = python self.version = version self.sitepackages = sitepackages - self.develop = develop + self.usedevelop = usedevelop self.deps = deps def writeconfig(self, path): lines = ["%s %s" % (self.md5, self.python)] - lines.append("%s %d %d" % (self.version, self.sitepackages, self.develop)) + lines.append("%s %d %d" % (self.version, self.sitepackages, self.usedevelop)) for dep in self.deps: lines.append("%s %s" % dep) path.ensure() @@ -32,14 +32,14 @@ lines = path.readlines(cr=0) value = lines.pop(0).split(None, 1) md5, python = value - version, sitepackages, develop = lines.pop(0).split(None, 3) + version, sitepackages, usedevelop = lines.pop(0).split(None, 3) sitepackages = bool(int(sitepackages)) - develop = bool(int(develop)) + usedevelop = bool(int(usedevelop)) deps = [] for line in lines: md5, depstring = line.split(None, 1) deps.append((md5, depstring)) - return CreationConfig(md5, python, version, sitepackages, develop, deps) + return CreationConfig(md5, python, version, sitepackages, usedevelop, deps) except Exception: return None @@ -48,7 +48,7 @@ and self.python == other.python and self.version == other.version and self.sitepackages == other.sitepackages - and self.develop == other.develop + and self.usedevelop == other.usedevelop and self.deps == other.deps) @@ -147,7 +147,7 @@ md5 = getdigest(python) version = tox.__version__ sitepackages = self.envconfig.sitepackages - develop = self.envconfig.develop + develop = self.envconfig.usedevelop deps = [] for dep in self._getresolvedeps(): raw_dep = dep.name @@ -321,11 +321,13 @@ for envname in self.envconfig.passenv: if envname in os.environ: env[envname] = os.environ[envname] - setenv = self.envconfig.setenv - if setenv: - env.update(setenv) + + env.update(self.envconfig.setenv) + env['VIRTUAL_ENV'] = str(self.path) + env.update(extraenv) + return env def test(self, redirect=False): https://bitbucket.org/hpk42/tox/commits/c2e0dbe2fddd/ Changeset: c2e0dbe2fddd Branch: pluggy User: hpk42 Date: 2015-05-11 10:19:03+00:00 Summary: merge default Affected #: 6 files diff -r 7f1bcc90f673122444ce1efac01206da661c84ff -r c2e0dbe2fddd4d505b9690071608b819af1a9a08 CHANGELOG --- a/CHANGELOG +++ b/CHANGELOG @@ -17,6 +17,12 @@ If platform is set and doesn't match the platform spec in the test environment the test environment is ignored, no setup or tests are attempted. +.. (new) add per-venv "ignore_errors" setting, which defaults to False. + If ``True``, a non-zero exit code from one command will be ignored and + further commands will be executed (which was the default behavior in tox < + 2.0). If ``False`` (the default), then a non-zero exit code from one command + will abort execution of commands for that environment. + - remove the long-deprecated "distribute" option as it has no effect these days. - fix issue233: avoid hanging with tox-setuptools integration example. Thanks simonb. diff -r 7f1bcc90f673122444ce1efac01206da661c84ff -r c2e0dbe2fddd4d505b9690071608b819af1a9a08 doc/config.txt --- a/doc/config.txt +++ b/doc/config.txt @@ -110,6 +110,26 @@ pip install {opts} {packages} +.. confval:: ignore_errors=True|False(default) + + .. versionadded:: 2.0 + + If ``True``, a non-zero exit code from one command will be ignored and + further commands will be executed (which was the default behavior in tox < + 2.0). If ``False`` (the default), then a non-zero exit code from one command + will abort execution of commands for that environment. + + It may be helpful to note that this setting is analogous to the ``-i`` or + ``ignore-errors`` option of GNU Make. A similar name was chosen to reflect the + similarity in function. + + Note that in tox 2.0, the default behavior of tox with respect to + treating errors from commands changed. Tox < 2.0 would ignore errors by + default. Tox >= 2.0 will abort on an error by default, which is safer and more + typical of CI and command execution tools, as it doesn't make sense to + run tests if installing some prerequisite failed and it doesn't make sense to + try to deploy if tests failed. + .. confval:: pip_pre=True|False(default) .. versionadded:: 1.9 diff -r 7f1bcc90f673122444ce1efac01206da661c84ff -r c2e0dbe2fddd4d505b9690071608b819af1a9a08 tests/test_config.py --- a/tests/test_config.py +++ b/tests/test_config.py @@ -589,6 +589,7 @@ assert envconfig.changedir == config.setupdir assert envconfig.sitepackages is False assert envconfig.usedevelop is False + assert envconfig.ignore_errors is False assert envconfig.envlogdir == envconfig.envdir.join("log") assert list(envconfig.setenv.keys()) == ['PYTHONHASHSEED'] hashseed = envconfig.setenv['PYTHONHASHSEED'] @@ -649,6 +650,15 @@ assert envconfig.changedir.basename == "xyz" assert envconfig.changedir == config.toxinidir.join("xyz") + def test_ignore_errors(self, tmpdir, newconfig): + config = newconfig(""" + [testenv] + ignore_errors=True + """) + assert len(config.envconfigs) == 1 + envconfig = config.envconfigs['python'] + assert envconfig.ignore_errors is True + def test_envbindir(self, tmpdir, newconfig): config = newconfig(""" [testenv] diff -r 7f1bcc90f673122444ce1efac01206da661c84ff -r c2e0dbe2fddd4d505b9690071608b819af1a9a08 tox/_cmdline.py --- a/tox/_cmdline.py +++ b/tox/_cmdline.py @@ -179,7 +179,7 @@ raise tox.exception.InvocationError( "%s (see %s)" % (invoked, outpath), ret) else: - raise tox.exception.InvocationError("%r" % (invoked, )) + raise tox.exception.InvocationError("%r" % (invoked, ), ret) if not out and outpath: out = outpath.read() if hasattr(self, "commandlog"): diff -r 7f1bcc90f673122444ce1efac01206da661c84ff -r c2e0dbe2fddd4d505b9690071608b819af1a9a08 tox/_config.py --- a/tox/_config.py +++ b/tox/_config.py @@ -335,6 +335,11 @@ "you need the virtualenv management but do not want to install " "the current package") + parser.add_testenv_attribute( + name="ignore_errors", type="bool", default=False, + help="if set to True all commands will be executed irrespective of their " + "result error status.") + def recreate(config, reader, section_val): if config.option.recreate: return True @@ -644,15 +649,6 @@ if env_attr.name == "install_command": reader.addsubstitutions(envbindir=vc.envbindir, envpython=vc.envpython, envsitepackagesdir=vc.envsitepackagesdir) - - # XXX introduce some testenv verification like this: - # try: - # sec = self._cfg[section] - # except KeyError: - # sec = self._cfg["testenv"] - # for name in sec: - # if name not in names: - # print ("unknown testenv attribute: %r" % (name,)) return vc def _getenvdata(self, reader): diff -r 7f1bcc90f673122444ce1efac01206da661c84ff -r c2e0dbe2fddd4d505b9690071608b819af1a9a08 tox/_venv.py --- a/tox/_venv.py +++ b/tox/_venv.py @@ -363,6 +363,14 @@ val = sys.exc_info()[1] self.session.report.error(str(val)) self.status = "commands failed" + if not self.envconfig.ignore_errors: + self.session.report.error( + 'Stopping processing of commands for env %s ' + 'because `%s` failed with exit code %s' + % (self.name, + ' '.join([str(x) for x in argv]), + val.args[1])) + break # Don't process remaining commands except KeyboardInterrupt: self.status = "keyboardinterrupt" self.session.report.error(self.status) https://bitbucket.org/hpk42/tox/commits/d7d35b623979/ Changeset: d7d35b623979 User: hpk42 Date: 2015-05-11 10:35:22+00:00 Summary: remove erroring code / somewhat superflous error reporting Affected #: 1 file diff -r 3d7925f2ef6c716de3bc3e84f649ce9b5f15b498 -r d7d35b623979d2e8186798c5bd29a679ef825c56 tox/_venv.py --- a/tox/_venv.py +++ b/tox/_venv.py @@ -357,17 +357,10 @@ try: self._pcall(argv, cwd=cwd, action=action, redirect=redirect, ignore_ret=ignore_ret) - except tox.exception.InvocationError: - val = sys.exc_info()[1] - self.session.report.error(str(val)) + except tox.exception.InvocationError as err: + self.session.report.error(str(err)) self.status = "commands failed" if not self.envconfig.ignore_errors: - self.session.report.error( - 'Stopping processing of commands for env %s ' - 'because `%s` failed with exit code %s' - % (self.name, - ' '.join([str(x) for x in argv]), - val.args[1])) break # Don't process remaining commands except KeyboardInterrupt: self.status = "keyboardinterrupt" https://bitbucket.org/hpk42/tox/commits/06bb44d51a4c/ Changeset: 06bb44d51a4c User: hpk42 Date: 2015-05-11 10:35:56+00:00 Summary: merge pluggy branch Affected #: 15 files diff -r d7d35b623979d2e8186798c5bd29a679ef825c56 -r 06bb44d51a4c4d02f643da800d47a7197cfab32c CHANGELOG --- a/CHANGELOG +++ b/CHANGELOG @@ -38,6 +38,13 @@ - fix issue240: allow to specify empty argument list without it being rewritten to ".". Thanks Daniel Hahler. +- introduce experimental (not much documented yet) plugin system + based on pytest's externalized "pluggy" system. + See tox/hookspecs.py for the current hooks. + +- introduce parser.add_testenv_attribute() to register an ini-variable + for testenv sections. Can be used from plugins through the + tox_add_option hook. 1.9.2 ----------- diff -r d7d35b623979d2e8186798c5bd29a679ef825c56 -r 06bb44d51a4c4d02f643da800d47a7197cfab32c doc/conf.py --- a/doc/conf.py +++ b/doc/conf.py @@ -48,8 +48,8 @@ # built documents. # # The short X.Y version. -release = "1.9" -version = "1.9.0" +release = "2.0" +version = "2.0.0" # The full version, including alpha/beta/rc tags. # The language for content autogenerated by Sphinx. Refer to documentation diff -r d7d35b623979d2e8186798c5bd29a679ef825c56 -r 06bb44d51a4c4d02f643da800d47a7197cfab32c doc/index.txt --- a/doc/index.txt +++ b/doc/index.txt @@ -5,7 +5,7 @@ --------------------------------------------- ``tox`` aims to automate and standardize testing in Python. It is part -of a larger vision of easing the packaging, testing and release process +of a larger vision of easing the packaging, testing and release process of Python software. What is Tox? @@ -21,6 +21,7 @@ * acting as a frontend to Continuous Integration servers, greatly reducing boilerplate and merging CI and shell-based testing. + Basic example ----------------- @@ -62,10 +63,10 @@ - test-tool agnostic: runs py.test, nose or unittests in a uniform manner -* supports :ref:`using different / multiple PyPI index servers <multiindex>` +* :doc:`(new in 2.0) plugin system <plugins>` to modify tox execution with simple hooks. * uses pip_ and setuptools_ by default. Experimental - support for configuring the installer command + support for configuring the installer command through :confval:`install_command=ARGV`. * **cross-Python compatible**: CPython-2.6, 2.7, 3.2 and higher, @@ -74,11 +75,11 @@ * **cross-platform**: Windows and Unix style environments * **integrates with continuous integration servers** like Jenkins_ - (formerly known as Hudson) and helps you to avoid boilerplatish + (formerly known as Hudson) and helps you to avoid boilerplatish and platform-specific build-step hacks. * **full interoperability with devpi**: is integrated with and - is used for testing in the devpi_ system, a versatile pypi + is used for testing in the devpi_ system, a versatile pypi index server and release managing tool. * **driven by a simple ini-style config file** @@ -89,6 +90,9 @@ * **professionally** :doc:`supported <support>` +* supports :ref:`using different / multiple PyPI index servers <multiindex>` + + .. _pypy: http://pypy.org .. _`tox.ini`: :doc:configfile diff -r d7d35b623979d2e8186798c5bd29a679ef825c56 -r 06bb44d51a4c4d02f643da800d47a7197cfab32c doc/plugins.txt --- /dev/null +++ b/doc/plugins.txt @@ -0,0 +1,69 @@ +.. be in -*- rst -*- mode! + +tox plugins +=========== + +.. versionadded:: 2.0 + +With tox-2.0 a few aspects of tox running can be experimentally modified +by writing hook functions. We expect the list of hook function to grow +over time. + +writing a setuptools entrypoints plugin +--------------------------------------- + +If you have a ``tox_MYPLUGIN.py`` module you could use the following +rough ``setup.py`` to make it into a package which you can upload to the +Python packaging index:: + + # content of setup.py + from setuptools import setup + + if __name__ == "__main__": + setup( + name='tox-MYPLUGIN', + description='tox plugin decsription', + license="MIT license", + version='0.1', + py_modules=['tox_MYPLUGIN'], + entry_points={'tox': ['MYPLUGIN = tox_MYPLUGIN']}, + install_requires=['tox>=2.0'], + ) + +You can then install the plugin to develop it via:: + + pip install -e . + +and later publish it. + +The ``entry_points`` part allows tox to see your plugin during startup. + + +Writing hook implementations +---------------------------- + +A plugin module needs can define one or more hook implementation functions:: + + from tox import hookimpl + + @hookimpl + def tox_addoption(parser): + # add your own command line options + + + @hookimpl + def tox_configure(config): + # post process tox configuration after cmdline/ini file have + # been parsed + +If you put this into a module and make it pypi-installable with the ``tox`` +entry point you'll get your code executed as part of a tox run. + + + +tox hook specifications +---------------------------- + +.. automodule:: tox.hookspecs + :members: + diff -r d7d35b623979d2e8186798c5bd29a679ef825c56 -r 06bb44d51a4c4d02f643da800d47a7197cfab32c setup.py --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def main(): version = sys.version_info[:2] - install_requires = ['virtualenv>=1.11.2', 'py>=1.4.17', ] + install_requires = ['virtualenv>=1.11.2', 'py>=1.4.17', 'pluggy>=0.3.0,<0.4.0'] if version < (2, 7): install_requires += ['argparse'] setup( diff -r d7d35b623979d2e8186798c5bd29a679ef825c56 -r 06bb44d51a4c4d02f643da800d47a7197cfab32c tests/test_config.py --- a/tests/test_config.py +++ b/tests/test_config.py @@ -6,7 +6,6 @@ import tox import tox._config from tox._config import * # noqa -from tox._config import _split_env from tox._venv import VirtualEnv @@ -19,7 +18,7 @@ assert config.toxworkdir.realpath() == tmpdir.join(".tox").realpath() assert config.envconfigs['py1'].basepython == sys.executable assert config.envconfigs['py1'].deps == [] - assert not config.envconfigs['py1'].platform + assert config.envconfigs['py1'].platform == ".*" def test_config_parsing_multienv(self, tmpdir, newconfig): config = newconfig([], """ @@ -93,12 +92,12 @@ """ Ensure correct parseini._is_same_dep is working with a few samples. """ - assert parseini._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3') - assert parseini._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3>=2.0') - assert parseini._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3>2.0') - assert parseini._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3<2.0') - assert parseini._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3<=2.0') - assert not parseini._is_same_dep('pkg_hello-world3==1.0', 'otherpkg>=2.0') + assert DepOption._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3') + assert DepOption._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3>=2.0') + assert DepOption._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3>2.0') + assert DepOption._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3<2.0') + assert DepOption._is_same_dep('pkg_hello-world3==1.0', 'pkg_hello-world3<=2.0') + assert not DepOption._is_same_dep('pkg_hello-world3==1.0', 'otherpkg>=2.0') class TestConfigPlatform: @@ -220,8 +219,8 @@ commands = echo {[section]key} """) - reader = IniReader(config._cfg) - x = reader.getargvlist("testenv", "commands") + reader = SectionReader("testenv", config._cfg) + x = reader.getargvlist("commands") assert x == [["echo", "whatever"]] def test_command_substitution_from_other_section_multiline(self, newconfig): @@ -245,8 +244,8 @@ # comment is omitted echo {[base]commands} """) - reader = IniReader(config._cfg) - x = reader.getargvlist("testenv", "commands") + reader = SectionReader("testenv", config._cfg) + x = reader.getargvlist("commands") assert x == [ "cmd1 param11 param12".split(), "cmd2 param21 param22".split(), @@ -257,16 +256,16 @@ class TestIniParser: - def test_getdefault_single(self, tmpdir, newconfig): + def test_getstring_single(self, tmpdir, newconfig): config = newconfig(""" [section] key=value """) - reader = IniReader(config._cfg) - x = reader.getdefault("section", "key") + reader = SectionReader("section", config._cfg) + x = reader.getstring("key") assert x == "value" - assert not reader.getdefault("section", "hello") - x = reader.getdefault("section", "hello", "world") + assert not reader.getstring("hello") + x = reader.getstring("hello", "world") assert x == "world" def test_missing_substitution(self, tmpdir, newconfig): @@ -274,40 +273,40 @@ [mydefault] key2={xyz} """) - reader = IniReader(config._cfg, fallbacksections=['mydefault']) + reader = SectionReader("mydefault", config._cfg, fallbacksections=['mydefault']) assert reader is not None - py.test.raises(tox.exception.ConfigError, - 'reader.getdefault("mydefault", "key2")') + with py.test.raises(tox.exception.ConfigError): + reader.getstring("key2") - def test_getdefault_fallback_sections(self, tmpdir, newconfig): + def test_getstring_fallback_sections(self, tmpdir, newconfig): config = newconfig(""" [mydefault] key2=value2 [section] key=value """) - reader = IniReader(config._cfg, fallbacksections=['mydefault']) - x = reader.getdefault("section", "key2") + reader = SectionReader("section", config._cfg, fallbacksections=['mydefault']) + x = reader.getstring("key2") assert x == "value2" - x = reader.getdefault("section", "key3") + x = reader.getstring("key3") assert not x - x = reader.getdefault("section", "key3", "world") + x = reader.getstring("key3", "world") assert x == "world" - def test_getdefault_substitution(self, tmpdir, newconfig): + def test_getstring_substitution(self, tmpdir, newconfig): config = newconfig(""" [mydefault] key2={value2} [section] key={value} """) - reader = IniReader(config._cfg, fallbacksections=['mydefault']) + reader = SectionReader("section", config._cfg, fallbacksections=['mydefault']) reader.addsubstitutions(value="newvalue", value2="newvalue2") - x = reader.getdefault("section", "key2") + x = reader.getstring("key2") assert x == "newvalue2" - x = reader.getdefault("section", "key3") + x = reader.getstring("key3") assert not x - x = reader.getdefault("section", "key3", "{value2}") + x = reader.getstring("key3", "{value2}") assert x == "newvalue2" def test_getlist(self, tmpdir, newconfig): @@ -317,9 +316,9 @@ item1 {item2} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions(item1="not", item2="grr") - x = reader.getlist("section", "key2") + x = reader.getlist("key2") assert x == ['item1', 'grr'] def test_getdict(self, tmpdir, newconfig): @@ -329,28 +328,31 @@ key1=item1 key2={item2} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions(item1="not", item2="grr") - x = reader.getdict("section", "key2") + x = reader.getdict("key2") assert 'key1' in x assert 'key2' in x assert x['key1'] == 'item1' assert x['key2'] == 'grr' - def test_getdefault_environment_substitution(self, monkeypatch, newconfig): + x = reader.getdict("key3", {1: 2}) + assert x == {1: 2} + + def test_getstring_environment_substitution(self, monkeypatch, newconfig): monkeypatch.setenv("KEY1", "hello") config = newconfig(""" [section] key1={env:KEY1} key2={env:KEY2} """) - reader = IniReader(config._cfg) - x = reader.getdefault("section", "key1") + reader = SectionReader("section", config._cfg) + x = reader.getstring("key1") assert x == "hello" - py.test.raises(tox.exception.ConfigError, - 'reader.getdefault("section", "key2")') + with py.test.raises(tox.exception.ConfigError): + reader.getstring("key2") - def test_getdefault_environment_substitution_with_default(self, monkeypatch, newconfig): + def test_getstring_environment_substitution_with_default(self, monkeypatch, newconfig): monkeypatch.setenv("KEY1", "hello") config = newconfig(""" [section] @@ -358,12 +360,12 @@ key2={env:KEY2:DEFAULT_VALUE} key3={env:KEY3:} """) - reader = IniReader(config._cfg) - x = reader.getdefault("section", "key1") + reader = SectionReader("section", config._cfg) + x = reader.getstring("key1") assert x == "hello" - x = reader.getdefault("section", "key2") + x = reader.getstring("key2") assert x == "DEFAULT_VALUE" - x = reader.getdefault("section", "key3") + x = reader.getstring("key3") assert x == "" def test_value_matches_section_substituion(self): @@ -374,15 +376,15 @@ assert is_section_substitution("{[setup]}") is None assert is_section_substitution("{[setup] commands}") is None - def test_getdefault_other_section_substitution(self, newconfig): + def test_getstring_other_section_substitution(self, newconfig): config = newconfig(""" [section] key = rue [testenv] key = t{[section]key} """) - reader = IniReader(config._cfg) - x = reader.getdefault("testenv", "key") + reader = SectionReader("testenv", config._cfg) + x = reader.getstring("key") assert x == "true" def test_argvlist(self, tmpdir, newconfig): @@ -392,12 +394,12 @@ cmd1 {item1} {item2} cmd2 {item2} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions(item1="with space", item2="grr") # py.test.raises(tox.exception.ConfigError, - # "reader.getargvlist('section', 'key1')") - assert reader.getargvlist('section', 'key1') == [] - x = reader.getargvlist("section", "key2") + # "reader.getargvlist('key1')") + assert reader.getargvlist('key1') == [] + x = reader.getargvlist("key2") assert x == [["cmd1", "with", "space", "grr"], ["cmd2", "grr"]] @@ -406,9 +408,9 @@ [section] comm = py.test {posargs} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions([r"hello\this"]) - argv = reader.getargv("section", "comm") + argv = reader.getargv("comm") assert argv == ["py.test", "hello\\this"] def test_argvlist_multiline(self, tmpdir, newconfig): @@ -418,12 +420,12 @@ cmd1 {item1} \ # a comment {item2} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions(item1="with space", item2="grr") # py.test.raises(tox.exception.ConfigError, - # "reader.getargvlist('section', 'key1')") - assert reader.getargvlist('section', 'key1') == [] - x = reader.getargvlist("section", "key2") + # "reader.getargvlist('key1')") + assert reader.getargvlist('key1') == [] + x = reader.getargvlist("key2") assert x == [["cmd1", "with", "space", "grr"]] def test_argvlist_quoting_in_command(self, tmpdir, newconfig): @@ -433,8 +435,8 @@ cmd1 'with space' \ # a comment 'after the comment' """) - reader = IniReader(config._cfg) - x = reader.getargvlist("section", "key1") + reader = SectionReader("section", config._cfg) + x = reader.getargvlist("key1") assert x == [["cmd1", "with space", "after the comment"]] def test_argvlist_positional_substitution(self, tmpdir, newconfig): @@ -445,22 +447,22 @@ cmd2 {posargs:{item2} \ other} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) posargs = ['hello', 'world'] reader.addsubstitutions(posargs, item2="value2") # py.test.raises(tox.exception.ConfigError, - # "reader.getargvlist('section', 'key1')") - assert reader.getargvlist('section', 'key1') == [] - argvlist = reader.getargvlist("section", "key2") + # "reader.getargvlist('key1')") + assert reader.getargvlist('key1') == [] + argvlist = reader.getargvlist("key2") assert argvlist[0] == ["cmd1"] + posargs assert argvlist[1] == ["cmd2"] + posargs - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions([], item2="value2") # py.test.raises(tox.exception.ConfigError, - # "reader.getargvlist('section', 'key1')") - assert reader.getargvlist('section', 'key1') == [] - argvlist = reader.getargvlist("section", "key2") + # "reader.getargvlist('key1')") + assert reader.getargvlist('key1') == [] + argvlist = reader.getargvlist("key2") assert argvlist[0] == ["cmd1"] assert argvlist[1] == ["cmd2", "value2", "other"] @@ -472,10 +474,10 @@ cmd2 -f '{posargs}' cmd3 -f {posargs} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions(["foo", "bar"]) - assert reader.getargvlist('section', 'key1') == [] - x = reader.getargvlist("section", "key2") + assert reader.getargvlist('key1') == [] + x = reader.getargvlist("key2") assert x == [["cmd1", "--foo-args=foo bar"], ["cmd2", "-f", "foo bar"], ["cmd3", "-f", "foo", "bar"]] @@ -486,10 +488,10 @@ key2= cmd1 -f {posargs} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions(["foo", "'bar", "baz'"]) - assert reader.getargvlist('section', 'key1') == [] - x = reader.getargvlist("section", "key2") + assert reader.getargvlist('key1') == [] + x = reader.getargvlist("key2") assert x == [["cmd1", "-f", "foo", "bar baz"]] def test_positional_arguments_are_only_replaced_when_standing_alone(self, tmpdir, newconfig): @@ -501,11 +503,11 @@ cmd2 -m '\'something\'' [] cmd3 something[]else """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) posargs = ['hello', 'world'] reader.addsubstitutions(posargs) - argvlist = reader.getargvlist('section', 'key') + argvlist = reader.getargvlist('key') assert argvlist[0] == ['cmd0'] + posargs assert argvlist[1] == ['cmd1', '-m', '[abc]'] assert argvlist[2] == ['cmd2', '-m', "something"] + posargs @@ -517,32 +519,32 @@ key = py.test -n5 --junitxml={envlogdir}/junit-{envname}.xml [] """ config = newconfig(inisource) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) posargs = ['hello', 'world'] reader.addsubstitutions(posargs, envlogdir='ENV_LOG_DIR', envname='ENV_NAME') expected = [ 'py.test', '-n5', '--junitxml=ENV_LOG_DIR/junit-ENV_NAME.xml', 'hello', 'world' ] - assert reader.getargvlist('section', 'key')[0] == expected + assert reader.getargvlist('key')[0] == expected def test_getargv(self, newconfig): config = newconfig(""" [section] key=some command "with quoting" """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) expected = ['some', 'command', 'with quoting'] - assert reader.getargv('section', 'key') == expected + assert reader.getargv('key') == expected def test_getpath(self, tmpdir, newconfig): config = newconfig(""" [section] path1={HELLO} """) - reader = IniReader(config._cfg) + reader = SectionReader("section", config._cfg) reader.addsubstitutions(toxinidir=tmpdir, HELLO="mypath") - x = reader.getpath("section", "path1", tmpdir) + x = reader.getpath("path1", tmpdir) assert x == tmpdir.join("mypath") def test_getbool(self, tmpdir, newconfig): @@ -554,13 +556,13 @@ key2a=falsE key5=yes """) - reader = IniReader(config._cfg) - assert reader.getbool("section", "key1") is True - assert reader.getbool("section", "key1a") is True - assert reader.getbool("section", "key2") is False - assert reader.getbool("section", "key2a") is False - py.test.raises(KeyError, 'reader.getbool("section", "key3")') - py.test.raises(tox.exception.ConfigError, 'reader.getbool("section", "key5")') + reader = SectionReader("section", config._cfg) + assert reader.getbool("key1") is True + assert reader.getbool("key1a") is True + assert reader.getbool("key2") is False + assert reader.getbool("key2a") is False + py.test.raises(KeyError, 'reader.getbool("key3")') + py.test.raises(tox.exception.ConfigError, 'reader.getbool("key5")') class TestConfigTestEnv: @@ -586,7 +588,7 @@ assert envconfig.commands == [["xyz", "--abc"]] assert envconfig.changedir == config.setupdir assert envconfig.sitepackages is False - assert envconfig.develop is False + assert envconfig.usedevelop is False assert envconfig.ignore_errors is False assert envconfig.envlogdir == envconfig.envdir.join("log") assert list(envconfig.setenv.keys()) == ['PYTHONHASHSEED'] @@ -607,7 +609,7 @@ [testenv] usedevelop = True """) - assert not config.envconfigs["python"].develop + assert not config.envconfigs["python"].usedevelop def test_specific_command_overrides(self, tmpdir, newconfig): config = newconfig(""" @@ -1382,7 +1384,7 @@ def test_noset(self, tmpdir, newconfig): args = ['--hashseed', 'noset'] envconfig = self._get_envconfig(newconfig, args=args) - assert envconfig.setenv is None + assert envconfig.setenv == {} def test_noset_with_setenv(self, tmpdir, newconfig): tox_ini = """ @@ -1571,31 +1573,16 @@ ]) -class TestArgumentParser: - - def test_dash_e_single_1(self): - parser = prepare_parse('testpkg') - args = parser.parse_args('-e py26'.split()) - envlist = _split_env(args.env) - assert envlist == ['py26'] - - def test_dash_e_single_2(self): - parser = prepare_parse('testpkg') - args = parser.parse_args('-e py26,py33'.split()) - envlist = _split_env(args.env) - assert envlist == ['py26', 'py33'] - - def test_dash_e_same(self): - parser = prepare_parse('testpkg') - args = parser.parse_args('-e py26,py26'.split()) - envlist = _split_env(args.env) - assert envlist == ['py26', 'py26'] - - def test_dash_e_combine(self): - parser = prepare_parse('testpkg') - args = parser.parse_args('-e py26,py25,py33 -e py33,py27'.split()) - envlist = _split_env(args.env) - assert envlist == ['py26', 'py25', 'py33', 'py33', 'py27'] +@pytest.mark.parametrize("cmdline,envlist", [ + ("-e py26", ['py26']), + ("-e py26,py33", ['py26', 'py33']), + ("-e py26,py26", ['py26', 'py26']), + ("-e py26,py33 -e py33,py27", ['py26', 'py33', 'py33', 'py27']) +]) +def test_env_spec(cmdline, envlist): + args = cmdline.split() + config = parseconfig(args) + assert config.envlist == envlist class TestCommandParser: diff -r d7d35b623979d2e8186798c5bd29a679ef825c56 -r 06bb44d51a4c4d02f643da800d47a7197cfab32c tests/test_interpreters.py --- a/tests/test_interpreters.py +++ b/tests/test_interpreters.py @@ -3,11 +3,13 @@ import pytest from tox.interpreters import * # noqa +from tox._config import get_plugin_manager @pytest.fixture def interpreters(): - return Interpreters() + pm = get_plugin_manager() + return Interpreters(hook=pm.hook) @pytest.mark.skipif("sys.platform != 'win32'") @@ -28,8 +30,11 @@ assert locate_via_py('3', '2') == sys.executable -def test_find_executable(): - p = find_executable(sys.executable) +def test_tox_get_python_executable(): + class envconfig: + basepython = sys.executable + envname = "pyxx" + p = tox_get_python_executable(envconfig) assert p == py.path.local(sys.executable) for ver in [""] + "2.4 2.5 2.6 2.7 3.0 3.1 3.2 3.3".split(): name = "python%s" % ver @@ -42,7 +47,8 @@ else: if not py.path.local.sysfind(name): continue - p = find_executable(name) + envconfig.basepython = name + p = tox_get_python_executable(envconfig) assert p popen = py.std.subprocess.Popen([str(p), '-V'], stderr=py.std.subprocess.PIPE) @@ -55,7 +61,12 @@ def sysfind(x): return "hello" monkeypatch.setattr(py.path.local, "sysfind", sysfind) - t = find_executable("qweqwe") + + class envconfig: + basepython = "1lk23j" + envname = "pyxx" + + t = tox_get_python_executable(envconfig) assert t == "hello" @@ -69,31 +80,33 @@ class TestInterpreters: - def test_get_info_self_exceptions(self, interpreters): - pytest.raises(ValueError, lambda: - interpreters.get_info()) - pytest.raises(ValueError, lambda: - interpreters.get_info(name="12", executable="123")) + def test_get_executable(self, interpreters): + class envconfig: + basepython = sys.executable + envname = "pyxx" - def test_get_executable(self, interpreters): - x = interpreters.get_executable(sys.executable) + x = interpreters.get_executable(envconfig) assert x == sys.executable - assert not interpreters.get_executable("12l3k1j23") - - def test_get_info__name(self, interpreters): - info = interpreters.get_info(executable=sys.executable) + info = interpreters.get_info(envconfig) assert info.version_info == tuple(sys.version_info) assert info.executable == sys.executable assert info.runnable - def test_get_info__name_not_exists(self, interpreters): - info = interpreters.get_info("qlwkejqwe") + def test_get_executable_no_exist(self, interpreters): + class envconfig: + basepython = "1lkj23" + envname = "pyxx" + assert not interpreters.get_executable(envconfig) + info = interpreters.get_info(envconfig) assert not info.version_info - assert info.name == "qlwkejqwe" + assert info.name == "1lkj23" assert not info.executable assert not info.runnable def test_get_sitepackagesdir_error(self, interpreters): - info = interpreters.get_info(sys.executable) + class envconfig: + basepython = sys.executable + envname = "123" + info = interpreters.get_info(envconfig) s = interpreters.get_sitepackagesdir(info, "") assert s diff -r d7d35b623979d2e8186798c5bd29a679ef825c56 -r 06bb44d51a4c4d02f643da800d47a7197cfab32c tests/test_venv.py --- a/tests/test_venv.py +++ b/tests/test_venv.py @@ -35,6 +35,7 @@ py.test.raises(tox.exception.UnsupportedInterpreter, venv.getsupportedinterpreter) monkeypatch.undo() + monkeypatch.setattr(venv.envconfig, "envname", "py1") monkeypatch.setattr(venv.envconfig, 'basepython', 'notexistingpython') py.test.raises(tox.exception.InterpreterNotFound, venv.getsupportedinterpreter) @@ -42,7 +43,7 @@ # check that we properly report when no version_info is present info = NoInterpreterInfo(name=venv.name) info.executable = "something" - monkeypatch.setattr(config.interpreters, "get_info", lambda *args: info) + monkeypatch.setattr(config.interpreters, "get_info", lambda *args, **kw: info) pytest.raises(tox.exception.InvocationError, venv.getsupportedinterpreter) @@ -65,7 +66,7 @@ # assert Envconfig.toxworkdir in args assert venv.getcommandpath("easy_install", cwd=py.path.local()) interp = venv._getliveconfig().python - assert interp == venv.envconfig._basepython_info.executable + assert interp == venv.envconfig.python_info.executable assert venv.path_config.check(exists=False) @@ -463,7 +464,7 @@ venv = VirtualEnv(envconfig, session=mocksession) venv.update() cconfig = venv._getliveconfig() - cconfig.develop = True + cconfig.usedevelop = True cconfig.writeconfig(venv.path_config) mocksession._clearmocks() venv.update() diff -r d7d35b623979d2e8186798c5bd29a679ef825c56 -r 06bb44d51a4c4d02f643da800d47a7197cfab32c tox.ini --- a/tox.ini +++ b/tox.ini @@ -5,8 +5,10 @@ commands=echo {posargs} [testenv] -commands=py.test --junitxml={envlogdir}/junit-{envname}.xml {posargs} +commands= py.test --timeout=60 {posargs} + deps=pytest>=2.3.5 + pytest-timeout [testenv:docs] basepython=python @@ -14,15 +16,16 @@ deps=sphinx {[testenv]deps} commands= - py.test -v \ - --junitxml={envlogdir}/junit-{envname}.xml \ - check_sphinx.py {posargs} + py.test -v check_sphinx.py {posargs} [testenv:flakes] +qwe = 123 deps = pytest-flakes>=0.2 pytest-pep8 -commands = py.test -x --flakes --pep8 tox tests +commands = + py.test --flakes -m flakes tox tests + py.test --pep8 -m pep8 tox tests [testenv:dev] # required to make looponfail reload on every source code change diff -r d7d35b623979d2e8186798c5bd29a679ef825c56 -r 06bb44d51a4c4d02f643da800d47a7197cfab32c tox/__init__.py --- a/tox/__init__.py +++ b/tox/__init__.py @@ -1,6 +1,8 @@ # __version__ = '2.0.0.dev1' +from .hookspecs import hookspec, hookimpl # noqa + class exception: class Error(Exception): diff -r d7d35b623979d2e8186798c5bd29a679ef825c56 -r 06bb44d51a4c4d02f643da800d47a7197cfab32c tox/_cmdline.py --- a/tox/_cmdline.py +++ b/tox/_cmdline.py @@ -24,13 +24,35 @@ def main(args=None): try: - config = parseconfig(args, 'tox') + config = parseconfig(args) + if config.option.help: + show_help(config) + raise SystemExit(0) + elif config.option.helpini: + show_help_ini(config) + raise SystemExit(0) retcode = Session(config).runcommand() raise SystemExit(retcode) except KeyboardInterrupt: raise SystemExit(2) +def show_help(config): + tw = py.io.TerminalWriter() + tw.write(config._parser.format_help()) + tw.line() + + +def show_help_ini(config): + tw = py.io.TerminalWriter() + tw.sep("-", "per-testenv attributes") + for env_attr in config._testenv_attr: + tw.line("%-15s %-8s default: %s" % + (env_attr.name, "<" + env_attr.type + ">", env_attr.default), bold=True) + tw.line(env_attr.help) + tw.line() + + class Action(object): def __init__(self, session, venv, msg, args): self.venv = venv @@ -487,7 +509,7 @@ venv.status = "platform mismatch" continue # we simply omit non-matching platforms if self.setupenv(venv): - if venv.envconfig.develop: + if venv.envconfig.usedevelop: self.developpkg(venv, self.config.setupdir) elif self.config.skipsdist or venv.envconfig.skip_install: self.finishvenv(venv) @@ -551,8 +573,7 @@ for envconfig in self.config.envconfigs.values(): self.report.line("[testenv:%s]" % envconfig.envname, bold=True) self.report.line(" basepython=%s" % envconfig.basepython) - self.report.line(" _basepython_info=%s" % - envconfig._basepython_info) + self.report.line(" pythoninfo=%s" % (envconfig.python_info,)) self.report.line(" envpython=%s" % envconfig.envpython) self.report.line(" envtmpdir=%s" % envconfig.envtmpdir) self.report.line(" envbindir=%s" % envconfig.envbindir) @@ -567,7 +588,7 @@ self.report.line(" deps=%s" % envconfig.deps) self.report.line(" envdir= %s" % envconfig.envdir) self.report.line(" downloadcache=%s" % envconfig.downloadcache) - self.report.line(" usedevelop=%s" % envconfig.develop) + self.report.line(" usedevelop=%s" % envconfig.usedevelop) def showenvs(self): for env in self.config.envlist: diff -r d7d35b623979d2e8186798c5bd29a679ef825c56 -r 06bb44d51a4c4d02f643da800d47a7197cfab32c tox/_config.py --- a/tox/_config.py +++ b/tox/_config.py @@ -8,8 +8,10 @@ import string import pkg_resources import itertools +import pluggy -from tox.interpreters import Interpreters +import tox.interpreters +from tox import hookspecs import py @@ -22,20 +24,161 @@ for version in '24,25,26,27,30,31,32,33,34,35'.split(','): default_factors['py' + version] = 'python%s.%s' % tuple(version) +hookimpl = pluggy.HookimplMarker("tox") -def parseconfig(args=None, pkg=None): + +def get_plugin_manager(): + # initialize plugin manager + pm = pluggy.PluginManager("tox") + pm.add_hookspecs(hookspecs) + pm.register(tox._config) + pm.register(tox.interpreters) + pm.load_setuptools_entrypoints("tox") + pm.check_pending() + return pm + + +class MyParser: + def __init__(self): + self.argparser = argparse.ArgumentParser( + description="tox options", add_help=False) + self._testenv_attr = [] + + def add_argument(self, *args, **kwargs): + return self.argparser.add_argument(*args, **kwargs) + + def add_testenv_attribute(self, name, type, help, default=None, postprocess=None): + self._testenv_attr.append(VenvAttribute(name, type, default, help, postprocess)) + + def add_testenv_attribute_obj(self, obj): + assert hasattr(obj, "name") + assert hasattr(obj, "type") + assert hasattr(obj, "help") + assert hasattr(obj, "postprocess") + self._testenv_attr.append(obj) + + def parse_args(self, args): + return self.argparser.parse_args(args) + + def format_help(self): + return self.argparser.format_help() + + +class VenvAttribute: + def __init__(self, name, type, default, help, postprocess): + self.name = name + self.type = type + self.default = default + self.help = help + self.postprocess = postprocess + + +class DepOption: + name = "deps" + type = "line-list" + help = "each line specifies a dependency in pip/setuptools format." + default = () + + def postprocess(self, config, reader, section_val): + deps = [] + for depline in section_val: + m = re.match(r":(\w+):\s*(\S+)", depline) + if m: + iname, name = m.groups() + ixserver = config.indexserver[iname] + else: + name = depline.strip() + ixserver = None + name = self._replace_forced_dep(name, config) + deps.append(DepConfig(name, ixserver)) + return deps + + def _replace_forced_dep(self, name, config): + """ + Override the given dependency config name taking --force-dep-version + option into account. + + :param name: dep config, for example ["pkg==1.0", "other==2.0"]. + :param config: Config instance + :return: the new dependency that should be used for virtual environments + """ + if not config.option.force_dep: + return name + for forced_dep in config.option.force_dep: + if self._is_same_dep(forced_dep, name): + return forced_dep + return name + + @classmethod + def _is_same_dep(cls, dep1, dep2): + """ + Returns True if both dependency definitions refer to the + same package, even if versions differ. + """ + dep1_name = pkg_resources.Requirement.parse(dep1).project_name + dep2_name = pkg_resources.Requirement.parse(dep2).project_name + return dep1_name == dep2_name + + +class PosargsOption: + name = "args_are_paths" + type = "bool" + default = True + help = "treat positional args in commands as paths" + + def postprocess(self, config, reader, section_val): + args = config.option.args + if args: + if section_val: + args = [] + for arg in config.option.args: + if arg: + origpath = config.invocationcwd.join(arg, abs=True) + if origpath.check(): + arg = reader.getpath("changedir", ".").bestrelpath(origpath) + args.append(arg) + reader.addsubstitutions(args) + return section_val + + +class InstallcmdOption: + name = "install_command" + type = "argv" + default = "pip install {opts} {packages}" + help = "install command for dependencies and package under test." + + def postprocess(self, config, reader, section_val): + if '{packages}' not in section_val: + raise tox.exception.ConfigError( + "'install_command' must contain '{packages}' substitution") + return section_val + + +def parseconfig(args=None): """ :param list[str] args: Optional list of arguments. :type pkg: str :rtype: :class:`Config` :raise SystemExit: toxinit file is not found """ + + pm = get_plugin_manager() + if args is None: args = sys.argv[1:] - parser = prepare_parse(pkg) - opts = parser.parse_args(args) - config = Config() - config.option = opts + + # prepare command line options + parser = MyParser() + pm.hook.tox_addoption(parser=parser) + + # parse command line options + option = parser.parse_args(args) + interpreters = tox.interpreters.Interpreters(hook=pm.hook) + config = Config(pluginmanager=pm, option=option, interpreters=interpreters) + config._parser = parser + config._testenv_attr = parser._testenv_attr + + # parse ini file basename = config.option.configfile if os.path.isabs(basename): inipath = py.path.local(basename) @@ -52,6 +195,10 @@ exn = sys.exc_info()[1] # Use stdout to match test expectations py.builtin.print_("ERROR: " + str(exn)) + + # post process config object + pm.hook.tox_configure(config=config) + return config @@ -63,10 +210,8 @@ class VersionAction(argparse.Action): def __call__(self, argparser, *args, **kwargs): - name = argparser.pkgname - mod = __import__(name) - version = mod.__version__ - py.builtin.print_("%s imported from %s" % (version, mod.__file__)) + version = tox.__version__ + py.builtin.print_("%s imported from %s" % (version, tox.__file__)) raise SystemExit(0) @@ -78,13 +223,16 @@ setattr(namespace, self.dest, 0) -def prepare_parse(pkgname): - parser = argparse.ArgumentParser(description=__doc__,) +@hookimpl +def tox_addoption(parser): # formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.pkgname = pkgname parser.add_argument("--version", nargs=0, action=VersionAction, dest="version", help="report version information to stdout.") + parser.add_argument("-h", "--help", action="store_true", dest="help", + help="show help about options") + parser.add_argument("--help-ini", "--hi", action="store_true", dest="helpini", + help="show help about ini-names") parser.add_argument("-v", nargs=0, action=CountAction, default=0, dest="verbosity", help="increase verbosity of reporting output.") @@ -130,6 +278,7 @@ "all commands and results involved. This will turn off " "pass-through output from running test commands which is " "instead captured into the json result file.") + # We choose 1 to 4294967295 because it is the range of PYTHONHASHSEED. parser.add_argument("--hashseed", action="store", metavar="SEED", default=None, @@ -149,14 +298,144 @@ parser.add_argument("args", nargs="*", help="additional arguments available to command positional substitution") - return parser + + # add various core venv interpreter attributes + + parser.add_testenv_attribute( + name="envdir", type="path", default="{toxworkdir}/{envname}", + help="venv directory") + + parser.add_testenv_attribute( + name="envtmpdir", type="path", default="{envdir}/tmp", + help="venv temporary directory") + + parser.add_testenv_attribute( + name="envlogdir", type="path", default="{envdir}/log", + help="venv log directory") + + def downloadcache(config, reader, section_val): + if section_val: + # env var, if present, takes precedence + downloadcache = os.environ.get("PIP_DOWNLOAD_CACHE", section_val) + return py.path.local(downloadcache) + + parser.add_testenv_attribute( + name="downloadcache", type="string", default=None, postprocess=downloadcache, + help="(deprecated) set PIP_DOWNLOAD_CACHE.") + + parser.add_testenv_attribute( + name="changedir", type="path", default="{toxinidir}", + help="directory to change to when running commands") + + parser.add_testenv_attribute_obj(PosargsOption()) + + parser.add_testenv_attribute( + name="skip_install", type="bool", default=False, + help="Do not install the current package. This can be used when " + "you need the virtualenv management but do not want to install " + "the current package") + + parser.add_testenv_attribute( + name="ignore_errors", type="bool", default=False, + help="if set to True all commands will be executed irrespective of their " + "result error status.") + + def recreate(config, reader, section_val): + if config.option.recreate: + return True + return section_val + + parser.add_testenv_attribute( + name="recreate", type="bool", default=False, postprocess=recreate, + help="always recreate this test environment.") + + def setenv(config, reader, section_val): + setenv = section_val + if "PYTHONHASHSEED" not in setenv and config.hashseed is not None: + setenv['PYTHONHASHSEED'] = config.hashseed + return setenv + + parser.add_testenv_attribute( + name="setenv", type="dict", postprocess=setenv, + help="list of X=Y lines with environment variable settings") + + def passenv(config, reader, section_val): + passenv = set(["PATH"]) + if sys.platform == "win32": + passenv.add("SYSTEMROOT") # needed for python's crypto module + passenv.add("PATHEXT") # needed for discovering executables + for spec in section_val: + for name in os.environ: + if fnmatchcase(name.upper(), spec.upper()): + passenv.add(name) + return passenv + + parser.add_testenv_attribute( + name="passenv", type="space-separated-list", postprocess=passenv, + help="environment variables names which shall be passed " + "from tox invocation to test environment when executing commands.") + + parser.add_testenv_attribute( + name="whitelist_externals", type="line-list", + help="each lines specifies a path or basename for which tox will not warn " + "about it coming from outside the test environment.") + + parser.add_testenv_attribute( + name="platform", type="string", default=".*", + help="regular expression which must match against ``sys.platform``. " + "otherwise testenv will be skipped.") + + def sitepackages(config, reader, section_val): + return config.option.sitepackages or section_val + + parser.add_testenv_attribute( + name="sitepackages", type="bool", default=False, postprocess=sitepackages, + help="Set to ``True`` if you want to create virtual environments that also " + "have access to globally installed packages.") + + def pip_pre(config, reader, section_val): + return config.option.pre or section_val + + parser.add_testenv_attribute( + name="pip_pre", type="bool", default=False, postprocess=pip_pre, + help="If ``True``, adds ``--pre`` to the ``opts`` passed to " + "the install command. ") + + def develop(config, reader, section_val): + return not config.option.installpkg and (section_val or config.option.develop) + + parser.add_testenv_attribute( + name="usedevelop", type="bool", postprocess=develop, default=False, + help="install package in develop/editable mode") + + def basepython_default(config, reader, section_val): + if section_val is None: + for f in reader.factors: + if f in default_factors: + return default_factors[f] + return sys.executable + return str(section_val) + + parser.add_testenv_attribute( + name="basepython", type="string", default=None, postprocess=basepython_default, + help="executable name or path of interpreter used to create a " + "virtual test environment.") + + parser.add_testenv_attribute_obj(InstallcmdOption()) + parser.add_testenv_attribute_obj(DepOption()) + + parser.add_testenv_attribute( + name="commands", type="argvlist", default="", + help="each line specifies a test command and can use substitution.") class Config(object): - def __init__(self): + def __init__(self, pluginmanager, option, interpreters): self.envconfigs = {} self.invocationcwd = py.path.local() - self.interpreters = Interpreters() + self.interpreters = interpreters + self.pluginmanager = pluginmanager + self.option = option @property def homedir(self): @@ -192,16 +471,20 @@ def envsitepackagesdir(self): self.getsupportedinterpreter() # for throwing exceptions x = self.config.interpreters.get_sitepackagesdir( - info=self._basepython_info, + info=self.python_info, envdir=self.envdir) return x + @property + def python_info(self): + return self.config.interpreters.get_info(envconfig=self) + def getsupportedinterpreter(self): if sys.platform == "win32" and self.basepython and \ "jython" in self.basepython: raise tox.exception.UnsupportedInterpreter( "Jython/Windows does not support installing scripts") - info = self.config.interpreters.get_info(self.basepython) + info = self.config.interpreters.get_info(envconfig=self) if not info.executable: raise tox.exception.InterpreterNotFound(self.basepython) if not info.version_info: @@ -240,12 +523,10 @@ self.config = config ctxname = getcontextname() if ctxname == "jenkins": - reader = IniReader(self._cfg, fallbacksections=['tox']) - toxsection = "tox:%s" % ctxname + reader = SectionReader("tox:jenkins", self._cfg, fallbacksections=['tox']) distshare_default = "{toxworkdir}/distshare" elif not ctxname: - reader = IniReader(self._cfg) - toxsection = "tox" + reader = SectionReader("tox", self._cfg) distshare_default = "{homedir}/.tox/distshare" else: raise ValueError("invalid context") @@ -260,18 +541,17 @@ reader.addsubstitutions(toxinidir=config.toxinidir, homedir=config.homedir) - config.toxworkdir = reader.getpath(toxsection, "toxworkdir", - "{toxinidir}/.tox") - config.minversion = reader.getdefault(toxsection, "minversion", None) + config.toxworkdir = reader.getpath("toxworkdir", "{toxinidir}/.tox") + config.minversion = reader.getstring("minversion", None) if not config.option.skip_missing_interpreters: config.option.skip_missing_interpreters = \ - reader.getbool(toxsection, "skip_missing_interpreters", False) + reader.getbool("skip_missing_interpreters", False) # determine indexserver dictionary config.indexserver = {'default': IndexServerConfig('default')} prefix = "indexserver" - for line in reader.getlist(toxsection, prefix): + for line in reader.getlist(prefix): name, url = map(lambda x: x.strip(), line.split("=", 1)) config.indexserver[name] = IndexServerConfig(name, url) @@ -296,16 +576,15 @@ config.indexserver[name] = IndexServerConfig(name, override) reader.addsubstitutions(toxworkdir=config.toxworkdir) - config.distdir = reader.getpath(toxsection, "distdir", "{toxworkdir}/dist") + config.distdir = reader.getpath("distdir", "{toxworkdir}/dist") reader.addsubstitutions(distdir=config.distdir) - config.distshare = reader.getpath(toxsection, "distshare", - distshare_default) + config.distshare = reader.getpath("distshare", distshare_default) reader.addsubstitutions(distshare=config.distshare) - config.sdistsrc = reader.getpath(toxsection, "sdistsrc", None) - config.setupdir = reader.getpath(toxsection, "setupdir", "{toxinidir}") + config.sdistsrc = reader.getpath("sdistsrc", None) + config.setupdir = reader.getpath("setupdir", "{toxinidir}") config.logdir = config.toxworkdir.join("log") - config.envlist, all_envs = self._getenvdata(reader, toxsection) + config.envlist, all_envs = self._getenvdata(reader) # factors used in config or predefined known_factors = self._list_section_factors("testenv") @@ -313,7 +592,7 @@ known_factors.add("python") # factors stated in config envlist - stated_envlist = reader.getdefault(toxsection, "envlist", replace=False) + stated_envlist = reader.getstring("envlist", replace=False) if stated_envlist: for env in _split_env(stated_envlist): known_factors.update(env.split('-')) @@ -324,13 +603,13 @@ factors = set(name.split('-')) if section in self._cfg or factors <= known_factors: config.envconfigs[name] = \ - self._makeenvconfig(name, section, reader._subs, config) + self.make_envconfig(name, section, reader._subs, config) all_develop = all(name in config.envconfigs - and config.envconfigs[name].develop + and config.envconfigs[name].usedevelop for name in config.envlist) - config.skipsdist = reader.getbool(toxsection, "skipsdist", all_develop) + config.skipsdist = reader.getbool("skipsdist", all_develop) def _list_section_factors(self, section): factors = set() @@ -340,116 +619,42 @@ factors.update(*mapcat(_split_factor_expr, exprs)) return factors - def _makeenvconfig(self, name, section, subs, config): + def make_envconfig(self, name, section, subs, config): vc = VenvConfig(config=config, envname=name) factors = set(name.split('-')) - reader = IniReader(self._cfg, fallbacksections=["testenv"], factors=factors) + reader = SectionReader(section, self._cfg, fallbacksections=["testenv"], + factors=factors) reader.addsubstitutions(**subs) - vc.develop = ( - not config.option.installpkg - and reader.getbool(section, "usedevelop", config.option.develop)) - vc.envdir = reader.getpath(section, "envdir", "{toxworkdir}/%s" % name) - vc.args_are_paths = reader.getbool(section, "args_are_paths", True) - if reader.getdefault(section, "python", None): - raise tox.exception.ConfigError( - "'python=' key was renamed to 'basepython='") - bp = next((default_factors[f] for f in factors if f in default_factors), - sys.executable) - vc.basepython = reader.getdefault(section, "basepython", bp) - vc._basepython_info = config.interpreters.get_info(vc.basepython) - reader.addsubstitutions(envdir=vc.envdir, envname=vc.envname, - envbindir=vc.envbindir, envpython=vc.envpython, - envsitepackagesdir=vc.envsitepackagesdir) - vc.envtmpdir = reader.getpath(section, "tmpdir", "{envdir}/tmp") - vc.envlogdir = reader.getpath(section, "envlogdir", "{envdir}/log") - reader.addsubstitutions(envlogdir=vc.envlogdir, envtmpdir=vc.envtmpdir) - vc.changedir = reader.getpath(section, "changedir", "{toxinidir}") - if config.option.recreate: - vc.recreate = True - else: - vc.recreate = reader.getbool(section, "recreate", False) - args = config.option.args - if args: - if vc.args_are_paths: - args = [] - for arg in config.option.args: - if arg: - origpath = config.invocationcwd.join(arg, abs=True) - if origpath.check(): - arg = vc.changedir.bestrelpath(origpath) - args.append(arg) - reader.addsubstitutions(args) - setenv = {} - if config.hashseed is not None: - setenv['PYTHONHASHSEED'] = config.hashseed - setenv.update(reader.getdict(section, 'setenv')) + reader.addsubstitutions(envname=name) - # read passenv - vc.passenv = set(["PATH"]) - if sys.platform == "win32": - vc.passenv.add("SYSTEMROOT") # needed for python's crypto module - vc.passenv.add("PATHEXT") # needed for discovering executables - for spec in reader.getlist(section, "passenv", sep=" "): - for name in os.environ: - if fnmatchcase(name.lower(), spec.lower()): - vc.passenv.add(name) + for env_attr in config._testenv_attr: + atype = env_attr.type + if atype in ("bool", "path", "string", "dict", "argv", "argvlist"): + meth = getattr(reader, "get" + atype) + res = meth(env_attr.name, env_attr.default) + elif atype == "space-separated-list": + res = reader.getlist(env_attr.name, sep=" ") + elif atype == "line-list": + res = reader.getlist(env_attr.name, sep="\n") + else: + raise ValueError("unknown type %r" % (atype,)) - vc.setenv = setenv - if not vc.setenv: - vc.setenv = None + if env_attr.postprocess: + res = env_attr.postprocess(config, reader, res) + setattr(vc, env_attr.name, res) - vc.commands = reader.getargvlist(section, "commands") - vc.whitelist_externals = reader.getlist(section, - "whitelist_externals") - vc.deps = [] - for depline in reader.getlist(section, "deps"): - m = re.match(r":(\w+):\s*(\S+)", depline) - if m: - iname, name = m.groups() - ixserver = config.indexserver[iname] - else: - name = depline.strip() - ixserver = None - name = self._replace_forced_dep(name, config) - vc.deps.append(DepConfig(name, ixserver)) + if atype == "path": + reader.addsubstitutions(**{env_attr.name: res}) - platform = "" - for platform in reader.getlist(section, "platform"): - if platform.strip(): - break - vc.platform = platform - - vc.sitepackages = ( - self.config.option.sitepackages - or reader.getbool(section, "sitepackages", False)) - - vc.downloadcache = None - downloadcache = reader.getdefault(section, "downloadcache") - if downloadcache: - # env var, if present, takes precedence - downloadcache = os.environ.get("PIP_DOWNLOAD_CACHE", downloadcache) - vc.downloadcache = py.path.local(downloadcache) - - vc.install_command = reader.getargv( - section, - "install_command", - "pip install {opts} {packages}", - ) - if '{packages}' not in vc.install_command: - raise tox.exception.ConfigError( - "'install_command' must contain '{packages}' substitution") - vc.pip_pre = config.option.pre or reader.getbool( - section, "pip_pre", False) - - vc.skip_install = reader.getbool(section, "skip_install", False) - vc.ignore_errors = reader.getbool(section, "ignore_errors", False) - + if env_attr.name == "install_command": + reader.addsubstitutions(envbindir=vc.envbindir, envpython=vc.envpython, + envsitepackagesdir=vc.envsitepackagesdir) return vc - def _getenvdata(self, reader, toxsection): + def _getenvdata(self, reader): envstr = self.config.option.env \ or os.environ.get("TOXENV") \ - or reader.getdefault(toxsection, "envlist", replace=False) \ + or reader.getstring("envlist", replace=False) \ or [] envlist = _split_env(envstr) @@ -466,32 +671,6 @@ return envlist, all_envs - def _replace_forced_dep(self, name, config): - """ - Override the given dependency config name taking --force-dep-version - option into account. - - :param name: dep config, for example ["pkg==1.0", "other==2.0"]. - :param config: Config instance - :return: the new dependency that should be used for virtual environments - """ - if not config.option.force_dep: - return name - for forced_dep in config.option.force_dep: - if self._is_same_dep(forced_dep, name): - return forced_dep - return name - - @classmethod - def _is_same_dep(cls, dep1, dep2): - """ - Returns True if both dependency definitions refer to the - same package, even if versions differ. - """ - dep1_name = pkg_resources.Requirement.parse(dep1).project_name - dep2_name = pkg_resources.Requirement.parse(dep2).project_name - return dep1_name == dep2_name - def _split_env(env): """if handed a list, action="append" was used for -e """ @@ -558,8 +737,9 @@ re.VERBOSE) -class IniReader: - def __init__(self, cfgparser, fallbacksections=None, factors=()): +class SectionReader: + def __init__(self, section_name, cfgparser, fallbacksections=None, factors=()): + self.section_name = section_name self._cfg = cfgparser self.fallbacksections = fallbacksections or [] self.factors = factors @@ -571,129 +751,39 @@ if _posargs: self.posargs = _posargs - def getpath(self, section, name, defaultpath): + def getpath(self, name, defaultpath): toxinidir = self._subs['toxinidir'] - path = self.getdefault(section, name, defaultpath) + path = self.getstring(name, defaultpath) if path is None: return path return toxinidir.join(path, abs=True) - def getlist(self, section, name, sep="\n"): - s = self.getdefault(section, name, None) + def getlist(self, name, sep="\n"): + s = self.getstring(name, None) if s is None: return [] return [x.strip() for x in s.split(sep) if x.strip()] - def getdict(self, section, name, sep="\n"): - s = self.getdefault(section, name, None) + def getdict(self, name, default=None, sep="\n"): + s = self.getstring(name, None) if s is None: - return {} + return default or {} value = {} for line in s.split(sep): - if not line.strip(): - continue - name, rest = line.split('=', 1) - value[name.strip()] = rest.strip() + if line.strip(): + name, rest = line.split('=', 1) + value[name.strip()] = rest.strip() return value - def getargvlist(self, section, name): - """Get arguments for every parsed command. - - :param str section: Section name in the configuration. - :param str name: Key name in a section. - :rtype: list[list[str]] - :raise :class:`tox.exception.ConfigError`: - line-continuation ends nowhere while resolving for specified section - """ - content = self.getdefault(section, name, '', replace=False) - return self._parse_commands(section, name, content) - - def _parse_commands(self, section, name, content): - """Parse commands from key content in specified section. - - :param str section: Section name in the configuration. - :param str name: Key name in a section. - :param str content: Content stored by key. - - :rtype: list[list[str]] - :raise :class:`tox.exception.ConfigError`: - line-continuation ends nowhere while resolving for specified section - """ - commands = [] - current_command = "" - for line in content.splitlines(): - line = line.rstrip() - i = line.find("#") - if i != -1: - line = line[:i].rstrip() - if not line: - continue - if line.endswith("\\"): - current_command += " " + line[:-1] - continue - current_command += line - - if is_section_substitution(current_command): - replaced = self._replace(current_command) - commands.extend(self._parse_commands(section, name, replaced)) - else: - commands.append(self._processcommand(current_command)) - current_command = "" - else: - if current_command: - raise tox.exception.ConfigError( - "line-continuation ends nowhere while resolving for [%s] %s" % - (section, name)) - return commands - - def _processcommand(self, command): - posargs = getattr(self, "posargs", None) - - # Iterate through each word of the command substituting as - # appropriate to construct the new command string. This - # string is then broken up into exec argv components using - # shlex. - newcommand = "" - for word in CommandParser(command).words(): - if word == "{posargs}" or word == "[]": - if posargs: - newcommand += " ".join(posargs) - continue - elif word.startswith("{posargs:") and word.endswith("}"): - if posargs: - newcommand += " ".join(posargs) - continue - else: - word = word[9:-1] - new_arg = "" - new_word = self._replace(word) - new_word = self._replace(new_word) - new_arg += new_word - newcommand += new_arg - - # Construct shlex object that will not escape any values, - # use all values as is in argv. - shlexer = shlex.shlex(newcommand, posix=True) - shlexer.whitespace_split = True - shlexer.escape = '' - shlexer.commenters = '' - argv = list(shlexer) - return argv - - def getargv(self, section, name, default=None, replace=True): - command = self.getdefault( - section, name, default=default, replace=False) - return self._processcommand(command.strip()) - - def getbool(self, section, name, default=None): - s = self.getdefault(section, name, default) + def getbool(self, name, default=None): + s = self.getstring(name, default) if not s: s = default if s is None: raise KeyError("no config value [%s] %s found" % ( - section, name)) + self.section_name, name)) if not isinstance(s, bool): if s.lower() == "true": @@ -705,9 +795,16 @@ "boolean value %r needs to be 'True' or 'False'") return s - def getdefault(self, section, name, default=None, replace=True): + def getargvlist(self, name, default=""): + s = self.getstring(name, default, replace=False) + return _ArgvlistReader.getargvlist(self, s) + + def getargv(self, name, default=""): + return self.getargvlist(name, default)[0] + + def getstring(self, name, default=None, replace=True): x = None - for s in [section] + self.fallbacksections: + for s in [self.section_name] + self.fallbacksections: try: x = self._cfg[s][name] break @@ -720,12 +817,12 @@ x = self._apply_factors(x) if replace and x and hasattr(x, 'replace'): - self._subststack.append((section, name)) + self._subststack.append((self.section_name, name)) try: x = self._replace(x) finally: - assert self._subststack.pop() == (section, name) - # print "getdefault", section, name, "returned", repr(x) + assert self._subststack.pop() == (self.section_name, name) + # print "getstring", self.section_name, name, "returned", repr(x) return x def _apply_factors(self, s): @@ -821,8 +918,80 @@ return RE_ITEM_REF.sub(self._replace_match, x) return x - def _parse_command(self, command): - pass + +class _ArgvlistReader: + @classmethod + def getargvlist(cls, reader, section_val): + """Parse ``commands`` argvlist multiline string. + + :param str name: Key name in a section. + :param str section_val: Content stored by key. + + :rtype: list[list[str]] + :raise :class:`tox.exception.ConfigError`: + line-continuation ends nowhere while resolving for specified section + """ + commands = [] + current_command = "" + for line in section_val.splitlines(): + line = line.rstrip() + i = line.find("#") + if i != -1: + line = line[:i].rstrip() + if not line: + continue + if line.endswith("\\"): + current_command += " " + line[:-1] + continue + current_command += line + + if is_section_substitution(current_command): + replaced = reader._replace(current_command) + commands.extend(cls.getargvlist(reader, replaced)) + else: + commands.append(cls.processcommand(reader, current_command)) + current_command = "" + else: + if current_command: + raise tox.exception.ConfigError( + "line-continuation ends nowhere while resolving for [%s] %s" % + (reader.section_name, "commands")) + return commands + + @classmethod + def processcommand(cls, reader, command): + posargs = getattr(reader, "posargs", None) + + # Iterate through each word of the command substituting as + # appropriate to construct the new command string. This + # string is then broken up into exec argv components using + # shlex. + newcommand = "" + for word in CommandParser(command).words(): + if word == "{posargs}" or word == "[]": + if posargs: + newcommand += " ".join(posargs) + continue + elif word.startswith("{posargs:") and word.endswith("}"): + if posargs: + newcommand += " ".join(posargs) + continue + else: + word = word[9:-1] + new_arg = "" + new_word = reader._replace(word) + new_word = reader._replace(new_word) + new_arg += new_word + newcommand += new_arg + + # Construct shlex object that will not escape any values, + # use all values as is in argv. + shlexer = shlex.shlex(newcommand, posix=True) + shlexer.whitespace_split = True + shlexer.escape = '' + shlexer.commenters = '' + argv = list(shlexer) + return argv class CommandParser(object): diff -r d7d35b623979d2e8186798c5bd29a679ef825c56 -r 06bb44d51a4c4d02f643da800d47a7197cfab32c tox/_venv.py --- a/tox/_venv.py +++ b/tox/_venv.py @@ -10,17 +10,17 @@ class CreationConfig: def __init__(self, md5, python, version, sitepackages, - develop, deps): + usedevelop, deps): self.md5 = md5 self.python = python self.version = version self.sitepackages = sitepackages - self.develop = develop + self.usedevelop = usedevelop self.deps = deps def writeconfig(self, path): lines = ["%s %s" % (self.md5, self.python)] - lines.append("%s %d %d" % (self.version, self.sitepackages, self.develop)) + lines.append("%s %d %d" % (self.version, self.sitepackages, self.usedevelop)) for dep in self.deps: lines.append("%s %s" % dep) path.ensure() @@ -32,14 +32,14 @@ lines = path.readlines(cr=0) value = lines.pop(0).split(None, 1) md5, python = value - version, sitepackages, develop = lines.pop(0).split(None, 3) + version, sitepackages, usedevelop = lines.pop(0).split(None, 3) sitepackages = bool(int(sitepackages)) - develop = bool(int(develop)) + usedevelop = bool(int(usedevelop)) deps = [] for line in lines: md5, depstring = line.split(None, 1) deps.append((md5, depstring)) - return CreationConfig(md5, python, version, sitepackages, develop, deps) + return CreationConfig(md5, python, version, sitepackages, usedevelop, deps) except Exception: return None @@ -48,7 +48,7 @@ and self.python == other.python and self.version == other.version and self.sitepackages == other.sitepackages - and self.develop == other.develop + and self.usedevelop == other.usedevelop and self.deps == other.deps) @@ -143,11 +143,11 @@ self.envconfig.deps, v) def _getliveconfig(self): - python = self.envconfig._basepython_info.executable + python = self.envconfig.python_info.executable md5 = getdigest(python) version = tox.__version__ sitepackages = self.envconfig.sitepackages - develop = self.envconfig.develop + develop = self.envconfig.usedevelop deps = [] for dep in self._getresolvedeps(): raw_dep = dep.name @@ -321,11 +321,13 @@ for envname in self.envconfig.passenv: if envname in os.environ: env[envname] = os.environ[envname] - setenv = self.envconfig.setenv - if setenv: - env.update(setenv) + + env.update(self.envconfig.setenv) + env['VIRTUAL_ENV'] = str(self.path) + env.update(extraenv) + return env def test(self, redirect=False): diff -r d7d35b623979d2e8186798c5bd29a679ef825c56 -r 06bb44d51a4c4d02f643da800d47a7197cfab32c tox/hookspecs.py --- /dev/null +++ b/tox/hookspecs.py @@ -0,0 +1,32 @@ +""" Hook specifications for tox. + +""" + +from pluggy import HookspecMarker, HookimplMarker + +hookspec = HookspecMarker("tox") +hookimpl = HookimplMarker("tox") + + +@hookspec +def tox_addoption(parser): + """ add command line options to the argparse-style parser object.""" + + +@hookspec +def tox_configure(config): + """ called after command line options have been parsed and the ini-file has + been read. Please be aware that the config object layout may change as its + API was not designed yet wrt to providing stability (it was an internal + thing purely before tox-2.0). """ + + +@hookspec(firstresult=True) +def tox_get_python_executable(envconfig): + """ return a python executable for the given python base name. + The first plugin/hook which returns an executable path will determine it. + + ``envconfig`` is the testenv configuration which contains + per-testenv configuration, notably the ``.envname`` and ``.basepython`` + setting. + """ This diff is so big that we needed to truncate the remainder. Repository URL: https://bitbucket.org/hpk42/tox/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email. _______________________________________________ pytest-commit mailing list pytest-commit@python.org https://mail.python.org/mailman/listinfo/pytest-commit
