2 new commits in pytest:
https://bitbucket.org/hpk42/pytest/changeset/9a022cbc021d/ changeset: 9a022cbc021d user: hpk42 date: 2012-07-23 10:55:09 summary: allow funcarg factories to receive funcargs affected #: 4 files diff -r 01d1731a8542b591ebb6ba0cdad31b2c497a0b6b -r 9a022cbc021de5518b3036877e852b314469a15b _pytest/main.py --- a/_pytest/main.py +++ b/_pytest/main.py @@ -444,17 +444,26 @@ self.pytest_plugin_registered(plugin) def pytest_generate_tests(self, metafunc): - for argname in metafunc.funcargnames: + funcargnames = list(metafunc.funcargnames) + seen = set() + while funcargnames: + argname = funcargnames.pop(0) + if argname in seen: + continue + seen.add(argname) faclist = self.getfactorylist(argname, metafunc.parentid, metafunc.function, raising=False) if faclist is None: - continue # will raise at setup time + continue # will raise FuncargLookupError at setup time for fac in faclist: marker = getattr(fac, "funcarg", None) if marker is not None: params = marker.kwargs.get("params") if params is not None: metafunc.parametrize(argname, params, indirect=True) + newfuncargnames = getfuncargnames(fac) + newfuncargnames.remove("request") + funcargnames.extend(newfuncargnames) def _parsefactories(self, holderobj, nodeid): if holderobj in self._holderobjseen: @@ -773,3 +782,15 @@ tw.line(" " + line.strip(), red=True) tw.line() tw.line("%s:%d" % (self.filename, self.firstlineno+1)) + +def getfuncargnames(function, startindex=None): + # XXX merge with main.py's varnames + argnames = py.std.inspect.getargs(py.code.getrawcode(function))[0] + if startindex is None: + startindex = py.std.inspect.ismethod(function) and 1 or 0 + defaults = getattr(function, 'func_defaults', + getattr(function, '__defaults__', None)) or () + numdefaults = len(defaults) + if numdefaults: + return argnames[startindex:-numdefaults] + return argnames[startindex:] diff -r 01d1731a8542b591ebb6ba0cdad31b2c497a0b6b -r 9a022cbc021de5518b3036877e852b314469a15b _pytest/python.py --- a/_pytest/python.py +++ b/_pytest/python.py @@ -3,7 +3,7 @@ import inspect import sys import pytest -from _pytest.main import getfslineno +from _pytest.main import getfslineno, getfuncargnames from _pytest.monkeypatch import monkeypatch import _pytest @@ -475,17 +475,6 @@ return True -def getfuncargnames(function, startindex=None): - # XXX merge with main.py's varnames - argnames = py.std.inspect.getargs(py.code.getrawcode(function))[0] - if startindex is None: - startindex = py.std.inspect.ismethod(function) and 1 or 0 - defaults = getattr(function, 'func_defaults', - getattr(function, '__defaults__', None)) or () - numdefaults = len(defaults) - if numdefaults: - return argnames[startindex:-numdefaults] - return argnames[startindex:] def fillfuncargs(function): """ fill missing funcargs. """ @@ -887,6 +876,7 @@ self.funcargnames = getfuncargnames(self.function) self.parentid = pyfuncitem.parent.nodeid self.scope = "function" + self._factorystack = [] def _getfaclist(self, argname): faclist = self._name2factory.get(argname, None) @@ -947,7 +937,8 @@ if self.funcargnames: assert not getattr(self._pyfuncitem, '_args', None), ( "yielded functions cannot have funcargs") - for argname in self.funcargnames: + while self.funcargnames: + argname = self.funcargnames.pop(0) if argname not in self._pyfuncitem.funcargs: self._pyfuncitem.funcargs[argname] = \ self.getfuncargvalue(argname) @@ -984,7 +975,10 @@ val = cache[cachekey] except KeyError: __tracebackhide__ = True - check_scope(self.scope, scope) + if scopemismatch(self.scope, scope): + raise ScopeMismatchError("You tried to access a %r scoped " + "resource with a %r scoped request object" %( + (scope, self.scope))) __tracebackhide__ = False val = setup() cache[cachekey] = val @@ -1010,6 +1004,21 @@ pass factorylist = self._getfaclist(argname) funcargfactory = factorylist.pop() + self._factorystack.append(funcargfactory) + try: + return self._getfuncargvalue(funcargfactory, argname) + finally: + self._factorystack.pop() + + def _getfuncargvalue(self, funcargfactory, argname): + # collect funcargs from the factory + newnames = getfuncargnames(funcargfactory) + newnames.remove("request") + factory_kwargs = {"request": self} + def fillfactoryargs(): + for newname in newnames: + factory_kwargs[newname] = self.getfuncargvalue(newname) + node = self._pyfuncitem mp = monkeypatch() mp.setattr(self, '_currentarg', argname) @@ -1027,16 +1036,30 @@ scope = marker.kwargs.get("scope") if scope is not None: __tracebackhide__ = True - check_scope(self.scope, scope) + if scopemismatch(self.scope, scope): + # try to report something helpful + lines = [] + for factory in self._factorystack: + fs, lineno = getfslineno(factory) + p = self._pyfuncitem.session.fspath.bestrelpath(fs) + args = inspect.formatargspec(*inspect.getargspec(factory)) + lines.append("%s:%d\n def %s%s" %( + p, lineno, factory.__name__, args)) + raise ScopeMismatchError("You tried to access the %r scoped " + "funcarg %r with a %r scoped request object, " + "involved factories\n%s" %( + (scope, argname, self.scope, "\n".join(lines)))) __tracebackhide__ = False mp.setattr(self, "scope", scope) kwargs = {} if hasattr(self, "param"): kwargs["extrakey"] = param - val = self.cached_setup(lambda: funcargfactory(request=self), + fillfactoryargs() + val = self.cached_setup(lambda: funcargfactory(**factory_kwargs), scope=scope, **kwargs) else: - val = funcargfactory(request=self) + fillfactoryargs() + val = funcargfactory(**factory_kwargs) mp.undo() self._funcargs[argname] = val return val @@ -1072,11 +1095,14 @@ """ A funcarg factory tries to access a funcargvalue/factory which has a lower scope (e.g. a Session one calls a function one) """ + scopes = "session module class function".split() -def check_scope(currentscope, newscope): - __tracebackhide__ = True - i_currentscope = scopes.index(currentscope) - i_newscope = scopes.index(newscope) - if i_newscope > i_currentscope: - raise ScopeMismatchError("You tried to access a %r scoped funcarg " - "from a %r scoped one." % (newscope, currentscope)) +def scopemismatch(currentscope, newscope): + return scopes.index(newscope) > scopes.index(currentscope) + +def slice_kwargs(names, kwargs): + new_kwargs = {} + for name in names: + new_kwargs[name] = kwargs[name] + return new_kwargs + diff -r 01d1731a8542b591ebb6ba0cdad31b2c497a0b6b -r 9a022cbc021de5518b3036877e852b314469a15b doc/en/resources.txt --- a/doc/en/resources.txt +++ b/doc/en/resources.txt @@ -248,10 +248,11 @@ def modes(tmpdir, request): # ... -This would execute the ``modes`` function once for each parameter. -In addition to normal funcargs you can also receive the "request" -funcarg which represents a takes on each of the values in the -``params=[1,2,3]`` decorator argument. +This would execute the ``modes`` function once for each parameter +which will be put at ``request.param``. This request object offers +the ``addfinalizer(func)`` helper which allows to register a function +which will be executed when test functions within the specified scope +finished execution. .. note:: diff -r 01d1731a8542b591ebb6ba0cdad31b2c497a0b6b -r 9a022cbc021de5518b3036877e852b314469a15b testing/test_python.py --- a/testing/test_python.py +++ b/testing/test_python.py @@ -619,7 +619,7 @@ def test_request_attributes_method(self, testdir): item, = testdir.getitems(""" class TestB: - def pytest_funcarg__something(request): + def pytest_funcarg__something(self, request): return 1 def test_func(self, something): pass @@ -1616,6 +1616,70 @@ "*1 passed*", ]) +class TestFuncargFactory: + def test_receives_funcargs(self, testdir): + testdir.makepyfile(""" + import pytest + @pytest.mark.funcarg + def arg1(request): + return 1 + + @pytest.mark.funcarg + def arg2(request, arg1): + return arg1 + 1 + + def test_add(arg2): + assert arg2 == 2 + def test_all(arg1, arg2): + assert arg1 == 1 + assert arg2 == 2 + """) + reprec = testdir.inline_run() + reprec.assertoutcome(passed=2) + + def test_receives_funcargs_scope_mismatch(self, testdir): + testdir.makepyfile(""" + import pytest + @pytest.mark.funcarg(scope="function") + def arg1(request): + return 1 + + @pytest.mark.funcarg(scope="module") + def arg2(request, arg1): + return arg1 + 1 + + def test_add(arg2): + assert arg2 == 2 + """) + result = testdir.runpytest() + result.stdout.fnmatch_lines([ + "*ScopeMismatch*involved factories*", + "* def arg2*", + "* def arg1*", + "*1 error*" + ]) + + def test_funcarg_parametrized_and_used_twice(self, testdir): + testdir.makepyfile(""" + import pytest + l = [] + @pytest.mark.funcarg(params=[1,2]) + def arg1(request): + l.append(1) + return request.param + + @pytest.mark.funcarg + def arg2(request, arg1): + return arg1 + 1 + + def test_add(arg1, arg2): + assert arg2 == arg1 + 1 + assert len(l) == arg1 + """) + result = testdir.runpytest() + result.stdout.fnmatch_lines([ + "*2 passed*" + ]) class TestResourceIntegrationFunctional: @@ -1802,7 +1866,7 @@ result = testdir.runpytest() assert result.ret != 0 result.stdout.fnmatch_lines([ - "*ScopeMismatch*You tried*function*from*session*", + "*ScopeMismatch*You tried*function*session*request*", ]) def test_register_only_with_mark(self, testdir): https://bitbucket.org/hpk42/pytest/changeset/7bbcf3b9fff3/ changeset: 7bbcf3b9fff3 user: hpk42 date: 2012-07-24 12:10:04 summary: introduce @pytest.mark.setup decorated function, extend newexamples.txt and draft a V4 resources API doc. affected #: 7 files diff -r 9a022cbc021de5518b3036877e852b314469a15b -r 7bbcf3b9fff3ea413cc2f4d7f862ba1fe355c93a _pytest/__init__.py --- a/_pytest/__init__.py +++ b/_pytest/__init__.py @@ -1,2 +1,2 @@ # -__version__ = '2.3.0.dev3' +__version__ = '2.3.0.dev4' diff -r 9a022cbc021de5518b3036877e852b314469a15b -r 7bbcf3b9fff3ea413cc2f4d7f862ba1fe355c93a _pytest/main.py --- a/_pytest/main.py +++ b/_pytest/main.py @@ -8,6 +8,8 @@ from _pytest.monkeypatch import monkeypatch from py._code.code import TerminalRepr +from _pytest.mark import MarkInfo + tracebackcutdir = py.path.local(_pytest.__file__).dirpath() # exitcodes for the command line @@ -422,6 +424,7 @@ self.arg2facspec = {} session.config.pluginmanager.register(self, "funcmanage") self._holderobjseen = set() + self.setuplist = [] ### XXX this hook should be called for historic events like pytest_configure ### so that we don't have to do the below pytest_collection hook @@ -445,6 +448,9 @@ def pytest_generate_tests(self, metafunc): funcargnames = list(metafunc.funcargnames) + setuplist, allargnames = self.getsetuplist(metafunc.parentid) + #print "setuplist, allargnames", setuplist, allargnames + funcargnames.extend(allargnames) seen = set() while funcargnames: argname = funcargnames.pop(0) @@ -465,6 +471,8 @@ newfuncargnames.remove("request") funcargnames.extend(newfuncargnames) + + def _parsefactories(self, holderobj, nodeid): if holderobj in self._holderobjseen: return @@ -473,20 +481,36 @@ for name in dir(holderobj): #print "check", holderobj, name obj = getattr(holderobj, name) + if not callable(obj): + continue # funcarg factories either have a pytest_funcarg__ prefix # or are "funcarg" marked if hasattr(obj, "funcarg"): - if name.startswith(self._argprefix): - argname = name[len(self._argprefix):] - else: - argname = name + assert not name.startswith(self._argprefix) + argname = name elif name.startswith(self._argprefix): argname = name[len(self._argprefix):] else: + # no funcargs. check if we have a setup function. + setup = getattr(obj, "setup", None) + if setup is not None and isinstance(setup, MarkInfo): + self.setuplist.append((nodeid, obj)) continue faclist = self.arg2facspec.setdefault(argname, []) faclist.append((nodeid, obj)) + def getsetuplist(self, nodeid): + l = [] + allargnames = set() + for baseid, setup in self.setuplist: + #print "check", baseid, setup + if nodeid.startswith(baseid): + funcargnames = getfuncargnames(setup) + l.append((setup, funcargnames)) + allargnames.update(funcargnames) + return l, allargnames + + def getfactorylist(self, argname, nodeid, function, raising=True): try: factorydef = self.arg2facspec[argname] diff -r 9a022cbc021de5518b3036877e852b314469a15b -r 7bbcf3b9fff3ea413cc2f4d7f862ba1fe355c93a _pytest/python.py --- a/_pytest/python.py +++ b/_pytest/python.py @@ -834,6 +834,8 @@ def setup(self): super(Function, self).setup() fillfuncargs(self) + if hasattr(self, "_request"): + self._request._callsetup() def __eq__(self, other): try: @@ -990,6 +992,22 @@ return val + def _callsetup(self): + setuplist, allnames = self.funcargmanager.getsetuplist( + self._pyfuncitem.nodeid) + for setupfunc, funcargnames in setuplist: + kwargs = {} + for name in funcargnames: + if name == "request": + kwargs[name] = self + else: + kwargs[name] = self.getfuncargvalue(name) + scope = readscope(setupfunc, "setup") + if scope is None: + setupfunc(**kwargs) + else: + self.cached_setup(lambda: setupfunc(**kwargs), scope=scope) + def getfuncargvalue(self, argname): """ Retrieve a function argument by name for this test function invocation. This allows one function argument factory @@ -1030,10 +1048,8 @@ mp.setattr(self, 'param', param, raising=False) # implemenet funcarg marker scope - marker = getattr(funcargfactory, "funcarg", None) - scope = None - if marker is not None: - scope = marker.kwargs.get("scope") + scope = readscope(funcargfactory, "funcarg") + if scope is not None: __tracebackhide__ = True if scopemismatch(self.scope, scope): @@ -1106,3 +1122,7 @@ new_kwargs[name] = kwargs[name] return new_kwargs +def readscope(func, markattr): + marker = getattr(func, markattr, None) + if marker is not None: + return marker.kwargs.get("scope") diff -r 9a022cbc021de5518b3036877e852b314469a15b -r 7bbcf3b9fff3ea413cc2f4d7f862ba1fe355c93a doc/en/example/newexamples.txt --- a/doc/en/example/newexamples.txt +++ b/doc/en/example/newexamples.txt @@ -48,7 +48,7 @@ ================================= FAILURES ================================= ________________________________ test_ehlo _________________________________ - smtp = <smtplib.SMTP instance at 0x28599e0> + smtp = <smtplib.SMTP instance at 0x20ba7e8> def test_ehlo(smtp): response = smtp.ehlo() @@ -60,7 +60,7 @@ test_module.py:5: AssertionError ________________________________ test_noop _________________________________ - smtp = <smtplib.SMTP instance at 0x28599e0> + smtp = <smtplib.SMTP instance at 0x20ba7e8> def test_noop(smtp): response = smtp.noop() @@ -69,7 +69,7 @@ E assert 0 test_module.py:10: AssertionError - 2 failed in 0.14 seconds + 2 failed in 0.27 seconds you will see the two ``assert 0`` failing and can see that the same (session-scoped) object was passed into the two test functions. @@ -100,7 +100,7 @@ ================================= FAILURES ================================= __________________________ test_ehlo[merlinux.eu] __________________________ - smtp = <smtplib.SMTP instance at 0x2bf3d40> + smtp = <smtplib.SMTP instance at 0x2a51830> def test_ehlo(smtp): response = smtp.ehlo() @@ -112,7 +112,7 @@ test_module.py:5: AssertionError ________________________ test_ehlo[mail.python.org] ________________________ - smtp = <smtplib.SMTP instance at 0x2bf9170> + smtp = <smtplib.SMTP instance at 0x2a56c20> def test_ehlo(smtp): response = smtp.ehlo() @@ -123,7 +123,7 @@ test_module.py:4: AssertionError __________________________ test_noop[merlinux.eu] __________________________ - smtp = <smtplib.SMTP instance at 0x2bf3d40> + smtp = <smtplib.SMTP instance at 0x2a51830> def test_noop(smtp): response = smtp.noop() @@ -134,7 +134,7 @@ test_module.py:10: AssertionError ________________________ test_noop[mail.python.org] ________________________ - smtp = <smtplib.SMTP instance at 0x2bf9170> + smtp = <smtplib.SMTP instance at 0x2a56c20> def test_noop(smtp): response = smtp.noop() @@ -143,9 +143,9 @@ E assert 0 test_module.py:10: AssertionError - 4 failed in 5.70 seconds - closing <smtplib.SMTP instance at 0x2bf9170> - closing <smtplib.SMTP instance at 0x2bf3d40> + 4 failed in 6.91 seconds + closing <smtplib.SMTP instance at 0x2a56c20> + closing <smtplib.SMTP instance at 0x2a51830> We get four failures because we are running the two tests twice with different ``smtp`` instantiations as defined on the factory. @@ -157,7 +157,7 @@ $ py.test --collectonly =========================== test session starts ============================ - platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev3 + platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev4 plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov collecting ... collected 4 items <Module 'test_module.py'> @@ -174,10 +174,113 @@ collecting ... collected 4 items FFFF ================================= FAILURES ================================= - /home/hpk/tmp/doc-exec-330/test_module.py:5: assert 0 - /home/hpk/tmp/doc-exec-330/test_module.py:4: assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN' - /home/hpk/tmp/doc-exec-330/test_module.py:10: assert 0 - /home/hpk/tmp/doc-exec-330/test_module.py:10: assert 0 - 4 failed in 6.02 seconds - closing <smtplib.SMTP instance at 0x1f5ef38> - closing <smtplib.SMTP instance at 0x1f5acf8> + /home/hpk/tmp/doc-exec-361/test_module.py:5: assert 0 + /home/hpk/tmp/doc-exec-361/test_module.py:4: assert 'merlinux' in 'mail.python.org\nSIZE 10240000\nETRN\nSTARTTLS\nENHANCEDSTATUSCODES\n8BITMIME\nDSN' + /home/hpk/tmp/doc-exec-361/test_module.py:10: assert 0 + /home/hpk/tmp/doc-exec-361/test_module.py:10: assert 0 + 4 failed in 6.83 seconds + closing <smtplib.SMTP instance at 0x236da28> + closing <smtplib.SMTP instance at 0x23687e8> + +.. _`new_setup`: + +``@pytest.mark.setup``: xUnit on steroids +-------------------------------------------------------------------- + +.. regendoc:wipe + +.. versionadded:: 2.3 + +The ``@pytest.mark.setup`` marker allows + +* to mark a function as a setup/fixture method; the function can itself + receive funcargs +* to set a scope which determines the level of caching and how often + the setup function is going to be called. + +Here is a simple example which configures a global funcarg without +the test needing to have it in its signature:: + + # content of conftest.py + import pytest + + @pytest.mark.funcarg(scope="module") + def resource(request, tmpdir): + def fin(): + print "finalize", tmpdir + request.addfinalizer(fin) + print "created resource", tmpdir + return tmpdir + +And the test file contains a setup function using this resource:: + + # content of test_module.py + import pytest + + @pytest.mark.setup(scope="function") + def setresource(resource): + global myresource + myresource = resource + + def test_1(): + assert myresource + print "using myresource", myresource + + def test_2(): + assert myresource + print "using myresource", myresource + +Let's run this module:: + + $ py.test -qs + collecting ... collected 2 items + .. + 2 passed in 0.24 seconds + created resource /home/hpk/tmp/pytest-3715/test_10 + using myresource /home/hpk/tmp/pytest-3715/test_10 + using myresource /home/hpk/tmp/pytest-3715/test_10 + finalize /home/hpk/tmp/pytest-3715/test_10 + +The two test functions will see the same resource instance because it has +a module life cycle or scope. + +The resource funcarg can later add parametrization without any test +or setup code needing to change:: + + # content of conftest.py + import pytest + + @pytest.mark.funcarg(scope="module", params=["aaa", "bbb"]) + def resource(request, tmpdir): + newtmp = tmpdir.join(request.param) + def fin(): + print "finalize", newtmp + request.addfinalizer(fin) + print "created resource", newtmp + return newtmp + +Running this will run four tests:: + + $ py.test -qs + collecting ... collected 4 items + .... + 4 passed in 0.24 seconds + created resource /home/hpk/tmp/pytest-3716/test_1_aaa_0/aaa + using myresource /home/hpk/tmp/pytest-3716/test_1_aaa_0/aaa + created resource /home/hpk/tmp/pytest-3716/test_1_bbb_0/bbb + using myresource /home/hpk/tmp/pytest-3716/test_1_bbb_0/bbb + using myresource /home/hpk/tmp/pytest-3716/test_1_aaa_0/aaa + using myresource /home/hpk/tmp/pytest-3716/test_1_bbb_0/bbb + finalize /home/hpk/tmp/pytest-3716/test_1_bbb_0/bbb + finalize /home/hpk/tmp/pytest-3716/test_1_aaa_0/aaa + +Each parameter causes the creation of a respective resource and the +unchanged test module uses it in its ``@setup`` decorated method. + +.. note:: + + Currently, parametrized tests are sorted by test function location + so a test function will execute multiple times with different parametrized + funcargs. If you have class/module/session scoped funcargs and + they cause global side effects this can cause problems because the + code under test may not be prepared to deal with it. diff -r 9a022cbc021de5518b3036877e852b314469a15b -r 7bbcf3b9fff3ea413cc2f4d7f862ba1fe355c93a doc/en/resources.txt --- a/doc/en/resources.txt +++ b/doc/en/resources.txt @@ -1,43 +1,55 @@ -V3: Creating and working with parametrized test resources +V4: Creating and working with parametrized resources =============================================================== **Target audience**: Reading this document requires basic knowledge of python testing, xUnit setup methods and the basic pytest funcarg mechanism, see http://pytest.org/latest/funcargs.html -**Abstract**: pytest-2.X provides more powerful and more flexible funcarg -and setup machinery. It does so by introducing a new @funcarg and a -new @setup marker which allows to define scoping and parametrization -parameters. If using ``@funcarg``, following the ``pytest_funcarg__`` -naming pattern becomes optional. Functions decorated with ``@setup`` -are called independenlty from the definition of funcargs but can -access funcarg values if needed. This allows for ultimate flexibility -in designing your test fixtures and their parametrization. Also, -you can now use ``py.test --collectonly`` to inspect your fixture -setup. Nonwithstanding these extensions, pre-existing test suites -and plugins written to work for previous pytest versions shall run unmodified. +**Abstract**: pytest-2.X provides yet more powerful and flexible +fixture machinery by introducing: +* a new ``@pytest.mark.funcarg`` marker to define funcarg factories and their + scoping and parametrization. No special ``pytest_funcarg__`` naming there. -**Changes**: This V3 draft is based on incorporating and thinking about -feedback provided by Floris Bruynooghe, Carl Meyer and Samuele Pedroni. -It remains as draft documentation, pending further refinements and -changes according to implementation or backward compatibility issues. -The main changes to V2 are: +* a new ``@pytest.mark.setup`` marker to define setup functions and their + scoping. -* Collapse funcarg factory decorator into a single "@funcarg" one. - You can specify scopes and params with it. Moreover, if you supply - a "name" you do not need to follow the "pytest_funcarg__NAME" naming - pattern. Keeping with "funcarg" naming arguable now makes more - sense since the main interface using these resources are test and - setup functions. Keeping it probably causes the least semantic friction. +* directly use funcargs through funcarg factory signatures -* Drop setup_directory/setup_session and introduce a new @setup - decorator similar to the @funcarg one but accepting funcargs. +Both funcarg factories and setup functions can be defined in test modules, +classes, conftest.py files and installed plugins. -* cosnider the extended setup_X funcargs for dropping because - the new @setup decorator probably is more flexible and introduces - less implementation complexity. +The introduction of these two markers lifts several prior limitations +and allows to easily define and implement complex testing scenarios. + +Nonwithstanding these extensions, already existing test suites and plugins +written to work for previous pytest versions shall run unmodified. + + +**Changes**: This V4 draft is based on incorporating and thinking about +feedback on previous versions provided by Floris Bruynooghe, Carl Meyer, +Ronny Pfannschmidt and Samuele Pedroni. It remains as draft +documentation, pending further refinements and changes according to +implementation or backward compatibility issues. The main changes are: + +* Collapse funcarg factory decorators into a single "@funcarg" one. + You can specify scopes and params with it. When using the decorator + the "pytest_funcarg__" prefix becomes optional. + +* funcarg factories can now use funcargs themselves + +* Drop setup/directory scope from this draft + +* introduce a new @setup decorator similar to the @funcarg one + except that setup-markers cannot define parametriation themselves. + Instead they can easily depend on a parametrized funcarg (which + must not be visible at test function signatures). + +* drop consideration of setup_X support for funcargs because + it is less flexible and probably causes more implementation + troubles than the current @setup approach which can share + a lot of logic with the @funcarg one. .. currentmodule:: _pytest @@ -78,17 +90,13 @@ ``extrakey`` parameter containing ``request.param`` to the :py:func:`~python.Request.cached_setup` call. -3. the current implementation is inefficient: it performs factory discovery - each time a "db" argument is required. This discovery wrongly happens at - setup-time. +3. there is no way how you can make use of funcarg factories + in xUnit setup methods. -4. there is no way how you can use funcarg factories, let alone - parametrization, when your tests use the xUnit setup_X approach. +4. A non-parametrized funcarg factory cannot use a parametrized + funcarg resource if it isn't stated in the test function signature. -5. there is no way to specify a per-directory scope for caching. - -In the following sections, API extensions are presented to solve -each of these problems. +The following sections address the advances which solve all of these problems. Direct scoping of funcarg factories @@ -158,7 +166,7 @@ Direct usage of funcargs with funcargs factories ---------------------------------------------------------- -.. note:: Not Implemented - unclear if to. +.. note:: Implemented. You can now directly use funcargs in funcarg factories. Example:: @@ -168,33 +176,39 @@ Apart from convenience it also solves an issue when your factory depends on a parametrized funcarg. Previously, a call to -``request.getfuncargvalue()`` would not allow pytest to know -at collection time about the fact that a required resource is -actually parametrized. +``request.getfuncargvalue()`` happens at test execution time and +thus pytest would not know at collection time about the fact that +a required resource is parametrized. -The "pytest_funcarg__" prefix becomes optional ------------------------------------------------------ +No ``pytest_funcarg__`` prefix when using @funcarg decorator +------------------------------------------------------------------- + .. note:: Implemented -When using the ``@funcarg`` decorator you do not need to use -the ``pytest_funcarg__`` prefix any more:: +When using the ``@funcarg`` decorator the name of the function +does not need to (and in fact cannot) use the ``pytest_funcarg__`` +naming:: @pytest.mark.funcarg def db(request): ... The name under which the funcarg resource can be requested is ``db``. -Any ``pytest_funcarg__`` prefix will be stripped. Note that a an -unqualified funcarg-marker implies a scope of "function" meaning -that the funcarg factory will be called for each test function invocation. +You can also use the "old" non-decorator way of specifying funcarg factories +aka:: + def pytest_funcarg__db(request): + ... -support for a new @setup marker ------------------------------------------------------- +It is recommended to use the funcarg-decorator, however. -.. note:: Not-Implemented, still under consideration if to. + +solving per-session setup / the new @setup marker +-------------------------------------------------------------- + +.. note:: Implemented, at least working for basic situations. pytest for a long time offered a pytest_configure and a pytest_sessionstart hook which are often used to setup global resources. This suffers from @@ -212,9 +226,7 @@ fact that this hook is actually used for reporting, in particular the test-header with platform/custom information. -4. there is no direct way how you can restrict setup to a directory scope. - -Moreover, it is today not easy to define scoped setup from plugins or +Moreover, it is today not easy to define a scoped setup from plugins or conftest files other than to implement a ``pytest_runtest_setup()`` hook and caring for scoping/caching yourself. And it's virtually impossible to do this with parametrization as ``pytest_runtest_setup()`` is called @@ -222,222 +234,76 @@ It follows that pytest_configure/session/runtest_setup are often not appropriate for implementing common fixture needs. Therefore, -pytest-2.X introduces a new "@pytest.mark.setup" marker, accepting -the same parameters as the @funcargs decorator. The difference is -that the decorated function can accept function arguments itself -Example:: - - # content of conftest.py - import pytest - @pytest.mark.setup(scope="session") - def mysetup(db): - ... +pytest-2.X introduces a new "@pytest.mark.setup" marker which takes +an optional "scope" parameter. -This ``mysetup`` function is going to be executed when the first -test in the directory tree executes. It is going to be executed once -per-session and it receives the ``db`` funcarg which must be of same -of higher scope; you e. g. generally cannot use a per-module or per-function -scoped resource in a session-scoped setup function. - -You can also use ``@setup`` inside a test module or class:: - - # content of test_module.py - import pytest - - @pytest.mark.setup(scope="module", params=[1,2,3]) - def modes(tmpdir, request): - # ... - -This would execute the ``modes`` function once for each parameter -which will be put at ``request.param``. This request object offers -the ``addfinalizer(func)`` helper which allows to register a function -which will be executed when test functions within the specified scope -finished execution. - -.. note:: - - For each scope, the funcargs will be setup and then the setup functions - will be called. This allows @setup-decorated functions to depend - on already setup funcarg values by accessing ``request.funcargs``. - -Using funcarg resources in xUnit setup methods ------------------------------------------------------------- - -.. note:: Not implemented. Not clear if to. - -XXX Consider this feature in contrast to the @setup feature - probably -introducing one of them is better and the @setup decorator is more flexible. - -For a long time, pytest has recommended the usage of funcarg -factories as a primary means for managing resources in your test run. -It is a better approach than the jUnit-based approach in many cases, even -more with the new pytest-2.X features, because the funcarg resource factory -provides a single place to determine scoping and parametrization. Your tests -do not need to encode setup/teardown details in every test file's -setup_module/class/method. - -However, the jUnit methods originally introduced by pytest to Python, -remain popoular with nose and unittest-based test suites. Without question, -there are large existing test suites using this paradigm. pytest-2.X -recognizes this fact and now offers direct integration with funcarg resources. Here is a basic example for getting a per-module tmpdir:: - - def setup_module(mod, tmpdir): - mod.tmpdir = tmpdir - -This will trigger pytest's funcarg mechanism to create a value of -"tmpdir" which can then be used throughout the module as a global. - -The new extension to setup_X methods also works in case a resource is -parametrized. For example, let's consider an setup_class example using -our "db" resource:: - - class TestClass: - def setup_class(cls, db): - cls.db = db - # perform some extra things on db - # so that test methods can work with it - -With pytest-2.X the setup* methods will be discovered at collection-time, -allowing to seemlessly integrate this approach with parametrization, -allowing the factory specification to determine all details. The -setup_class itself does not itself need to be aware of the fact that -"db" might be a mysql/PG database. -Note that if the specified resource is provided only as a per-testfunction -resource, collection would early on report a ScopingMismatch error. - - -the "directory" caching scope --------------------------------------------- - -.. note:: Not implemented. - -All API accepting a scope (:py:func:`cached_setup()` and -the new funcarg/setup decorators) now also accept a "directory" -specification. This allows to restrict/cache resource values on a -per-directory level. +See :ref:`new_setup` for examples. funcarg and setup discovery now happens at collection time --------------------------------------------------------------------- -.. note:: Partially implemented - collectonly shows no extra information +.. note:: + Partially implemented - collectonly shows no extra information however. -pytest-2.X takes care to discover funcarg factories and setup_X methods +pytest-2.X takes care to discover funcarg factories and @setup methods at collection time. This is more efficient especially for large test suites. Moreover, a call to "py.test --collectonly" should be able to show a lot of setup-information and thus presents a nice method to get an overview of resource management in your project. -Implementation level -=================================================================== -To implement the above new features, pytest-2.X grows some new hooks and -methods. At the time of writing V2 and without actually implementing -it, it is not clear how much of this new internal API will also be -exposed and advertised e. g. for plugin writers. +Sorting tests by funcarg scopes +------------------------------------------- -The main effort, however, will lie in revising what is done at -collection and what at test setup time. All funcarg factories and -xUnit setup methods need to be discovered at collection time -for the above mechanism to work. Additionally all test function -signatures need to be parsed in order to know which resources are -used. On the plus side, all previously collected fixtures and -test functions only need to be called, no discovery is neccessary -is required anymore. +.. note:: Not implemented, Under consideration. -the "request" object incorporates scope-specific behaviour ------------------------------------------------------------------- +pytest by default sorts test items by their source location. +For class/module/session scoped funcargs it is not always +desirable to have multiple active funcargs. Sometimes, +the application under test may not even be able to handle it +because it relies on global state/side effects related to those +resources. -funcarg factories receive a request object to help with implementing -finalization and inspection of the requesting-context. If there is -no scoping is in effect, nothing much will change of the API behaviour. -However, with scoping the request object represents the according context. -Let's consider this example:: +Therefore, pytest-2.3 tries to minimize the number of active +resources and re-orders test items accordingly. Consider the following +example:: - @pytest.mark.factory_scope("class") - def pytest_funcarg__db(request): - # ... - request.getfuncargvalue(...) - # - request.addfinalizer(db) + @pytest.mark.funcarg(scope="module", params=[1,2]) + def arg(request): + ... + @pytest.mark.funcarg(scope="function", params=[1,2]) + def otherarg(request): + ... -Due to the class-scope, the request object will: + def test_0(otherarg): + pass + def test_1(arg): + pass + def test_2(arg, otherarg): + pass -- provide a ``None`` value for the ``request.function`` attribute. -- default to per-class finalization with the addfinalizer() call. -- raise a ScopeMismatchError if a more broadly scoped factory - wants to use a more tighly scoped factory (e.g. per-function) +if arg.1, arg.2, otherarg.1, otherarg.2 denote the respective +parametrized funcarg instances this will re-order test +execution like follows:: -In fact, the request object is likely going to provide a "node" -attribute, denoting the current collection node on which it internally -operates. (Prior to pytest-2.3 there already was an internal -_pyfuncitem). + test_0(otherarg.1) + test_0(otherarg.2) + test_1(arg.1) + test_2(arg.1, otherarg.1) + test_2(arg.1, otherarg.2) + test_1(arg.2) + test_2(arg.2, otherarg.1) + test_2(arg.2, otherarg.2) -As these are rather intuitive extensions, not much friction is expected -for test/plugin writers using the new scoping and parametrization mechanism. -It's, however, a serious internal effort to reorganize the pytest -implementation. +Moreover, test_2(arg.1) will execute any registered teardowns for +the arg.1 resource after the test finished execution. +.. note:: -node.register_factory/getresource() methods --------------------------------------------------------- + XXX it's quite unclear at the moment how to implement. + If we have a 1000 tests requiring different sets of parametrized + resources with different scopes, how to re-order accordingly? + It even seems difficult to express the expectation in a + concise manner. -In order to implement factory- and setup-method discovery at -collection time, a new node API will be introduced to allow -for factory registration and a getresource() call to obtain -created values. The exact details of this API remain subject -to experimentation. The basic idea is to introduce two new -methods to the Session class which is already available on all nodes -through the ``node.session`` attribute:: - class Session: - def register_resource_factory(self, name, factory_or_list, scope): - """ register a resource factory for the given name. - - :param name: Name of the resource. - :factory_or_list: a function or a list of functions creating - one or multiple resource values. - :param scope: a node instance. The factory will be only visisble - available for all descendant nodes. - specify the "session" instance for global availability - """ - - def getresource(self, name, node): - """ get a named resource for the give node. - - This method looks up a matching funcarg resource factory - and calls it. - """ - -.. todo:: - - XXX While this new API (or some variant of it) may suffices to implement - all of the described new usage-level features, it remains unclear how the - existing "@parametrize" or "metafunc.parametrize()" calls will map to it. - These parametrize-approaches tie resource parametrization to the - function/funcargs-usage rather than to the factories. - - - -ISSUES --------------------------- - -decorating a parametrized funcarg factory:: - - @pytest.mark.funcarg(scope="session", params=["mysql", "pg"]) - def db(request): - ... - class TestClass: - @pytest.mark.funcarg(scope="function") - def something(self, request): - session_db = request.getfuncargvalue("db") - ... - -Here the function-scoped "something" factory uses the session-scoped -"db" factory to perform some additional steps. The dependency, however, -is only visible at setup-time, when the factory actually gets called. - -In order to allow parametrization at collection-time I see two ways: - -- allow specifying dependencies in the funcarg-marker -- allow funcargs for factories as well - diff -r 9a022cbc021de5518b3036877e852b314469a15b -r 7bbcf3b9fff3ea413cc2f4d7f862ba1fe355c93a setup.py --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ name='pytest', description='py.test: simple powerful testing with Python', long_description = long_description, - version='2.3.0.dev3', + version='2.3.0.dev4', url='http://pytest.org', license='MIT license', platforms=['unix', 'linux', 'osx', 'cygwin', 'win32'], diff -r 9a022cbc021de5518b3036877e852b314469a15b -r 7bbcf3b9fff3ea413cc2f4d7f862ba1fe355c93a testing/test_python.py --- a/testing/test_python.py +++ b/testing/test_python.py @@ -1746,12 +1746,121 @@ reprec = testdir.inline_run("-s") reprec.assertoutcome(passed=1) +class TestSetupDiscovery: + def pytest_funcarg__testdir(self, request): + testdir = request.getfuncargvalue("testdir") + testdir.makeconftest(""" + import pytest + @pytest.mark.setup + def perfunction(request): + pass + @pytest.mark.setup + def perfunction2(request): + pass + + def pytest_funcarg__fm(request): + return request.funcargmanager + + def pytest_funcarg__item(request): + return request._pyfuncitem + """) + return testdir + + def test_parsefactories_conftest(self, testdir): + testdir.makepyfile(""" + def test_check_setup(item, fm): + setuplist, allnames = fm.getsetuplist(item.nodeid) + assert len(setuplist) == 2 + assert setuplist[0][0].__name__ == "perfunction" + assert "request" in setuplist[0][1] + assert setuplist[1][0].__name__ == "perfunction2" + assert "request" in setuplist[1][1] + """) + reprec = testdir.inline_run("-s") + reprec.assertoutcome(passed=1) + + +class TestSetupManagement: + def test_funcarg_and_setup(self, testdir): + testdir.makepyfile(""" + import pytest + l = [] + @pytest.mark.funcarg(scope="module") + def arg(request): + l.append(1) + return 0 + @pytest.mark.setup(scope="class") + def something(request, arg): + l.append(2) + + def test_hello(arg): + assert len(l) == 2 + assert l == [1,2] + assert arg == 0 + + def test_hello2(arg): + assert len(l) == 2 + assert l == [1,2] + assert arg == 0 + """) + reprec = testdir.inline_run() + reprec.assertoutcome(passed=2) + + def test_setup_uses_parametrized_resource(self, testdir): + testdir.makepyfile(""" + import pytest + l = [] + @pytest.mark.funcarg(params=[1,2]) + def arg(request): + return request.param + + @pytest.mark.setup + def something(request, arg): + l.append(arg) + + def test_hello(): + if len(l) == 1: + assert l == [1] + elif len(l) == 2: + assert l == [1, 2] + else: + 0/0 + + """) + reprec = testdir.inline_run("-s") + reprec.assertoutcome(passed=2) + + def test_session_parametrized_function_setup(self, testdir): + testdir.makepyfile(""" + import pytest + + l = [] + + @pytest.mark.funcarg(scope="session", params=[1,2]) + def arg(request): + return request.param + + @pytest.mark.setup(scope="function") + def append(request, arg): + if request.function.__name__ == "test_some": + l.append(arg) + + def test_some(): + pass + + def test_result(arg): + assert len(l) == 2 + assert l == [1,2] + """) + reprec = testdir.inline_run("-s") + reprec.assertoutcome(passed=4) + class TestFuncargMarker: def test_parametrize(self, testdir): testdir.makepyfile(""" import pytest @pytest.mark.funcarg(params=["a", "b", "c"]) - def pytest_funcarg__arg(request): + def arg(request): return request.param l = [] def test_param(arg): @@ -1767,7 +1876,7 @@ import pytest l = [] @pytest.mark.funcarg(scope="module") - def pytest_funcarg__arg(request): + def arg(request): l.append(1) return 1 @@ -1789,7 +1898,7 @@ import pytest l = [] @pytest.mark.funcarg(scope="module") - def pytest_funcarg__arg(request): + def arg(request): l.append(1) return 1 @@ -1812,7 +1921,7 @@ finalized = [] created = [] @pytest.mark.funcarg(scope="module") - def pytest_funcarg__arg(request): + def arg(request): created.append(1) assert request.scope == "module" request.addfinalizer(lambda: finalized.append(1)) @@ -1851,14 +1960,14 @@ finalized = [] created = [] @pytest.mark.funcarg(scope="function") - def pytest_funcarg__arg(request): + def arg(request): pass """) testdir.makepyfile( test_mod1=""" import pytest @pytest.mark.funcarg(scope="session") - def pytest_funcarg__arg(request): + def arg(request): %s def test_1(arg): pass @@ -1894,7 +2003,7 @@ testdir.makepyfile(""" import pytest @pytest.mark.funcarg(scope="module", params=["a", "b", "c"]) - def pytest_funcarg__arg(request): + def arg(request): return request.param l = [] def test_param(arg): Repository URL: https://bitbucket.org/hpk42/pytest/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email. _______________________________________________ py-svn mailing list py-svn@codespeak.net http://codespeak.net/mailman/listinfo/py-svn
