This replaces the make_profile function with a subclass of TestProfile. It's a little cleaner and easier to understand, and much better documented than make_profile.
It will be further extended in later patches in this series to do more than just override the constructor. Signed-off-by: Dylan Baker <[email protected]> --- framework/test/deqp.py | 96 ++++++++++++++++++++++++++++++++++--------------- tests/cts.py | 42 ++++++++++++++-------- tests/deqp_gles2.py | 11 +++--- tests/deqp_gles3.py | 12 ++++--- tests/deqp_gles31.py | 11 +++--- tests/deqp_vk.py | 11 +++--- unittests/deqp_tests.py | 63 +++++++++++++++++++++----------- 7 files changed, 162 insertions(+), 84 deletions(-) diff --git a/framework/test/deqp.py b/framework/test/deqp.py index 1ed594a..de6aab4 100644 --- a/framework/test/deqp.py +++ b/framework/test/deqp.py @@ -22,6 +22,7 @@ from __future__ import ( absolute_import, division, print_function, unicode_literals ) import abc +import functools import os import re import subprocess @@ -36,10 +37,8 @@ from framework.options import OPTIONS __all__ = [ 'DEQPBaseTest', 'DEQPGroupTest', - 'gen_caselist_txt', + 'DEQPProfile', 'get_option', - 'iter_deqp_test_cases', - 'make_profile', ] @@ -63,25 +62,7 @@ _EXTRA_ARGS = get_option('PIGLIT_DEQP_EXTRA_ARGS', default='').split() -def make_profile(test_list, single_class=None, multi_class=None): - """Create a TestProfile instance.""" - if OPTIONS.deqp_mode == 'group': - assert multi_class is not None - _class = multi_class - elif OPTIONS.deqp_mode == 'test': - assert single_class is not None - _class = single_class - - profile = TestProfile() - for testname in test_list: - # deqp uses '.' as the testgroup separator. - piglit_name = testname.replace('.', grouptools.SEPARATOR) - profile.test_list[piglit_name] = _class(testname) - - return profile - - -def gen_caselist_txt(bin_, caselist, extra_args): +def _gen_caselist_txt(bin_, caselist, extra_args): """Generate a caselist.txt and return its path. Extra args should be a list of extra arguments to pass to deqp. @@ -106,7 +87,8 @@ def gen_caselist_txt(bin_, caselist, extra_args): subprocess.check_call( [bin_, '--deqp-runmode=txt-caselist'] + extra_args, cwd=basedir, stdout=d, stderr=d) - assert os.path.exists(caselist_path) + assert os.path.exists(caselist_path), \ + 'Could not find {}'.format(caselist_path) return caselist_path @@ -119,7 +101,7 @@ def _iterate_file(file_): yield line -def _iter_deqp_test_groups(case_file): +def _iter_test_groups(case_file): """Iterate over original dEQP testcase groups. This generator yields the name of each leaf group (that is, a group which @@ -141,7 +123,7 @@ def _iter_deqp_test_groups(case_file): 'deqp: {}:{}: ill-formed line'.format(case_file, i)) -def _iter_deqp_test_single(case_file): +def _iter_test_single(case_file): """Iterate over original dEQP testcase names.""" with open(case_file, 'r') as caselist_file: for i, line in enumerate(_iterate_file(caselist_file)): @@ -154,12 +136,70 @@ def _iter_deqp_test_single(case_file): 'deqp: {}:{}: ill-formed line'.format(case_file, i)) -def iter_deqp_test_cases(case_file): +def _iter_test_cases(case_file): """Wrapper that sets the iterator based on the mode.""" if OPTIONS.deqp_mode == 'group': - return _iter_deqp_test_groups(case_file) + return _iter_test_groups(case_file) elif OPTIONS.deqp_mode == 'test': - return _iter_deqp_test_single(case_file) + return _iter_test_single(case_file) + + +def _pop(kwargs, name, default=None): + """A guard function for DEQPProfile.""" + try: + return kwargs.pop(name) + except KeyError: + if default is not None: + return default + raise TypeError('Required keyword argument {} was not set'.format(name)) + + +class DEQPProfile(TestProfile): + """A profile specifically for dEQP tests. + + This profile provides much of the necessary setup bits for dEQP + integration, including generating a valid test list. + + All arguments not listed will be passed to the parent class. + + Keyword Arguments: + single_class -- the class to use for 'test at a time' mode. Required. + multi_class -- the class to use for 'group at a time' mode. Required. + filter_ -- A function that filters that test cases. By default this is a + no-op function. + bin_ -- the dEQP binary to use. Required. + extra_args -- the extra arguments to pass to the dEQP binary for each test. + Default: []. + filename -- the name of the txt file containing all of the test and group + information that dEQP generates. Required. + + """ + + def __init__(self, *args, **kwargs): + # PYTHON3: This could all be done with explict keyword arguments + # instead of this...madness + pop = functools.partial(_pop, kwargs) + + single_class = pop('single_class') + multi_class = pop('multi_class') + bin_ = pop('bin_') + filename = pop('filename') + extra_args = pop('extra_args', default=[]) + filter_ = pop('filter_', default=lambda x: x) + + super(DEQPProfile, self).__init__(*args, **kwargs) + + iter_ = _iter_test_cases(filter_( + _gen_caselist_txt(bin_, filename, extra_args))) + + if OPTIONS.deqp_mode == 'group': + class_ = multi_class + elif OPTIONS.deqp_mode == 'test': + class_ = single_class + + for testname in iter_: + piglit_name = testname.replace('.', grouptools.SEPARATOR) + self.test_list[piglit_name] = class_(testname) @six.add_metaclass(abc.ABCMeta) diff --git a/tests/cts.py b/tests/cts.py index e0b05d1..afbb842 100644 --- a/tests/cts.py +++ b/tests/cts.py @@ -45,9 +45,10 @@ PIGLIT_CTS_EXTRA_ARGS -- environment equivalent of [cts]:extra_args from __future__ import ( absolute_import, division, print_function, unicode_literals ) -import itertools +import functools from framework.test import deqp +from framework import profile __all__ = ['profile'] @@ -76,17 +77,28 @@ class DEQPCTSGroupTest(_Mixin, deqp.DEQPGroupTest): pass -# Add all of the suites by default, users can use filters to remove them. -profile = deqp.make_profile( # pylint: disable=invalid-name - itertools.chain( - deqp.iter_deqp_test_cases( - deqp.gen_caselist_txt(_CTS_BIN, 'ES2-CTS-cases.txt', _EXTRA_ARGS)), - deqp.iter_deqp_test_cases( - deqp.gen_caselist_txt(_CTS_BIN, 'ES3-CTS-cases.txt', _EXTRA_ARGS)), - deqp.iter_deqp_test_cases( - deqp.gen_caselist_txt(_CTS_BIN, 'ES31-CTS-cases.txt', _EXTRA_ARGS)), - deqp.iter_deqp_test_cases( - deqp.gen_caselist_txt(_CTS_BIN, 'ESEXT-CTS-cases.txt', - _EXTRA_ARGS)), - ), - single_class=DEQPCTSTest, multi_class=DEQPCTSGroupTest) +def _make_profile(): + """Make a single profile for the GLES CTS. + + The GLES CTS is wierd. It's 4 distinct test suites, that need to be run as + one. The profile mechanism for dEQP integration doesn't really support + this, so instead what is done is that 4 profiles are created, then merged. + + This can easily be trimmed using the standard test filters from the command + line. + + """ + partial = functools.partial(deqp.DEQPProfile, + single_class=DEQPCTSTest, + multi_class=DEQPCTSGroupTest, + bin_=_CTS_BIN, + extra_args=_EXTRA_ARGS) + profile_ = partial(filename='ES2-CTS-cases.txt') + profile_.update(partial(filename='ES3-CTS-cases.txt')) + profile_.update(partial(filename='ES31-CTS-cases.txt')) + profile_.update(partial(filename='ESEXT-CTS-cases.txt')) + + return profile_ + + +profile = _make_profile() # pylint: disable=invalid-name diff --git a/tests/deqp_gles2.py b/tests/deqp_gles2.py index 5bf3672..56279d8 100644 --- a/tests/deqp_gles2.py +++ b/tests/deqp_gles2.py @@ -57,8 +57,9 @@ class DEQPGLES2GroupTest(_Mixin, deqp.DEQPGroupTest): pass -profile = deqp.make_profile( # pylint: disable=invalid-name - deqp.iter_deqp_test_cases( - deqp.gen_caselist_txt(_DEQP_GLES2_BIN, 'dEQP-GLES2-cases.txt', - _EXTRA_ARGS)), - single_class=DEQPGLES2Test, multi_class=DEQPGLES2GroupTest) +profile = deqp.DEQPProfile( # pylint: disable=invalid-name + single_class=DEQPGLES2Test, + multi_class=DEQPGLES2GroupTest, + bin_=_DEQP_GLES2_BIN, + filename='dEQP-GLES2-cases.txt', + extra_args=_EXTRA_ARGS) diff --git a/tests/deqp_gles3.py b/tests/deqp_gles3.py index c50033b..506ef09 100644 --- a/tests/deqp_gles3.py +++ b/tests/deqp_gles3.py @@ -99,8 +99,10 @@ class DEQPGLES3GroupTest(_Mixin, deqp.DEQPGroupTest): pass -profile = deqp.make_profile( # pylint: disable=invalid-name - deqp.iter_deqp_test_cases(filter_mustpass( - deqp.gen_caselist_txt(_DEQP_GLES3_EXE, 'dEQP-GLES3-cases.txt', - _EXTRA_ARGS))), - single_class=DEQPGLES3Test, multi_class=DEQPGLES3GroupTest) +profile = deqp.DEQPProfile( # pylint: disable=invalid-name + single_class=DEQPGLES3Test, + multi_class=DEQPGLES3GroupTest, + bin_=_DEQP_GLES3_EXE, + filename='dEQP-GLES3-cases.txt', + filter_=filter_mustpass, + extra_args=_EXTRA_ARGS) diff --git a/tests/deqp_gles31.py b/tests/deqp_gles31.py index 666dcab..8691939 100644 --- a/tests/deqp_gles31.py +++ b/tests/deqp_gles31.py @@ -57,8 +57,9 @@ class DEQPGLES31GroupTest(_Mixin, deqp.DEQPGroupTest): pass -profile = deqp.make_profile( # pylint: disable=invalid-name - deqp.iter_deqp_test_cases( - deqp.gen_caselist_txt(_DEQP_GLES31_BIN, 'dEQP-GLES31-cases.txt', - _EXTRA_ARGS)), - single_class=DEQPGLES31Test, multi_class=DEQPGLES31GroupTest) +profile = deqp.DEQPProfile( # pylint: disable=invalid-name + single_class=DEQPGLES31Test, + multi_class=DEQPGLES31GroupTest, + bin_=_DEQP_GLES31_BIN, + filename='dEQP-GLES31-cases.txt', + extra_args=_EXTRA_ARGS) diff --git a/tests/deqp_vk.py b/tests/deqp_vk.py index 9a31c46..1cfce6e 100644 --- a/tests/deqp_vk.py +++ b/tests/deqp_vk.py @@ -77,8 +77,9 @@ class DEQPVKGroupTest(_Mixin, deqp.DEQPGroupTest): pass -profile = deqp.make_profile( # pylint: disable=invalid-name - deqp.iter_deqp_test_cases( - deqp.gen_caselist_txt(_DEQP_VK_BIN, 'dEQP-VK-cases.txt', - _EXTRA_ARGS)), - single_class=DEQPVKTest, multi_class=DEQPVKGroupTest) +profile = deqp.DEQPProfile( # pylint: disable=invalid-name + single_class=DEQPVKTest, + multi_class=DEQPVKGroupTest, + bin_=_DEQP_VK_BIN, + filename='dEQP-VK-cases.txt', + extra_args=_EXTRA_ARGS) diff --git a/unittests/deqp_tests.py b/unittests/deqp_tests.py index f3b243e..2d347a8 100644 --- a/unittests/deqp_tests.py +++ b/unittests/deqp_tests.py @@ -28,10 +28,12 @@ tests from __future__ import ( absolute_import, division, print_function, unicode_literals ) +import copy import sys import textwrap import nose.tools as nt +import six # There is a bug in mock < 1.2 or python 3.4 that we'd like to avoid, otherwise # some tests will skip. @@ -105,34 +107,17 @@ def test_get_option_conf_no_option(): None) -class TestMakeProfile(object): - """Test deqp.make_profile.""" - @classmethod - def setup_class(cls): - cls.profile = deqp.make_profile(['this.is.a.deqp.test'], _DEQPTestTest) - - def test_returns_profile(self): - """deqp.make_profile: returns a TestProfile""" - nt.assert_is_instance(self.profile, profile.TestProfile) - - @doc_formatter - def test_grouptools(self): - """deqp.make_profile: replaces '.' with '{separator}'""" - nt.assert_in(grouptools.join('this', 'is', 'a', 'deqp', 'test'), - self.profile.test_list) - - def test_iter_deqp_test_cases_test(): """deqp.iter_deqp_test_cases: correctly detects a TEST: line""" with utils.tempfile('TEST: a.deqp.test') as tfile: - gen = deqp.iter_deqp_test_cases(tfile) + gen = deqp._iter_test_cases(tfile) nt.eq_('a.deqp.test', next(gen)) def test_iter_deqp_test_cases_group(): """deqp.iter_deqp_test_casesgen_caselist_txt: correctly detects a GROUP: line""" with utils.tempfile('GROUP: a group\nTEST: a.deqp.test') as tfile: - gen = deqp.iter_deqp_test_cases(tfile) + gen = deqp._iter_test_cases(tfile) nt.eq_('a.deqp.test', next(gen)) @@ -141,7 +126,7 @@ def test_iter_deqp_test_cases_bad(): """deqp.iter_deqp_test_casesgen_caselist_txt: PiglitFatalException is raised if line is not TEST: or GROUP: """ with utils.tempfile('this will fail') as tfile: - gen = deqp.iter_deqp_test_cases(tfile) + gen = deqp._iter_test_cases(tfile) nt.eq_('a.deqp.test', next(gen)) @@ -552,6 +537,42 @@ def test_iter_deqp_test_groups(): with mock.patch('framework.test.deqp.open', create=True, new=mock.mock_open(read_data=text)): - actual = list(deqp._iter_deqp_test_groups(None)) + actual = list(deqp._iter_test_groups(None)) nt.assert_list_equal(actual, expected) + + [email protected]_generator +def test_DEQPProfile_init_required(): + """Generate tests for DEQPPRofile's constructor required arguments. + + Basically this builds a valid list of arguments, and then erases them one + by one looking for an exception. + + """ + description = 'deqp.DEQPProfile: requires kwarg {}' + + kwargs = { + 'single_class': None, + 'multi_class': None, + 'bin_': None, + 'filename': None, + } + + def test(arg): + """The actual test.""" + test_args = copy.copy(kwargs) + del test_args[arg] + + with nt.assert_raises(TypeError) as e: + deqp.DEQPProfile(**test_args) + + # Ensure that we're getting the right error. + # An initial version of the test "passed" because of a typo in + # DEQPProfile + nt.eq_(six.text_type(e.exception), + 'Required keyword argument {} was not set'.format(arg)) + + for arg in six.iterkeys(kwargs): + test.description = description.format(arg) + yield test, arg -- 2.7.4 _______________________________________________ Piglit mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/piglit
