This patches adds all of the boilerplate code that is required to get group-at-a-time working. It adds the command line option, and all of the switches to choose between creating a profile full of single tests, or tests that run group at a time.
Signed-off-by: Dylan Baker <[email protected]> --- framework/options.py | 1 +- framework/test/deqp.py | 93 +++++++++++-- unittests/framework/test/test_deqp.py | 196 +++++++++++++++++++++------ 3 files changed, 235 insertions(+), 55 deletions(-) diff --git a/framework/options.py b/framework/options.py index 5cb88aa..8bf5d9d 100644 --- a/framework/options.py +++ b/framework/options.py @@ -181,6 +181,7 @@ class _Options(object): # pylint: disable=too-many-instance-attributes monitored -- True if monitoring is desired. This forces concurrency off env -- environment variables set for each test before run deqp_mustpass -- True to enable the use of the deqp mustpass list feature. + process_isolation -- Whether to enforce process isolation. Default: True """ include_filter = _ReListDescriptor('_include_filter', type_=_FilterReList) diff --git a/framework/test/deqp.py b/framework/test/deqp.py index 25dd077..1240ee2 100644 --- a/framework/test/deqp.py +++ b/framework/test/deqp.py @@ -90,13 +90,24 @@ def select_source(bin_, filename, mustpass, extra_args): gen_caselist_txt(bin_, filename, extra_args)) -def make_profile(test_list, test_class): +def make_profile(test_list, single=None, group=None): """Create a TestProfile instance.""" + if options.OPTIONS.process_isolation: + assert single is not None + if isinstance(single, DEQPUnsupportedMode): + raise exceptions.PiglitFatalError(single) + test_class = single + else: + assert group is not None + if isinstance(group, DEQPUnsupportedMode): + raise exceptions.PiglitFatalError(group) + test_class = group + profile = TestProfile() for testname in test_list: # deqp uses '.' as the testgroup separator. - piglit_name = testname.replace('.', grouptools.SEPARATOR) - profile.test_list[piglit_name] = test_class(testname) + piglit_name = testname[0].replace('.', grouptools.SEPARATOR) + profile.test_list[piglit_name] = test_class(*testname) return profile @@ -104,17 +115,34 @@ def make_profile(test_list, test_class): def gen_mustpass_tests(mp_list): """Return a testlist from the mustpass list.""" root = et.parse(mp_list).getroot() - group = [] + cur_group = [] - def gen(root): + def single(root): for elem in root: if elem.tag == 'Test': - yield '{}.{}'.format('.'.join(group), elem.get('name')) + yield ('{}.{}'.format('.'.join(cur_group), elem.get('name')), ) else: - group.append(elem.get('name')) - for test in gen(elem): + cur_group.append(elem.get('name')) + for test in single(elem): + yield test + del cur_group[-1] + + def group(root): + for elem in root: + if elem.tag == 'TestCase': + case = '{}.{}'.format('.'.join(cur_group), elem.get('name')) + yield (case, ['{}.{}'.format(case, t.get('name')) + for t in elem.findall('.//Test')]) + elif elem.tag == 'TestSuite': + cur_group.append(elem.get('name')) + for test in group(elem): yield test - del group[-1] + del cur_group[-1] + + if options.OPTIONS.process_isolation: + gen = single + else: + gen = group for test in gen(root): yield test @@ -150,15 +178,54 @@ def gen_caselist_txt(bin_, caselist, extra_args): def iter_deqp_test_cases(case_file): """Iterate over original dEQP testcase names.""" - with open(case_file, 'r') as caselist_file: - for i, line in enumerate(caselist_file): + def single(f): + """Iterate over the txt file, and yield each test instance.""" + for i, line in enumerate(f): if line.startswith('GROUP:'): continue elif line.startswith('TEST:'): - yield line[len('TEST:'):].strip() + # The group mode yields a tuple, so the single mode needs to as + # well. + yield (line[len('TEST:'):].strip(), ) else: raise exceptions.PiglitFatalError( - 'deqp: {}:{}: ill-formed line'.format(case_file, i)) + 'deqp: {}:{}: ill-formed line'.format(f.name, i)) + + def group(f): + """Iterate over the txt file, and yield each group and its members. + + The group must contain actual members to be yielded. + """ + group = '' + tests = [] + + for i, line in enumerate(f): + if line.startswith('GROUP:'): + new = line[len('GROUP:'):].strip() + if group != new and tests: + yield (group, tests) + tests = [] + group = new + elif line.startswith('TEST:'): + tests.append(line[len('TEST:'):].strip()) + else: + raise exceptions.PiglitFatalError( + 'deqp: {}:{}: ill-formed line'.format(f.name, i)) + # If we get to the end of the file and we have new tests (the would + # have been cleared if there weren't any. + if tests: + yield (group, tests) + + adder = None + if options.OPTIONS.process_isolation: + adder = single + else: + adder = group + assert adder is not None + + with open(case_file, 'r') as f: + for x in adder(f): + yield x def format_trie_list(classname, testnames): diff --git a/unittests/framework/test/test_deqp.py b/unittests/framework/test/test_deqp.py index 58504fd..985f2c4 100644 --- a/unittests/framework/test/test_deqp.py +++ b/unittests/framework/test/test_deqp.py @@ -39,11 +39,12 @@ import six from framework import exceptions from framework import grouptools +from framework import options from framework import profile from framework import status from framework.test import deqp -# pylint:disable=invalid-name,no-self-use +# pylint:disable=no-self-use,protected-access class _DEQPTestTest(deqp.DEQPSingleTest): @@ -116,44 +117,123 @@ class TestGetOptions(object): class TestMakeProfile(object): """Test deqp.make_profile.""" - @classmethod - def setup_class(cls): - cls.profile = deqp.make_profile(['this.is.a.deqp.test'], _DEQPTestTest) + class TestSingle(object): + """Tests for the single mode.""" + + @classmethod + def setup_class(cls): + with mock.patch('framework.test.deqp.options.OPTIONS', + new=options._Options()) as mocked: + mocked.process_isolation = True + cls.profile = deqp.make_profile([('this.is.a.deqp.test', )], + single=_DEQPTestTest) - def test_returns_profile(self): - """deqp.make_profile: returns a TestProfile.""" - assert isinstance(self.profile, profile.TestProfile) + def test_returns_profile(self): + """deqp.make_profile: returns a TestProfile.""" + assert isinstance(self.profile, profile.TestProfile) - def test_replaces_separator(self): - """deqp.make_profile: replaces '.' with grouptools.separator""" - expected = grouptools.join('this', 'is', 'a', 'deqp', 'test') - assert expected in self.profile.test_list + def test_replaces_separator(self): + """deqp.make_profile: replaces '.' with grouptools.separator""" + expected = grouptools.join('this', 'is', 'a', 'deqp', 'test') + assert expected in self.profile.test_list + class TestGroup(object): + """Tests for group mode.""" -class TestIterDeqpTestCases(object): - """Tests for iter_deqp_test_cases.""" + @classmethod + def setup_class(cls): + with mock.patch('framework.test.deqp.options.OPTIONS', + new=options._Options()) as mocked: + mocked.process_isolation = False + cls.profile = deqp.make_profile( + [('this.is.a.deqp', ['this.is.a.deqp.test', + 'this.is.a.deqp.thing'])], + group=_DEQPGroupTrieTest) - def _do_test(self, write, expect, tmpdir): - """Run the acutal test.""" - p = tmpdir.join('foo') - p.write(write) - gen = deqp.iter_deqp_test_cases(six.text_type(p)) - assert next(gen) == expect + def test_returns_profile(self): + """deqp.make_profile: returns a TestProfile.""" + assert isinstance(self.profile, profile.TestProfile) - def test_test_cases(self, tmpdir): - """Correctly detects a test line.""" - self._do_test('TEST: a.deqp.test', 'a.deqp.test', tmpdir) + def test_replaces_separator(self): + """deqp.make_profile: replaces '.' with grouptools.separator""" + expected = grouptools.join('this', 'is', 'a', 'deqp') + assert expected in self.profile.test_list - def test_test_group(self, tmpdir): - """Correctly detects a group line.""" - self._do_test('GROUP: a group\nTEST: a.deqp.test', 'a.deqp.test', - tmpdir) - def test_bad_entry(self, tmpdir): - """A PiglitFatalException is raised if a line is not a TEST or GROUP. - """ - with pytest.raises(exceptions.PiglitFatalError): - self._do_test('this will fail', None, tmpdir) +class TestIterDeqpTestCases(object): + """Tests for iter_deqp_test_cases.""" + + class TestSingle(object): + """Tests for the single mode.""" + + @pytest.yield_fixture(autouse=True, scope='class') + def group_setup(self): + with mock.patch('framework.test.deqp.options.OPTIONS', + new=options._Options()) as mocked: + mocked.process_isolation = True + yield + + def _do_test(self, write, expect, tmpdir): + """Run the acutal test.""" + p = tmpdir.join('foo') + p.write(write) + gen = deqp.iter_deqp_test_cases(six.text_type(p)) + actual = next(gen) + assert actual == expect + + def test_test_cases(self, tmpdir): + """Correctly detects a test line.""" + self._do_test('TEST: a.deqp.test', ('a.deqp.test', ), tmpdir) + + def test_test_group(self, tmpdir): + """Correctly detects a group line.""" + self._do_test('GROUP: a group\nTEST: a.deqp.test', + ('a.deqp.test', ), tmpdir) + + def test_bad_entry(self, tmpdir): + """A PiglitFatalException is raised if a line is not a TEST or + GROUP. + """ + with pytest.raises(exceptions.PiglitFatalError): + self._do_test('this will fail', None, tmpdir) + + class TestGroup(object): + """Tests for the group mode.""" + + @pytest.yield_fixture(autouse=True, scope='class') + def group_setup(self): + with mock.patch('framework.test.deqp.options.OPTIONS', + new=options._Options()) as mocked: + mocked.process_isolation = False + yield + + def _do_test(self, write, expect, tmpdir): + """Run the acutal test.""" + p = tmpdir.join('foo') + p.write(write) + gen = deqp.iter_deqp_test_cases(six.text_type(p)) + actual = next(gen) + assert actual == expect + + def test_test_cases(self, tmpdir): + """Correctly detects a test line.""" + self._do_test( + textwrap.dedent("""\ + GROUP: a + GROUP: a.deqp + TEST: a.deqp.test + TEST: a.deqp.failure + TEST: a.deqp.foo + """), + ('a.deqp', ['a.deqp.test', 'a.deqp.failure', 'a.deqp.foo']), + tmpdir) + + def test_bad_entry(self, tmpdir): + """A PiglitFatalException is raised if a line is not a TEST or + GROUP. + """ + with pytest.raises(exceptions.PiglitFatalError): + self._do_test('this will fail', None, tmpdir) class TestFormatTrieList(object): @@ -460,7 +540,7 @@ class TestDEQPGroupTest(object): class TestGenMustpassTests(object): """Tests for the gen_mustpass_tests function.""" - _xml = textwrap.dedent("""\ + xml = textwrap.dedent("""\ <?xml version="1.0" encoding="UTF-8"?> <TestPackage name="dEQP-piglit-test" appPackageName="com.freedesktop.org.piglit.deqp" testType="deqpTest" xmlns:deqp="http://drawelements.com/deqp"; deqp:glesVersion="196608"> <TestSuite name="dEQP.piglit"> @@ -478,13 +558,45 @@ class TestGenMustpassTests(object): </TestPackage> """) - def test_basic(self, tmpdir): - p = tmpdir.join('foo.xml') - p.write(self._xml) - tests = set(deqp.gen_mustpass_tests(six.text_type(p))) - assert tests == { - 'dEQP.piglit.group1.test1', - 'dEQP.piglit.group1.test2', - 'dEQP.piglit.nested.group2.test3', - 'dEQP.piglit.nested.group2.test4', - } + class TestSingle(object): + """Tests for single mode.""" + + @pytest.yield_fixture(autouse=True, scope='class') + def group_setup(self): + with mock.patch('framework.test.deqp.options.OPTIONS', + new=options._Options()) as mocked: + mocked.process_isolation = True + yield + + def test_basic(self, tmpdir): + p = tmpdir.join('foo.xml') + p.write(TestGenMustpassTests.xml) + tests = list(deqp.gen_mustpass_tests(six.text_type(p))) + assert tests == [ + ('dEQP.piglit.group1.test1', ), + ('dEQP.piglit.group1.test2', ), + ('dEQP.piglit.nested.group2.test3', ), + ('dEQP.piglit.nested.group2.test4', ), + ] + + class TestGroup(object): + """Tests for groupmode.""" + + @pytest.yield_fixture(autouse=True, scope='class') + def group_setup(self): + with mock.patch('framework.test.deqp.options.OPTIONS', + new=options._Options()) as mocked: + mocked.process_isolation = False + yield + + def test_basic(self, tmpdir): + p = tmpdir.join('foo.xml') + p.write(TestGenMustpassTests.xml) + tests = list(deqp.gen_mustpass_tests(six.text_type(p))) + assert tests == [ + ('dEQP.piglit.group1', + ['dEQP.piglit.group1.test1', 'dEQP.piglit.group1.test2']), + ('dEQP.piglit.nested.group2', + ['dEQP.piglit.nested.group2.test3', + 'dEQP.piglit.nested.group2.test4']), + ] -- git-series 0.8.10 _______________________________________________ Piglit mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/piglit
