Currently, if a test from provided testlist fails to be discovered by the framework, piglit blows up with an exception.
This is both good - for consistency/early errors - and bad - for handling some CI/automation scenarios (e.g autobisecting the tests). So let's keep the current default, but allow some flexibility with the new option that reports the missing tests as 'notrun'. v2: - change switch name to 'ignore', as skip is too suggestive - use DummyTest to get 'notrun' result instead of warnings v3: don't use OPTIONS bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=99649 Cc: Dylan Baker <[email protected]> Cc: Tomi Sarvela <[email protected]> Cc: Martin Peres <[email protected]> Signed-off-by: Arkadiusz Hiler <[email protected]> --- framework/profile.py | 10 +++++++--- framework/programs/run.py | 12 ++++++++++++ framework/test/base.py | 12 ++++++++++++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/framework/profile.py b/framework/profile.py index a625318..cf0e298 100644 --- a/framework/profile.py +++ b/framework/profile.py @@ -42,11 +42,11 @@ import re import six -from framework import grouptools, exceptions +from framework import grouptools, exceptions, status from framework.dmesg import get_dmesg from framework.log import LogManager from framework.monitoring import Monitoring -from framework.test.base import Test +from framework.test.base import Test, DummyTest __all__ = [ 'RegexFilter', @@ -285,6 +285,7 @@ class TestProfile(object): self.options = { 'dmesg': get_dmesg(False), 'monitor': Monitoring(False), + 'ignore_missing': False, } def setup(self): @@ -314,7 +315,10 @@ class TestProfile(object): if self.forced_test_list: opts = collections.OrderedDict() for n in self.forced_test_list: - opts[n] = self.test_list[n] + if self.options['ignore_missing'] and n not in self.test_list: + opts[n] = DummyTest(name=n, result=status.NOTRUN) + else: + opts[n] = self.test_list[n] else: opts = self.test_list # pylint: disable=redefined-variable-type diff --git a/framework/programs/run.py b/framework/programs/run.py index 6444cfe..4524f17 100644 --- a/framework/programs/run.py +++ b/framework/programs/run.py @@ -208,6 +208,10 @@ def _run_parser(input_): 'isolation. This allows, but does not require, ' 'tests to run multiple tests per process. ' 'This value can also be set in piglit.conf.') + parser.add_argument("--ignore-missing", + dest="ignore_missing", + action="store_true", + help="missing tests are considered as 'notrun'") parser.add_argument("test_profile", metavar="<Profile path(s)>", nargs='+', @@ -234,6 +238,7 @@ def _create_metadata(args, name, forced_test_list): if args.platform: opts['platform'] = args.platform opts['forced_test_list'] = forced_test_list + opts['ignore_missing'] = args.ignore_missing metadata = {'options': opts} metadata['name'] = name @@ -346,6 +351,10 @@ def run(input_): for p in profiles: p.options['monitor'] = monitoring.Monitoring(args.monitored) + if args.ignore_missing: + for p in profiles: + p.options['ignore_missing'] = args.ignore_missing + for p in profiles: if args.exclude_tests: p.filters.append(profile.RegexFilter(args.exclude_tests, @@ -422,6 +431,9 @@ def resume(input_): p.options['monitor'] = monitoring.Monitoring( results.options['monitoring']) + if results.options['ignore_missing']: + p.options['ignore_missing'] = results.options['ignore_missing'] + if exclude_tests: p.filters.append(lambda n, _: n not in exclude_tests) if results.options['exclude_filter']: diff --git a/framework/test/base.py b/framework/test/base.py index d73dee9..e74ea3d 100644 --- a/framework/test/base.py +++ b/framework/test/base.py @@ -381,6 +381,18 @@ class Test(object): return not self == other +class DummyTest(Test): + def __init__(self, name, result): + super(DummyTest, self).__init__([name]) + self.result.result = result + + def execute(self, path, log, options): + pass + + def interpret_result(self): + pass + + class WindowResizeMixin(object): """ Mixin class that deals with spurious window resizes -- 2.9.4 _______________________________________________ Piglit mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/piglit
