tatyana-krasnukha created this revision.
tatyana-krasnukha added reviewers: labath, teemperor.
Herald added subscribers: lldb-commits, JDevlieghere.
Herald added a project: LLDB.
There already are decorators and "--excluded" option to mark test-cases/files
as expected to fail. However, when a new test file is added and it relates
to a feature that a target doesn't support, this requires either adding
decorators
to that file or modifying the file provided as "--excluded" option value.
The purpose of this patch is to avoid any modifications in such cases.
E.g. if a target doesn't support "watchpoints" and passes "--xfail-category
watchpoint"
to dotest, a testing job will not fail after a new watchpoint-related test file
is added.
Repository:
rLLDB LLDB
https://reviews.llvm.org/D71906
Files:
packages/Python/lldbsuite/test/configuration.py
packages/Python/lldbsuite/test/dotest.py
packages/Python/lldbsuite/test/dotest_args.py
packages/Python/lldbsuite/test/test_result.py
Index: packages/Python/lldbsuite/test/test_result.py
===================================================================
--- packages/Python/lldbsuite/test/test_result.py
+++ packages/Python/lldbsuite/test/test_result.py
@@ -156,6 +156,10 @@
return True
return False
+ def checkCategoryExclusion(self, exclusion_list, test):
+ return not set(exclusion_list).isdisjoint(
+ self.getCategoriesForTest(test))
+
def startTest(self, test):
if configuration.shouldSkipBecauseOfCategories(
self.getCategoriesForTest(test)):
@@ -174,8 +178,10 @@
EventBuilder.event_for_start(test))
def addSuccess(self, test):
- if self.checkExclusion(
- configuration.xfail_tests, test.id()):
+ if (self.checkExclusion(
+ configuration.xfail_tests, test.id()) or
+ self.checkCategoryExclusion(
+ configuration.xfail_categories, test)):
self.addUnexpectedSuccess(test, None)
return
@@ -245,8 +251,10 @@
test, err))
def addFailure(self, test, err):
- if self.checkExclusion(
- configuration.xfail_tests, test.id()):
+ if (self.checkExclusion(
+ configuration.xfail_tests, test.id()) or
+ self.checkCategoryExclusion(
+ configuration.xfail_categories, test)):
self.addExpectedFailure(test, err, None)
return
Index: packages/Python/lldbsuite/test/dotest_args.py
===================================================================
--- packages/Python/lldbsuite/test/dotest_args.py
+++ packages/Python/lldbsuite/test/dotest_args.py
@@ -84,6 +84,12 @@
action='append',
dest='skip_categories',
help=textwrap.dedent('''Specify categories of test cases to skip.
Takes precedence over -G. Can be specified more than once.'''))
+ group.add_argument(
+ '--xfail-category',
+ metavar='category',
+ action='append',
+ dest='xfail_categories',
+ help=textwrap.dedent('''Specify categories of test cases that are
expected to fail. Can be specified more than once.'''))
# Configuration options
group = parser.add_argument_group('Configuration options')
Index: packages/Python/lldbsuite/test/dotest.py
===================================================================
--- packages/Python/lldbsuite/test/dotest.py
+++ packages/Python/lldbsuite/test/dotest.py
@@ -329,6 +329,10 @@
configuration.skip_categories += test_categories.validate(
args.skip_categories, False)
+ if args.xfail_categories:
+ configuration.xfail_categories += test_categories.validate(
+ args.xfail_categories, False)
+
if args.E:
os.environ['CFLAGS_EXTRAS'] = args.E
Index: packages/Python/lldbsuite/test/configuration.py
===================================================================
--- packages/Python/lldbsuite/test/configuration.py
+++ packages/Python/lldbsuite/test/configuration.py
@@ -30,6 +30,8 @@
use_categories = False
# Categories we want to skip
skip_categories = ["darwin-log"]
+# Categories we expect to fail
+xfail_categories = []
# use this to track per-category failures
failures_per_category = {}
Index: packages/Python/lldbsuite/test/test_result.py
===================================================================
--- packages/Python/lldbsuite/test/test_result.py
+++ packages/Python/lldbsuite/test/test_result.py
@@ -156,6 +156,10 @@
return True
return False
+ def checkCategoryExclusion(self, exclusion_list, test):
+ return not set(exclusion_list).isdisjoint(
+ self.getCategoriesForTest(test))
+
def startTest(self, test):
if configuration.shouldSkipBecauseOfCategories(
self.getCategoriesForTest(test)):
@@ -174,8 +178,10 @@
EventBuilder.event_for_start(test))
def addSuccess(self, test):
- if self.checkExclusion(
- configuration.xfail_tests, test.id()):
+ if (self.checkExclusion(
+ configuration.xfail_tests, test.id()) or
+ self.checkCategoryExclusion(
+ configuration.xfail_categories, test)):
self.addUnexpectedSuccess(test, None)
return
@@ -245,8 +251,10 @@
test, err))
def addFailure(self, test, err):
- if self.checkExclusion(
- configuration.xfail_tests, test.id()):
+ if (self.checkExclusion(
+ configuration.xfail_tests, test.id()) or
+ self.checkCategoryExclusion(
+ configuration.xfail_categories, test)):
self.addExpectedFailure(test, err, None)
return
Index: packages/Python/lldbsuite/test/dotest_args.py
===================================================================
--- packages/Python/lldbsuite/test/dotest_args.py
+++ packages/Python/lldbsuite/test/dotest_args.py
@@ -84,6 +84,12 @@
action='append',
dest='skip_categories',
help=textwrap.dedent('''Specify categories of test cases to skip. Takes precedence over -G. Can be specified more than once.'''))
+ group.add_argument(
+ '--xfail-category',
+ metavar='category',
+ action='append',
+ dest='xfail_categories',
+ help=textwrap.dedent('''Specify categories of test cases that are expected to fail. Can be specified more than once.'''))
# Configuration options
group = parser.add_argument_group('Configuration options')
Index: packages/Python/lldbsuite/test/dotest.py
===================================================================
--- packages/Python/lldbsuite/test/dotest.py
+++ packages/Python/lldbsuite/test/dotest.py
@@ -329,6 +329,10 @@
configuration.skip_categories += test_categories.validate(
args.skip_categories, False)
+ if args.xfail_categories:
+ configuration.xfail_categories += test_categories.validate(
+ args.xfail_categories, False)
+
if args.E:
os.environ['CFLAGS_EXTRAS'] = args.E
Index: packages/Python/lldbsuite/test/configuration.py
===================================================================
--- packages/Python/lldbsuite/test/configuration.py
+++ packages/Python/lldbsuite/test/configuration.py
@@ -30,6 +30,8 @@
use_categories = False
# Categories we want to skip
skip_categories = ["darwin-log"]
+# Categories we expect to fail
+xfail_categories = []
# use this to track per-category failures
failures_per_category = {}
_______________________________________________
lldb-commits mailing list
lldb-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits
