Modified: trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py (106590 => 106591)
--- trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py 2012-02-02 21:49:50 UTC (rev 106590)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py 2012-02-02 21:55:08 UTC (rev 106591)
@@ -108,15 +108,14 @@
class ParseError(Exception):
- def __init__(self, fatal, errors):
- self.fatal = fatal
- self.errors = errors
+ def __init__(self, warnings):
+ self.warnings = warnings
def __str__(self):
- return '\n'.join(map(str, self.errors))
+ return '\n'.join(map(str, self.warnings))
def __repr__(self):
- return 'ParseError(fatal=%s, errors=%s)' % (self.fatal, self.errors)
+ return 'ParseError(warnings=%s)' % self.warnings
class TestExpectationSerializer(object):
@@ -126,7 +125,7 @@
self._parsed_expectation_to_string = dict([[parsed_expectation, expectation_string] for expectation_string, parsed_expectation in TestExpectations.EXPECTATIONS.items()])
def to_string(self, expectation_line):
- if expectation_line.is_malformed():
+ if expectation_line.is_invalid():
return expectation_line.original_string or ''
if expectation_line.name is None:
@@ -248,7 +247,7 @@
elif modifier.startswith(self.BUG_MODIFIER_PREFIX):
has_bugid = True
if re.match(self.BUG_MODIFIER_REGEX, modifier):
- expectation_line.errors.append('BUG\d+ is not allowed, must be one of BUGCR\d+, BUGWK\d+, BUGV8_\d+, or a non-numeric bug identifier.')
+ expectation_line.warnings.append('BUG\d+ is not allowed, must be one of BUGCR\d+, BUGWK\d+, BUGV8_\d+, or a non-numeric bug identifier.')
else:
expectation_line.parsed_bug_modifiers.append(modifier)
else:
@@ -258,23 +257,23 @@
expectation_line.warnings.append('Test lacks BUG modifier.')
if self._allow_rebaseline_modifier and self.REBASELINE_MODIFIER in expectation_line.modifiers:
- expectation_line.errors.append('REBASELINE should only be used for running rebaseline.py. Cannot be checked in.')
+ expectation_line.warnings.append('REBASELINE should only be used for running rebaseline.py. Cannot be checked in.')
- expectation_line.matching_configurations = self._test_configuration_converter.to_config_set(parsed_specifiers, expectation_line.errors)
+ expectation_line.matching_configurations = self._test_configuration_converter.to_config_set(parsed_specifiers, expectation_line.warnings)
def _parse_expectations(self, expectation_line):
result = set()
for part in expectation_line.expectations:
expectation = TestExpectations.expectation_from_string(part)
if expectation is None: # Careful, PASS is currently 0.
- expectation_line.errors.append('Unsupported expectation: %s' % part)
+ expectation_line.warnings.append('Unsupported expectation: %s' % part)
continue
result.add(expectation)
expectation_line.parsed_expectations = result
def _check_modifiers_against_expectations(self, expectation_line):
if self.SLOW_MODIFIER in expectation_line.modifiers and self.TIMEOUT_EXPECTATION in expectation_line.expectations:
- expectation_line.errors.append('A test can not be both SLOW and TIMEOUT. If it times out indefinitely, then it should be just TIMEOUT.')
+ expectation_line.warnings.append('A test can not be both SLOW and TIMEOUT. If it times out indefinitely, then it should be just TIMEOUT.')
def _check_path_does_not_exist(self, expectation_line):
# WebKit's way of skipping tests is to add a -disabled suffix.
@@ -338,13 +337,13 @@
parts = remaining_string.split(':')
if len(parts) != 2:
- expectation_line.errors.append("Missing a ':'" if len(parts) < 2 else "Extraneous ':'")
+ expectation_line.warnings.append("Missing a ':'" if len(parts) < 2 else "Extraneous ':'")
else:
test_and_expectation = parts[1].split('=')
if len(test_and_expectation) != 2:
- expectation_line.errors.append("Missing expectations" if len(test_and_expectation) < 2 else "Extraneous '='")
+ expectation_line.warnings.append("Missing expectations" if len(test_and_expectation) < 2 else "Extraneous '='")
- if not expectation_line.is_malformed():
+ if not expectation_line.is_invalid():
expectation_line.modifiers = cls._split_space_separated(parts[0])
expectation_line.name = test_and_expectation[0].strip()
expectation_line.expectations = cls._split_space_separated(test_and_expectation[1])
@@ -385,14 +384,10 @@
self.parsed_expectations = set()
self.comment = None
self.matching_tests = []
- self.errors = []
self.warnings = []
- def is_malformed(self):
- return len(self.errors) > 0
-
def is_invalid(self):
- return self.is_malformed() or len(self.warnings) > 0
+ return len(self.warnings) > 0
def is_flaky(self):
return len(self.parsed_expectations) > 1
@@ -476,7 +471,7 @@
return self._test_to_expectations[test]
def add_expectation_line(self, expectation_line, overrides_allowed):
- """Returns a list of errors, encountered while matching modifiers."""
+ """Returns a list of warnings encountered while matching modifiers."""
if expectation_line.is_invalid():
return
@@ -592,20 +587,20 @@
# to be warnings and return False".
if prev_expectation_line.matching_configurations == expectation_line.matching_configurations:
- expectation_line.errors.append('Duplicate or ambiguous %s.' % expectation_source)
+ expectation_line.warnings.append('Duplicate or ambiguous %s.' % expectation_source)
return True
if prev_expectation_line.matching_configurations >= expectation_line.matching_configurations:
- expectation_line.errors.append('More specific entry on line %d overrides line %d' % (expectation_line.line_number, prev_expectation_line.line_number))
+ expectation_line.warnings.append('More specific entry on line %d overrides line %d' % (expectation_line.line_number, prev_expectation_line.line_number))
# FIXME: return False if we want more specific to win.
return True
if prev_expectation_line.matching_configurations <= expectation_line.matching_configurations:
- expectation_line.errors.append('More specific entry on line %d overrides line %d' % (prev_expectation_line.line_number, expectation_line.line_number))
+ expectation_line.warnings.append('More specific entry on line %d overrides line %d' % (prev_expectation_line.line_number, expectation_line.line_number))
return True
if prev_expectation_line.matching_configurations & expectation_line.matching_configurations:
- expectation_line.errors.append('Entries on line %d and line %d match overlapping sets of configurations' % (prev_expectation_line.line_number, expectation_line.line_number))
+ expectation_line.warnings.append('Entries on line %d and line %d match overlapping sets of configurations' % (prev_expectation_line.line_number, expectation_line.line_number))
return True
# Configuration sets are disjoint, then.
@@ -706,8 +701,8 @@
test_config: specific values to check against when
parsing the file (usually port.test_config(),
but may be different when linting or doing other things).
- is_lint_mode: If True, just parse the expectations string
- looking for errors.
+ is_lint_mode: If True, parse the expectations string and raise
+ an exception if warnings are encountered.
overrides: test expectations that are allowed to override any
entries in |expectations|. This is used by callers
that need to manage two sets of expectations (e.g., upstream
@@ -731,7 +726,7 @@
self._expectations += overrides_expectations
self._has_warnings = False
- self._report_errors()
+ self._report_warnings()
self._process_tests_without_expectations()
# TODO(ojan): Allow for removing skipped tests when getting the list of
@@ -798,28 +793,22 @@
def is_rebaselining(self, test):
return self._model.has_modifier(test, REBASELINE)
- def _report_errors(self):
- errors = []
+ def _report_warnings(self):
warnings = []
test_expectation_path = self._port.path_to_test_expectations_file()
if test_expectation_path.startswith(self._port.path_from_webkit_base()):
test_expectation_path = self._port.host.filesystem.relpath(test_expectation_path, self._port.path_from_webkit_base())
for expectation in self._expectations:
- for error in expectation.errors:
- errors.append('%s:%d %s %s' % (test_expectation_path, expectation.line_number, error, expectation.name if expectation.expectations else expectation.original_string))
for warning in expectation.warnings:
warnings.append('%s:%d %s %s' % (test_expectation_path, expectation.line_number, warning, expectation.name if expectation.expectations else expectation.original_string))
for warning in self._skipped_tests_warnings:
warnings.append('%s%s' % (test_expectation_path, warning))
- if errors or warnings:
- if errors:
- raise ParseError(fatal=True, errors=sorted(errors + warnings))
- if warnings:
- self._has_warnings = True
- if self._is_lint_mode:
- raise ParseError(fatal=False, errors=warnings)
+ if warnings:
+ self._has_warnings = True
+ if self._is_lint_mode:
+ raise ParseError(warnings)
def _process_tests_without_expectations(self):
if self._full_test_list:
@@ -833,7 +822,7 @@
def remove_rebaselined_tests(self, except_these_tests):
"""Returns a copy of the expectations with the tests removed."""
def without_rebaseline_modifier(expectation):
- return not (not expectation.is_malformed() and expectation.name in except_these_tests and "rebaseline" in expectation.modifiers)
+ return not (not expectation.is_invalid() and expectation.name in except_these_tests and "rebaseline" in expectation.modifiers)
return TestExpectationSerializer.list_to_string(filter(without_rebaseline_modifier, self._expectations))
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py (106590 => 106591)
--- trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py 2012-02-02 21:49:50 UTC (rev 106590)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py 2012-02-02 21:55:08 UTC (rev 106591)
@@ -131,7 +131,10 @@
self.assertEquals(self._exp.get_expectations(self.get_test(test)),
set([result]))
+ def assert_bad_expectations(self, expectations, overrides=None):
+ self.assertRaises(ParseError, self.parse_exp, expectations, is_lint_mode=True, overrides=overrides)
+
class BasicTests(Base):
def test_basic(self):
self.parse_exp(self.get_basic_expectations())
@@ -192,26 +195,23 @@
s = self._exp.get_test_set(WONTFIX, CRASH, include_skips=False)
self.assertEqual(s, set([]))
- def test_parse_error_fatal(self):
+ def test_parse_warning(self):
try:
self.parse_exp("""FOO : failures/expected/text.html = TEXT
-SKIP : failures/expected/image.html""")
+SKIP : failures/expected/image.html""", is_lint_mode=True)
self.assertFalse(True, "ParseError wasn't raised")
except ParseError, e:
- self.assertTrue(e.fatal)
- exp_errors = [u":1 Test lacks BUG modifier. failures/expected/text.html",
- u":1 Unrecognized modifier 'foo' failures/expected/text.html",
- u":2 Missing expectations SKIP : failures/expected/image.html"]
- self.assertEqual(str(e), '\n'.join(self._port.path_to_test_expectations_file() + str(error) for error in exp_errors))
+ warnings = [u":1 Test lacks BUG modifier. failures/expected/text.html",
+ u":1 Unrecognized modifier 'foo' failures/expected/text.html",
+ u":2 Missing expectations SKIP : failures/expected/image.html"]
+ self.assertEqual(str(e), '\n'.join(self._port.path_to_test_expectations_file() + str(warning) for warning in warnings))
- def test_parse_error_nonfatal(self):
try:
self.parse_exp('SKIP : failures/expected/text.html = TEXT', is_lint_mode=True)
self.assertFalse(True, "ParseError wasn't raised")
except ParseError, e:
- self.assertFalse(e.fatal)
- exp_errors = [u':1 Test lacks BUG modifier. failures/expected/text.html']
- self.assertEqual(str(e), '\n'.join(self._port.path_to_test_expectations_file() + str(error) for error in exp_errors))
+ warnings = [u':1 Test lacks BUG modifier. failures/expected/text.html']
+ self.assertEqual(str(e), '\n'.join(self._port.path_to_test_expectations_file() + str(warning) for warning in warnings))
def test_error_on_different_platform(self):
# parse_exp uses a Windows port. Assert errors on Mac show up in lint mode.
@@ -237,12 +237,9 @@
self.assert_exp('failures/expected/text.html', IMAGE)
def test_overrides__duplicate(self):
- self.assertRaises(ParseError, self.parse_exp,
- "BUG_EXP: failures/expected/text.html = TEXT",
- """
-BUG_OVERRIDE : failures/expected/text.html = IMAGE
-BUG_OVERRIDE : failures/expected/text.html = CRASH
-""")
+ self.assert_bad_expectations("BUG_EXP: failures/expected/text.html = TEXT",
+ "BUG_OVERRIDE : failures/expected/text.html = IMAGE\n"
+ "BUG_OVERRIDE : failures/expected/text.html = CRASH\n")
def test_pixel_tests_flag(self):
def match(test, result, pixel_tests_enabled):
@@ -287,26 +284,20 @@
class ExpectationSyntaxTests(Base):
def test_missing_expectation(self):
# This is missing the expectation.
- self.assertRaises(ParseError, self.parse_exp,
- 'BUG_TEST: failures/expected/text.html')
+ self.assert_bad_expectations('BUG_TEST: failures/expected/text.html')
def test_missing_colon(self):
# This is missing the modifiers and the ':'
- self.assertRaises(ParseError, self.parse_exp,
- 'failures/expected/text.html = TEXT')
+ self.assert_bad_expectations('failures/expected/text.html = TEXT')
- def disabled_test_too_many_colons(self):
- # FIXME: Enable this test and fix the underlying bug.
- self.assertRaises(ParseError, self.parse_exp,
- 'BUG_TEST: failures/expected/text.html = PASS :')
+ def test_too_many_colons(self):
+ self.assert_bad_expectations('BUG_TEST: failures/expected/text.html = PASS :')
def test_too_many_equals_signs(self):
- self.assertRaises(ParseError, self.parse_exp,
- 'BUG_TEST: failures/expected/text.html = TEXT = IMAGE')
+ self.assert_bad_expectations('BUG_TEST: failures/expected/text.html = TEXT = IMAGE')
def test_unrecognized_expectation(self):
- self.assertRaises(ParseError, self.parse_exp,
- 'BUG_TEST: failures/expected/text.html = UNKNOWN')
+ self.assert_bad_expectations('BUG_TEST: failures/expected/text.html = UNKNOWN')
def test_macro(self):
exp_str = """
@@ -318,25 +309,23 @@
class SemanticTests(Base):
def test_bug_format(self):
- self.assertRaises(ParseError, self.parse_exp, 'BUG1234 : failures/expected/text.html = TEXT')
+ self.assertRaises(ParseError, self.parse_exp, 'BUG1234 : failures/expected/text.html = TEXT', is_lint_mode=True)
def test_bad_bugid(self):
try:
- self.parse_exp('BUG1234 SLOW : failures/expected/text.html = TEXT')
+ self.parse_exp('BUG1234 SLOW : failures/expected/text.html = TEXT', is_lint_mode=True)
self.fail('should have raised an error about a bad bug identifier')
except ParseError, exp:
- self.assertEquals(exp.fatal, True)
- self.assertEquals(len(exp.errors), 1)
+ self.assertEquals(len(exp.warnings), 1)
def test_missing_bugid(self):
- # This should log a non-fatal error.
self.parse_exp('SLOW : failures/expected/text.html = TEXT')
self.assertTrue(self._exp.has_warnings())
def test_slow_and_timeout(self):
# A test cannot be SLOW and expected to TIMEOUT.
self.assertRaises(ParseError, self.parse_exp,
- 'BUG_TEST SLOW : failures/expected/timeout.html = TIMEOUT')
+ 'BUG_TEST SLOW : failures/expected/timeout.html = TIMEOUT', is_lint_mode=True)
def test_rebaseline(self):
# Can't lint a file w/ 'REBASELINE' in it.
@@ -347,12 +336,12 @@
def test_duplicates(self):
self.assertRaises(ParseError, self.parse_exp, """
BUG_EXP : failures/expected/text.html = TEXT
-BUG_EXP : failures/expected/text.html = IMAGE""")
+BUG_EXP : failures/expected/text.html = IMAGE""", is_lint_mode=True)
self.assertRaises(ParseError, self.parse_exp,
self.get_basic_expectations(), overrides="""
BUG_OVERRIDE : failures/expected/text.html = TEXT
-BUG_OVERRIDE : failures/expected/text.html = IMAGE""", )
+BUG_OVERRIDE : failures/expected/text.html = IMAGE""", is_lint_mode=True)
def test_missing_file(self):
# This should log a non-fatal error.
@@ -381,37 +370,26 @@
self.assert_exp('failures/expected/crash.html', IMAGE)
def test_ambiguous(self):
- self.assertRaises(ParseError, self.parse_exp, """
-BUG_TEST RELEASE : passes/text.html = PASS
-BUG_TEST WIN : passes/text.html = FAIL
-""")
+ self.assert_bad_expectations("BUG_TEST RELEASE : passes/text.html = PASS\n"
+ "BUG_TEST WIN : passes/text.html = FAIL\n")
def test_more_modifiers(self):
- exp_str = """
-BUG_TEST RELEASE : passes/text.html = PASS
-BUG_TEST WIN RELEASE : passes/text.html = TEXT
-"""
- self.assertRaises(ParseError, self.parse_exp, exp_str)
+ self.assert_bad_expectations("BUG_TEST RELEASE : passes/text.html = PASS\n"
+ "BUG_TEST WIN RELEASE : passes/text.html = TEXT\n")
def test_order_in_file(self):
- exp_str = """
-BUG_TEST WIN RELEASE : passes/text.html = TEXT
-BUG_TEST RELEASE : passes/text.html = PASS
-"""
- self.assertRaises(ParseError, self.parse_exp, exp_str)
+ self.assert_bad_expectations("BUG_TEST WIN RELEASE : passes/text.html = TEXT\n"
+ "BUG_TEST RELEASE : passes/text.html = PASS\n")
def test_macro_overrides(self):
- exp_str = """
-BUG_TEST WIN : passes/text.html = PASS
-BUG_TEST XP : passes/text.html = TEXT
-"""
- self.assertRaises(ParseError, self.parse_exp, exp_str)
+ self.assert_bad_expectations("BUG_TEST WIN : passes/text.html = PASS\n"
+ "BUG_TEST XP : passes/text.html = TEXT\n")
class RebaseliningTest(Base):
"""Test rebaselining-specific functionality."""
def assertRemove(self, input_expectations, tests, expected_expectations):
- self.parse_exp(input_expectations)
+ self.parse_exp(input_expectations, is_lint_mode=False)
actual_expectations = self._exp.remove_rebaselined_tests(tests)
self.assertEqual(expected_expectations, actual_expectations)
@@ -431,63 +409,53 @@
class TestExpectationParserTests(unittest.TestCase):
def test_tokenize_blank(self):
expectation = TestExpectationParser._tokenize('')
- self.assertEqual(expectation.is_malformed(), False)
self.assertEqual(expectation.comment, None)
- self.assertEqual(len(expectation.errors), 0)
+ self.assertEqual(len(expectation.warnings), 0)
def test_tokenize_missing_colon(self):
expectation = TestExpectationParser._tokenize('Qux.')
- self.assertEqual(expectation.is_malformed(), True)
- self.assertEqual(str(expectation.errors), '["Missing a \':\'"]')
+ self.assertEqual(str(expectation.warnings), '["Missing a \':\'"]')
def test_tokenize_extra_colon(self):
expectation = TestExpectationParser._tokenize('FOO : : bar')
- self.assertEqual(expectation.is_malformed(), True)
- self.assertEqual(str(expectation.errors), '["Extraneous \':\'"]')
+ self.assertEqual(str(expectation.warnings), '["Extraneous \':\'"]')
def test_tokenize_empty_comment(self):
expectation = TestExpectationParser._tokenize('//')
- self.assertEqual(expectation.is_malformed(), False)
self.assertEqual(expectation.comment, '')
- self.assertEqual(len(expectation.errors), 0)
+ self.assertEqual(len(expectation.warnings), 0)
def test_tokenize_comment(self):
expectation = TestExpectationParser._tokenize('//Qux.')
- self.assertEqual(expectation.is_malformed(), False)
self.assertEqual(expectation.comment, 'Qux.')
- self.assertEqual(len(expectation.errors), 0)
+ self.assertEqual(len(expectation.warnings), 0)
def test_tokenize_missing_equal(self):
expectation = TestExpectationParser._tokenize('FOO : bar')
- self.assertEqual(expectation.is_malformed(), True)
- self.assertEqual(str(expectation.errors), "['Missing expectations\']")
+ self.assertEqual(str(expectation.warnings), "['Missing expectations\']")
def test_tokenize_extra_equal(self):
expectation = TestExpectationParser._tokenize('FOO : bar = BAZ = Qux.')
- self.assertEqual(expectation.is_malformed(), True)
- self.assertEqual(str(expectation.errors), '["Extraneous \'=\'"]')
+ self.assertEqual(str(expectation.warnings), '["Extraneous \'=\'"]')
def test_tokenize_valid(self):
expectation = TestExpectationParser._tokenize('FOO : bar = BAZ')
- self.assertEqual(expectation.is_malformed(), False)
self.assertEqual(expectation.comment, None)
- self.assertEqual(len(expectation.errors), 0)
+ self.assertEqual(len(expectation.warnings), 0)
def test_tokenize_valid_with_comment(self):
expectation = TestExpectationParser._tokenize('FOO : bar = BAZ //Qux.')
- self.assertEqual(expectation.is_malformed(), False)
self.assertEqual(expectation.comment, 'Qux.')
self.assertEqual(str(expectation.modifiers), '[\'foo\']')
self.assertEqual(str(expectation.expectations), '[\'baz\']')
- self.assertEqual(len(expectation.errors), 0)
+ self.assertEqual(len(expectation.warnings), 0)
def test_tokenize_valid_with_multiple_modifiers(self):
expectation = TestExpectationParser._tokenize('FOO1 FOO2 : bar = BAZ //Qux.')
- self.assertEqual(expectation.is_malformed(), False)
self.assertEqual(expectation.comment, 'Qux.')
self.assertEqual(str(expectation.modifiers), '[\'foo1\', \'foo2\']')
self.assertEqual(str(expectation.expectations), '[\'baz\']')
- self.assertEqual(len(expectation.errors), 0)
+ self.assertEqual(len(expectation.warnings), 0)
def test_parse_empty_string(self):
host = MockHost()
@@ -538,7 +506,7 @@
self.assertEqual(serializer.to_string(expectation), 'FOO : bar = BAZ1 BAZ2 //Qux.')
expectation.modifiers = ['foo1', 'foO2']
self.assertEqual(serializer.to_string(expectation), 'FOO1 FOO2 : bar = BAZ1 BAZ2 //Qux.')
- expectation.errors.append('Oh the horror.')
+ expectation.warnings.append('Oh the horror.')
self.assertEqual(serializer.to_string(expectation), '')
expectation.original_string = 'Yes it is!'
self.assertEqual(serializer.to_string(expectation), 'Yes it is!')
