Modified: trunk/Tools/ChangeLog (129147 => 129148)
--- trunk/Tools/ChangeLog 2012-09-20 18:27:46 UTC (rev 129147)
+++ trunk/Tools/ChangeLog 2012-09-20 18:41:46 UTC (rev 129148)
@@ -1,3 +1,28 @@
+2012-09-20 Dirk Pranke <[email protected]>
+
+ REGRESSION: layout test results doesn't show diffs
+ https://bugs.webkit.org/show_bug.cgi?id=97182
+
+ Reviewed by Ojan Vafai.
+
+ Go back to storing TEXT, AUDIO, and IMAGE+TEXT in results.json
+ so that results.html (and hopefully garden-o-matic) can
+ determine which things actually failed. However, we keep mapping
+ these results to Failure so that we still only have a single
+ expectation type for them.
+
+ * Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py:
+ (JSONLayoutResultsGenerator):
+ * Scripts/webkitpy/layout_tests/models/test_expectations.py:
+ (TestExpectationParser):
+ (TestExpectations):
+ (TestExpectations.result_was_expected):
+ * Scripts/webkitpy/layout_tests/models/test_failures.py:
+ (determine_result_type):
+ * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
+ (MainTest.test_missing_and_unexpected_results):
+ (MainTest.test_retrying_and_flaky_tests):
+
2012-09-20 Tommy Widenflycht <[email protected]>
MediaStream API: Extend UserMediaRequest with a ownerDocument method
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py (129147 => 129148)
--- trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py 2012-09-20 18:27:46 UTC (rev 129147)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py 2012-09-20 18:41:46 UTC (rev 129148)
@@ -40,15 +40,15 @@
# Additional JSON fields.
WONTFIX = "wontfixCounts"
- # Note that there used to be an "A" for audio failures and a "Z" for IMAGE+TEXT failures.
-
FAILURE_TO_CHAR = {test_expectations.PASS: json_results_generator.JSONResultsGeneratorBase.PASS_RESULT,
test_expectations.SKIP: json_results_generator.JSONResultsGeneratorBase.SKIP_RESULT,
test_expectations.CRASH: "C",
test_expectations.TIMEOUT: "T",
- test_expectations.FAIL: "F",
test_expectations.IMAGE: "I",
- test_expectations.MISSING: "O"}
+ test_expectations.TEXT: "F",
+ test_expectations.AUDIO: "A",
+ test_expectations.MISSING: "O",
+ test_expectations.IMAGE_PLUS_TEXT: "Z"}
def __init__(self, port, builder_name, build_name, build_number,
results_file_base_path, builder_base_url,
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py (129147 => 129148)
--- trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py 2012-09-20 18:27:46 UTC (rev 129147)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py 2012-09-20 18:41:46 UTC (rev 129148)
@@ -40,9 +40,6 @@
# Test expectation and modifier constants.
-# TEXT, IMAGE_PLUS_TEXT, and AUDIO are no longer used in new test runs but
-# we keep them around for now so we can parse old results.json entries and to
-# avoid changing the numbering for the constants.
#
# FIXME: range() starts with 0 which makes if expectation checks harder
# as PASS is 0.
@@ -286,7 +283,8 @@
'WontFix': 'WONTFIX',
}
- _inverted_expectation_tokens = dict((value, name) for name, value in _expectation_tokens.iteritems())
+ _inverted_expectation_tokens = dict([(value, name) for name, value in _expectation_tokens.iteritems()] +
+ [('TEXT', 'Failure'), ('IMAGE+TEXT', 'Failure'), ('AUDIO', 'Failure')])
@classmethod
def _tokenize_line_using_new_format(cls, filename, expectation_string, line_number):
@@ -791,8 +789,11 @@
# FIXME: Update to new syntax once the old format is no longer supported.
EXPECTATIONS = {'pass': PASS,
+ 'audio': AUDIO,
'fail': FAIL,
'image': IMAGE,
+ 'image+text': IMAGE_PLUS_TEXT,
+ 'text': TEXT,
'timeout': TIMEOUT,
'crash': CRASH,
'missing': MISSING}
@@ -802,6 +803,9 @@
PASS: ('passes', 'passed', ''),
FAIL: ('failures', 'failed', ''),
IMAGE: ('image-only failures', 'failed', ' (image diff)'),
+ TEXT: ('text-only failures', 'failed', ' (text diff)'),
+ IMAGE_PLUS_TEXT: ('image and text failures', 'failed', ' (image and text diff)'),
+ AUDIO: ('audio failures', 'failed', ' (audio diff)'),
CRASH: ('crashes', 'crashed', ''),
TIMEOUT: ('timeouts', 'timed out', ''),
MISSING: ('no expected results found', 'no expected result found', '')}
@@ -839,6 +843,8 @@
test_is_skipped: whether test was marked as SKIP"""
if result in expected_results:
return True
+ if result in (TEXT, IMAGE_PLUS_TEXT, AUDIO) and (FAIL in expected_results):
+ return True
if result == MISSING and test_needs_rebaselining:
return True
if result == SKIP and test_is_skipped:
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py (129147 => 129148)
--- trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2012-09-20 18:27:46 UTC (rev 129147)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2012-09-20 18:41:46 UTC (rev 129148)
@@ -549,7 +549,7 @@
file_list = host.filesystem.written_files.keys()
file_list.remove('/tmp/layout-test-results/tests_run0.txt')
self.assertEquals(res, 1)
- expected_token = '"unexpected":{"text-image-checksum.html":{"expected":"PASS","actual":"FAIL"},"missing_text.html":{"expected":"PASS","is_missing_text":true,"actual":"MISSING"}'
+ expected_token = '"unexpected":{"text-image-checksum.html":{"expected":"PASS","actual":"TEXT"},"missing_text.html":{"expected":"PASS","is_missing_text":true,"actual":"MISSING"}'
json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
self.assertTrue(json_string.find(expected_token) != -1)
self.assertTrue(json_string.find('"num_regressions":1') != -1)
@@ -764,7 +764,7 @@
self.assertEquals(res, 1)
self.assertTrue('Clobbering old results' in err.getvalue())
self.assertTrue('flaky/text.html' in err.getvalue())
- self.assertTrue('Unexpected failures' in out.getvalue())
+ self.assertTrue('Unexpected text-only failures' in out.getvalue())
self.assertFalse('Unexpected flakiness' in out.getvalue())
self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/failures/flaky/text-actual.txt'))
self.assertFalse(host.filesystem.exists('retries'))
