Title: [97339] trunk/Tools
- Revision
- 97339
- Author
- [email protected]
- Date
- 2011-10-12 20:18:40 -0700 (Wed, 12 Oct 2011)
Log Message
[NRWT] New tests without expected files are reported as flakey tests
https://bugs.webkit.org/show_bug.cgi?id=67268
Reviewed by Dirk Pranke.
The bug was caused by summarize_results's regarding all tests that have an entry in result_summary
and no entry in retry_summary as flaky. Since we don't retry missing tests, there will be no entries
for missing results in retry_summary.
Fixed the bug by treating missing results separately.
* Scripts/webkitpy/layout_tests/controllers/manager.py:
* Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
Modified Paths
Diff
Modified: trunk/Tools/ChangeLog (97338 => 97339)
--- trunk/Tools/ChangeLog 2011-10-13 02:51:19 UTC (rev 97338)
+++ trunk/Tools/ChangeLog 2011-10-13 03:18:40 UTC (rev 97339)
@@ -1,3 +1,19 @@
+2011-10-12 Ryosuke Niwa <[email protected]>
+
+ [NRWT] New tests without expected files are reported as flakey tests
+ https://bugs.webkit.org/show_bug.cgi?id=67268
+
+ Reviewed by Dirk Pranke.
+
+ The bug was caused by summarize_results's regarding all tests that have an entry in result_summary
+ and no entry in retry_summary as flaky. Since we don't retry missing tests, there will be no entries
+ for missing results in retry_summary.
+
+ Fixed the bug by treating missing results separately.
+
+ * Scripts/webkitpy/layout_tests/controllers/manager.py:
+ * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
+
2011-10-12 Adam Barth <[email protected]>
Remove ENABLE(XHTMLMP) and associated code
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py (97338 => 97339)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py 2011-10-13 02:51:19 UTC (rev 97338)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py 2011-10-13 03:18:40 UTC (rev 97339)
@@ -138,6 +138,12 @@
continue
elif result_type == test_expectations.CRASH:
num_regressions += 1
+ elif result_type == test_expectations.MISSING:
+ # We count missing results as flaky not to turn buildbot red
+ # This is a huge hack should be fixed by adding new category for MISSING results.
+ # See also: https://bugs.webkit.org/show_bug.cgi?id=64812
+ if test_name in result_summary.unexpected_results:
+ num_flaky += 1
elif test_name in result_summary.unexpected_results:
if test_name not in retry_summary.unexpected_results:
actual.extend(expectations.get_expectations_string(test_name).split(" "))
@@ -147,9 +153,6 @@
if result_type != retry_result_type:
actual.append(keywords[retry_result_type])
num_flaky += 1
- # FIXME: break MISSING results into a different category
- elif 'MISSING' in actual:
- num_flaky += 1
else:
num_regressions += 1
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py (97338 => 97339)
--- trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2011-10-13 02:51:19 UTC (rev 97338)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2011-10-13 03:18:40 UTC (rev 97339)
@@ -426,6 +426,23 @@
self.assertFalse(err.empty())
self.assertEqual(user.opened_urls, ['/tmp/layout-test-results/results.html'])
+ def test_missing_and_unexpected_results(self):
+ # Test that we update expectations in place. If the expectation
+ # is missing, update the expected generic location.
+ fs = unit_test_filesystem()
+ res, out, err, _ = logging_run(['--no-show-results',
+ 'failures/expected/missing_image.html',
+ 'failures/unexpected/missing_text.html',
+ 'failures/unexpected/text-image-checksum.html'],
+ tests_included=True, filesystem=fs, record_results=True)
+ file_list = fs.written_files.keys()
+ file_list.remove('/tmp/layout-test-results/tests_run0.txt')
+ self.assertEquals(res, 1)
+ expected_token = '"unexpected":{"text-image-checksum.html":{"expected":"PASS","actual":"TEXT"},"missing_text.html":{"expected":"PASS","is_missing_text":true,"actual":"MISSING"}'
+ json_string = fs.read_text_file('/tmp/layout-test-results/full_results.json')
+ self.assertTrue(json_string.find(expected_token) != -1)
+ self.assertTrue(json_string.find('"num_flaky":1') != -1)
+
def test_crash_with_stderr(self):
fs = unit_test_filesystem()
res, buildbot_output, regular_output, user = logging_run([
_______________________________________________
webkit-changes mailing list
[email protected]
http://lists.webkit.org/mailman/listinfo.cgi/webkit-changes
