- Revision
- 235771
- Author
- [email protected]
- Date
- 2018-09-06 17:24:51 -0700 (Thu, 06 Sep 2018)
Log Message
Log when leak detection changes the test result
https://bugs.webkit.org/show_bug.cgi?id=189293
Reviewed by Jon Lee.
When you have an "Leak" expectation in TestExpectations, it's confusing to see a test unexpectedly pass,
but then show up as an expected fail at the end (because leak detection happens at the end of a shard).
So log when leak detection changes a test result.
* Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py:
(LayoutTestRunner._annotate_results_with_additional_failures):
* Scripts/webkitpy/layout_tests/models/test_expectations.py:
(TestExpectations):
Modified Paths
Diff
Modified: trunk/Tools/ChangeLog (235770 => 235771)
--- trunk/Tools/ChangeLog 2018-09-07 00:21:14 UTC (rev 235770)
+++ trunk/Tools/ChangeLog 2018-09-07 00:24:51 UTC (rev 235771)
@@ -1,3 +1,19 @@
+2018-09-06 Simon Fraser <[email protected]>
+
+ Log when leak detection changes the test result
+ https://bugs.webkit.org/show_bug.cgi?id=189293
+
+ Reviewed by Jon Lee.
+
+ When you have an "Leak" expectation in TestExpectations, it's confusing to see a test unexpectedly pass,
+ but then show up as an expected fail at the end (because leak detection happens at the end of a shard).
+ So log when leak detection changes a test result.
+
+ * Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py:
+ (LayoutTestRunner._annotate_results_with_additional_failures):
+ * Scripts/webkitpy/layout_tests/models/test_expectations.py:
+ (TestExpectations):
+
2018-09-06 Wenson Hsieh <[email protected]>
[macOS] [WK2] Support changing attributes for selected text (text shadow, underline, strike-through)
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py (235770 => 235771)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py 2018-09-07 00:21:14 UTC (rev 235770)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py 2018-09-07 00:24:51 UTC (rev 235771)
@@ -202,6 +202,11 @@
expectations = self._expectations.filtered_expectations_for_test(new_result.test_name, self._options.pixel_tests or bool(new_result.reftest_type), self._options.world_leaks)
was_expected = self._expectations.matches_an_expected_result(new_result.test_name, existing_result.type, expectations)
now_expected = self._expectations.matches_an_expected_result(new_result.test_name, new_result.type, expectations)
+ if was_expected != now_expected:
+ # When annotation is not just about leaks, this logging should be changed.
+ _log.warning(' %s -> changed by leak detection from a %s (%s) to a %s (%s)' % (new_result.test_name,
+ TestExpectations.EXPECTATION_DESCRIPTION[existing_result.type], 'expected' if was_expected else 'unexpected',
+ TestExpectations.EXPECTATION_DESCRIPTION[new_result.type], 'expected' if now_expected else 'unexpected'))
run_results.change_result_to_failure(existing_result, new_result, was_expected, now_expected)
def start_servers(self):
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py (235770 => 235771)
--- trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py 2018-09-07 00:21:14 UTC (rev 235770)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py 2018-09-07 00:24:51 UTC (rev 235771)
@@ -815,6 +815,19 @@
'leak': LEAK,
'skip': SKIP}
+ # Singulars
+ EXPECTATION_DESCRIPTION = {SKIP: 'skipped',
+ PASS: 'pass',
+ FAIL: 'failure',
+ IMAGE: 'image-only failure',
+ TEXT: 'text-only failure',
+ IMAGE_PLUS_TEXT: 'image and text failure',
+ AUDIO: 'audio failure',
+ CRASH: 'crash',
+ TIMEOUT: 'timeout',
+ MISSING: 'missing',
+ LEAK: 'leak'}
+
# (aggregated by category, pass/fail/skip, type)
EXPECTATION_DESCRIPTIONS = {SKIP: 'skipped',
PASS: 'passes',
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/views/printing.py (235770 => 235771)
--- trunk/Tools/Scripts/webkitpy/layout_tests/views/printing.py 2018-09-07 00:21:14 UTC (rev 235770)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/views/printing.py 2018-09-07 00:24:51 UTC (rev 235771)
@@ -321,7 +321,7 @@
def print_finished_test(self, result, expected, exp_str, got_str):
test_name = result.test_name
- result_message = self._result_message(result.type, result.failures, expected, self._options.verbose)
+ result_message = self._result_message(result.type, result.failures, expected, exp_str, self._options.verbose)
if self._options.details:
self._print_test_trace(result, exp_str, got_str)
@@ -340,8 +340,11 @@
self._completed_tests = []
self._running_tests.remove(test_name)
- def _result_message(self, result_type, failures, expected, verbose):
- exp_string = ' unexpectedly' if not expected else ''
+ def _result_message(self, result_type, failures, expected, exp_str, verbose):
+ exp_string = ''
+ if not expected:
+ exp_string = ' (leak detection is pending)' if 'LEAK' in exp_str else ' unexpectedly'
+
if result_type == test_expectations.PASS:
return ' passed%s' % exp_string
else:
