Diff
Modified: trunk/Tools/ChangeLog (136766 => 136767)
--- trunk/Tools/ChangeLog 2012-12-05 23:14:14 UTC (rev 136766)
+++ trunk/Tools/ChangeLog 2012-12-05 23:14:42 UTC (rev 136767)
@@ -1,3 +1,32 @@
+2012-12-05 Dirk Pranke <[email protected]>
+
+ nrwt: only summarize the results one way
+ https://bugs.webkit.org/show_bug.cgi?id=104064
+
+ Reviewed by Ojan Vafai.
+
+ Change manager.summarize_results so that we always (and only)
+ include both expected and unexpected results. Previously we
+ would summarize the results twice, once with expected results
+ and once without. This made the code much more confusing than
+ it needed to be.
+
+ * Scripts/webkitpy/layout_tests/controllers/manager.py:
+ (summarize_results):
+ (Manager.run):
+ * Scripts/webkitpy/layout_tests/controllers/manager_unittest.py:
+ (ResultSummaryTest.summarized_results):
+ (ResultSummaryTest.test_no_svn_revision):
+ (ResultSummaryTest.test_svn_revision):
+ (ResultSummaryTest.test_summarized_results_wontfix):
+ * Scripts/webkitpy/layout_tests/views/printing.py:
+ (Printer.print_results):
+ (Printer._print_unexpected_results):
+ (Printer._print_unexpected_results.add_result):
+ * Scripts/webkitpy/layout_tests/views/printing_unittest.py:
+ (Testprinter.test_print_unexpected_results.get_unexpected_results):
+ (Testprinter):
+
2012-12-05 Filip Pizlo <[email protected]>
DFG profiler should display more information
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py (136766 => 136767)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py 2012-12-05 23:14:14 UTC (rev 136766)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py 2012-12-05 23:14:42 UTC (rev 136767)
@@ -87,20 +87,8 @@
# or split off from Manager onto another helper class, but should not be a free function.
# Most likely this should be made into its own class, and this super-long function
# split into many helper functions.
-def summarize_results(port_obj, expectations, result_summary, retry_summary, only_unexpected):
- """Summarize failing results as a dict.
-
- FIXME: split this data structure into a separate class?
-
- Args:
- port_obj: interface to port-specific hooks
- expectations: test_expectations.TestExpectations object
- result_summary: summary object from initial test runs
- retry_summary: summary object from final test run of retried tests (if any)
- only_unexpected: whether to return a summary only for the unexpected results
- Returns:
- A dictionary containing a summary of the unexpected results from the
- run, with the following fields:
+def summarize_results(port_obj, expectations, result_summary, retry_summary):
+ """Returns a dictionary containing a summary of the test runs, with the following fields:
'version': a version indicator
'fixable': The number of fixable tests (NOW - PASS)
'skipped': The number of skipped tests (NOW & SKIPPED)
@@ -130,9 +118,8 @@
keywords[modifier_enum] = modifier_string.upper()
tests = {}
- original_results = result_summary.unexpected_results if only_unexpected else result_summary.results
- for test_name, result in original_results.iteritems():
+ for test_name, result in result_summary.results.iteritems():
# Note that if a test crashed in the original run, we ignore
# whether or not it crashed when we retried it (if we retried it),
# and always consider the result not flaky.
@@ -159,7 +146,8 @@
if expected == 'PASS':
continue
elif result_type == test_expectations.CRASH:
- num_regressions += 1
+ if test_name in result_summary.unexpected_results:
+ num_regressions += 1
elif result_type == test_expectations.MISSING:
if test_name in result_summary.unexpected_results:
num_missing += 1
@@ -392,16 +380,14 @@
if retry_summary:
self._look_for_new_crash_logs(retry_summary, start_time)
- unexpected_results = summarize_results(self._port, self._expectations, result_summary, retry_summary, _only_unexpected_=True)
+ summarized_results = summarize_results(self._port, self._expectations, result_summary, retry_summary)
+ self._printer.print_results(end_time - start_time, result_summary, summarized_results)
- self._printer.print_results(end_time - start_time, result_summary, unexpected_results)
-
# FIXME: remove record_results. It's just used for testing. There's no need
# for it to be a commandline argument.
if self._options.record_results and not self._options.dry_run:
self._port.print_leaks_summary()
# Write the same data to log files and upload generated JSON files to appengine server.
- summarized_results = summarize_results(self._port, self._expectations, result_summary, retry_summary, _only_unexpected_=False)
self._upload_json_files(summarized_results, result_summary)
# Write the summary to disk (results.html) and display it if requested.
@@ -411,7 +397,7 @@
if self._options.show_results and result_summary.unexpected_results or (self._options.full_results_html and result_summary.total_failures):
self._port.show_results_html_file(results_path)
- return self._port.exit_code_from_summarized_results(unexpected_results)
+ return self._port.exit_code_from_summarized_results(summarized_results)
def _run_tests(self, tests_to_run, tests_to_skip, repeat_each, iterations, num_workers, retrying):
needs_http = self._port.requires_http_server() or any(self._is_http_test(test) for test in tests_to_run)
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager_unittest.py (136766 => 136767)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager_unittest.py 2012-12-05 23:14:14 UTC (rev 136766)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager_unittest.py 2012-12-05 23:14:42 UTC (rev 136767)
@@ -161,33 +161,33 @@
for test in extra_tests:
rs.add(self.get_result(test, test_expectations.CRASH), expected, test_is_slow)
- retry = rs
if flaky:
paths, retry, exp = self.get_result_summary(port, tests, expectations)
retry.add(self.get_result('passes/text.html'), True, test_is_slow)
retry.add(self.get_result('failures/expected/timeout.html'), True, test_is_slow)
retry.add(self.get_result('failures/expected/crash.html'), True, test_is_slow)
- unexpected_results = summarize_results(port, exp, rs, retry, _only_unexpected_=True)
- expected_results = summarize_results(port, exp, rs, retry, _only_unexpected_=False)
- return expected_results, unexpected_results
+ else:
+ retry = None
+ return summarize_results(port, exp, rs, retry)
+
def test_no_svn_revision(self):
host = MockHost(initialize_scm_by_default=False)
port = host.port_factory.get('test')
- expected_results, unexpected_results = self.summarized_results(port, expected=False, passing=False, flaky=False)
- self.assertTrue('revision' not in unexpected_results)
+ summarized_results = self.summarized_results(port, expected=False, passing=False, flaky=False)
+ self.assertTrue('revision' not in summarized_results)
def test_svn_revision(self):
host = MockHost(initialize_scm_by_default=False)
port = host.port_factory.get('test')
port._options.builder_name = 'dummy builder'
- expected_results, unexpected_results = self.summarized_results(port, expected=False, passing=False, flaky=False)
- self.assertNotEquals(unexpected_results['revision'], '')
+ summarized_results = self.summarized_results(port, expected=False, passing=False, flaky=False)
+ self.assertNotEquals(summarized_results['revision'], '')
def test_summarized_results_wontfix(self):
host = MockHost()
port = host.port_factory.get('test')
port._options.builder_name = 'dummy builder'
port._filesystem.write_text_file(port._filesystem.join(port.layout_tests_dir(), "failures/expected/wontfix.html"), "Dummy test contents")
- expected_results, unexpected_results = self.summarized_results(port, expected=False, passing=False, flaky=False, extra_tests=['failures/expected/wontfix.html'], extra_expectations='Bug(x) failures/expected/wontfix.html [ WontFix ]\n')
- self.assertTrue(expected_results['tests']['failures']['expected']['wontfix.html']['wontfix'])
+ summarized_results = self.summarized_results(port, expected=False, passing=False, flaky=False, extra_tests=['failures/expected/wontfix.html'], extra_expectations='Bug(x) failures/expected/wontfix.html [ WontFix ]\n')
+ self.assertTrue(summarized_results['tests']['failures']['expected']['wontfix.html']['wontfix'])
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/views/printing.py (136766 => 136767)
--- trunk/Tools/Scripts/webkitpy/layout_tests/views/printing.py 2012-12-05 23:14:14 UTC (rev 136766)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/views/printing.py 2012-12-05 23:14:42 UTC (rev 136767)
@@ -148,13 +148,13 @@
ndigits = int(math.log10(len(num))) + 1
return ndigits
- def print_results(self, run_time, result_summary, unexpected_results):
+ def print_results(self, run_time, result_summary, summarized_results):
self._print_timing_statistics(run_time, result_summary)
self._print_result_summary(result_summary)
self._print_one_line_summary(result_summary.total - result_summary.expected_skips,
result_summary.expected - result_summary.expected_skips,
result_summary.unexpected)
- self._print_unexpected_results(unexpected_results)
+ self._print_unexpected_results(summarized_results)
def _print_timing_statistics(self, total_time, result_summary):
self._print_debug("Test timing:")
@@ -423,7 +423,7 @@
relpath = '<none>'
self._print_default(' %s: %s' % (extension[1:], relpath))
- def _print_unexpected_results(self, unexpected_results):
+ def _print_unexpected_results(self, summarized_results):
# Prints to the buildbot stream
passes = {}
flaky = {}
@@ -435,6 +435,14 @@
def add_result(test, results, passes=passes, flaky=flaky, regressions=regressions):
actual = results['actual'].split(" ")
expected = results['expected'].split(" ")
+
+ def is_expected(result):
+ return (result in expected) or (result in ('TEXT', 'IMAGE+TEXT') and 'FAIL' in expected)
+
+ if all(is_expected(actual_result) for actual_result in actual):
+ # Don't print anything for tests that ran as expected.
+ return
+
if actual == ['PASS']:
if 'CRASH' in expected:
add_to_dict_of_lists(passes, 'Expected to crash, but passed', test)
@@ -448,7 +456,7 @@
else:
add_to_dict_of_lists(regressions, results['actual'], test)
- resultsjsonparser.for_each_test(unexpected_results['tests'], add_result)
+ resultsjsonparser.for_each_test(summarized_results['tests'], add_result)
if len(passes) or len(flaky) or len(regressions):
self._print_for_bot("")
@@ -469,7 +477,7 @@
tests.sort()
for test in tests:
- result = resultsjsonparser.result_for_test(unexpected_results['tests'], test)
+ result = resultsjsonparser.result_for_test(summarized_results['tests'], test)
actual = result['actual'].split(" ")
expected = result['expected'].split(" ")
result = TestExpectations.EXPECTATIONS[key.lower()]
@@ -489,7 +497,7 @@
self._print_for_bot(" %s [ %s ]" % (test, TestExpectationParser._inverted_expectation_tokens[key]))
self._print_for_bot("")
- if len(unexpected_results['tests']) and self._options.debug_rwt_logging:
+ if len(summarized_results['tests']) and self._options.debug_rwt_logging:
self._print_for_bot("%s" % ("-" * 78))
def _print_quiet(self, msg):
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/views/printing_unittest.py (136766 => 136767)
--- trunk/Tools/Scripts/webkitpy/layout_tests/views/printing_unittest.py 2012-12-05 23:14:14 UTC (rev 136766)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/views/printing_unittest.py 2012-12-05 23:14:42 UTC (rev 136767)
@@ -193,14 +193,13 @@
rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), expected, test_is_slow)
rs.add(self.get_result('failures/expected/timeout.html', test_expectations.CRASH), expected, test_is_slow)
rs.add(self.get_result('failures/expected/crash.html', test_expectations.TIMEOUT), expected, test_is_slow)
- retry = rs
+ retry = None
if flaky:
paths, retry, exp = self.get_result_summary(tests, expectations)
retry.add(self.get_result('passes/text.html'), True, test_is_slow)
retry.add(self.get_result('failures/expected/timeout.html'), True, test_is_slow)
retry.add(self.get_result('failures/expected/crash.html'), True, test_is_slow)
- unexpected_results = manager.summarize_results(self._port, exp, rs, retry, _only_unexpected_=True)
- return unexpected_results
+ return manager.summarize_results(self._port, exp, rs, retry)
tests = ['passes/text.html', 'failures/expected/timeout.html', 'failures/expected/crash.html']
expectations = ''