Title: [136767] trunk/Tools
Revision
136767
Author
[email protected]
Date
2012-12-05 15:14:42 -0800 (Wed, 05 Dec 2012)

Log Message

nrwt: only summarize the results one way
https://bugs.webkit.org/show_bug.cgi?id=104064

Reviewed by Ojan Vafai.

Change manager.summarize_results so that we always (and only)
include both expected and unexpected results. Previously we
would summarize the results twice, once with expected results
and once without. This made the code much more confusing than
it needed to be.

* Scripts/webkitpy/layout_tests/controllers/manager.py:
(summarize_results):
(Manager.run):
* Scripts/webkitpy/layout_tests/controllers/manager_unittest.py:
(ResultSummaryTest.summarized_results):
(ResultSummaryTest.test_no_svn_revision):
(ResultSummaryTest.test_svn_revision):
(ResultSummaryTest.test_summarized_results_wontfix):
* Scripts/webkitpy/layout_tests/views/printing.py:
(Printer.print_results):
(Printer._print_unexpected_results):
(Printer._print_unexpected_results.add_result):
* Scripts/webkitpy/layout_tests/views/printing_unittest.py:
(Testprinter.test_print_unexpected_results.get_unexpected_results):
(Testprinter):

Modified Paths

Diff

Modified: trunk/Tools/ChangeLog (136766 => 136767)


--- trunk/Tools/ChangeLog	2012-12-05 23:14:14 UTC (rev 136766)
+++ trunk/Tools/ChangeLog	2012-12-05 23:14:42 UTC (rev 136767)
@@ -1,3 +1,32 @@
+2012-12-05  Dirk Pranke  <[email protected]>
+
+        nrwt: only summarize the results one way
+        https://bugs.webkit.org/show_bug.cgi?id=104064
+
+        Reviewed by Ojan Vafai.
+
+        Change manager.summarize_results so that we always (and only)
+        include both expected and unexpected results. Previously we
+        would summarize the results twice, once with expected results
+        and once without. This made the code much more confusing than
+        it needed to be.
+
+        * Scripts/webkitpy/layout_tests/controllers/manager.py:
+        (summarize_results):
+        (Manager.run):
+        * Scripts/webkitpy/layout_tests/controllers/manager_unittest.py:
+        (ResultSummaryTest.summarized_results):
+        (ResultSummaryTest.test_no_svn_revision):
+        (ResultSummaryTest.test_svn_revision):
+        (ResultSummaryTest.test_summarized_results_wontfix):
+        * Scripts/webkitpy/layout_tests/views/printing.py:
+        (Printer.print_results):
+        (Printer._print_unexpected_results):
+        (Printer._print_unexpected_results.add_result):
+        * Scripts/webkitpy/layout_tests/views/printing_unittest.py:
+        (Testprinter.test_print_unexpected_results.get_unexpected_results):
+        (Testprinter):
+
 2012-12-05  Filip Pizlo  <[email protected]>
 
         DFG profiler should display more information

Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py (136766 => 136767)


--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py	2012-12-05 23:14:14 UTC (rev 136766)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py	2012-12-05 23:14:42 UTC (rev 136767)
@@ -87,20 +87,8 @@
 # or split off from Manager onto another helper class, but should not be a free function.
 # Most likely this should be made into its own class, and this super-long function
 # split into many helper functions.
-def summarize_results(port_obj, expectations, result_summary, retry_summary, only_unexpected):
-    """Summarize failing results as a dict.
-
-    FIXME: split this data structure into a separate class?
-
-    Args:
-        port_obj: interface to port-specific hooks
-        expectations: test_expectations.TestExpectations object
-        result_summary: summary object from initial test runs
-        retry_summary: summary object from final test run of retried tests (if any)
-        only_unexpected: whether to return a summary only for the unexpected results
-    Returns:
-        A dictionary containing a summary of the unexpected results from the
-        run, with the following fields:
+def summarize_results(port_obj, expectations, result_summary, retry_summary):
+    """Returns a dictionary containing a summary of the test runs, with the following fields:
         'version': a version indicator
         'fixable': The number of fixable tests (NOW - PASS)
         'skipped': The number of skipped tests (NOW & SKIPPED)
@@ -130,9 +118,8 @@
         keywords[modifier_enum] = modifier_string.upper()
 
     tests = {}
-    original_results = result_summary.unexpected_results if only_unexpected else result_summary.results
 
-    for test_name, result in original_results.iteritems():
+    for test_name, result in result_summary.results.iteritems():
         # Note that if a test crashed in the original run, we ignore
         # whether or not it crashed when we retried it (if we retried it),
         # and always consider the result not flaky.
@@ -159,7 +146,8 @@
             if expected == 'PASS':
                 continue
         elif result_type == test_expectations.CRASH:
-            num_regressions += 1
+            if test_name in result_summary.unexpected_results:
+                num_regressions += 1
         elif result_type == test_expectations.MISSING:
             if test_name in result_summary.unexpected_results:
                 num_missing += 1
@@ -392,16 +380,14 @@
         if retry_summary:
             self._look_for_new_crash_logs(retry_summary, start_time)
 
-        unexpected_results = summarize_results(self._port, self._expectations, result_summary, retry_summary, _only_unexpected_=True)
+        summarized_results = summarize_results(self._port, self._expectations, result_summary, retry_summary)
+        self._printer.print_results(end_time - start_time, result_summary, summarized_results)
 
-        self._printer.print_results(end_time - start_time, result_summary, unexpected_results)
-
         # FIXME: remove record_results. It's just used for testing. There's no need
         # for it to be a commandline argument.
         if self._options.record_results and not self._options.dry_run:
             self._port.print_leaks_summary()
             # Write the same data to log files and upload generated JSON files to appengine server.
-            summarized_results = summarize_results(self._port, self._expectations, result_summary, retry_summary, _only_unexpected_=False)
             self._upload_json_files(summarized_results, result_summary)
 
         # Write the summary to disk (results.html) and display it if requested.
@@ -411,7 +397,7 @@
             if self._options.show_results and result_summary.unexpected_results or (self._options.full_results_html and result_summary.total_failures):
                 self._port.show_results_html_file(results_path)
 
-        return self._port.exit_code_from_summarized_results(unexpected_results)
+        return self._port.exit_code_from_summarized_results(summarized_results)
 
     def _run_tests(self, tests_to_run, tests_to_skip, repeat_each, iterations, num_workers, retrying):
         needs_http = self._port.requires_http_server() or any(self._is_http_test(test) for test in tests_to_run)

Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager_unittest.py (136766 => 136767)


--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager_unittest.py	2012-12-05 23:14:14 UTC (rev 136766)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager_unittest.py	2012-12-05 23:14:42 UTC (rev 136767)
@@ -161,33 +161,33 @@
         for test in extra_tests:
             rs.add(self.get_result(test, test_expectations.CRASH), expected, test_is_slow)
 
-        retry = rs
         if flaky:
             paths, retry, exp = self.get_result_summary(port, tests, expectations)
             retry.add(self.get_result('passes/text.html'), True, test_is_slow)
             retry.add(self.get_result('failures/expected/timeout.html'), True, test_is_slow)
             retry.add(self.get_result('failures/expected/crash.html'), True, test_is_slow)
-        unexpected_results = summarize_results(port, exp, rs, retry, _only_unexpected_=True)
-        expected_results = summarize_results(port, exp, rs, retry, _only_unexpected_=False)
-        return expected_results, unexpected_results
+        else:
+            retry = None
 
+        return summarize_results(port, exp, rs, retry)
+
     def test_no_svn_revision(self):
         host = MockHost(initialize_scm_by_default=False)
         port = host.port_factory.get('test')
-        expected_results, unexpected_results = self.summarized_results(port, expected=False, passing=False, flaky=False)
-        self.assertTrue('revision' not in unexpected_results)
+        summarized_results = self.summarized_results(port, expected=False, passing=False, flaky=False)
+        self.assertTrue('revision' not in summarized_results)
 
     def test_svn_revision(self):
         host = MockHost(initialize_scm_by_default=False)
         port = host.port_factory.get('test')
         port._options.builder_name = 'dummy builder'
-        expected_results, unexpected_results = self.summarized_results(port, expected=False, passing=False, flaky=False)
-        self.assertNotEquals(unexpected_results['revision'], '')
+        summarized_results = self.summarized_results(port, expected=False, passing=False, flaky=False)
+        self.assertNotEquals(summarized_results['revision'], '')
 
     def test_summarized_results_wontfix(self):
         host = MockHost()
         port = host.port_factory.get('test')
         port._options.builder_name = 'dummy builder'
         port._filesystem.write_text_file(port._filesystem.join(port.layout_tests_dir(), "failures/expected/wontfix.html"), "Dummy test contents")
-        expected_results, unexpected_results = self.summarized_results(port, expected=False, passing=False, flaky=False, extra_tests=['failures/expected/wontfix.html'], extra_expectations='Bug(x) failures/expected/wontfix.html [ WontFix ]\n')
-        self.assertTrue(expected_results['tests']['failures']['expected']['wontfix.html']['wontfix'])
+        summarized_results = self.summarized_results(port, expected=False, passing=False, flaky=False, extra_tests=['failures/expected/wontfix.html'], extra_expectations='Bug(x) failures/expected/wontfix.html [ WontFix ]\n')
+        self.assertTrue(summarized_results['tests']['failures']['expected']['wontfix.html']['wontfix'])

Modified: trunk/Tools/Scripts/webkitpy/layout_tests/views/printing.py (136766 => 136767)


--- trunk/Tools/Scripts/webkitpy/layout_tests/views/printing.py	2012-12-05 23:14:14 UTC (rev 136766)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/views/printing.py	2012-12-05 23:14:42 UTC (rev 136767)
@@ -148,13 +148,13 @@
             ndigits = int(math.log10(len(num))) + 1
         return ndigits
 
-    def print_results(self, run_time, result_summary, unexpected_results):
+    def print_results(self, run_time, result_summary, summarized_results):
         self._print_timing_statistics(run_time, result_summary)
         self._print_result_summary(result_summary)
         self._print_one_line_summary(result_summary.total - result_summary.expected_skips,
                                      result_summary.expected - result_summary.expected_skips,
                                      result_summary.unexpected)
-        self._print_unexpected_results(unexpected_results)
+        self._print_unexpected_results(summarized_results)
 
     def _print_timing_statistics(self, total_time, result_summary):
         self._print_debug("Test timing:")
@@ -423,7 +423,7 @@
             relpath = '<none>'
         self._print_default('  %s: %s' % (extension[1:], relpath))
 
-    def _print_unexpected_results(self, unexpected_results):
+    def _print_unexpected_results(self, summarized_results):
         # Prints to the buildbot stream
         passes = {}
         flaky = {}
@@ -435,6 +435,14 @@
         def add_result(test, results, passes=passes, flaky=flaky, regressions=regressions):
             actual = results['actual'].split(" ")
             expected = results['expected'].split(" ")
+
+            def is_expected(result):
+                return (result in expected) or (result in ('TEXT', 'IMAGE+TEXT') and 'FAIL' in expected)
+
+            if all(is_expected(actual_result) for actual_result in actual):
+                # Don't print anything for tests that ran as expected.
+                return
+
             if actual == ['PASS']:
                 if 'CRASH' in expected:
                     add_to_dict_of_lists(passes, 'Expected to crash, but passed', test)
@@ -448,7 +456,7 @@
             else:
                 add_to_dict_of_lists(regressions, results['actual'], test)
 
-        resultsjsonparser.for_each_test(unexpected_results['tests'], add_result)
+        resultsjsonparser.for_each_test(summarized_results['tests'], add_result)
 
         if len(passes) or len(flaky) or len(regressions):
             self._print_for_bot("")
@@ -469,7 +477,7 @@
                 tests.sort()
 
                 for test in tests:
-                    result = resultsjsonparser.result_for_test(unexpected_results['tests'], test)
+                    result = resultsjsonparser.result_for_test(summarized_results['tests'], test)
                     actual = result['actual'].split(" ")
                     expected = result['expected'].split(" ")
                     result = TestExpectations.EXPECTATIONS[key.lower()]
@@ -489,7 +497,7 @@
                     self._print_for_bot("  %s [ %s ]" % (test, TestExpectationParser._inverted_expectation_tokens[key]))
                 self._print_for_bot("")
 
-        if len(unexpected_results['tests']) and self._options.debug_rwt_logging:
+        if len(summarized_results['tests']) and self._options.debug_rwt_logging:
             self._print_for_bot("%s" % ("-" * 78))
 
     def _print_quiet(self, msg):

Modified: trunk/Tools/Scripts/webkitpy/layout_tests/views/printing_unittest.py (136766 => 136767)


--- trunk/Tools/Scripts/webkitpy/layout_tests/views/printing_unittest.py	2012-12-05 23:14:14 UTC (rev 136766)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/views/printing_unittest.py	2012-12-05 23:14:42 UTC (rev 136767)
@@ -193,14 +193,13 @@
                 rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), expected, test_is_slow)
                 rs.add(self.get_result('failures/expected/timeout.html', test_expectations.CRASH), expected, test_is_slow)
                 rs.add(self.get_result('failures/expected/crash.html', test_expectations.TIMEOUT), expected, test_is_slow)
-            retry = rs
+            retry = None
             if flaky:
                 paths, retry, exp = self.get_result_summary(tests, expectations)
                 retry.add(self.get_result('passes/text.html'), True, test_is_slow)
                 retry.add(self.get_result('failures/expected/timeout.html'), True, test_is_slow)
                 retry.add(self.get_result('failures/expected/crash.html'), True, test_is_slow)
-            unexpected_results = manager.summarize_results(self._port, exp, rs, retry, _only_unexpected_=True)
-            return unexpected_results
+            return manager.summarize_results(self._port, exp, rs, retry)
 
         tests = ['passes/text.html', 'failures/expected/timeout.html', 'failures/expected/crash.html']
         expectations = ''
_______________________________________________
webkit-changes mailing list
[email protected]
http://lists.webkit.org/mailman/listinfo/webkit-changes

Reply via email to