[webkit-changes] [119766] trunk/Tools

Thu, 07 Jun 2012 16:10:44 -0700

Title: [119766] trunk/Tools
Revision
119766
Author
[email protected]
Date
2012-06-07 16:10:23 -0700 (Thu, 07 Jun 2012)

Log Message

webkit-patch rebaseline-expectations should only rebaseline the appropriate suffixes for the failure in question
https://bugs.webkit.org/show_bug.cgi?id=88581

Reviewed by Ojan Vafai.

webkit-paptch rebaseline-expectations was ignoring what the
expected results for a test were and pulling down all the result
files from a bot. This could lead to us pulling down old,
incorrect .txt or .png files. With this patch we will now only
pull down expectations of the appropriate file types (so an
IMAGE failure will only pull the png and not the txt file).

* Scripts/webkitpy/layout_tests/models/test_expectations.py:
(has_pixel_failures):
(suffixes_for_expectations):
* Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py:
(FunctionsTest.test_suffixes_for_expectations):
* Scripts/webkitpy/tool/commands/rebaseline.py:
(AbstractRebaseliningCommand.__init__):
(RebaselineExpectations._is_supported_port):
(RebaselineExpectations._update_expectations_file):
(RebaselineExpectations._tests_to_rebaseline):
(RebaselineExpectations._rebaseline_port):
* Scripts/webkitpy/tool/commands/rebaseline_unittest.py:
(test_overrides_are_included_correctly):

Modified Paths

Diff

Modified: trunk/Tools/ChangeLog (119765 => 119766)


--- trunk/Tools/ChangeLog	2012-06-07 23:07:48 UTC (rev 119765)
+++ trunk/Tools/ChangeLog	2012-06-07 23:10:23 UTC (rev 119766)
@@ -1,3 +1,31 @@
+2012-06-07  Dirk Pranke  <[email protected]>
+
+        webkit-patch rebaseline-expectations should only rebaseline the appropriate suffixes for the failure in question
+        https://bugs.webkit.org/show_bug.cgi?id=88581
+
+        Reviewed by Ojan Vafai.
+
+        webkit-paptch rebaseline-expectations was ignoring what the
+        expected results for a test were and pulling down all the result
+        files from a bot. This could lead to us pulling down old,
+        incorrect .txt or .png files. With this patch we will now only
+        pull down expectations of the appropriate file types (so an
+        IMAGE failure will only pull the png and not the txt file).
+
+        * Scripts/webkitpy/layout_tests/models/test_expectations.py:
+        (has_pixel_failures):
+        (suffixes_for_expectations):
+        * Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py:
+        (FunctionsTest.test_suffixes_for_expectations):
+        * Scripts/webkitpy/tool/commands/rebaseline.py:
+        (AbstractRebaseliningCommand.__init__):
+        (RebaselineExpectations._is_supported_port):
+        (RebaselineExpectations._update_expectations_file):
+        (RebaselineExpectations._tests_to_rebaseline):
+        (RebaselineExpectations._rebaseline_port):
+        * Scripts/webkitpy/tool/commands/rebaseline_unittest.py:
+        (test_overrides_are_included_correctly):
+
 2012-06-07  Ojan Vafai  <[email protected]>
 
         Fix recent null pointer error regression in the flakiness dashboard

Modified: trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py (119765 => 119766)


--- trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py	2012-06-07 23:07:48 UTC (rev 119765)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py	2012-06-07 23:10:23 UTC (rev 119766)
@@ -85,6 +85,21 @@
     return IMAGE in actual_results or IMAGE_PLUS_TEXT in actual_results
 
 
+# FIXME: Perhas these two routines should be part of the Port instead?
+BASELINE_SUFFIX_LIST = ('png', 'wav', 'txt')
+
+
+def suffixes_for_expectations(expectations):
+    suffixes = set()
+    if expectations.intersection(set([TEXT, IMAGE_PLUS_TEXT, FAIL])):
+        suffixes.add('txt')
+    if expectations.intersection(set([IMAGE, IMAGE_PLUS_TEXT, FAIL])):
+        suffixes.add('png')
+    if AUDIO in expectations:
+        suffixes.add('wav')
+    return set(suffixes)
+
+
 # FIXME: This method is no longer used here in this module. Remove remaining callsite in manager.py and this method.
 def strip_comments(line):
     """Strips comments from a line and return None if the line is empty

Modified: trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py (119765 => 119766)


--- trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py	2012-06-07 23:07:48 UTC (rev 119765)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py	2012-06-07 23:10:23 UTC (rev 119766)
@@ -86,7 +86,16 @@
         self.assertEquals(remove_pixel_failures(set([PASS, IMAGE, CRASH])),
                           set([PASS, CRASH]))
 
+    def test_suffixes_for_expectations(self):
+        self.assertEquals(suffixes_for_expectations(set([TEXT])), set(['txt']))
+        self.assertEquals(suffixes_for_expectations(set([FAIL])), set(['txt', 'png']))
+        self.assertEquals(suffixes_for_expectations(set([IMAGE_PLUS_TEXT])), set(['txt', 'png']))
+        self.assertEquals(suffixes_for_expectations(set([IMAGE])), set(['png']))
+        self.assertEquals(suffixes_for_expectations(set([AUDIO])), set(['wav']))
+        self.assertEquals(suffixes_for_expectations(set([TEXT, FAIL, CRASH])), set(['txt', 'png']))
+        self.assertEquals(suffixes_for_expectations(set()), set())
 
+
 class Base(unittest.TestCase):
     # Note that all of these tests are written assuming the configuration
     # being tested is Windows XP, Release build.

Modified: trunk/Tools/Scripts/webkitpy/tool/commands/rebaseline.py (119765 => 119766)


--- trunk/Tools/Scripts/webkitpy/tool/commands/rebaseline.py	2012-06-07 23:07:48 UTC (rev 119765)
+++ trunk/Tools/Scripts/webkitpy/tool/commands/rebaseline.py	2012-06-07 23:10:23 UTC (rev 119766)
@@ -43,16 +43,12 @@
 from webkitpy.layout_tests.controllers.test_result_writer import TestResultWriter
 from webkitpy.layout_tests.models import test_failures
 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
-from webkitpy.layout_tests.models.test_expectations import TestExpectations
+from webkitpy.layout_tests.models.test_expectations import TestExpectations, suffixes_for_expectations, BASELINE_SUFFIX_LIST
 from webkitpy.layout_tests.port import builders
 from webkitpy.tool.grammar import pluralize
 from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand
 
 
-# FIXME: Pull this from Port.baseline_extensions().
-_baseline_suffix_list = ['png', 'wav', 'txt']
-
-
 _log = logging.getLogger(__name__)
 
 # FIXME: Should TestResultWriter know how to compute this string?
@@ -64,10 +60,10 @@
     def __init__(self, options=None):
         options = options or []
         options.extend([
-            optparse.make_option('--suffixes', default=','.join(_baseline_suffix_list), action='',
+            optparse.make_option('--suffixes', default=','.join(BASELINE_SUFFIX_LIST), action='',
                                  help='file types to rebaseline')])
         AbstractDeclarativeCommand.__init__(self, options=options)
-        self._baseline_suffix_list = _baseline_suffix_list
+        self._baseline_suffix_list = BASELINE_SUFFIX_LIST
 
 
 class RebaselineTest(AbstractRebaseliningCommand):
@@ -260,9 +256,6 @@
         # FIXME: Support non-Chromium ports.
         return port_name.startswith('chromium-')
 
-    def _expectations(self, port, include_overrides):
-        return TestExpectations(port, include_overrides=include_overrides)
-
     def _update_expectations_file(self, port_name):
         if not self._is_supported_port(port_name):
             return
@@ -272,12 +265,16 @@
         # This is not good, but avoids having the overrides getting written into the main file.
         # See https://bugs.webkit.org/show_bug.cgi?id=88456 for context. This will no longer be needed
         # once we properly support cascading expectations files.
-        expectations = self._expectations(port, include_overrides=False)
+        expectations = TestExpectations(port, include_overrides=False)
         path = port.path_to_test_expectations_file()
         self._tool.filesystem.write_text_file(path, expectations.remove_rebaselined_tests(expectations.get_rebaselining_failures()))
 
     def _tests_to_rebaseline(self, port):
-        return self._expectations(port, include_overrides=True).get_rebaselining_failures()
+        tests_to_rebaseline = {}
+        expectations = TestExpectations(port, include_overrides=True)
+        for test in expectations.get_rebaselining_failures():
+            tests_to_rebaseline[test] = suffixes_for_expectations(expectations.get_expectations(test))
+        return tests_to_rebaseline
 
     def _rebaseline_port(self, port_name):
         if not self._is_supported_port(port_name):
@@ -286,11 +283,11 @@
         if not builder_name:
             return
         _log.info("Retrieving results for %s from %s." % (port_name, builder_name))
-        for test_name in self._tests_to_rebaseline(self._tool.port_factory.get(port_name)):
+        for test_name, suffixes in self._tests_to_rebaseline(self._tool.port_factory.get(port_name)).iteritems():
             self._touched_test_names.add(test_name)
-            _log.info("    %s" % test_name)
-            # FIXME: need to extract the correct list of suffixes here.
-            self._run_webkit_patch(['rebaseline-test', builder_name, test_name])
+            _log.info("    %s (%s)" % (test_name, ','.join(suffixes)))
+            # FIXME: we should use executive.run_in_parallel() to speed this up.
+            self._run_webkit_patch(['rebaseline-test', '--suffixes', ','.join(suffixes), builder_name, test_name])
 
     def execute(self, options, args, tool):
         self._touched_test_names = set([])

Modified: trunk/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py (119765 => 119766)


--- trunk/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py	2012-06-07 23:07:48 UTC (rev 119765)
+++ trunk/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py	2012-06-07 23:10:23 UTC (rev 119766)
@@ -185,50 +185,50 @@
         tool.executive = MockExecutive(should_log=True)
 
         expected_logs = """Retrieving results for chromium-linux-x86 from Webkit Linux 32.
-    userscripts/another-test.html
-    userscripts/images.svg
+    userscripts/another-test.html (txt)
+    userscripts/images.svg (png)
 Retrieving results for chromium-linux-x86_64 from Webkit Linux.
-    userscripts/another-test.html
-    userscripts/images.svg
+    userscripts/another-test.html (txt)
+    userscripts/images.svg (png)
 Retrieving results for chromium-mac-leopard from Webkit Mac10.5.
-    userscripts/another-test.html
-    userscripts/images.svg
+    userscripts/another-test.html (txt)
+    userscripts/images.svg (png)
 Retrieving results for chromium-mac-lion from Webkit Mac10.7.
-    userscripts/another-test.html
-    userscripts/images.svg
+    userscripts/another-test.html (txt)
+    userscripts/images.svg (png)
 Retrieving results for chromium-mac-snowleopard from Webkit Mac10.6.
-    userscripts/another-test.html
-    userscripts/images.svg
+    userscripts/another-test.html (txt)
+    userscripts/images.svg (png)
 Retrieving results for chromium-win-vista from Webkit Vista.
-    userscripts/another-test.html
-    userscripts/images.svg
+    userscripts/another-test.html (txt)
+    userscripts/images.svg (png)
 Retrieving results for chromium-win-win7 from Webkit Win7.
-    userscripts/another-test.html
-    userscripts/images.svg
+    userscripts/another-test.html (txt)
+    userscripts/images.svg (png)
 Retrieving results for chromium-win-xp from Webkit Win.
-    userscripts/another-test.html
-    userscripts/images.svg
+    userscripts/another-test.html (txt)
+    userscripts/images.svg (png)
 """
 
-        expected_stderr = """MOCK run_command: ['echo', 'rebaseline-test', 'Webkit Linux 32', 'userscripts/another-test.html'], cwd=/mock-checkout
-MOCK run_command: ['echo', 'rebaseline-test', 'Webkit Linux 32', 'userscripts/images.svg'], cwd=/mock-checkout
-MOCK run_command: ['echo', 'rebaseline-test', 'Webkit Linux', 'userscripts/another-test.html'], cwd=/mock-checkout
-MOCK run_command: ['echo', 'rebaseline-test', 'Webkit Linux', 'userscripts/images.svg'], cwd=/mock-checkout
-MOCK run_command: ['echo', 'rebaseline-test', 'Webkit Mac10.5', 'userscripts/another-test.html'], cwd=/mock-checkout
-MOCK run_command: ['echo', 'rebaseline-test', 'Webkit Mac10.5', 'userscripts/images.svg'], cwd=/mock-checkout
-MOCK run_command: ['echo', 'rebaseline-test', 'Webkit Mac10.7', 'userscripts/another-test.html'], cwd=/mock-checkout
-MOCK run_command: ['echo', 'rebaseline-test', 'Webkit Mac10.7', 'userscripts/images.svg'], cwd=/mock-checkout
-MOCK run_command: ['echo', 'rebaseline-test', 'Webkit Mac10.6', 'userscripts/another-test.html'], cwd=/mock-checkout
-MOCK run_command: ['echo', 'rebaseline-test', 'Webkit Mac10.6', 'userscripts/images.svg'], cwd=/mock-checkout
-MOCK run_command: ['echo', 'rebaseline-test', 'Webkit Vista', 'userscripts/another-test.html'], cwd=/mock-checkout
-MOCK run_command: ['echo', 'rebaseline-test', 'Webkit Vista', 'userscripts/images.svg'], cwd=/mock-checkout
-MOCK run_command: ['echo', 'rebaseline-test', 'Webkit Win7', 'userscripts/another-test.html'], cwd=/mock-checkout
-MOCK run_command: ['echo', 'rebaseline-test', 'Webkit Win7', 'userscripts/images.svg'], cwd=/mock-checkout
-MOCK run_command: ['echo', 'rebaseline-test', 'Webkit Win', 'userscripts/another-test.html'], cwd=/mock-checkout
-MOCK run_command: ['echo', 'rebaseline-test', 'Webkit Win', 'userscripts/images.svg'], cwd=/mock-checkout
+        expected_stderr = """MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'txt', 'Webkit Linux 32', 'userscripts/another-test.html'], cwd=/mock-checkout
+MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'png', 'Webkit Linux 32', 'userscripts/images.svg'], cwd=/mock-checkout
+MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'txt', 'Webkit Linux', 'userscripts/another-test.html'], cwd=/mock-checkout
+MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'png', 'Webkit Linux', 'userscripts/images.svg'], cwd=/mock-checkout
+MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'txt', 'Webkit Mac10.5', 'userscripts/another-test.html'], cwd=/mock-checkout
+MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'png', 'Webkit Mac10.5', 'userscripts/images.svg'], cwd=/mock-checkout
+MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'txt', 'Webkit Mac10.7', 'userscripts/another-test.html'], cwd=/mock-checkout
+MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'png', 'Webkit Mac10.7', 'userscripts/images.svg'], cwd=/mock-checkout
+MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'txt', 'Webkit Mac10.6', 'userscripts/another-test.html'], cwd=/mock-checkout
+MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'png', 'Webkit Mac10.6', 'userscripts/images.svg'], cwd=/mock-checkout
+MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'txt', 'Webkit Vista', 'userscripts/another-test.html'], cwd=/mock-checkout
+MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'png', 'Webkit Vista', 'userscripts/images.svg'], cwd=/mock-checkout
+MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'txt', 'Webkit Win7', 'userscripts/another-test.html'], cwd=/mock-checkout
+MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'png', 'Webkit Win7', 'userscripts/images.svg'], cwd=/mock-checkout
+MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'txt', 'Webkit Win', 'userscripts/another-test.html'], cwd=/mock-checkout
+MOCK run_command: ['echo', 'rebaseline-test', '--suffixes', 'png', 'Webkit Win', 'userscripts/images.svg'], cwd=/mock-checkout
 """
 
-        command._tests_to_rebaseline = lambda port: ['userscripts/another-test.html', 'userscripts/images.svg']
+        command._tests_to_rebaseline = lambda port: {'userscripts/another-test.html': set(['txt']), 'userscripts/images.svg': set(['png'])}
         OutputCapture().assert_outputs(self, command.execute, [MockOptions(optimize=False), [], tool], expected_logs=expected_logs, expected_stderr=expected_stderr)
 
         expected_logs_with_optimize = expected_logs + (
@@ -238,7 +238,7 @@
             "MOCK run_command: ['echo', 'optimize-baselines', 'userscripts/another-test.html'], cwd=/mock-checkout\n"
             "MOCK run_command: ['echo', 'optimize-baselines', 'userscripts/images.svg'], cwd=/mock-checkout\n")
 
-        command._tests_to_rebaseline = lambda port: ['userscripts/another-test.html', 'userscripts/images.svg']
+        command._tests_to_rebaseline = lambda port: {'userscripts/another-test.html': set(['txt']), 'userscripts/images.svg': set(['png'])}
         OutputCapture().assert_outputs(self, command.execute, [MockOptions(optimize=True), [], tool], expected_logs=expected_logs_with_optimize, expected_stderr=expected_stderr_with_optimize)
 
     def test_overrides_are_included_correctly(self):
@@ -255,5 +255,5 @@
         port._filesystem.write_text_file(port.path_to_test_expectations_file(), '')
         port._filesystem.write_text_file(port.layout_tests_dir() + '/userscripts/another-test.html', '')
         port.test_expectations_overrides = lambda: self.overrides
-        self.assertEquals(command._tests_to_rebaseline(port), set(['userscripts/another-test.html']))
+        self.assertEquals(command._tests_to_rebaseline(port), {'userscripts/another-test.html': set(['txt'])})
         self.assertEquals(port._filesystem.read_text_file(port.path_to_test_expectations_file()), '')

_______________________________________________
webkit-changes mailing list
[email protected]
http://lists.webkit.org/mailman/listinfo.cgi/webkit-changes

Reply via email to