Diff
Modified: trunk/Tools/ChangeLog (110782 => 110783)
--- trunk/Tools/ChangeLog 2012-03-14 23:34:13 UTC (rev 110782)
+++ trunk/Tools/ChangeLog 2012-03-14 23:36:53 UTC (rev 110783)
@@ -1,3 +1,32 @@
+2012-03-13 Ojan Vafai <[email protected]>
+
+ Have webkit-patch rebaseline-test update test_expectations.txt
+ https://bugs.webkit.org/show_bug.cgi?id=81054
+
+ Reviewed by Dirk Pranke.
+
+ This will allow us to use the All Failures tab in garden-o-matic
+ to rebaseline tests without making test_expectations.txt stale.
+
+ It's conservative and only removes lines if the test is
+ not flaky and fails in a way the rebaseline would fix (e.g.
+ CRASH/TIMEOUT will not be touched).
+
+ * Scripts/webkitpy/layout_tests/models/test_expectations.py:
+ (TestExpectations.remove_configuration_from_test):
+ * Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py:
+ (RemoveConfigurationsTest):
+ (RemoveConfigurationsTest.test_remove):
+ (test_remove_line):
+ * Scripts/webkitpy/tool/commands/rebaseline.py:
+ (RebaselineTest._is_supported_port):
+ (RebaselineTest):
+ (RebaselineTest._update_expectations_file):
+ (RebaselineTest._rebaseline_test_and_update_expectations):
+ (RebaselineTest.execute):
+ * Scripts/webkitpy/tool/commands/rebaseline_unittest.py:
+ (TestRebaseline.test_rebaseline_updates_expectations_file):
+
2012-03-14 Jessie Berlin <[email protected]>
Crash in WKTR under addChromeInputField due to using 0 as a key in a HashMap
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py (110782 => 110783)
--- trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py 2012-03-14 23:34:13 UTC (rev 110782)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py 2012-03-14 23:36:53 UTC (rev 110783)
@@ -816,6 +816,25 @@
def has_warnings(self):
return self._has_warnings
+ def remove_configuration_from_test(self, test, test_configuration):
+ expectations_to_remove = []
+ for expectation in self._expectations:
+ if expectation.name != test or expectation.is_flaky() or not expectation.parsed_expectations:
+ continue
+ if iter(expectation.parsed_expectations).next() not in (FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO):
+ continue
+ if test_configuration not in expectation.matching_configurations:
+ continue
+
+ expectation.matching_configurations.remove(test_configuration)
+ if not expectation.matching_configurations:
+ expectations_to_remove.append(expectation)
+
+ for expectation in expectations_to_remove:
+ self._expectations.remove(expectation)
+
+ return TestExpectationSerializer.list_to_string(self._expectations, self._parser._test_configuration_converter)
+
def remove_rebaselined_tests(self, except_these_tests):
"""Returns a copy of the expectations with the tests removed."""
def without_rebaseline_modifier(expectation):
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py (110782 => 110783)
--- trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py 2012-03-14 23:34:13 UTC (rev 110782)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py 2012-03-14 23:36:53 UTC (rev 110783)
@@ -378,6 +378,53 @@
"BUG_TEST XP : passes/text.html = TEXT\n")
+class RemoveConfigurationsTest(Base):
+ def test_remove(self):
+ host = MockHost()
+ test_port = host.port_factory.get('test-win-xp', None)
+ test_port.test_exists = lambda test: True
+ test_port.test_isfile = lambda test: True
+
+ test_config = test_port.test_configuration()
+ expectations = TestExpectations(test_port,
+ tests=self.get_basic_tests(),
+ expectations="""BUGX LINUX WIN RELEASE : failures/expected/foo.html = TEXT
+BUGY WIN DEBUG : failures/expected/foo.html = CRASH
+""",
+ test_config=test_config,
+ is_lint_mode=False,
+ overrides=None)
+
+ actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
+
+ self.assertEqual("""BUGX LINUX VISTA WIN7 RELEASE : failures/expected/foo.html = TEXT
+BUGY WIN DEBUG : failures/expected/foo.html = CRASH
+""", actual_expectations)
+
+ def test_remove_line(self):
+ host = MockHost()
+ test_port = host.port_factory.get('test-win-xp', None)
+ test_port.test_exists = lambda test: True
+ test_port.test_isfile = lambda test: True
+
+ test_config = test_port.test_configuration()
+ expectations = TestExpectations(test_port,
+ tests=None,
+ expectations="""BUGX WIN RELEASE : failures/expected/foo.html = TEXT
+BUGY WIN DEBUG : failures/expected/foo.html = CRASH
+""",
+ test_config=test_config,
+ is_lint_mode=False,
+ overrides=None)
+
+ actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config)
+ actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-vista', None).test_configuration())
+ actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration())
+
+ self.assertEqual("""BUGY WIN DEBUG : failures/expected/foo.html = CRASH
+""", actual_expectations)
+
+
class RebaseliningTest(Base):
"""Test rebaselining-specific functionality."""
def assertRemove(self, input_expectations, tests, expected_expectations):
Modified: trunk/Tools/Scripts/webkitpy/tool/commands/rebaseline.py (110782 => 110783)
--- trunk/Tools/Scripts/webkitpy/tool/commands/rebaseline.py 2012-03-14 23:34:13 UTC (rev 110782)
+++ trunk/Tools/Scripts/webkitpy/tool/commands/rebaseline.py 2012-03-14 23:36:53 UTC (rev 110783)
@@ -40,6 +40,7 @@
from webkitpy.common.system.user import User
from webkitpy.layout_tests.controllers.test_result_writer import TestResultWriter
from webkitpy.layout_tests.models import test_failures
+from webkitpy.layout_tests.models.test_configuration import TestConfiguration
from webkitpy.layout_tests.models.test_expectations import TestExpectations
from webkitpy.layout_tests.port import builders
from webkitpy.tool.grammar import pluralize
@@ -108,6 +109,17 @@
if not self._tool.scm().exists(target_baseline):
self._tool.scm().add(target_baseline)
+ def _update_expectations_file(self, builder_name, test_name):
+ port = self._tool.port_factory.get_from_builder_name(builder_name)
+ expectationsString = port.test_expectations()
+ expectations = TestExpectations(port, None, expectationsString, port.test_configuration())
+
+ for test_configuration in port.all_test_configurations():
+ if test_configuration.version == port.test_configuration().version:
+ expectationsString = expectations.remove_configuration_from_test(test_name, test_configuration)
+
+ self._tool.filesystem.write_text_file(port.path_to_test_expectations_file(), expectationsString)
+
def _test_root(self, test_name):
return os.path.splitext(test_name)[0]
@@ -130,15 +142,19 @@
print "Retrieving %s." % source_baseline
self._save_baseline(self._tool.web.get_binary(source_baseline, convert_404_to_None=True), target_baseline)
- def execute(self, options, args, tool):
+ def _rebaseline_test_and_update_expectations(self, builder_name, test_name, platforms_to_move_existing_baselines_to):
for suffix in _baseline_suffix_list:
- if len(args) > 2:
- platforms_to_move_existing_baselines_to = args[2:]
- else:
- platforms_to_move_existing_baselines_to = None
- self._rebaseline_test(args[0], args[1], platforms_to_move_existing_baselines_to, suffix)
+ self._rebaseline_test(builder_name, test_name, platforms_to_move_existing_baselines_to, suffix)
+ self._update_expectations_file(builder_name, test_name)
+ def execute(self, options, args, tool):
+ if len(args) > 2:
+ platforms_to_move_existing_baselines_to = args[2:]
+ else:
+ platforms_to_move_existing_baselines_to = None
+ self._rebaseline_test_and_update_expectations(args[0], args[1], platforms_to_move_existing_baselines_to)
+
class OptimizeBaselines(AbstractDeclarativeCommand):
name = "optimize-baselines"
help_text = "Reshuffles the baselines for the given tests to use as litte space on disk as possible."
Modified: trunk/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py (110782 => 110783)
--- trunk/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py 2012-03-14 23:34:13 UTC (rev 110782)
+++ trunk/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py 2012-03-14 23:36:53 UTC (rev 110783)
@@ -42,6 +42,21 @@
build = Mock()
OutputCapture().assert_outputs(self, command._tests_to_update, [build])
+ def test_rebaseline_updates_expectations_file(self):
+ command = RebaselineTest()
+ tool = MockTool()
+ command.bind_to_tool(tool)
+
+ lion_port = tool.port_factory.get_from_builder_name("Webkit Mac10.7")
+ tool.filesystem.write_text_file(lion_port.path_to_test_expectations_file(), "BUGX MAC : userscripts/another-test.html = IMAGE\nBUGZ LINUX : userscripts/another-test.html\n")
+ tool.filesystem.write_text_file(os.path.join(lion_port.layout_tests_dir(), "userscripts/another-test.html"), "Dummy test contents")
+
+ expected_stdout = "Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.png.\nRetrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt.\n"
+ OutputCapture().assert_outputs(self, command._rebaseline_test_and_update_expectations, ["Webkit Mac10.7", "userscripts/another-test.html", None], expected_stdout=expected_stdout)
+
+ new_expectations = tool.filesystem.read_text_file(lion_port.path_to_test_expectations_file())
+ self.assertEqual(new_expectations, "BUGX SNOWLEOPARD : userscripts/another-test.html = IMAGE\nBUGX LEOPARD : userscripts/another-test.html = IMAGE\nBUGZ LINUX : userscripts/another-test.html\n")
+
def test_rebaseline_test(self):
command = RebaselineTest()
command.bind_to_tool(MockTool())
