Diff
Modified: trunk/Tools/ChangeLog (105168 => 105169)
--- trunk/Tools/ChangeLog 2012-01-17 18:28:31 UTC (rev 105168)
+++ trunk/Tools/ChangeLog 2012-01-17 18:55:53 UTC (rev 105169)
@@ -1,3 +1,19 @@
+2012-01-17 Hao Zheng <[email protected]>
+
+ Do not remove pixel failures for ref tests.
+ https://bugs.webkit.org/show_bug.cgi?id=76243
+
+ Reviewed by Tony Chang.
+
+ 'NRWT --no-pixel-tests' complains when reftests are expected to
+ be image mismatch.
+
+ * Scripts/webkitpy/layout_tests/controllers/manager.py:
+ (Manager._update_summary_with_result):
+ * Scripts/webkitpy/layout_tests/models/test_failures.py:
+ (is_reftest_failure):
+ (determine_result_type):
+
2012-01-17 Balazs Ankes <[email protected]>
REGRESSION(r81225): ORWT should ignore reftests
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py (105168 => 105169)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py 2012-01-17 18:28:31 UTC (rev 105168)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py 2012-01-17 18:55:53 UTC (rev 105169)
@@ -1024,7 +1024,7 @@
if result.type == test_expectations.SKIP:
result_summary.add(result, expected=True)
else:
- expected = self._expectations.matches_an_expected_result(result.test_name, result.type, self._options.pixel_tests)
+ expected = self._expectations.matches_an_expected_result(result.test_name, result.type, self._options.pixel_tests or test_failures.is_reftest_failure(result.failures))
result_summary.add(result, expected)
exp_str = self._expectations.get_expectations_string(result.test_name)
got_str = self._expectations.expectation_to_string(result.type)
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager_unittest.py (105168 => 105169)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager_unittest.py 2012-01-17 18:28:31 UTC (rev 105168)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager_unittest.py 2012-01-17 18:55:53 UTC (rev 105169)
@@ -45,6 +45,8 @@
from webkitpy.layout_tests.controllers.manager import interpret_test_failures, Manager, natural_sort_key, test_key, TestRunInterruptedException, TestShard
from webkitpy.layout_tests.models import test_failures
from webkitpy.layout_tests.models.result_summary import ResultSummary
+from webkitpy.layout_tests.models.test_expectations import TestExpectations
+from webkitpy.layout_tests.models.test_results import TestResult
from webkitpy.layout_tests.views import printing
from webkitpy.tool.mocktool import MockOptions
from webkitpy.common.system.executive_mock import MockExecutive
@@ -245,6 +247,22 @@
manager._options.exit_after_n_failures = 10
exception = self.assertRaises(TestRunInterruptedException, manager._interrupt_if_at_failure_limits, result_summary)
+ def test_update_summary_with_result(self):
+ host = MockHost()
+ port = host.port_factory.get('test-win-xp')
+ test = 'failures/expected/reftest.html'
+ expectations = TestExpectations(port, tests=[test],
+ expectations='WONTFIX : failures/expected/reftest.html = IMAGE',
+ test_config=port.test_configuration())
+ # Reftests expected to be image mismatch should be respected when pixel_tests=False.
+ manager = Manager(port=port, options=MockOptions(pixel_tests=False, exit_after_n_failures=None, exit_after_n_crashes_or_timeouts=None), printer=Mock())
+ manager._expectations = expectations
+ result_summary = ResultSummary(expectations=expectations, test_files=[test])
+ result = TestResult(test_name=test, failures=[test_failures.FailureReftestMismatchDidNotOccur()])
+ manager._update_summary_with_result(result_summary, result)
+ self.assertEquals(1, result_summary.expected)
+ self.assertEquals(0, result_summary.unexpected)
+
def test_needs_servers(self):
def get_manager_with_tests(test_names):
port = Mock() # FIXME: Use a tighter mock.
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/models/test_failures.py (105168 => 105169)
--- trunk/Tools/Scripts/webkitpy/layout_tests/models/test_failures.py 2012-01-17 18:28:31 UTC (rev 105168)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/models/test_failures.py 2012-01-17 18:55:53 UTC (rev 105169)
@@ -32,6 +32,10 @@
from webkitpy.layout_tests.models import test_expectations
+def is_reftest_failure(failure_list):
+ failure_types = [type(f) for f in failure_list]
+ return set((FailureReftestMismatch, FailureReftestMismatchDidNotOccur, FailureReftestNoImagesGenerated)).intersection(failure_types)
+
# FIXME: This is backwards. Each TestFailure subclass should know what
# test_expectation type it corresponds too. Then this method just
# collects them all from the failure list and returns the worst one.
@@ -59,13 +63,12 @@
is_text_failure = FailureTextMismatch in failure_types
is_image_failure = (FailureImageHashIncorrect in failure_types or
FailureImageHashMismatch in failure_types)
- is_reftest_failure = set((FailureReftestMismatch, FailureReftestMismatchDidNotOccur, FailureReftestNoImagesGenerated)).intersection(failure_types)
is_audio_failure = (FailureAudioMismatch in failure_types)
if is_text_failure and is_image_failure:
return test_expectations.IMAGE_PLUS_TEXT
elif is_text_failure:
return test_expectations.TEXT
- elif is_image_failure or is_reftest_failure:
+ elif is_image_failure or is_reftest_failure(failure_list):
return test_expectations.IMAGE
elif is_audio_failure:
return test_expectations.AUDIO
