- Revision
- 129303
- Author
- [email protected]
- Date
- 2012-09-22 01:10:38 -0700 (Sat, 22 Sep 2012)
Log Message
Unexpected reftest passes are only reported when pixel testing is enabled
https://bugs.webkit.org/show_bug.cgi?id=97242
Reviewed by Dirk Pranke.
LayoutTestRunner should know whether the finished test it's handling is
a reftest. This is necessary when updating result summary for reftests
when pixel testing is not enabled - in these circumstances an unexpectedly
passing reftest was not reported as such due to pixel testing then being
determined by searching for reftest failures in failures list, which were
not found in an empty failure list of a passing test. Now the TestResult
objects have a member variable indicating the test is a reftest whenever
that's the case.
* Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py:
(LayoutTestRunner._update_summary_with_result):
* Scripts/webkitpy/layout_tests/controllers/layout_test_runner_unittest.py:
(LayoutTestRunnerTests.test_update_summary_with_result):
* Scripts/webkitpy/layout_tests/controllers/single_test_runner.py:
(SingleTestRunner.run):
(SingleTestRunner._run_reftest):
* Scripts/webkitpy/layout_tests/models/test_results.py:
(TestResult.__init__):
Modified Paths
Diff
Modified: trunk/Tools/ChangeLog (129302 => 129303)
--- trunk/Tools/ChangeLog 2012-09-22 08:00:21 UTC (rev 129302)
+++ trunk/Tools/ChangeLog 2012-09-22 08:10:38 UTC (rev 129303)
@@ -1,3 +1,29 @@
+2012-09-22 Zan Dobersek <[email protected]>
+
+ Unexpected reftest passes are only reported when pixel testing is enabled
+ https://bugs.webkit.org/show_bug.cgi?id=97242
+
+ Reviewed by Dirk Pranke.
+
+ LayoutTestRunner should know whether the finished test it's handling is
+ a reftest. This is necessary when updating result summary for reftests
+ when pixel testing is not enabled - in these circumstances an unexpectedly
+ passing reftest was not reported as such due to pixel testing then being
+ determined by searching for reftest failures in failures list, which were
+ not found in an empty failure list of a passing test. Now the TestResult
+ objects have a member variable indicating the test is a reftest whenever
+ that's the case.
+
+ * Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py:
+ (LayoutTestRunner._update_summary_with_result):
+ * Scripts/webkitpy/layout_tests/controllers/layout_test_runner_unittest.py:
+ (LayoutTestRunnerTests.test_update_summary_with_result):
+ * Scripts/webkitpy/layout_tests/controllers/single_test_runner.py:
+ (SingleTestRunner.run):
+ (SingleTestRunner._run_reftest):
+ * Scripts/webkitpy/layout_tests/models/test_results.py:
+ (TestResult.__init__):
+
2012-09-21 Sam Weinig <[email protected]>
WebProcess XPC services need have their environment set without disrupting all other XPC services
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py (129302 => 129303)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py 2012-09-22 08:00:21 UTC (rev 129302)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py 2012-09-22 08:10:38 UTC (rev 129303)
@@ -198,7 +198,7 @@
exp_str = got_str = 'SKIP'
expected = True
else:
- expected = self._expectations.matches_an_expected_result(result.test_name, result.type, self._options.pixel_tests or test_failures.is_reftest_failure(result.failures))
+ expected = self._expectations.matches_an_expected_result(result.test_name, result.type, self._options.pixel_tests or result.is_reftest)
exp_str = self._expectations.get_expectations_string(result.test_name)
got_str = self._expectations.expectation_to_string(result.type)
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner_unittest.py (129302 => 129303)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner_unittest.py 2012-09-22 08:00:21 UTC (rev 129302)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner_unittest.py 2012-09-22 08:10:38 UTC (rev 129303)
@@ -150,12 +150,19 @@
test = 'failures/expected/reftest.html'
expectations = TestExpectations(runner._port, tests=[test])
runner._expectations = expectations
+
result_summary = ResultSummary(expectations, [test], 1, set())
- result = TestResult(test_name=test, failures=[test_failures.FailureReftestMismatchDidNotOccur()])
+ result = TestResult(test_name=test, failures=[test_failures.FailureReftestMismatchDidNotOccur()], is_reftest=True)
runner._update_summary_with_result(result_summary, result)
self.assertEquals(1, result_summary.expected)
self.assertEquals(0, result_summary.unexpected)
+ result_summary = ResultSummary(expectations, [test], 1, set())
+ result = TestResult(test_name=test, failures=[], is_reftest=True)
+ runner._update_summary_with_result(result_summary, result)
+ self.assertEquals(0, result_summary.expected)
+ self.assertEquals(1, result_summary.unexpected)
+
def test_servers_started(self):
def start_http_server(number_of_servers=None):
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py (129302 => 129303)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py 2012-09-22 08:00:21 UTC (rev 129302)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py 2012-09-22 08:10:38 UTC (rev 129303)
@@ -94,7 +94,7 @@
def run(self):
if self._reference_files:
if self._port.get_option('no_ref_tests') or self._options.reset_results:
- result = TestResult(self._test_name)
+ result = TestResult(self._test_name, is_reftest=True)
result.type = test_expectations.SKIP
return result
return self._run_reftest()
@@ -303,7 +303,7 @@
assert(reference_output)
test_result_writer.write_test_result(self._filesystem, self._port, self._test_name, test_output, reference_output, test_result.failures)
- return TestResult(self._test_name, test_result.failures, total_test_time + test_result.test_run_time, test_result.has_stderr)
+ return TestResult(self._test_name, test_result.failures, total_test_time + test_result.test_run_time, test_result.has_stderr, is_reftest=True)
def _compare_output_with_reference(self, reference_driver_output, actual_driver_output, reference_filename, mismatch):
total_test_time = reference_driver_output.test_time + actual_driver_output.test_time
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/models/test_results.py (129302 => 129303)
--- trunk/Tools/Scripts/webkitpy/layout_tests/models/test_results.py 2012-09-22 08:00:21 UTC (rev 129302)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/models/test_results.py 2012-09-22 08:10:38 UTC (rev 129303)
@@ -38,11 +38,12 @@
def loads(string):
return cPickle.loads(string)
- def __init__(self, test_name, failures=None, test_run_time=None, has_stderr=False):
+ def __init__(self, test_name, failures=None, test_run_time=None, has_stderr=False, is_reftest=False):
self.test_name = test_name
self.failures = failures or []
self.test_run_time = test_run_time or 0
self.has_stderr = has_stderr
+ self.is_reftest = is_reftest
# FIXME: Setting this in the constructor makes this class hard to mutate.
self.type = test_failures.determine_result_type(failures)
