Diff
Modified: trunk/LayoutTests/ChangeLog (99817 => 99818)
--- trunk/LayoutTests/ChangeLog 2011-11-10 07:12:38 UTC (rev 99817)
+++ trunk/LayoutTests/ChangeLog 2011-11-10 07:39:50 UTC (rev 99818)
@@ -1,3 +1,17 @@
+2011-11-09 Hayato Ito <[email protected]>
+
+ [NRWT] Make results.html aware of reference filename in reftests.
+ https://bugs.webkit.org/show_bug.cgi?id=71574
+
+ Reviewed by Ryosuke Niwa.
+
+ If a reference filename is different from the default one, include that in result summary
+ so results.html can link to the correct reference file.
+
+ * fast/harness/resources/results-test.js:
+ * fast/harness/results.html:
+ * platform/chromium/fast/harness/results-expected.txt:
+
2011-11-09 Andrew Scherkus <[email protected]>
Rebaselining media/video-frame-accurate-seek-expected.html now that it passes consistently.
Modified: trunk/LayoutTests/fast/harness/resources/results-test.js (99817 => 99818)
--- trunk/LayoutTests/fast/harness/resources/results-test.js 2011-11-10 07:12:38 UTC (rev 99817)
+++ trunk/LayoutTests/fast/harness/resources/results-test.js 2011-11-10 07:39:50 UTC (rev 99818)
@@ -265,6 +265,24 @@
runSingleRowTest(results, false, '', 'ref mismatch html actual ');
results = mockResults();
+ results.tests['bar-reftest.html'] = mockExpectation('PASS', 'IMAGE', 1);
+ results.tests['bar-reftest.html'].is_reftest = true;
+ results.tests['bar-reftest.html'].ref_file = 'common.html';
+ runSingleRowTest(results, false, '', 'ref html images diff (1%) ');
+ runTest(results, function() {
+ assertTrue(document.getElementsByClassName('result-link')[0].getAttribute('href') == 'common.html');
+ });
+
+ results = mockResults();
+ results.tests['bar-reftest.html'] = mockExpectation('PASS', 'IMAGE');
+ results.tests['bar-reftest.html'].is_mismatch_reftest = true;
+ results.tests['bar-reftest.html'].ref_file = 'common.html';
+ runSingleRowTest(results, false, '', 'ref mismatch html actual ');
+ runTest(results, function() {
+ assertTrue(document.getElementsByClassName('result-link')[0].getAttribute('href') == 'common.html');
+ });
+
+ results = mockResults();
var subtree = results.tests['foo'] = {}
subtree['bar-flaky-pass.html'] = mockExpectation('PASS TEXT', 'PASS');
runTest(results, function() {
Modified: trunk/LayoutTests/fast/harness/results.html (99817 => 99818)
--- trunk/LayoutTests/fast/harness/results.html 2011-11-10 07:12:38 UTC (rev 99817)
+++ trunk/LayoutTests/fast/harness/results.html 2011-11-10 07:39:50 UTC (rev 99818)
@@ -406,9 +406,14 @@
function resultLink(testPrefix, suffix, contents)
{
- return '<a class=result-link href="" + testPrefix + suffix + '" data-prefix="' + testPrefix + '">' + contents + '</a> ';
+ return referenceLink(testPrefix, testPrefix + suffix, contents);
}
+function referenceLink(testPrefix, reference_filename, contents)
+{
+ return '<a class=result-link href="" + reference_filename + '" data-prefix="' + testPrefix + '">' + contents + '</a> ';
+}
+
function isFailureExpected(expected, actual)
{
var isExpected = true;
@@ -541,12 +546,18 @@
globalState().hasImageFailures = true;
if (testObject.is_mismatch_reftest) {
- row += resultLink(test_prefix, '-expected-mismatch.html', 'ref mismatch html') +
- resultLink(test_prefix, '-actual.png', 'actual');
+ if (testObject.ref_file)
+ row += referenceLink(test_prefix, testObject.ref_file, 'ref mismatch html');
+ else
+ row += resultLink(test_prefix, '-expected-mismatch.html', 'ref mismatch html');
+ row += resultLink(test_prefix, '-actual.png', 'actual');
} else {
- if (testObject.is_reftest)
- row += resultLink(test_prefix, '-expected.html', 'ref html');
-
+ if (testObject.is_reftest) {
+ if (testObject.ref_file)
+ row += referenceLink(test_prefix, testObject.ref_file, 'ref html');
+ else
+ row += resultLink(test_prefix, '-expected.html', 'ref html');
+ }
if (globalState().shouldToggleImages) {
row += resultLink(test_prefix, '-diffs.html', 'images');
} else {
Modified: trunk/LayoutTests/platform/chromium/fast/harness/results-expected.txt (99817 => 99818)
--- trunk/LayoutTests/platform/chromium/fast/harness/results-expected.txt 2011-11-10 07:12:38 UTC (rev 99817)
+++ trunk/LayoutTests/platform/chromium/fast/harness/results-expected.txt 2011-11-10 07:39:50 UTC (rev 99818)
@@ -96,70 +96,84 @@
TEST-16: PASS
TEST-16: PASS
TEST-16: PASS
+TEST-16: PASS
+TEST-16: PASS
+TEST-16: PASS
TEST-17: PASS
-TEST-17: PASS
-TEST-17: PASS
TEST-18: PASS
TEST-18: PASS
TEST-18: PASS
TEST-18: PASS
+TEST-18: PASS
+TEST-18: PASS
TEST-19: PASS
-TEST-19: PASS
TEST-20: PASS
TEST-20: PASS
TEST-20: PASS
-TEST-20: PASS
-TEST-20: PASS
TEST-21: PASS
TEST-21: PASS
+TEST-21: PASS
TEST-22: PASS
TEST-22: PASS
+TEST-22: PASS
+TEST-22: PASS
TEST-23: PASS
+TEST-23: PASS
TEST-24: PASS
TEST-24: PASS
+TEST-24: PASS
+TEST-24: PASS
+TEST-24: PASS
TEST-25: PASS
+TEST-25: PASS
TEST-26: PASS
+TEST-26: PASS
TEST-27: PASS
TEST-28: PASS
TEST-28: PASS
-TEST-28: PASS
TEST-29: PASS
-TEST-29: PASS
TEST-30: PASS
-TEST-30: PASS
-TEST-30: PASS
-TEST-30: PASS
-TEST-30: PASS
-TEST-30: PASS
-TEST-30: PASS
-TEST-30: PASS
-TEST-30: PASS
-TEST-30: PASS
-TEST-30: PASS
-TEST-30: PASS
-TEST-30: PASS
-TEST-30: PASS
-TEST-30: PASS
-TEST-30: PASS
TEST-31: PASS
-TEST-31: PASS
-TEST-31: PASS
-TEST-31: PASS
-TEST-31: PASS
-TEST-31: PASS
-TEST-31: PASS
-TEST-31: PASS
-TEST-31: PASS
-TEST-31: PASS
TEST-32: PASS
TEST-32: PASS
TEST-32: PASS
-TEST-32: PASS
TEST-33: PASS
TEST-33: PASS
-TEST-33: PASS
-TEST-33: PASS
-TEST-33: PASS
-TEST-33: PASS
-TEST-33: PASS
-TEST-33: PASS
+TEST-34: PASS
+TEST-34: PASS
+TEST-34: PASS
+TEST-34: PASS
+TEST-34: PASS
+TEST-34: PASS
+TEST-34: PASS
+TEST-34: PASS
+TEST-34: PASS
+TEST-34: PASS
+TEST-34: PASS
+TEST-34: PASS
+TEST-34: PASS
+TEST-34: PASS
+TEST-34: PASS
+TEST-34: PASS
+TEST-35: PASS
+TEST-35: PASS
+TEST-35: PASS
+TEST-35: PASS
+TEST-35: PASS
+TEST-35: PASS
+TEST-35: PASS
+TEST-35: PASS
+TEST-35: PASS
+TEST-35: PASS
+TEST-36: PASS
+TEST-36: PASS
+TEST-36: PASS
+TEST-36: PASS
+TEST-37: PASS
+TEST-37: PASS
+TEST-37: PASS
+TEST-37: PASS
+TEST-37: PASS
+TEST-37: PASS
+TEST-37: PASS
+TEST-37: PASS
Modified: trunk/Tools/ChangeLog (99817 => 99818)
--- trunk/Tools/ChangeLog 2011-11-10 07:12:38 UTC (rev 99817)
+++ trunk/Tools/ChangeLog 2011-11-10 07:39:50 UTC (rev 99818)
@@ -1,3 +1,16 @@
+2011-11-09 Hayato Ito <[email protected]>
+
+ [NRWT] Make results.html aware of reference filename in reftests.
+ https://bugs.webkit.org/show_bug.cgi?id=71574
+
+ Reviewed by Ryosuke Niwa.
+
+ If a reference filename is different from the default one, include that in result summary
+ so results.html can link to the correct reference file.
+
+ * Scripts/webkitpy/layout_tests/controllers/manager.py:
+ * Scripts/webkitpy/layout_tests/controllers/manager_unittest.py:
+
2011-11-09 Kevin Ollivier <[email protected]>
[wx] Unreviewed build fix. Update project files and
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py (99817 => 99818)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py 2011-11-10 07:12:38 UTC (rev 99817)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py 2011-11-10 07:39:50 UTC (rev 99818)
@@ -68,6 +68,43 @@
TestExpectations = test_expectations.TestExpectations
+def interpret_test_failures(port, test_name, failures):
+ """Interpret test failures and returns a test result as dict.
+
+ Args:
+ port: interface to port-specific hooks
+ test_name: test name relative to layout_tests directory
+ failures: list of test failures
+ Returns:
+ A dictionary like {'is_reftest': True, ...}
+ """
+ test_dict = {}
+ failure_types = [type(failure) for failure in failures]
+ # FIXME: get rid of all this is_* values once there is a 1:1 map between
+ # TestFailure type and test_expectations.EXPECTATION.
+ if test_failures.FailureMissingAudio in failure_types:
+ test_dict['is_missing_audio'] = True
+
+ for failure in failures:
+ if isinstance(failure, test_failures.FailureImageHashMismatch):
+ test_dict['image_diff_percent'] = failure.diff_percent
+ elif isinstance(failure, test_failures.FailureReftestMismatch):
+ test_dict['is_reftest'] = True
+ if failure.reference_filename != port.reftest_expected_filename(test_name):
+ test_dict['ref_file'] = port.relative_test_filename(failure.reference_filename)
+ elif isinstance(failure, test_failures.FailureReftestMismatchDidNotOccur):
+ test_dict['is_mismatch_reftest'] = True
+ if failure.reference_filename != port.reftest_expected_mismatch_filename(test_name):
+ test_dict['ref_file'] = port.relative_test_filename(failure.reference_filename)
+
+ if test_failures.FailureMissingResult in failure_types:
+ test_dict['is_missing_text'] = True
+
+ if test_failures.FailureMissingImage in failure_types or test_failures.FailureMissingImageHash in failure_types:
+ test_dict['is_missing_image'] = True
+ return test_dict
+
+
# FIXME: This should be on the Manager class (since that's the only caller)
# or split off from Manager onto another helper class, but should not be a free function.
# Most likely this should be made into its own class, and this super-long function
@@ -160,30 +197,8 @@
# FIXME: Set this correctly once https://webkit.org/b/37739 is fixed
# and only set it if there actually is stderr data.
- failure_types = [type(f) for f in result.failures]
- # FIXME: get rid of all this is_* values once there is a 1:1 map between
- # TestFailure type and test_expectations.EXPECTATION.
- if test_failures.FailureMissingAudio in failure_types:
- test_dict['is_missing_audio'] = True
+ test_dict.update(interpret_test_failures(port_obj, test_name, result.failures))
- if test_failures.FailureReftestMismatch in failure_types:
- test_dict['is_reftest'] = True
-
- for f in result.failures:
- if 'is_reftest' in result.failures:
- test_dict['is_reftest'] = True
- if type(f) is test_failures.FailureImageHashMismatch:
- test_dict['image_diff_percent'] = f.diff_percent
-
- if test_failures.FailureReftestMismatchDidNotOccur in failure_types:
- test_dict['is_mismatch_reftest'] = True
-
- if test_failures.FailureMissingResult in failure_types:
- test_dict['is_missing_text'] = True
-
- if test_failures.FailureMissingImage in failure_types or test_failures.FailureMissingImageHash in failure_types:
- test_dict['is_missing_image'] = True
-
# Store test hierarchically by directory. e.g.
# foo/bar/baz.html: test_dict
# foo/bar/baz1.html: test_dict
Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager_unittest.py (99817 => 99818)
--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager_unittest.py 2011-11-10 07:12:38 UTC (rev 99817)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager_unittest.py 2011-11-10 07:39:50 UTC (rev 99818)
@@ -42,7 +42,8 @@
from webkitpy import layout_tests
from webkitpy.layout_tests import run_webkit_tests
-from webkitpy.layout_tests.controllers.manager import Manager, natural_sort_key, test_key, TestRunInterruptedException, TestShard
+from webkitpy.layout_tests.controllers.manager import interpret_test_failures, Manager, natural_sort_key, test_key, TestRunInterruptedException, TestShard
+from webkitpy.layout_tests.models import test_failures
from webkitpy.layout_tests.models.result_summary import ResultSummary
from webkitpy.layout_tests.views import printing
from webkitpy.tool.mocktool import MockOptions
@@ -304,5 +305,37 @@
self.assert_cmp('/foo-bar/baz', '/foo/baz', -1)
+class ResultSummaryTest(unittest.TestCase):
+
+ def setUp(self):
+ host = MockHost()
+ self.port = host.port_factory.get(port_name='test')
+
+ def test_interpret_test_failures(self):
+ test_dict = interpret_test_failures(self.port, 'foo/reftest.html',
+ [test_failures.FailureReftestMismatch(self.port.abspath_for_test('foo/reftest-expected.html'))])
+ self.assertTrue('is_reftest' in test_dict)
+ self.assertFalse('is_mismatch_reftest' in test_dict)
+ self.assertFalse('ref_file' in test_dict)
+
+ test_dict = interpret_test_failures(self.port, 'foo/reftest.html',
+ [test_failures.FailureReftestMismatch(self.port.abspath_for_test('foo/common.html'))])
+ self.assertTrue('is_reftest' in test_dict)
+ self.assertFalse('is_mismatch_reftest' in test_dict)
+ self.assertEqual(test_dict['ref_file'], 'foo/common.html')
+
+ test_dict = interpret_test_failures(self.port, 'foo/reftest.html',
+ [test_failures.FailureReftestMismatchDidNotOccur(self.port.abspath_for_test('foo/reftest-expected-mismatch.html'))])
+ self.assertFalse('is_reftest' in test_dict)
+ self.assertTrue(test_dict['is_mismatch_reftest'])
+ self.assertFalse('ref_file' in test_dict)
+
+ test_dict = interpret_test_failures(self.port, 'foo/reftest.html',
+ [test_failures.FailureReftestMismatchDidNotOccur(self.port.abspath_for_test('foo/common.html'))])
+ self.assertFalse('is_reftest' in test_dict)
+ self.assertTrue(test_dict['is_mismatch_reftest'])
+ self.assertEqual(test_dict['ref_file'], 'foo/common.html')
+
+
if __name__ == '__main__':
port_testcase.main()
