Modified: trunk/LayoutTests/ChangeLog (122863 => 122864)
--- trunk/LayoutTests/ChangeLog 2012-07-17 18:42:15 UTC (rev 122863)
+++ trunk/LayoutTests/ChangeLog 2012-07-17 18:44:37 UTC (rev 122864)
@@ -1,5 +1,19 @@
2012-07-17 Ojan Vafai <[email protected]>
+ results.html should handle flaky tests differently
+ https://bugs.webkit.org/show_bug.cgi?id=90892
+
+ Reviewed by Dirk Pranke.
+
+ -Put tests that failed both runs into the main failures table.
+ -Put flaky passes below the list of timeout tests.
+
+ * fast/harness/resources/results-test.js:
+ * fast/harness/results-expected.txt:
+ * fast/harness/results.html:
+
+2012-07-17 Ojan Vafai <[email protected]>
+
results.html doesn't hide the right data when not showing expected failures
https://bugs.webkit.org/show_bug.cgi?id=90889
Modified: trunk/LayoutTests/fast/harness/resources/results-test.js (122863 => 122864)
--- trunk/LayoutTests/fast/harness/resources/results-test.js 2012-07-17 18:42:15 UTC (rev 122863)
+++ trunk/LayoutTests/fast/harness/resources/results-test.js 2012-07-17 18:44:37 UTC (rev 122864)
@@ -685,7 +685,7 @@
runTest(results, function() {
var titles = document.getElementsByTagName('h1');
assertTrue(titles[0].textContent == 'Tests that crashed (1):');
- assertTrue(titles[1].textContent == 'Tests where results did not match expected results (3):');
+ assertTrue(titles[1].textContent == 'Tests that failed text/pixel/audio diff (3):');
assertTrue(titles[2].textContent == 'Tests that had no expected results (probably new) (1):');
assertTrue(titles[3].textContent == 'Tests that timed out (0):');
assertTrue(titles[4].textContent == 'Tests that had stderr output (1):');
@@ -695,13 +695,28 @@
document.getElementById('unexpected-results').onchange();
assertTrue(titles[0].textContent == 'Tests that crashed (2):');
- assertTrue(titles[1].textContent == 'Tests where results did not match expected results (5):');
+ assertTrue(titles[1].textContent == 'Tests that failed text/pixel/audio diff (5):');
assertTrue(titles[2].textContent == 'Tests that had no expected results (probably new) (1):');
assertTrue(titles[3].textContent == 'Tests that timed out (1):');
assertTrue(titles[4].textContent == 'Tests that had stderr output (1):');
assertTrue(titles[5].textContent == 'Tests expected to fail but passed (1):');
});
+ results = mockResults();
+ var subtree = results.tests['foo'] = {}
+ subtree['bar.html'] = mockExpectation('', 'PASS TEXT');
+ subtree['bar-1.html'] = mockExpectation('', 'TEXT IMAGE');
+ subtree['bar-2.html'] = mockExpectation('IMAGE TEXT', 'TEXT IMAGE');
+ subtree['bar-3.html'] = mockExpectation('PASS TEXT', 'TEXT PASS');
+ runTest(results, function() {
+ var titles = document.getElementsByTagName('h1');
+ assertTrue(titles[0].textContent == 'Tests that failed text/pixel/audio diff (1):');
+ assertTrue(titles[1].textContent =='Flaky tests (failed the first run and passed on retry) (1):');
+
+ assertTrue(document.querySelectorAll('#results-table tbody').length == 2);
+ assertTrue(document.querySelectorAll('#flaky-tests-table tbody').length == 2);
+ });
+
document.body.innerHTML = '<pre>' + g_log.join('\n') + '</pre>';
}
Modified: trunk/LayoutTests/fast/harness/results-expected.txt (122863 => 122864)
--- trunk/LayoutTests/fast/harness/results-expected.txt 2012-07-17 18:42:15 UTC (rev 122863)
+++ trunk/LayoutTests/fast/harness/results-expected.txt 2012-07-17 18:44:37 UTC (rev 122864)
@@ -217,3 +217,7 @@
TEST-37: PASS
TEST-37: PASS
TEST-37: PASS
+TEST-38: PASS
+TEST-38: PASS
+TEST-38: PASS
+TEST-38: PASS
Modified: trunk/LayoutTests/fast/harness/results.html (122863 => 122864)
--- trunk/LayoutTests/fast/harness/results.html 2012-07-17 18:42:15 UTC (rev 122863)
+++ trunk/LayoutTests/fast/harness/results.html 2012-07-17 18:44:37 UTC (rev 122864)
@@ -204,14 +204,14 @@
if (!g_state) {
g_state = {
crashTests: [],
- flakyTests: [],
+ flakyPassTests: [],
hasHttpTests: false,
hasImageFailures: false,
hasTextFailures: false,
missingResults: [],
results: {},
shouldToggleImages: true,
- nonFlakyFailingTests: [],
+ failingTests: [],
testsWithStderr: [],
timeoutTests: [],
unexpectedPassTests: []
@@ -475,8 +475,8 @@
return;
}
- if (actual.indexOf(' ') != -1) {
- globalState().flakyTests.push(testObject);
+ if (actual.indexOf(' ') != -1 && actual.indexOf('PASS') != -1) {
+ globalState().flakyPassTests.push(testObject);
return;
}
@@ -497,7 +497,7 @@
return;
}
- globalState().nonFlakyFailingTests.push(testObject);
+ globalState().failingTests.push(testObject);
}
function toggleImages()
@@ -1275,21 +1275,21 @@
if (globalState().crashTests.length)
html += testList(globalState().crashTests, 'Tests that crashed', 'crash-tests-table');
- html += failingTestsTable(globalState().nonFlakyFailingTests,
- 'Tests where results did not match expected results', 'results-table');
+ html += failingTestsTable(globalState().failingTests,
+ 'Tests that failed text/pixel/audio diff', 'results-table');
html += failingTestsTable(globalState().missingResults,
'Tests that had no expected results (probably new)', 'missing-table');
- html += failingTestsTable(globalState().flakyTests,
- 'Flaky tests (failed the first run and got a different result on retry)', 'flaky-tests-table');
-
if (globalState().timeoutTests.length)
html += testList(globalState().timeoutTests, 'Tests that timed out', 'timeout-tests-table');
if (globalState().testsWithStderr.length)
html += testList(globalState().testsWithStderr, 'Tests that had stderr output', 'stderr-table');
+ html += failingTestsTable(globalState().flakyPassTests,
+ 'Flaky tests (failed the first run and passed on retry)', 'flaky-tests-table');
+
if (globalState().results.uses_expectations_file && globalState().unexpectedPassTests.length)
html += testList(globalState().unexpectedPassTests, 'Tests expected to fail but passed', 'passes-table');
