Diff
Modified: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector.js (90921 => 90922)
--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector.js 2011-07-13 16:06:52 UTC (rev 90921)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector.js 2011-07-13 16:30:22 UTC (rev 90922)
@@ -70,43 +70,11 @@
return newFlakyTests;
},
- flakinessExamples: function(testName) {
- if (!(testName in this._tests) || this._tests[testName].state !== this._states.PossiblyFlaky)
+ allFailures: function(testName) {
+ if (!(testName in this._tests))
return null;
- var history = this._tests[testName].history;
-
- var examples = [];
- for (var i = 0; i < history.length - 1; ++i) {
- var thisIsPassing = history[i].result.failureType === 'pass';
- var nextIsPassing = history[i + 1].result.failureType === 'pass';
- if (thisIsPassing === nextIsPassing)
- continue;
- var last = examples.last();
- if (!last || last.build !== history[i].build)
- examples.push(history[i]);
- examples.push(history[i + 1]);
- }
-
- // The list of examples can get quite long. Instead of showing the whole list, we abbreviate
- // by replacing the middle items with a separator.
- const startAndEndAbbreviatedExamplesCount = 3;
- console.assert(startAndEndAbbreviatedExamplesCount > 1);
- var abbreviatedExamplesToShow = 2 * startAndEndAbbreviatedExamplesCount;
- if (examples.length > abbreviatedExamplesToShow) {
- var examplesBeforeSeparator = examples.slice(0, startAndEndAbbreviatedExamplesCount);
- var examplesAfterSeparator = examples.slice(-startAndEndAbbreviatedExamplesCount);
-
- // There's no real use in having two "pass" examples in a row immediately next to the
- // separator.
- if (examplesBeforeSeparator[examplesBeforeSeparator.length - 1].result.failureType === 'pass' && examplesBeforeSeparator[examplesBeforeSeparator.length - 2].result.failureType === 'pass')
- examplesBeforeSeparator.splice(examplesBeforeSeparator.length - 1, 1);
- if (examplesAfterSeparator[0].result.failureType === 'pass' && examplesAfterSeparator[1].result.failureType === 'pass')
- examplesAfterSeparator.splice(0, 1);
- examples = examplesBeforeSeparator.concat({ isSeparator: true }, examplesAfterSeparator);
- }
-
- return examples;
+ return this._tests[testName].history.filter(function(historyItem) { return historyItem.result.failureType !== 'pass' });
},
get possiblyFlakyTests() {
Added: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector_unittests.js (0 => 90922)
--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector_unittests.js (rev 0)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector_unittests.js 2011-07-13 16:30:22 UTC (rev 90922)
@@ -0,0 +1,102 @@
+/*
+ * Copyright (C) 2011 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+(function() {
+
+module('FlakyLayoutTestDetector');
+
+const runs = [
+ {
+ buildName: 'build6',
+ failingTests: {
+ b: 'fail',
+ },
+ tooManyFailures: false,
+ },
+ {
+ buildName: 'build5',
+ failingTests: {
+ a: 'fail',
+ b: 'fail',
+ },
+ tooManyFailures: false,
+ },
+ {
+ buildName: 'build4',
+ failingTests: {
+ b: 'fail',
+ },
+ tooManyFailures: false,
+ },
+ {
+ buildName: 'build3',
+ failingTests: {
+ a: 'crash',
+ b: 'fail',
+ },
+ tooManyFailures: false,
+ },
+ {
+ buildName: 'build2',
+ failingTests: {
+ a: 'fail',
+ },
+ tooManyFailures: false,
+ },
+ {
+ buildName: 'build1',
+ failingTests: {},
+ tooManyFailures: false,
+ },
+];
+
+test('possiblyFlakyTests', 1, function() {
+ var detector = new FlakyLayoutTestDetector();
+ runs.forEach(function(run) { detector.incorporateTestResults(run.buildName, run.failingTests, run.tooManyFailures) });
+
+ deepEqual(detector.possiblyFlakyTests, ['a']);
+});
+
+test('allFailures', 3, function() {
+ var detector = new FlakyLayoutTestDetector();
+ runs.forEach(function(run) { detector.incorporateTestResults(run.buildName, run.failingTests, run.tooManyFailures) });
+
+ deepEqual(detector.allFailures('a'), [
+ { build: 'build5', result: 'fail' },
+ { build: 'build3', result: 'crash' },
+ { build: 'build2', result: 'fail' },
+ ]);
+
+ deepEqual(detector.allFailures('b'), [
+ { build: 'build6', result: 'fail' },
+ { build: 'build5', result: 'fail' },
+ { build: 'build4', result: 'fail' },
+ { build: 'build3', result: 'fail' },
+ ]);
+
+ equal(detector.allFailures('c'), null);
+});
+
+})();
Modified: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestHistoryAnalyzer.js (90921 => 90922)
--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestHistoryAnalyzer.js 2011-07-13 16:06:52 UTC (rev 90921)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestHistoryAnalyzer.js 2011-07-13 16:30:22 UTC (rev 90922)
@@ -56,10 +56,8 @@
* },
* 'possiblyFlaky': {
* 'fast/workers/worker-test.html': [
- * { 'build': 'r12345 (679)', 'result': 'pass' },
* { 'build': 'r12344 (678)', 'result': 'fail' },
- * { 'build': 'r12340 (676)', 'result': 'fail' },
- * { 'build': 'r12338 (675)', 'result': 'pass' },
+ * { 'build': 'r12340 (676)', 'result': 'crash' },
* ],
* },
* }
@@ -86,7 +84,7 @@
possiblyFlaky: {},
};
self._flakinessDetector.possiblyFlakyTests.forEach(function(testName) {
- data.possiblyFlaky[testName] = self._flakinessDetector.flakinessExamples(testName);
+ data.possiblyFlaky[testName] = self._flakinessDetector.allFailures(testName);
});
var nextIndex = buildIndex + 1;
Modified: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/TestFailures.css (90921 => 90922)
--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/TestFailures.css 2011-07-13 16:06:52 UTC (rev 90921)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/TestFailures.css 2011-07-13 16:30:22 UTC (rev 90922)
@@ -52,7 +52,7 @@
margin-bottom: 0;
}
-.existing-bugs-list, .suspect-revisions-list {
+.existing-bugs-list, .suspect-revisions-list, .flakiness-examples-list {
font-size: smaller;
}
@@ -64,3 +64,21 @@
padding-left: 5em;
font-size: larger;
}
+
+.disclosure-triangle {
+ -webkit-transition: -webkit-transform 0.1s;
+ -webkit-user-select: none;
+ cursor: default;
+ display: inline-block;
+ padding: 0px 3px;
+}
+
+.expanded > .disclosure-triangle {
+ -webkit-transform: rotateZ(90deg);
+}
+
+.flakiness-examples-list {
+ -webkit-transition: height 0.25s;
+ height: 0px;
+ overflow: hidden;
+}
Modified: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/ViewController.js (90921 => 90922)
--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/ViewController.js 2011-07-13 16:06:52 UTC (rev 90921)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/ViewController.js 2011-07-13 16:30:22 UTC (rev 90922)
@@ -70,7 +70,9 @@
var list = document.createElement('ol');
list.id = 'failure-history';
- Object.keys(data.history).forEach(function(buildName, buildIndex, buildNameArray) {
+
+ var buildNames = Object.keys(data.history)
+ buildNames.forEach(function(buildName, buildIndex, buildNameArray) {
var failingTestNames = Object.keys(data.history[buildName].tests);
if (!failingTestNames.length)
return;
@@ -107,7 +109,7 @@
self._mainContentElement.removeAllChildren();
self._mainContentElement.appendChild(list);
- self._mainContentElement.appendChild(self._domForPossiblyFlakyTests(builder, data.possiblyFlaky));
+ self._mainContentElement.appendChild(self._domForPossiblyFlakyTests(builder, data.possiblyFlaky, buildNames.length));
if (!stillFetchingData)
PersistentCache.prune();
@@ -398,7 +400,7 @@
return result;
},
- _domForPossiblyFlakyTests: function(builder, possiblyFlakyTestData) {
+ _domForPossiblyFlakyTests: function(builder, possiblyFlakyTestData, buildCount) {
var result = document.createDocumentFragment();
var flakyTests = Object.keys(possiblyFlakyTestData);
if (!flakyTests.length)
@@ -416,22 +418,46 @@
var self = this;
flakyList.appendChildren(sorted(flakyTests).map(function(testName) {
var item = document.createElement('li');
- item.appendChild(document.createTextNode(testName));
- var historyList = document.createElement('ol');
- item.appendChild(historyList);
- historyList.appendChildren(possiblyFlakyTestData[testName].map(function(historyItem) {
- var item = document.createElement('li');
- if (historyItem.isSeparator) {
- const verticalEllipsis = '\u22ee';
- item.appendChild(document.createTextNode(verticalEllipsis));
- item.className = 'flakiness-example-separator';
- return item;
+
+ var disclosureTriangle = document.createElement('span');
+ item.appendChild(disclosureTriangle);
+
+ disclosureTriangle.className = 'disclosure-triangle';
+ const blackRightPointingSmallTriangle = '\u25b8';
+ disclosureTriangle.appendChild(document.createTextNode(blackRightPointingSmallTriangle));
+
+ var failures = possiblyFlakyTestData[testName];
+
+ item.appendChild(document.createTextNode(testName + ' (failed ' + failures.length + ' out of ' + buildCount + ' times)'));
+
+ var failureList = document.createElement('ol');
+ item.appendChild(failureList);
+
+ failureList.className = 'flakiness-examples-list';
+
+ disclosureTriangle.addEventListener('click', function() {
+ item.classList.toggle('expanded');
+ if (!item.classList.contains('expanded')) {
+ failureList.style.height = '';
+ return;
}
- item.appendChild(self._domForBuildName(builder, historyItem.build));
- item.appendChild(document.createTextNode(': '));
- item.appendChild(self._domForFailureDiagnosis(builder, historyItem.build, testName, historyItem.result));
- return item;
- }));
+
+ if (!failureList.firstChild) {
+ failureList.appendChildren(failures.map(function(historyItem) {
+ var item = document.createElement('li');
+ item.appendChild(self._domForBuildName(builder, historyItem.build));
+ item.appendChild(document.createTextNode(': '));
+ item.appendChild(self._domForFailureDiagnosis(builder, historyItem.build, testName, historyItem.result));
+ return item;
+ }));
+ }
+
+ // CSS transitions can't transition to a value of 'auto', so we find out the actual
+ // value using getComputedStyle and transition to that.
+ failureList.style.height = 'auto';
+ failureList.style.height = getComputedStyle(failureList).height;
+ });
+
return item;
}));
Modified: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/run-unittests.html (90921 => 90922)
--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/run-unittests.html 2011-07-13 16:06:52 UTC (rev 90921)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/run-unittests.html 2011-07-13 16:30:22 UTC (rev 90922)
@@ -19,6 +19,8 @@
<script src=""
<script src=""
<script src=""
+<script src=""
+<script src=""
<script src=""
<script src=""
<script src=""
Modified: trunk/Tools/ChangeLog (90921 => 90922)
--- trunk/Tools/ChangeLog 2011-07-13 16:06:52 UTC (rev 90921)
+++ trunk/Tools/ChangeLog 2011-07-13 16:30:22 UTC (rev 90922)
@@ -1,4 +1,50 @@
2011-07-13 Adam Roben <[email protected]>
+
+ Make TestFailures show every time a possibly-flaky test failed, but hide it by default
+
+ It's useful to be able to see every time a flaky test failed to see whether it failed the
+ same way every time. But doing so takes a lot of space, so the list of failures is now
+ collapsed by default and can be revealed using a disclosure triangle.
+
+ Fixes <http://webkit.org/b/64455> TestFailures page doesn't show as much information for
+ flaky tests as I would like, even though the page is already so long
+
+ Reviewed by Dimitri Glazkov.
+
+ * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector.js:
+ (FlakyLayoutTestDetector.prototype.allFailures): Replaced flakinessExamples with this
+ function. Now returns all failures for the given test.
+
+ * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector_unittests.js:
+ Added. This just contains some simple tests of the FlakyLayoutTestDetector class. We'll add
+ more over time.
+
+ * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestHistoryAnalyzer.js:
+ (LayoutTestHistoryAnalyzer.prototype.start): Updated the documentation comment to reflect
+ that we no longer return passing builds for possibly-flaky tests.
+
+ * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/TestFailures.css:
+ (.existing-bugs-list, .suspect-revisions-list, .flakiness-examples-list): Make the list of
+ flakiness examples small, too, since it can get quite long.
+
+ (.disclosure-triangle):
+ (.expanded > .disclosure-triangle):
+ Simple styles for the disclosure triangle.
+
+ (.flakiness-examples-list): Collapse the list by default.
+
+ * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/ViewController.js:
+ (ViewController.prototype._displayBuilder): Pass the total number of builds analyzed to
+ _domForPossiblyFlakyTests.
+ (ViewController.prototype._domForPossiblyFlakyTests): Put a disclosure triangle to the left
+ of each test name, and the number of failures to the right. When the disclosure triangle is
+ clicked for the first time, we build up the list of failures and expand the element. After
+ that we just collapse or expand the element on subsequent clicks.
+
+ * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/run-unittests.html:
+ Pulled in new tests.
+
+2011-07-13 Adam Roben <[email protected]>
Teach TestFailures to understand NRWT's output when it exits early due to too many failures
