Diff
Modified: trunk/Tools/ChangeLog (90878 => 90879)
--- trunk/Tools/ChangeLog 2011-07-13 02:28:17 UTC (rev 90878)
+++ trunk/Tools/ChangeLog 2011-07-13 02:34:54 UTC (rev 90879)
@@ -1,3 +1,29 @@
+2011-07-12 Adam Barth <[email protected]>
+
+ garden-o-matic should display how many times we've seen a failure
+ https://bugs.webkit.org/show_bug.cgi?id=64417
+
+ Reviewed by Ojan Vafai.
+
+ This patch adds some UI to display how many times we've seen a given
+ failure, which can be helpful for determining whether that failure is a
+ real failure or a flaky test.
+
+ When a failure has only been seen once (i.e., only a single run on a
+ single bot), we set the opacity of to 50% to avoid distracting the
+ gardener.
+
+ This patch also refactors the failure walker to have a simpler API
+ internally by moving from an object-oriented paradigm to a functional
+ paradigm.
+
+ * Scripts/webkitpy/tool/servers/data/gardeningserver/main.css:
+ * Scripts/webkitpy/tool/servers/data/gardeningserver/main.js:
+ * Scripts/webkitpy/tool/servers/data/gardeningserver/results.js:
+ * Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js:
+ * Scripts/webkitpy/tool/servers/data/gardeningserver/ui.js:
+ * Scripts/webkitpy/tool/servers/data/gardeningserver/ui_unittests.js:
+
2011-07-12 Chris Rogers <[email protected]>
Enable Web Audio for chromium DRT
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/main.css (90878 => 90879)
--- trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/main.css 2011-07-13 02:28:17 UTC (rev 90878)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/main.css 2011-07-13 02:34:54 UTC (rev 90879)
@@ -40,6 +40,11 @@
margin-left: 20px;
}
+/* If we've only seen a given test failure once, we dim it so as not to distract the gardener. */
+.test[data-failure-count="1"] {
+ opacity: 0.5;
+}
+
.regression {
padding: 0 0 10px 0;
}
@@ -124,3 +129,12 @@
border-radius: 4px;
color: #555;
}
+
+.regression .when, .regression .how-many {
+ padding: 0px 2px;
+ display: inline-block;
+}
+
+.regression .how-many {
+ color: #555;
+}
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/main.js (90878 => 90879)
--- trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/main.js 2011-07-13 02:28:17 UTC (rev 90878)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/main.js 2011-07-13 02:34:54 UTC (rev 90879)
@@ -29,8 +29,12 @@
var builderNameList = base.keys(resultNodesByBuilder);
results.unifyRegressionRanges(builderNameList, testName, function(oldestFailingRevision, newestPassingRevision) {
- $('.when', regressions).append(ui.summarizeRegressionRange(oldestFailingRevision, newestPassingRevision));
+ $('.when', testSummary).append(ui.summarizeRegressionRange(oldestFailingRevision, newestPassingRevision));
});
+ results.countFailureOccurances(builderNameList, testName, function(failureCount) {
+ $(testSummary).attr('data-failure-count', failureCount);
+ $('.how-many', testSummary).text(ui.failureCount(failureCount));
+ });
});
$('.results').append(regressions);
}
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results.js (90878 => 90879)
--- trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results.js 2011-07-13 02:28:17 UTC (rev 90878)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results.js 2011-07-13 02:34:54 UTC (rev 90879)
@@ -138,6 +138,8 @@
function isUnexpectedFailure(resultNode)
{
+ if (!resultNode)
+ return false;
if (anyIsSuccess(resultNode.actual.split(' ')))
return false;
return anyIsFailure(unexpectedResults(resultNode));
@@ -182,52 +184,40 @@
return base.uniquifyArray(results);
};
-function TestHistoryWalker(builderName, testName)
+function walkHistory(builderName, testName, callback)
{
- this._builderName = builderName;
- this._testName = testName;
- this._indexOfNextKeyToFetch = 0;
- this._keyList = [];
-}
+ var indexOfNextKeyToFetch = 0;
+ var keyList = [];
-TestHistoryWalker.prototype.init = function(callback)
-{
- var self = this;
+ function continueWalk()
+ {
+ if (indexOfNextKeyToFetch >= keyList.length) {
+ processResultNode(0, null);
+ return;
+ }
- base.jsonp(directoryOfResultsSummaryURL(self._builderName, kResultsName), function(keyList) {
- self._keyList = keyList.map(function (element) { return element.key; });
- callback();
- });
-};
-
-TestHistoryWalker.prototype._fetchNextResultNode = function(callback)
-{
- var self = this;
-
- if (self._indexOfNextKeyToFetch >= self._keyList) {
- callback(0, null);
- return;
+ var key = keyList[indexOfNextKeyToFetch];
+ ++indexOfNextKeyToFetch;
+ g_resultsCache.get(key, function(resultsTree) {
+ var resultNode = results.resultNodeForTest(resultsTree, testName);
+ var revision = parseInt(resultsTree['revision'])
+ if (isNaN(revision))
+ revision = 0;
+ processResultNode(revision, resultNode);
+ });
}
- var key = self._keyList[self._indexOfNextKeyToFetch];
- ++self._indexOfNextKeyToFetch;
- g_resultsCache.get(key, function(resultsTree) {
- var resultNode = results.resultNodeForTest(resultsTree, self._testName);
- var revision = parseInt(resultsTree['revision'])
- if (isNaN(revision))
- revision = 0;
- callback(revision, resultNode);
- });
-};
-
-TestHistoryWalker.prototype.walkHistory = function(callback)
-{
- var self = this;
- self._fetchNextResultNode(function(revision, resultNode) {
+ function processResultNode(revision, resultNode)
+ {
var shouldContinue = callback(revision, resultNode);
if (!shouldContinue)
return;
- self.walkHistory(callback);
+ continueWalk();
+ }
+
+ base.jsonp(directoryOfResultsSummaryURL(builderName, kResultsName), function(directory) {
+ keyList = directory.map(function (element) { return element.key; });
+ continueWalk();
});
}
@@ -236,24 +226,21 @@
var oldestFailingRevision = 0;
var newestPassingRevision = 0;
- var historyWalker = new TestHistoryWalker(builderName, testName);
- historyWalker.init(function() {
- historyWalker.walkHistory(function(revision, resultNode) {
- if (!resultNode) {
- newestPassingRevision = revision;
- callback(oldestFailingRevision, newestPassingRevision);
- return false;
- }
- if (isUnexpectedFailure(resultNode)) {
- oldestFailingRevision = revision;
- return true;
- }
- if (!oldestFailingRevision)
- return true; // We need to keep looking for a failing revision.
+ walkHistory(builderName, testName, function(revision, resultNode) {
+ if (!resultNode) {
newestPassingRevision = revision;
callback(oldestFailingRevision, newestPassingRevision);
return false;
- });
+ }
+ if (isUnexpectedFailure(resultNode)) {
+ oldestFailingRevision = revision;
+ return true;
+ }
+ if (!oldestFailingRevision)
+ return true; // We need to keep looking for a failing revision.
+ newestPassingRevision = revision;
+ callback(oldestFailingRevision, newestPassingRevision);
+ return false;
});
};
@@ -301,6 +288,28 @@
});
};
+results.countFailureOccurances = function(builderNameList, testName, callback)
+{
+ var queriesInFlight = builderNameList.length;
+ if (!queriesInFlight)
+ callback(0);
+
+ var failureCount = 0;
+ $.each(builderNameList, function(index, builderName) {
+ walkHistory(builderName, testName, function(revision, resultNode) {
+ if (isUnexpectedFailure(resultNode)) {
+ ++failureCount;
+ return true;
+ }
+
+ --queriesInFlight;
+ if (!queriesInFlight)
+ callback(failureCount);
+ return false;
+ });
+ });
+};
+
results.resultNodeForTest = function(resultsTree, testName)
{
var testNamePath = testName.split('/');
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js (90878 => 90879)
--- trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js 2011-07-13 02:28:17 UTC (rev 90878)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js 2011-07-13 02:34:54 UTC (rev 90879)
@@ -137,7 +137,7 @@
equal(window.base, realBase, "Failed to restore real base!");
}
-test("regressionRangeForFailure", 5, function() {
+test("walkHistory", 6, function() {
var simulator = new NetworkSimulator();
var keyMap = {
@@ -245,6 +245,10 @@
equals(oldestFailingRevision, 90426);
equals(newestPassingRevision, 90425);
});
+
+ results.countFailureOccurances(["Mock Builder", "Another Builder"], "userscripts/another-test.html", function(failureCount) {
+ equals(failureCount, 4);
+ });
});
});
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/ui.js (90878 => 90879)
--- trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/ui.js 2011-07-13 02:28:17 UTC (rev 90878)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/ui.js 2011-07-13 02:34:54 UTC (rev 90879)
@@ -33,6 +33,7 @@
'<span>fails on</span>' +
'<ul class="where"></ul>' +
'<div class="when"></div>' +
+ '<div class="how-many"></div>' +
'</div>');
$('.what a', block).text(testName).attr('href', ui.urlForTest(testName)).attr('class', unexpectedResults.join(' '));
@@ -61,6 +62,15 @@
return block;
};
+ui.failureCount = function(failureCount)
+{
+ if (failureCount < 1)
+ return '';
+ if (failureCount == 1)
+ return '(Seen once.)';
+ return '(Seen ' + failureCount + ' times.)';
+};
+
ui.results = function(resultsURLs)
{
var block = $('<div class="results"></div>');
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/ui_unittests.js (90878 => 90879)
--- trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/ui_unittests.js 2011-07-13 02:28:17 UTC (rev 90878)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/ui_unittests.js 2011-07-13 02:34:54 UTC (rev 90879)
@@ -38,6 +38,13 @@
equal(summaryWithOneRevision.parent().html(), '<div class="regression-range">Regression Range: <a href=""
});
+test("failureCount", 4, function() {
+ equal(ui.failureCount(0), '');
+ equal(ui.failureCount(1), '(Seen once.)');
+ equal(ui.failureCount(2), '(Seen 2 times.)');
+ equal(ui.failureCount(3), '(Seen 3 times.)');
+});
+
test("results", 1, function() {
var testResults = ui.results([
'http://example.com/layout-test-results/foo-bar-expected.txt',
