Diff
Modified: trunk/Tools/ChangeLog (90865 => 90866)
--- trunk/Tools/ChangeLog 2011-07-12 22:35:39 UTC (rev 90865)
+++ trunk/Tools/ChangeLog 2011-07-12 23:31:05 UTC (rev 90866)
@@ -1,5 +1,28 @@
2011-07-12 Adam Barth <[email protected]>
+ garden-o-matic should display regression ranges
+ https://bugs.webkit.org/show_bug.cgi?id=64407
+
+ Reviewed by Dimitri Glazkov.
+
+ This patch computes a regression range for a failure by intersecting
+ the regression ranges seen by the various bots. We make the underlying
+ assumption that a test is only failing due to one revision at any given
+ moment. If that's not true, this code probably explodes.
+
+ The regression ranges appear asynchronously, which might be a jarring
+ UI. We'll have to experiment to see.
+
+ * Scripts/webkitpy/tool/servers/data/gardeningserver/base.js:
+ * Scripts/webkitpy/tool/servers/data/gardeningserver/base_unittests.js:
+ * Scripts/webkitpy/tool/servers/data/gardeningserver/main.js:
+ * Scripts/webkitpy/tool/servers/data/gardeningserver/results.js:
+ * Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js:
+ * Scripts/webkitpy/tool/servers/data/gardeningserver/ui.js:
+ * Scripts/webkitpy/tool/servers/data/gardeningserver/ui_unittests.js:
+
+2011-07-12 Adam Barth <[email protected]>
+
cr-linux-ews should run pixel tests
https://bugs.webkit.org/show_bug.cgi?id=64394
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/base.js (90865 => 90866)
--- trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/base.js 2011-07-12 22:35:39 UTC (rev 90865)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/base.js 2011-07-12 23:31:05 UTC (rev 90866)
@@ -38,6 +38,15 @@
return result;
};
+base.keys = function(dictionary)
+{
+ var keys = [];
+ $.each(dictionary, function(key, value) {
+ keys.push(key);
+ });
+ return keys;
+};
+
base.filterTree = function(tree, isLeaf, predicate)
{
var filteredTree = {};
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/base_unittests.js (90865 => 90866)
--- trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/base_unittests.js 2011-07-12 22:35:39 UTC (rev 90865)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/base_unittests.js 2011-07-12 23:31:05 UTC (rev 90866)
@@ -39,6 +39,13 @@
deepEqual(base.uniquifyArray(["a", "b", "b", "a"]), ["a", "b"]);
});
+test("keys", 4, function() {
+ deepEqual(base.keys({}), []);
+ deepEqual(base.keys({"a": 1}), ["a"]);
+ deepEqual(base.keys({"a": 1, "b": 0}), ["a", "b"]);
+ deepEqual(base.keys({"a": 1, "b": { "c" : 1}}), ["a", "b"]);
+});
+
test("filterTree", 2, function() {
var tree = {
'path': {
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/main.js (90865 => 90866)
--- trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/main.js 2011-07-12 22:35:39 UTC (rev 90865)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/main.js 2011-07-12 23:31:05 UTC (rev 90866)
@@ -22,8 +22,17 @@
$('.results').append(partyTime);
partyTime.fadeIn(1200).delay(7000).fadeOut();
} else {
- var resultsSummary = ui.summarizeResultsByTest(unexpectedFailures);
- $('.results').append($(resultsSummary).addClass('regression'));
+ var regressions = $('<div class="results-summary regression"></div>');
+ $.each(resultsByTest, function(testName, resultNodesByBuilder) {
+ var testSummary = ui.summarizeTest(testName, resultNodesByBuilder);
+ regressions.append(testSummary);
+
+ var builderNameList = base.keys(resultNodesByBuilder);
+ results.unifyRegressionRanges(builderNameList, testName, function(oldestFailingRevision, newestPassingRevision) {
+ $('.when', regressions).append(ui.summarizeRegressionRange(oldestFailingRevision, newestPassingRevision));
+ });
+ });
+ $('.results').append(regressions);
}
setIconState(hasFailures);
onsuccess();
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results.js (90865 => 90866)
--- trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results.js 2011-07-12 22:35:39 UTC (rev 90865)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results.js 2011-07-12 23:31:05 UTC (rev 90866)
@@ -257,6 +257,50 @@
});
};
+function mergeRegressionRanges(regressionRanges)
+{
+ var mergedRange = {};
+
+ mergedRange.oldestFailingRevision = 0;
+ mergedRange.newestPassingRevision = 0;
+
+ $.each(regressionRanges, function(builderName, range) {
+ if (!mergedRange.oldestFailingRevision)
+ mergedRange.oldestFailingRevision = range.oldestFailingRevision;
+ if (!mergedRange.newestPassingRevision)
+ mergedRange.newestPassingRevision = range.newestPassingRevision;
+
+ if (range.oldestFailingRevision < mergedRange.oldestFailingRevision)
+ mergedRange.oldestFailingRevision = range.oldestFailingRevision;
+ if (range.newestPassingRevision > mergedRange.newestPassingRevision)
+ mergedRange.newestPassingRevision = range.newestPassingRevision;
+ });
+ return mergedRange;
+}
+
+results.unifyRegressionRanges = function(builderNameList, testName, callback)
+{
+ var queriesInFlight = builderNameList.length;
+ if (!queriesInFlight)
+ callback(0, 0);
+
+ var regressionRanges = {};
+ $.each(builderNameList, function(index, builderName) {
+ results.regressionRangeForFailure(builderName, testName, function(oldestFailingRevision, newestPassingRevision) {
+ var range = {};
+ range.oldestFailingRevision = oldestFailingRevision;
+ range.newestPassingRevision = newestPassingRevision;
+ regressionRanges[builderName] = range;
+
+ --queriesInFlight;
+ if (!queriesInFlight) {
+ var mergedRange = mergeRegressionRanges(regressionRanges);
+ callback(mergedRange.oldestFailingRevision, mergedRange.newestPassingRevision);
+ }
+ });
+ });
+};
+
results.resultNodeForTest = function(resultsTree, testName)
{
var testNamePath = testName.split('/');
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js (90865 => 90866)
--- trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js 2011-07-12 22:35:39 UTC (rev 90865)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js 2011-07-12 23:31:05 UTC (rev 90866)
@@ -137,7 +137,7 @@
equal(window.base, realBase, "Failed to restore real base!");
}
-test("regressionRangeForFailure", 3, function() {
+test("regressionRangeForFailure", 5, function() {
var simulator = new NetworkSimulator();
var keyMap = {
@@ -188,22 +188,47 @@
},
},
"revision": "90424"
+ },
+ "abc":{
+ "tests": {
+ "userscripts": {
+ "another-test.html": {
+ "expected": "PASS",
+ "actual": "TEXT"
+ }
+ },
+ },
+ "revision": "90426"
+ },
+ "xyz":{
+ "tests": {
+ },
+ "revision": "90425"
}
};
simulator.jsonpHook = function(url, callback) {
simulator.scheduleCallback(function() {
if (/dir=1/.test(url)) {
- callback([
- { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGLncUAw" },
- { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGNfTUAw" },
- { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGJWCUQw" },
- { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGKbLUAw" },
- { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGOj5UAw" },
- { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGP-AUQw" },
- { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGPL3UAw" },
- { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGNHJQAw" },
- ]);
+ if (/builder=Mock/.test(url)) {
+ callback([
+ { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGLncUAw" },
+ { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGNfTUAw" },
+ { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGJWCUQw" },
+ { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGKbLUAw" },
+ { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGOj5UAw" },
+ { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGP-AUQw" },
+ { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGPL3UAw" },
+ { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGNHJQAw" },
+ ]);
+ } else if (/builder=Another/.test(url)) {
+ callback([
+ { "key": "abc" },
+ { "key": "xyz" },
+ ]);
+ } else {
+ ok(false, 'Unexpected URL: ' + url);
+ }
} else {
var key = url.match(/key=([^&]+)/)[1];
callback(keyMap[key]);
@@ -215,6 +240,11 @@
equals(oldestFailingRevision, 90426);
equals(newestPassingRevision, 90424);
});
+
+ results.unifyRegressionRanges(["Mock Builder", "Another Builder"], "userscripts/another-test.html", function(oldestFailingRevision, newestPassingRevision) {
+ equals(oldestFailingRevision, 90426);
+ equals(newestPassingRevision, 90425);
+ });
});
});
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/ui.js (90865 => 90866)
--- trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/ui.js 2011-07-12 22:35:39 UTC (rev 90865)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/ui.js 2011-07-12 23:31:05 UTC (rev 90866)
@@ -12,6 +12,13 @@
return 'http://trac.webkit.org/browser/trunk/LayoutTests/' + testName;
}
+ui.urlForRevisionRange = function(firstRevision, lastRevision)
+{
+ if (firstRevision != lastRevision)
+ return 'http://trac.webkit.org/log/trunk/?rev=' + firstRevision + '&stop_rev=' + lastRevision + '&limit=100&verbose=on';
+ return 'http://trac.webkit.org/changeset/' + firstRevision;
+};
+
ui.summarizeTest = function(testName, resultNodesByBuilder)
{
var unexpectedResults = results.collectUnexpectedResults(resultNodesByBuilder);
@@ -20,6 +27,7 @@
'<span class="what"><a draggable></a></span>' +
'<span>fails on</span>' +
'<ul class="where"></ul>' +
+ '<div class="when"></div>' +
'</div>');
$('.what a', block).text(testName).attr('href', ui.urlForTest(testName)).attr('class', unexpectedResults.join(' '));
@@ -31,13 +39,18 @@
return block;
};
-ui.summarizeResultsByTest = function(resultsByTest)
+ui.summarizeRegressionRange = function(oldestFailingRevision, newestPassingRevision)
{
- var block = $('<div class="results-summary"></div>');
- $.each(resultsByTest, function(testName, resultNodesByBuilder) {
- block.append(ui.summarizeTest(testName, resultNodesByBuilder));
- });
- return block;
+ if (!oldestFailingRevision || !newestPassingRevision)
+ return $();
+
+ var impliedFirstFailingRevision = newestPassingRevision + 1;
+
+ var href = "" oldestFailingRevision);
+ var textForRevisionRange = impliedFirstFailingRevision == oldestFailingRevision ? impliedFirstFailingRevision : impliedFirstFailingRevision + ':' + oldestFailingRevision;
+ var text = 'Regression ' + textForRevisionRange;
+
+ return $('<a class="regression-range"></a>').attr('href', href).text(text);
};
ui.results = function(resultsURLs)
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/ui_unittests.js (90865 => 90866)
--- trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/ui_unittests.js 2011-07-12 22:35:39 UTC (rev 90865)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/ui_unittests.js 2011-07-12 23:31:05 UTC (rev 90866)
@@ -19,14 +19,25 @@
}
}
-test("summarizeResultsByTest", 3, function() {
- var resultsSummary = ui.summarizeResultsByTest(kExampleResultsByTest);
- var resultsSummaryHTML = resultsSummary.html();
- ok(resultsSummaryHTML.indexOf('scrollbars/custom-scrollbar-with-incomplete-style.html') != -1);
- ok(resultsSummaryHTML.indexOf('userscripts/another-test.html') != -1);
- ok(resultsSummaryHTML.indexOf('Mock Builder') != -1);
+test("summarizeTest", 3, function() {
+ var testName = 'userscripts/another-test.html';
+ var summary = ui.summarizeTest(testName, kExampleResultsByTest[testName]);
+ var summaryHTML = summary.html();
+ ok(summaryHTML.indexOf('scrollbars/custom-scrollbar-with-incomplete-style.html') == -1);
+ ok(summaryHTML.indexOf('userscripts/another-test.html') != -1);
+ ok(summaryHTML.indexOf('Mock Builder') != -1);
});
+test("summarizeRegressionRange", 2, function() {
+ var summaryWithMultipleRevisions = ui.summarizeRegressionRange(90424, 90426);
+ summaryWithMultipleRevisions.wrap('<wrapper></wrapper>');
+ equal(summaryWithMultipleRevisions.parent().html(), '<a class="regression-range" href="" 90427:90424</a>');
+
+ var summaryWithOneRevision = ui.summarizeRegressionRange(90425, 90426);
+ summaryWithOneRevision.wrap('<wrapper></wrapper>');
+ equal(summaryWithOneRevision.parent().html(), '<a class="regression-range" href="" 90427:90425</a>');
+});
+
test("results", 1, function() {
var testResults = ui.results([
'http://example.com/layout-test-results/foo-bar-expected.txt',
