- Revision
- 97770
- Author
- [email protected]
- Date
- 2011-10-18 10:17:50 -0700 (Tue, 18 Oct 2011)
Log Message
garden-o-matic should not report flaking in expected failures as failures.
https://bugs.webkit.org/show_bug.cgi?id=70279
Reviewed by Adam Barth.
On the bots, flakes that are already expected to fail aren't reported as failures (they keep the bot green).
For example, if TEXT failure is expected, a TEXT TIMEOUT result will not cause the redness. We should do the
same for garden-o-matic.
* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/results.js: Spuriously changed to use Array iterators rather than jQuery APIs,
rewrote the logic for determining expected or unexpected failures as a central analysis tool.
* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/results_unittests.js: Added unit tests.
Modified Paths
Diff
Modified: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/results.js (97769 => 97770)
--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/results.js 2011-10-18 17:06:52 UTC (rev 97769)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/results.js 2011-10-18 17:17:50 UTC (rev 97770)
@@ -76,16 +76,6 @@
results.kTextType = 'text'
// FIXME: There are more types of tests.
-function isFailure(result)
-{
- return kFailingResults.indexOf(result) != -1;
-}
-
-function isSuccess(result)
-{
- return result === PASS;
-}
-
function possibleSuffixListFor(failureTypeList)
{
var suffixList = [];
@@ -201,61 +191,62 @@
net.jsonp(key, callback);
});
-function anyIsFailure(resultsList)
-{
- return $.grep(resultsList, isFailure).length > 0;
-}
+results.ResultAnalyzer = base.extends(Object, {
+ init: function(resultNode)
+ {
+ this._actual = resultNode ? results.failureTypeList(resultNode.actual) : [];
+ this._expected = resultNode ? this._addImpliedExpectations(results.failureTypeList(resultNode.expected)) : [];
+ },
+ _addImpliedExpectations: function(resultsList)
+ {
+ if (resultsList.indexOf('FAIL') == -1)
+ return resultsList;
+ return resultsList.concat(kFailingResults);
+ },
+ _hasPass: function(results)
+ {
+ return results.indexOf(PASS) != -1;
+ },
+ unexpectedResults: function()
+ {
+ return this._actual.filter(function(result) {
+ return this._expected.indexOf(result) == -1;
+ }, this);
+ },
+ succeeded: function()
+ {
+ return this._hasPass(this._actual);
+ },
+ expectedToSucceed: function()
+ {
+ return this._hasPass(this._expected);
+ },
+ flaky: function()
+ {
+ return this._actual.length > 1;
+ },
+ hasUnexpectedFailures: function()
+ {
+ var difference = {};
+ this._actual.forEach(function(actual) {
+ difference[actual] = actual !== PASS;
+ });
+ this._expected.forEach(function(expected) {
+ if (expected !== PASS)
+ delete difference[expected];
+ });
+ return Object.keys(difference).some(function(key) {
+ return difference[key];
+ });
+ }
+})
-function anyIsSuccess(resultsList)
-{
- return $.grep(resultsList, isSuccess).length > 0;
-}
-
-function addImpliedExpectations(resultsList)
-{
- if (resultsList.indexOf('FAIL') == -1)
- return resultsList;
- return resultsList.concat(kFailingResults);
-}
-
-results.unexpectedResults = function(resultNode)
-{
- var actualResults = results.failureTypeList(resultNode.actual);
- var expectedResults = addImpliedExpectations(results.failureTypeList(resultNode.expected))
-
- return $.grep(actualResults, function(result) {
- return expectedResults.indexOf(result) == -1;
- });
-};
-
-function isExpectedOrUnexpectedFailure(resultNode)
-{
- if (!resultNode)
- return false;
- var actualResults = results.failureTypeList(resultNode.actual);
- if (anyIsSuccess(actualResults))
- return false;
- return anyIsFailure(actualResults);
-}
-
function isUnexpectedFailure(resultNode)
{
- if (!resultNode)
- return false;
- if (anyIsSuccess(results.failureTypeList(resultNode.actual)))
- return false;
- return anyIsFailure(results.unexpectedResults(resultNode));
+ var analyzer = new results.ResultAnalyzer(resultNode);
+ return analyzer.hasUnexpectedFailures() && !analyzer.succeeded() && !analyzer.flaky();
}
-function isUnexpectedSuccesses(resultNode)
-{
- if (!resultNode)
- return false;
- if (anyIsFailure(results.failureTypeList(resultNode.actual)))
- return false;
- return anyIsSuccess(results.unexpectedResults(resultNode));
-}
-
function isResultNode(node)
{
return !!node.actual;
@@ -263,7 +254,9 @@
results.expectedOrUnexpectedFailures = function(resultsTree)
{
- return base.filterTree(resultsTree.tests, isResultNode, isExpectedOrUnexpectedFailure);
+ return base.filterTree(resultsTree.tests, isResultNode, function(resultNode) {
+ return !(new results.ResultAnalyzer(resultNode).succeeded());
+ });
};
results.unexpectedFailures = function(resultsTree)
@@ -273,7 +266,10 @@
results.unexpectedSuccesses = function(resultsTree)
{
- return base.filterTree(resultsTree.tests, isResultNode, isUnexpectedSuccesses);
+ return base.filterTree(resultsTree.tests, isResultNode, function(resultNode) {
+ var analyzer = new results.ResultAnalyzer(resultNode);
+ return !analyzer.expectedToSucceed() && analyzer.succeeded() && !analyzer.flaky();
+ });
};
function resultsByTest(resultsByBuilder, filter)
@@ -318,7 +314,8 @@
{
var collectedResults = [];
$.each(dictionaryOfResultNodes, function(key, resultNode) {
- collectedResults = collectedResults.concat(results.unexpectedResults(resultNode));
+ var analyzer = new results.ResultAnalyzer(resultNode);
+ collectedResults = collectedResults.concat(analyzer.unexpectedResults());
});
return base.uniquifyArray(collectedResults);
};
Modified: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/results_unittests.js (97769 => 97770)
--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/results_unittests.js 2011-10-18 17:06:52 UTC (rev 97769)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/results_unittests.js 2011-10-18 17:17:50 UTC (rev 97770)
@@ -40,6 +40,10 @@
"expected": "PASS",
"actual": "PASS TEXT"
},
+ "unexpected-failing-flaky-scrollbar.html": {
+ "expected": "TEXT",
+ "actual": "TIMEOUT TEXT"
+ },
"unexpected-pass.html": {
"expected": "FAIL",
"actual": "PASS"
@@ -70,6 +74,59 @@
"revision": "90430"
};
+test("ResultAnalyzer", 35, function() {
+ var analyzer;
+
+ analyzer = new results.ResultAnalyzer({expected: 'PASS', actual: 'TEXT'});
+ ok(analyzer.expectedToSucceed());
+ ok(analyzer.hasUnexpectedFailures());
+ deepEqual(analyzer.unexpectedResults(), ['TEXT']);
+ ok(!analyzer.succeeded());
+ ok(!analyzer.flaky());
+
+ analyzer = new results.ResultAnalyzer({expected: 'PASS TIMEOUT', actual: 'TEXT'});
+ ok(analyzer.expectedToSucceed());
+ ok(analyzer.hasUnexpectedFailures());
+ deepEqual(analyzer.unexpectedResults(), ['TEXT']);
+ ok(!analyzer.succeeded());
+ ok(!analyzer.flaky());
+
+ analyzer = new results.ResultAnalyzer({expected: 'TEXT', actual: 'TEXT TIMEOUT'});
+ ok(!analyzer.expectedToSucceed());
+ ok(analyzer.hasUnexpectedFailures());
+ deepEqual(analyzer.unexpectedResults(), ['TIMEOUT']);
+ ok(!analyzer.succeeded());
+ ok(analyzer.flaky());
+
+ analyzer = new results.ResultAnalyzer({expected: 'PASS', actual: 'TEXT TIMEOUT'});
+ ok(analyzer.expectedToSucceed());
+ ok(analyzer.hasUnexpectedFailures());
+ deepEqual(analyzer.unexpectedResults(), ['TEXT', 'TIMEOUT']);
+ ok(!analyzer.succeeded());
+ ok(analyzer.flaky());
+
+ analyzer = new results.ResultAnalyzer({expected: 'PASS TIMEOUT', actual: 'PASS TIMEOUT'});
+ ok(analyzer.expectedToSucceed());
+ ok(!analyzer.hasUnexpectedFailures());
+ deepEqual(analyzer.unexpectedResults(), []);
+ ok(analyzer.succeeded());
+ ok(analyzer.flaky());
+
+ analyzer = new results.ResultAnalyzer({expected: 'PASS TIMEOUT', actual: 'TIMEOUT PASS'});
+ ok(analyzer.expectedToSucceed());
+ ok(!analyzer.hasUnexpectedFailures());
+ deepEqual(analyzer.unexpectedResults(), []);
+ ok(analyzer.succeeded());
+ ok(analyzer.flaky());
+
+ analyzer = new results.ResultAnalyzer({expected: 'FAIL', actual: 'TIMEOUT'});
+ ok(!analyzer.expectedToSucceed());
+ ok(!analyzer.hasUnexpectedFailures());
+ deepEqual(analyzer.unexpectedResults(), []);
+ ok(!analyzer.succeeded());
+ ok(!analyzer.flaky());
+});
+
test("unexpectedFailures", 1, function() {
var unexpectedFailures = results.unexpectedFailures(unittest.kExampleResultsJSON);
deepEqual(unexpectedFailures, {
Modified: trunk/Tools/ChangeLog (97769 => 97770)
--- trunk/Tools/ChangeLog 2011-10-18 17:06:52 UTC (rev 97769)
+++ trunk/Tools/ChangeLog 2011-10-18 17:17:50 UTC (rev 97770)
@@ -1,3 +1,18 @@
+2011-10-18 Dimitri Glazkov <[email protected]>
+
+ garden-o-matic should not report flaking in expected failures as failures.
+ https://bugs.webkit.org/show_bug.cgi?id=70279
+
+ Reviewed by Adam Barth.
+
+ On the bots, flakes that are already expected to fail aren't reported as failures (they keep the bot green).
+ For example, if TEXT failure is expected, a TEXT TIMEOUT result will not cause the redness. We should do the
+ same for garden-o-matic.
+
+ * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/results.js: Spuriously changed to use Array iterators rather than jQuery APIs,
+ rewrote the logic for determining expected or unexpected failures as a central analysis tool.
+ * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/results_unittests.js: Added unit tests.
+
2011-10-18 Zan Dobersek <[email protected]>
[Gtk] Support for client-based geolocation
