Title: [90442] trunk/Tools
- Revision
- 90442
- Author
- [email protected]
- Date
- 2011-07-06 00:39:30 -0700 (Wed, 06 Jul 2011)
Log Message
2011-07-06 Adam Barth <[email protected]>
garden-o-matic shouldn't show flaky tests by default
https://bugs.webkit.org/show_bug.cgi?id=63979
Reviewed by Ojan Vafai.
If any of the actual test results are PASS, the test is flaky and we
can ignore it (at least for now). If we get awesome at dealing with
failing tests, we might want to tackle flaky tests by refering the user
to the flakiness dashboard, which is way more awesome at dealing with
flaky tests that this tool ever will be.
* Scripts/webkitpy/tool/servers/data/gardeningserver/results.js:
* Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js:
Modified Paths
Diff
Modified: trunk/Tools/ChangeLog (90441 => 90442)
--- trunk/Tools/ChangeLog 2011-07-06 07:04:02 UTC (rev 90441)
+++ trunk/Tools/ChangeLog 2011-07-06 07:39:30 UTC (rev 90442)
@@ -1,3 +1,19 @@
+2011-07-06 Adam Barth <[email protected]>
+
+ garden-o-matic shouldn't show flaky tests by default
+ https://bugs.webkit.org/show_bug.cgi?id=63979
+
+ Reviewed by Ojan Vafai.
+
+ If any of the actual test results are PASS, the test is flaky and we
+ can ignore it (at least for now). If we get awesome at dealing with
+ failing tests, we might want to tackle flaky tests by refering the user
+ to the flakiness dashboard, which is way more awesome at dealing with
+ flaky tests that this tool ever will be.
+
+ * Scripts/webkitpy/tool/servers/data/gardeningserver/results.js:
+ * Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js:
+
2011-07-05 Adam Barth <[email protected]>
garden-o-matic should list the failing tests
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results.js (90441 => 90442)
--- trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results.js 2011-07-06 07:04:02 UTC (rev 90441)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results.js 2011-07-06 07:39:30 UTC (rev 90442)
@@ -6,18 +6,36 @@
var kTestType = 'layout-tests';
var kResultsName = 'full_results.json';
var kMasterName = 'ChromiumWebkit';
-var kFailingResults = ['TIMEOUT', 'TEXT', 'CRASH', 'IMAGE','IMAGE+TEXT'];
+var PASS = 'PASS';
+var TIMEOUT = 'TIMEOUT';
+var TEXT = 'TEXT';
+var CRASH = 'CRASH';
+var IMAGE = 'IMAGE';
+var IMAGE_TEXT = 'IMAGE+TEXT';
+
+var kFailingResults = [TIMEOUT, TEXT, CRASH, IMAGE, IMAGE_TEXT];
+
function isFailure(result)
{
return kFailingResults.indexOf(result) != -1;
}
+function isSuccess(result)
+{
+ return result === PASS;
+}
+
function anyIsFailure(resultsList)
{
return $.grep(resultsList, isFailure).length > 0;
}
+function anyIsSuccess(resultsList)
+{
+ return $.grep(resultsList, isSuccess).length > 0;
+}
+
function addImpliedExpectations(resultsList)
{
if (resultsList.indexOf('FAIL') == -1)
@@ -37,6 +55,8 @@
function isUnexpectedFailure(resultNode)
{
+ if (anyIsSuccess(resultNode.actual.split(' ')))
+ return false;
return anyIsFailure(unexpectedResults(resultNode));
}
Modified: trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js (90441 => 90442)
--- trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js 2011-07-06 07:04:02 UTC (rev 90441)
+++ trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js 2011-07-06 07:39:30 UTC (rev 90442)
@@ -6,6 +6,10 @@
"custom-scrollbar-with-incomplete-style.html": {
"expected": "IMAGE",
"actual": "IMAGE"
+ },
+ "flaky-scrollbarhtml": {
+ "expected": "PASS",
+ "actual": "PASS TEXT"
}
},
"userscripts": {
_______________________________________________
webkit-changes mailing list
[email protected]
http://lists.webkit.org/mailman/listinfo.cgi/webkit-changes
