- Revision
- 90489
- Author
- [email protected]
- Date
- 2011-07-06 13:42:14 -0700 (Wed, 06 Jul 2011)
Log Message
Teach TestFailures how to load, parse, and interpret NRWT test results
Fixes <http://webkit.org/b/61877> TestFailures page doesn't show testers that use
new-run-webkit-tests
Reviewed by Adam Barth.
* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder.js:
(Builder.prototype.failureDiagnosisTextAndURL): Added support for the new 'flaky' failure
type. For now we don't account for image-only flakes (but TestFailures doesn't deal with
pixel tests at all currently).
(Builder.prototype.getNumberOfFailingTests): Relaxed the regex used for parsing the number
of failing tests from the buildbot output. Make sure not to count "new passes" (a new
category introduced by NRWT) as failures.
* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestResultsLoader.js:
(LayoutTestResultsLoader.prototype.start): Moved a bunch of code from here to
_fetchAndParseORWTResults. This function now attempts to load NRWT results, then falls back
to loading ORWT results.
(LayoutTestResultsLoader.prototype._fetchAndParseNRWTResults): Added. Tries to load and
parse the full_results.json file for the given build.
(LayoutTestResultsLoader.prototype._fetchAndParseORWTResults): Added. Code came from start.
Fixed a bug along the way where we were sometimes calling the error callback instead of the
success callback when all tests passed.
* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/NRWTResultsParser.js: Added.
(NRWTResultsParser): Do-nothing constructor.
(NRWTResultsParser.prototype.parse): Uses eval() (eww!) to get the test results out of the
JS string, then iterates over all the tests in the results data and builds up a data
structure matching what ORWTResultsParser returns.
* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Utilities.js:
(Array.prototype.contains):
(String.prototype.contains):
Added these simple helper functions.
* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/index.html: Pull in
NRWTResultsParser.js.
Modified Paths
Added Paths
Diff
Modified: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder.js (90488 => 90489)
--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder.js 2011-07-06 20:42:02 UTC (rev 90488)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder.js 2011-07-06 20:42:14 UTC (rev 90489)
@@ -41,6 +41,10 @@
text: 'pretty diff',
url: urlStem + '-pretty-diff.html',
},
+ flaky: {
+ text: 'pretty diff (flaky)',
+ url: urlStem + '-pretty-diff.html',
+ },
timeout: {
text: 'timed out',
},
@@ -136,11 +140,11 @@
result.tooManyFailures = true;
result.failureCount = layoutTestStep.results[1].reduce(function(sum, outputLine) {
- var match = /^(\d+) test case/.exec(outputLine);
+ var match = /^(\d+)/.exec(outputLine);
if (!match)
return sum;
- // Don't count new tests as failures.
- if (outputLine.indexOf('were new') >= 0)
+ // Don't count new tests or passes as failures.
+ if (outputLine.contains('were new') || outputLine.contains('new passes'))
return sum;
return sum + parseInt(match[1], 10);
}, 0);
Modified: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestResultsLoader.js (90488 => 90489)
--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestResultsLoader.js 2011-07-06 20:42:02 UTC (rev 90488)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestResultsLoader.js 2011-07-06 20:42:14 UTC (rev 90489)
@@ -41,32 +41,60 @@
var result = { tests: {}, tooManyFailures: false, version: currentCachedDataVersion };
+ function cacheParseResultsAndCallCallback(parseResult) {
+ result.tests = parseResult.tests;
+ result.tooManyFailures = parseResult.tooManyFailures;
+
+ PersistentCache.set(cacheKey, result);
+ callback(result.tests, result.tooManyFailures);
+ }
+
+ var self = this;
+ self._fetchAndParseNRWTResults(buildName, cacheParseResultsAndCallCallback, function() {
+ self._fetchAndParseORWTResults(buildName, cacheParseResultsAndCallCallback, function() {
+ // We couldn't fetch results for this build.
+ PersistentCache.set(cacheKey, result);
+ errorCallback(result.tests, result.tooManyFailures);
+ });
+ });
+ },
+
+ _fetchAndParseNRWTResults: function(buildName, successCallback, errorCallback) {
+ getResource(this._builder.resultsDirectoryURL(buildName) + 'full_results.json', function(xhr) {
+ successCallback((new NRWTResultsParser()).parse(xhr.responseText));
+ },
+ function(xhr) {
+ errorCallback();
+ });
+ },
+
+ _fetchAndParseORWTResults: function(buildName, successCallback, errorCallback) {
var parsedBuildName = this._builder.buildbot.parseBuildName(buildName);
// http://webkit.org/b/62380 was fixed in r89610.
var resultsHTMLSupportsTooManyFailuresInfo = parsedBuildName.revision >= 89610;
+ var result = { tests: {}, tooManyFailures: false };
+
var self = this;
- function fetchAndParseResultsHTMLAndCallCallback(callback) {
+ function fetchAndParseResultsHTMLAndCallCallback() {
getResource(self._builder.resultsPageURL(buildName), function(xhr) {
var parseResult = (new ORWTResultsParser()).parse(xhr.responseText);
result.tests = parseResult.tests;
if (resultsHTMLSupportsTooManyFailuresInfo)
result.tooManyFailures = parseResult.tooManyFailures;
- PersistentCache.set(cacheKey, result);
- callback(result.tests, result.tooManyFailures);
+ successCallback(result);
},
function(xhr) {
// We failed to fetch results.html. run-webkit-tests must have aborted early.
- PersistentCache.set(cacheKey, result);
- errorCallback(result.tests, result.tooManyFailures);
+ errorCallback();
});
}
if (resultsHTMLSupportsTooManyFailuresInfo) {
- fetchAndParseResultsHTMLAndCallCallback(callback);
+ fetchAndParseResultsHTMLAndCallCallback();
return;
}
@@ -75,20 +103,18 @@
if (failingTestCount < 0) {
// The number of failing tests couldn't be determined.
- PersistentCache.set(cacheKey, result);
- errorCallback(result.tests, result.tooManyFailures);
+ errorCallback();
return;
}
if (!failingTestCount) {
// All tests passed.
- PersistentCache.set(cacheKey, result);
- errorCallback(result.tests, result.tooManyFailures);
+ successCallback(result);
return;
}
// Find out which tests failed.
- fetchAndParseResultsHTMLAndCallCallback(callback);
+ fetchAndParseResultsHTMLAndCallCallback();
});
},
};
Added: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/NRWTResultsParser.js (0 => 90489)
--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/NRWTResultsParser.js (rev 0)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/NRWTResultsParser.js 2011-07-06 20:42:14 UTC (rev 90489)
@@ -0,0 +1,99 @@
+/*
+ * Copyright (C) 2011 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+function NRWTResultsParser() {
+}
+
+NRWTResultsParser.prototype = {
+ parse: function(unexpectedResultsJS) {
+ var data;
+ function ADD_RESULTS(x) {
+ data = ""
+ }
+
+ eval(unexpectedResultsJS);
+ console.assert(data);
+
+ var result = { tests: {}, tooManyFailures: data.interrupted };
+
+ function forEachTest(tree, handler, opt_prefix) {
+ var prefix = opt_prefix || '';
+
+ for (var key in tree) {
+ var newPrefix = prefix ? (prefix + '/' + key) : key;
+ if ('actual' in tree[key]) {
+ var testObject = tree[key];
+ testObject.name = newPrefix;
+ handler(testObject);
+ } else
+ forEachTest(tree[key], handler, newPrefix);
+ }
+ }
+
+ function isFailureExpected(expected, actual) {
+ if (actual === 'SKIP')
+ return true;
+
+ var expectedArray = expected.split(' ');
+ var actualArray = actual.split(' ');
+ for (var i = 0; i < actualArray.length; i++) {
+ var actualValue = actualArray[i];
+ if (expectedArray.contains(actualValue))
+ continue;
+ if (expectedArray.contains('FAIL') && ['IMAGE', 'TEXT', 'IMAGE+TEXT'].contains(actualValue))
+ continue;
+ return false;
+ }
+
+ return true;
+ }
+
+ function convertNRWTResultString(nrwtResult) {
+ const translations = {
+ CRASH: 'crash',
+ 'IMAGE+TEXT': 'fail',
+ IMAGE: 'fail',
+ TEXT: 'fail',
+ TIMEOUT: 'timeout',
+ };
+
+ if (nrwtResult in translations)
+ return translations[nrwtResult];
+
+ if (nrwtResult.contains(' '))
+ return 'flaky';
+
+ return 'unknown failure type ' + nrwtResult;
+ }
+
+ forEachTest(data.tests, function(test) {
+ if (isFailureExpected(test.expected, test.actual))
+ return;
+ result.tests[test.name] = { failureType: convertNRWTResultString(test.actual) };
+ });
+
+ return result;
+ },
+};
Modified: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Utilities.js (90488 => 90489)
--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Utilities.js 2011-07-06 20:42:02 UTC (rev 90488)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Utilities.js 2011-07-06 20:42:14 UTC (rev 90489)
@@ -114,6 +114,10 @@
return newArray;
}
+Array.prototype.contains = function(value) {
+ return this.indexOf(value) >= 0;
+}
+
Array.prototype.findFirst = function(predicate) {
for (var i = 0; i < this.length; ++i) {
if (predicate(this[i]))
@@ -145,3 +149,7 @@
while (this.firstChild)
this.removeChild(this.firstChild);
}
+
+String.prototype.contains = function(substring) {
+ return this.indexOf(substring) >= 0;
+}
Modified: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/index.html (90488 => 90489)
--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/index.html 2011-07-06 20:42:02 UTC (rev 90488)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/index.html 2011-07-06 20:42:14 UTC (rev 90489)
@@ -33,6 +33,7 @@
<script src=""
<script src=""
<script src=""
+ <script src=""
<script src=""
<script src=""
<script src=""
Modified: trunk/Tools/ChangeLog (90488 => 90489)
--- trunk/Tools/ChangeLog 2011-07-06 20:42:02 UTC (rev 90488)
+++ trunk/Tools/ChangeLog 2011-07-06 20:42:14 UTC (rev 90489)
@@ -1,5 +1,46 @@
2011-07-06 Adam Roben <[email protected]>
+ Teach TestFailures how to load, parse, and interpret NRWT test results
+
+ Fixes <http://webkit.org/b/61877> TestFailures page doesn't show testers that use
+ new-run-webkit-tests
+
+ Reviewed by Adam Barth.
+
+ * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder.js:
+ (Builder.prototype.failureDiagnosisTextAndURL): Added support for the new 'flaky' failure
+ type. For now we don't account for image-only flakes (but TestFailures doesn't deal with
+ pixel tests at all currently).
+ (Builder.prototype.getNumberOfFailingTests): Relaxed the regex used for parsing the number
+ of failing tests from the buildbot output. Make sure not to count "new passes" (a new
+ category introduced by NRWT) as failures.
+
+ * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestResultsLoader.js:
+ (LayoutTestResultsLoader.prototype.start): Moved a bunch of code from here to
+ _fetchAndParseORWTResults. This function now attempts to load NRWT results, then falls back
+ to loading ORWT results.
+ (LayoutTestResultsLoader.prototype._fetchAndParseNRWTResults): Added. Tries to load and
+ parse the full_results.json file for the given build.
+ (LayoutTestResultsLoader.prototype._fetchAndParseORWTResults): Added. Code came from start.
+ Fixed a bug along the way where we were sometimes calling the error callback instead of the
+ success callback when all tests passed.
+
+ * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/NRWTResultsParser.js: Added.
+ (NRWTResultsParser): Do-nothing constructor.
+ (NRWTResultsParser.prototype.parse): Uses eval() (eww!) to get the test results out of the
+ JS string, then iterates over all the tests in the results data and builds up a data
+ structure matching what ORWTResultsParser returns.
+
+ * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Utilities.js:
+ (Array.prototype.contains):
+ (String.prototype.contains):
+ Added these simple helper functions.
+
+ * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/index.html: Pull in
+ NRWTResultsParser.js.
+
+2011-07-06 Adam Roben <[email protected]>
+
Extract code to parse ORWT's results.html file into its own class
Prep work for fixing <http://webkit.org/b/61877> TestFailures page doesn't show testers that
