Modified: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder.js (93221 => 93222)
--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder.js 2011-08-17 18:07:33 UTC (rev 93221)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder.js 2011-08-17 18:13:12 UTC (rev 93222)
@@ -100,7 +100,7 @@
getNumberOfFailingTests: function(buildNumber, callback) {
var cacheKey = this.name + '_getNumberOfFailingTests_' + buildNumber;
- const currentCachedDataVersion = 2;
+ const currentCachedDataVersion = 3;
if (PersistentCache.contains(cacheKey)) {
var cachedData = PersistentCache.get(cacheKey);
if (cachedData.version === currentCachedDataVersion) {
@@ -128,6 +128,15 @@
}
if (!('results' in layoutTestStep) || layoutTestStep.results[0] === 0) {
+ if (!('times' in layoutTestStep) || layoutTestStep.times.length < 2 || layoutTestStep.times[1] - layoutTestStep.times[0] < self._minimumSuccessfulLayoutTestStepRunTime) {
+ // Either something caused the start/stop times not to be recorded, or
+ // run-webkit-tests ran so quickly that we can't believe there wasn't an error
+ // (e.g., a bug in the script that made it not find any tests to run).
+ PersistentCache.set(cacheKey, result);
+ callback(result.failureCount, result.tooManyFailures);
+ return;
+ }
+
// All tests passed.
result.failureCount = 0;
PersistentCache.set(cacheKey, result);
@@ -224,4 +233,8 @@
callback(buildNames);
});
},
+
+ // Any successful runs of run-webkit-tests that took less than this number of seconds are
+ // assumed to be errors.
+ _minimumSuccessfulLayoutTestStepRunTime: 20,
};
Modified: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder_unittests.js (93221 => 93222)
--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder_unittests.js 2011-08-17 18:07:33 UTC (rev 93221)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder_unittests.js 2011-08-17 18:13:12 UTC (rev 93222)
@@ -74,6 +74,10 @@
"2 test cases (<1%) crashed",
],
],
+ times: [
+ 1310599204.1231229,
+ 1310600152.973659,
+ ]
},
],
};
@@ -204,4 +208,62 @@
});
});
+test("getNumberOfFailingTests treats successful but unbelievably short test runs as errors", 4, function() {
+ const jsonData = {
+ steps: [
+ {
+ isFinished: true,
+ isStarted: true,
+ name: "layout-test",
+ step_number: 7,
+ text: [
+ "layout-test"
+ ],
+ times: [
+ 1311288797.7207019,
+ 1311288802.7791941
+ ]
+ },
+ ],
+ };
+
+ runGetNumberOfFailingTestsTest(jsonData, function(failureCount, tooManyFailures) {
+ equal(failureCount, -1);
+ equal(tooManyFailures, false);
+ });
+});
+
+test("getNumberOfFailingTests doesn't care if a failing run is unbelievably short", 4, function() {
+ const jsonData = {
+ steps: [
+ {
+ isFinished: true,
+ isStarted: true,
+ name: "layout-test",
+ results: [
+ 2,
+ [
+ "2011-07-13 04:38:46,315 11247 manager.py:780 WARNING Exiting early after 20 crashes and 0 timeouts. 2251 tests run.",
+ "20 failed"
+ ]
+ ],
+ step_number: 4,
+ text: [
+ "2011-07-13 04:38:46,315 11247 manager.py:780 WARNING Exiting early after 20 crashes and 0 timeouts. 2251 tests run.",
+ "20 failed"
+ ],
+ times: [
+ 1310557115.793082,
+ 1310557119.832104
+ ]
+ },
+ ],
+ };
+
+ runGetNumberOfFailingTestsTest(jsonData, function(failureCount, tooManyFailures) {
+ equal(failureCount, 20);
+ equal(tooManyFailures, true);
+ });
+});
+
})();
Modified: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestResultsLoader.js (93221 => 93222)
--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestResultsLoader.js 2011-08-17 18:07:33 UTC (rev 93221)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestResultsLoader.js 2011-08-17 18:13:12 UTC (rev 93222)
@@ -30,7 +30,7 @@
LayoutTestResultsLoader.prototype = {
start: function(buildName, callback, errorCallback) {
var cacheKey = 'LayoutTestResultsLoader.' + this._builder.name + '.' + buildName;
- const currentCachedDataVersion = 7;
+ const currentCachedDataVersion = 8;
if (PersistentCache.contains(cacheKey)) {
var cachedData = PersistentCache.get(cacheKey);
if (cachedData.version === currentCachedDataVersion) {
Modified: trunk/Tools/ChangeLog (93221 => 93222)
--- trunk/Tools/ChangeLog 2011-08-17 18:07:33 UTC (rev 93221)
+++ trunk/Tools/ChangeLog 2011-08-17 18:13:12 UTC (rev 93222)
@@ -1,3 +1,24 @@
+2011-08-17 Adam Roben <[email protected]>
+
+ Teach TestFailures to ignore unbelievably short test runs
+
+ Fixes <http://webkit.org/b/66385> TestFailures page thinks all tests passed in
+ http://build.webkit.org/builders/Windows%207%20Release%20(Tests)/builds/14956
+
+ Reviewed by Dan Bates.
+
+ * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder.js:
+ (Builder.prototype.getNumberOfFailingTests): If it looks like all tests passed, but
+ run-webkit-tests took less than 10 seconds to run, assume that some weird error occurred
+ that caused it not to run any tests at all (as happened for a while due to
+ <http://webkit.org/b/64988>). Bumped the cache version to evict old, buggy cached data.
+
+ * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder_unittests.js:
+ Test for the above.
+
+ * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestResultsLoader.js:
+ (LayoutTestResultsLoader.prototype.start): Bumped the cache version to evict old, buggy cached data.
+
2011-08-16 Adam Barth <[email protected]>
Add a CG qualifier similar to the GPU qualifier
