Modified: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/BuildbotIteration.js (173380 => 173381)
--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/BuildbotIteration.js 2014-09-08 16:11:51 UTC (rev 173380)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/BuildbotIteration.js 2014-09-08 17:05:10 UTC (rev 173381)
@@ -258,6 +258,29 @@
var internalRevisionProperty = data.properties.findFirst(function(property) { return property[0] === "internal_got_revision" || isMultiCodebaseGotRevisionProperty(property); });
this.internalRevision = parseRevisionProperty(internalRevisionProperty, "Internal");
+ function sourceStampChanges(sourceStamp) {
+ var result = [];
+ var changes = sourceStamp.changes;
+ for (var i = 0; i < changes.length; ++i) {
+ var change = { revisionNumber: parseInt(changes[i].revision, 10) }
+ if (changes[i].repository)
+ change.repository = changes[i].repository;
+ if (changes[i].branch)
+ change.branch = changes[i].branch;
+ // There is also a timestamp, but it's not accurate.
+ result.push(change);
+ }
+ return result;
+ }
+
+ // The changes array is generally meaningful for svn triggered queues (such as builders),
+ // but not for internally triggered ones (such as testers), due to coalescing.
+ this.changes = [];
+ if (data.sourceStamp)
+ this.changes = sourceStampChanges(data.sourceStamp);
+ else for (var i = 0; i < data.sourceStamps.length; ++i)
+ this.changes = this.changes.concat(sourceStampChanges(data.sourceStamps[i]));
+
this.startTime = new Date(data.times[0] * 1000);
this.endTime = new Date(data.times[1] * 1000);
Modified: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/MetricsAnalyzer.js (173380 => 173381)
--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/MetricsAnalyzer.js 2014-09-08 16:11:51 UTC (rev 173380)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/MetricsAnalyzer.js 2014-09-08 17:05:10 UTC (rev 173381)
@@ -54,11 +54,19 @@
this._hasTracData = false;
- this._queues = queues;
+ // A commit can start a build-test sequence, or it can be ignored following builder queue rules.
+ // Only the builder queue knows which commits triggered builds, and which were ignored.
+ // We need to know which commits were ignored when measuring tester queue performance,
+ // so we load and analyze builder queues first.
+ this._queues = queues.slice(0);
+ this._queues.sort(function(a, b) { return b.builder - a.builder; });
+
this._remainingQueues = {};
this._queuesReadyToAnalyze = [];
- queues.forEach(function(queue) {
+ this._triggeringCommitsByTriggeringQueue = {};
+
+ this._queues.forEach(function(queue) {
if (!queue.iterations.length) {
this._remainingQueues[queue.id] = queue;
if (!this._queueBeingLoaded) {
@@ -67,11 +75,41 @@
}
} else
this._queuesReadyToAnalyze.push(queue);
- }.bind(this));
+ }, this);
webkitTrac.load(this._rangeStartTime, this._rangeEndTime);
},
+ _triggeringQueue: function(queue)
+ {
+ if (queue.builder)
+ return queue;
+ for (var i = 0; i < this._queues.length; ++i) {
+ if (this._queues[i].tester)
+ continue;
+ if (this._queues[i].platform === queue.platform && this._queues[i].architecture === queue.architecture && this._queues[i].debug === queue.debug)
+ return this._queues[i];
+ }
+ // Efl bot both builds and tests, but is registered as tester.
+ return queue;
+ },
+
+ _recordTriggeringCommitsForTriggeringQueue: function(queue)
+ {
+ console.assert(!(queue.id in this._triggeringCommitsByTriggeringQueue));
+ console.assert(queue.id === this._triggeringQueue(queue).id);
+
+ var commits = {};
+ queue.iterations.forEach(function(iteration) {
+ iteration.changes.forEach(function(change) {
+ // FIXME: Support multiple source trees.
+ commits[change.revisionNumber] = 1;
+ });
+ });
+
+ this._triggeringCommitsByTriggeringQueue[queue.id] = commits;
+ },
+
// Iterations is an array of finished iterations ordered by time, iterations[0] being the newest.
// Returns an index of an iteration that defined the queue color at the start, or -1
// if there was none.
@@ -144,7 +182,7 @@
var i = queueIterations.length - 1;
while (i >= 0 && queueIterations[i].endTime <= this._rangeEndTime)
iterations.push(queueIterations[i--]);
- }.bind(this));
+ }, this);
iterations.sort(function(a, b) { return b.endTime - a.endTime; });
@@ -207,102 +245,75 @@
}
},
- _fullyTestedRevisionNumber: function(lastTestedRevisionByQueue)
- {
- var result = Infinity;
- this._queues.forEach(function(queue) {
- if (lastTestedRevisionByQueue[queue.id] < result)
- result = lastTestedRevisionByQueue[queue.id];
- });
- return result;
- },
-
_countTimes: function(queues, result)
{
- // Combine all iterations that started and finished within the range into one array.
- // These are the iterations that can have results for revisions in the range.
- var iterationsByRevision = [];
+ var relevantIterationsByQueue = {};
queues.forEach(function(queue) {
- iterationsByRevision = iterationsByRevision.concat(queue.iterations.filter(function(iteration) {
+ relevantIterationsByQueue[queue.id] = queue.iterations.filter(function(iteration) {
return iteration.productive && iteration.startTime > this._rangeStartTime && iteration.endTime < this._rangeEndTime;
- }.bind(this)));
- }.bind(this));
- iterationsByRevision.sort(function(a, b) { return a.endTime - b.endTime; }); // When there are multiple iterations building the same revision, the first one wins (as the other ones were probably manual retries).
- iterationsByRevision.sort(function(a, b) { return a.openSourceRevision - b.openSourceRevision; });
+ }, this);
+ relevantIterationsByQueue[queue.id].sort(function(a, b) { return a.endTime - b.endTime; });
+ }, this);
- // Revisions that landed within the time range.
- var revisions = webkitTrac.recordedCommits.reduce(function(result, revision) {
- if (revision.date >= this._rangeStartTime && revision.date < this._rangeEndTime)
- result[revision.revisionNumber] = revision;
- return result;
- }.bind(this), {});
+ var times = [];
+ var ownTimes = [];
+ var worstTime = 0;
+ var worstOwnTime = 0
+ var worstTimeRevision;
+ var worstOwnTimeRevision;
- // Find the oldest iteration for each queue. Revisions landed before a new bot was added are considered fully tested
- // even without results from the not yet existent bot.
- // Unfortunately, we don't know when the bot got added to dashboard, so we have to assume that it was there for as long as it had results.
- var lastTestedRevisionByQueue = {};
- queues.forEach(function(queue) {
- var queueIterations = queue.iterations.filter(function(iteration) { return iteration.finished; });
- queueIterations.sort(function(a, b) { return b.endTime - a.endTime; });
- if (queueIterations.length > 0)
- lastTestedRevisionByQueue[queue.id] = queueIterations[queueIterations.length - 1].openSourceRevision;
- });
+ webkitTrac.recordedCommits.forEach(function(revision) {
+ if (revision.date < this._rangeStartTime || revision.date >= this._rangeEndTime)
+ return;
- var previousFullyTestedRevisionNumber = -1;
-
- for (var i = 0; i < iterationsByRevision.length; ++i) {
- var iteration = iterationsByRevision[i];
-
- console.assert(lastTestedRevisionByQueue[iteration.queue.id] === undefined || lastTestedRevisionByQueue[iteration.queue.id] <= iteration.openSourceRevision);
- lastTestedRevisionByQueue[iteration.queue.id] = iteration.openSourceRevision;
-
- var newFullyTestedRevisionNumber = this._fullyTestedRevisionNumber(lastTestedRevisionByQueue);
- console.assert(newFullyTestedRevisionNumber >= previousFullyTestedRevisionNumber);
-
- if (newFullyTestedRevisionNumber === previousFullyTestedRevisionNumber)
- continue;
-
- for (var revisionNumber = newFullyTestedRevisionNumber; (revisionNumber > previousFullyTestedRevisionNumber) && (revisionNumber in revisions); --revisionNumber) {
- var revision = revisions[revisionNumber];
- console.assert(!("elapsedTime" in revision));
- revision.elapsedTime = iteration.endTime - revision.date;
- revision.elapsedOwnTime = iteration.endTime - iteration.startTime;
+ var endTime = -1;
+ var ownTime = -1;
+ queues.forEach(function(queue) {
+ if (!(revision.revisionNumber in this._triggeringCommitsByTriggeringQueue[this._triggeringQueue(queue).id]))
+ return;
+ for (var i = 0; i < relevantIterationsByQueue[queue.id].length; ++i) {
+ var iteration = relevantIterationsByQueue[queue.id][i];
+ if (iteration.openSourceRevision >= revision.revisionNumber) {
+ endTime = Math.max(endTime, iteration.endTime);
+ ownTime = Math.max(ownTime, iteration.endTime - iteration.startTime);
+ break;
+ }
+ }
+ }, this);
+ if (endTime >= 0) {
+ console.assert(ownTime >= 0);
+ var time = endTime - revision.date;
+ times.push(time);
+ ownTimes.push(ownTime);
+ if (time > worstTime) {
+ worstTime = time;
+ worstTimeCommit = revision.revisionNumber;
+ }
+ if (ownTime > worstOwnTime) {
+ worstOwnTime = ownTime;
+ worstOwnTimeCommit = revision.revisionNumber;
+ }
}
+ }, this);
- previousFullyTestedRevisionNumber = newFullyTestedRevisionNumber;
- }
-
- var times = [];
- var ownTimes = [];
- for (var revisionNumber in revisions) {
- var revision = revisions[revisionNumber];
- if (!("elapsedTime" in revision)) {
- // A revision that landed within the time range didn't necessarily get all results by the range end.
- continue;
- }
- // Changes on other branches are irrelevant, as they are not built or tested.
- // FIXME: Support metrics for non-trunk queues.
- if (!revision.containsBranchLocation || revision.branch === "trunk") {
- times.push(revision.elapsedTime);
- ownTimes.push(revision.elapsedOwnTime);
- }
- delete revision.elapsedTime;
- delete revision.elapsedOwnTime;
- }
-
result.averageSecondsFromCommit = times.average() / 1000;
result.medianSecondsFromCommit = times.median() / 1000;
- result.bestSecondsFromCommit = Math.min.apply(Math, times) / 1000;
- result.worstSecondsFromCommit = Math.max.apply(Math, times) / 1000;
+ console.assert(worstTime === Math.max.apply(Math, times));
+ result.worstSecondsFromCommit = worstTime / 1000;
+ result.revisionWithWorstTimeFromCommit = worstTimeCommit;
result.averageSecondsOwnTime = ownTimes.average() / 1000;
result.medianSecondsOwnTime = ownTimes.median() / 1000;
- result.bestSecondsOwnTime = Math.min.apply(Math, ownTimes) / 1000;
- result.worstSecondsOwnTime = Math.max.apply(Math, ownTimes) / 1000;
+ result.worstSecondsOwnTime = worstOwnTime / 1000;
+ console.assert(worstOwnTime === Math.max.apply(Math, ownTimes));
+ result.revisionWithWorstOwnTime = worstOwnTimeCommit;
},
_analyzeQueue: function(queue)
{
+ if (this._triggeringQueue(queue).id === queue.id && !(queue.id in this._triggeringCommitsByTriggeringQueue))
+ this._recordTriggeringCommitsForTriggeringQueue(queue);
+
var result = { queueID: queue.id };
this._countPercentageOfGreen([queue], result);
this._countTimes([queue], result);
@@ -331,7 +342,7 @@
this._queuesReadyToAnalyze.forEach(function(queue) {
this._analyzeQueue(queue);
- }.bind(this));
+ }, this);
this._queuesReadyToAnalyze = [];
Modified: trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/MetricsView.js (173380 => 173381)
--- trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/MetricsView.js 2014-09-08 16:11:51 UTC (rev 173380)
+++ trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/MetricsView.js 2014-09-08 17:05:10 UTC (rev 173381)
@@ -122,15 +122,13 @@
addLine(this.element, "Time from commit: ");
addLine(this.element, "Average: " + Math.round(result.averageSecondsFromCommit / 60) + " minutes");
addLine(this.element, "Median: " + Math.round(result.medianSecondsFromCommit / 60) + " minutes");
- addLine(this.element, "Best: " + Math.round(result.bestSecondsFromCommit / 60) + " minutes");
- addLine(this.element, "Worst: " + Math.round(result.worstSecondsFromCommit / 60) + " minutes");
+ addLine(this.element, "Worst: " + Math.round(result.worstSecondsFromCommit / 60) + " minutes (r" + result.revisionWithWorstTimeFromCommit + ")");
} else {
// Time from commit is pretty useless for tester bots.
addLine(this.element, "Time on the bot: ");
addLine(this.element, "Average: " + Math.round(result.averageSecondsOwnTime / 60) + " minutes");
addLine(this.element, "Median: " + Math.round(result.medianSecondsOwnTime / 60) + " minutes");
- addLine(this.element, "Best: " + Math.round(result.bestSecondsOwnTime / 60) + " minutes");
- addLine(this.element, "Worst: " + Math.round(result.worstSecondsOwnTime / 60) + " minutes");
+ addLine(this.element, "Worst: " + Math.round(result.worstSecondsOwnTime / 60) + " minutes (r" + result.revisionWithWorstOwnTime + ")");
}
}
Modified: trunk/Tools/ChangeLog (173380 => 173381)
--- trunk/Tools/ChangeLog 2014-09-08 16:11:51 UTC (rev 173380)
+++ trunk/Tools/ChangeLog 2014-09-08 17:05:10 UTC (rev 173381)
@@ -1,3 +1,29 @@
+2014-09-08 Alexey Proskuryakov <[email protected]>
+
+ Dashboard metrics should ignore commits that didn't trigger builds
+ https://bugs.webkit.org/show_bug.cgi?id=136618
+
+ Reviewed by Darin Adler.
+
+ Commits that didn't trigger builds are ones like ChangeLog updates, patches for
+ other platforms etc. It does not make sense to count wait time for these, as it
+ can be arbitrarily long.
+
+ The new algorithm is much slower asymptotically, but it's OK, computers are fast.
+
+ * BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/BuildbotIteration.js:
+ (BuildbotIteration.prototype._parseData): Record changes that triggered the iteration.
+
+ * BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/MetricsAnalyzer.js:
+ We used to walk the timeline to see which revisions are fully tested, but that's not
+ correct. A revision that's only tested by a subset of queues finishes independently
+ of another that's tested by another subset. Now, we just search for the answer for
+ each revision individually.
+
+ * BuildSlaveSupport/build.webkit.org-config/public_html/dashboard/Scripts/MetricsView.js:
+ (MetricsView.prototype._update.appendQueueResults): Added worst revision number, which
+ the analyzer now reports. Removed best time, which is more confusing than meaningful.
+
2014-09-08 Tibor Meszaros <[email protected]>
Remove EWebLauncher from webkitdirs.pm
