Diff
Modified: trunk/PerformanceTests/ChangeLog (232560 => 232561)
--- trunk/PerformanceTests/ChangeLog 2018-06-06 23:38:51 UTC (rev 232560)
+++ trunk/PerformanceTests/ChangeLog 2018-06-06 23:46:43 UTC (rev 232561)
@@ -1,5 +1,36 @@
2018-06-06 Jon Lee <[email protected]>
+ Remove unneeded data processing in MotionMark
+ https://bugs.webkit.org/show_bug.cgi?id=186256
+
+ Reviewed by Said Abou-Hallawa.
+
+ Remove calculation of average frame rate based on multiple samples at a given complexity.
+ It is not used in the calculation of the score, and is just extra work to do at the end
+ of a test.
+
+ * MotionMark/developer.html: Remove options to show average data and regression based on
+ that data.
+ * MotionMark/resources/debug-runner/graph.js: Remove data. Update the time graph that
+ shows the relationship between the break point in the regression against the complexity
+ graph.
+ * MotionMark/resources/debug-runner/motionmark.css: Make the confidence intervals easier
+ to see.
+ * MotionMark/resources/runner/motionmark.js: Simplify score calculation now that we are
+ only calculating regressions on the raw data. We were calculating it based on the averages,
+ but that was just for analysis, and not used in the actual score. Get rid of the
+ |complexitySamples| local variable.
+ * MotionMark/resources/strings.js: Remove unneeded string constant.
+ * MotionMark/tests/resources/main.js:
+ (processSamples): Refactor so that more of it can be shared between Controller and
+ RampController. It also avoids processing the raw samples (controllerSamples) twice for
+ the ramp controller.
+ (_processComplexitySamples): Remove logic to calculate averages based on complexity.
+ (_processMarks): Split this out from the old processSamples.
+ (_processControllerSamples): Split this out from the old processSamples.
+
+2018-06-06 Jon Lee <[email protected]>
+
Strip out more characters when creating permalinks
https://bugs.webkit.org/show_bug.cgi?id=186259
Modified: trunk/PerformanceTests/MotionMark/developer.html (232560 => 232561)
--- trunk/PerformanceTests/MotionMark/developer.html 2018-06-06 23:38:51 UTC (rev 232560)
+++ trunk/PerformanceTests/MotionMark/developer.html 2018-06-06 23:46:43 UTC (rev 232561)
@@ -179,13 +179,11 @@
<form name="complexity-graph-options">
<ul class="series">
<li><label><input type="checkbox" name="series-raw" checked> Series raw</label></li>
- <li><label><input type="checkbox" name="series-average"> Series average</label></li>
</ul>
<ul>
<li><label><input type="checkbox" name="regression-time-score"> Controller score</label></li>
<li><label><input type="checkbox" name="bootstrap-score" checked> Bootstrap score and histogram</label></li>
<li><label><input type="checkbox" name="complexity-regression-aggregate-raw" checked> Regression, series raw</label><span id="complexity-regression-aggregate-raw"></span></li>
- <li><label><input type="checkbox" name="complexity-regression-aggregate-average"> Regression, series average</label><span id="complexity-regression-aggregate-average"></span></li>
</ul>
</form>
</nav>
Modified: trunk/PerformanceTests/MotionMark/resources/debug-runner/graph.js (232560 => 232561)
--- trunk/PerformanceTests/MotionMark/resources/debug-runner/graph.js 2018-06-06 23:38:51 UTC (rev 232560)
+++ trunk/PerformanceTests/MotionMark/resources/debug-runner/graph.js 2018-06-06 23:46:43 UTC (rev 232561)
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2015-2017 Apple Inc. All rights reserved.
+ * Copyright (C) 2015-2018 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -37,7 +37,7 @@
// Convert from compact JSON output to propertied data
var samplesWithProperties = {};
- [Strings.json.controller, Strings.json.complexity, Strings.json.complexityAverage].forEach(function(seriesName) {
+ [Strings.json.controller, Strings.json.complexity].forEach(function(seriesName) {
var series = testData[Strings.json.samples][seriesName];
samplesWithProperties[seriesName] = series.toArray();
})
@@ -92,7 +92,7 @@
svg.append("circle")
.attr("cx", xScale(data.segment1[1][0]))
.attr("cy", yScale(data.segment1[1][1]))
- .attr("r", 5);
+ .attr("r", 3);
this._addRegressionLine(svg, xScale, yScale, data.segment1, data.stdev);
this._addRegressionLine(svg, xScale, yScale, data.segment2, data.stdev);
},
@@ -158,7 +158,6 @@
// regression
this._addRegression(result[Strings.json.complexity], svg.append("g").attr("class", "regression raw"), xScale, yScale);
- this._addRegression(result[Strings.json.complexityAverage], svg.append("g").attr("class", "regression average"), xScale, yScale);
var bootstrapResult = result[Strings.json.complexity][Strings.json.bootstrap];
if (bootstrapResult) {
@@ -203,21 +202,6 @@
.attr("y1", function(d) { return yScale(d.frameLength) + 3; })
.attr("y2", function(d) { return yScale(d.frameLength) - 3; });
- group = svg.append("g")
- .attr("class", "series average")
- .selectAll("circle")
- .data(data[Strings.json.complexityAverage])
- .enter();
- group.append("circle")
- .attr("cx", function(d) { return xScale(d.complexity); })
- .attr("cy", function(d) { return yScale(d.frameLength); })
- .attr("r", 3)
- group.append("line")
- .attr("x1", function(d) { return xScale(d.complexity); })
- .attr("x2", function(d) { return xScale(d.complexity); })
- .attr("y1", function(d) { return yScale(d.frameLength - d.stdev); })
- .attr("y2", function(d) { return yScale(d.frameLength + d.stdev); });
-
// Cursor
var cursorGroup = svg.append("g").attr("class", "cursor hidden");
cursorGroup.append("line")
@@ -466,6 +450,17 @@
}
// inflection point
regressionGroup.append("circle")
+ .attr("cx", x(regression.segment2[0][0]))
+ .attr("cy", yRight(regression.segment2[0][1]))
+ .attr("r", 3);
+ regressionGroup.append("line")
+ .attr("class", "association")
+ .attr("stroke-dasharray", "5, 3")
+ .attr("x1", x(regression.segment2[0][0]))
+ .attr("x2", x(regression.segment2[0][0]))
+ .attr("y1", yRight(regression.segment2[0][1]))
+ .attr("y2", yLeft(regression.complexity));
+ regressionGroup.append("circle")
.attr("cx", x(regression.segment1[1][0]))
.attr("cy", yLeft(regression.complexity))
.attr("r", 5);
@@ -568,11 +563,9 @@
onComplexityGraphOptionsChanged: function() {
var form = document.forms["complexity-graph-options"].elements;
benchmarkController._showOrHideNodes(form["series-raw"].checked, "#complexity-graph .series.raw");
- benchmarkController._showOrHideNodes(form["series-average"].checked, "#complexity-graph .series.average");
benchmarkController._showOrHideNodes(form["regression-time-score"].checked, "#complexity-graph .mean.complexity");
benchmarkController._showOrHideNodes(form["bootstrap-score"].checked, "#complexity-graph .bootstrap");
benchmarkController._showOrHideNodes(form["complexity-regression-aggregate-raw"].checked, "#complexity-graph .regression.raw");
- benchmarkController._showOrHideNodes(form["complexity-regression-aggregate-average"].checked, "#complexity-graph .regression.average");
},
onTimeGraphOptionsChanged: function() {
@@ -616,11 +609,7 @@
mean = mean.join("");
} else {
var complexityRegression = testResult[Strings.json.complexity];
- var complexityAverageRegression = testResult[Strings.json.complexityAverage];
-
document.getElementById("complexity-regression-aggregate-raw").textContent = complexityRegression.complexity.toFixed(2) + ", ±" + complexityRegression.stdev.toFixed(2) + "ms";
- document.getElementById("complexity-regression-aggregate-average").textContent = complexityAverageRegression.complexity.toFixed(2) + ", ±" + complexityAverageRegression.stdev.toFixed(2) + "ms";
-
var bootstrap = complexityRegression[Strings.json.bootstrap];
if (bootstrap) {
score = bootstrap.median.toFixed(2);
Modified: trunk/PerformanceTests/MotionMark/resources/debug-runner/motionmark.css (232560 => 232561)
--- trunk/PerformanceTests/MotionMark/resources/debug-runner/motionmark.css 2018-06-06 23:38:51 UTC (rev 232560)
+++ trunk/PerformanceTests/MotionMark/resources/debug-runner/motionmark.css 2018-06-06 23:46:43 UTC (rev 232561)
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2015-2017 Apple Inc. All rights reserved.
+ * Copyright (C) 2015-2018 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -704,6 +704,10 @@
stroke-width: 2px;
}
+#regressions line.association {
+ stroke-width: 1px;
+}
+
#regressions circle {
fill: rgba(200, 200, 200, .8);
}
@@ -755,7 +759,7 @@
}
#complexity-graph .regression polygon {
- fill: rgba(253, 253, 253, .05);
+ fill: rgba(253, 253, 253, .2);
}
#complexity-graph .raw.series line {
@@ -771,15 +775,6 @@
stroke: rgba(30, 96%, 86%, .05);
}
-#complexity-graph .average.series circle {
- fill: hsl(170, 96%, 56%);
-}
-
-#complexity-graph .average.series line {
- stroke: hsla(170, 96%, 56%, .2);
- stroke-width: 2px;
-}
-
#complexity-graph .bootstrap .bar {
fill: hsla(260, 56%, 66%, .4);
}
@@ -794,5 +789,5 @@
}
#complexity-graph .bootstrap .median polygon {
- fill: hsla(300, 56%, 66%, .05);
+ fill: hsla(240, 56%, 66%, .4);
}
Modified: trunk/PerformanceTests/MotionMark/resources/runner/motionmark.js (232560 => 232561)
--- trunk/PerformanceTests/MotionMark/resources/runner/motionmark.js 2018-06-06 23:38:51 UTC (rev 232560)
+++ trunk/PerformanceTests/MotionMark/resources/runner/motionmark.js 2018-06-06 23:46:43 UTC (rev 232561)
@@ -92,8 +92,6 @@
data[Strings.json.result] = result;
var samples = data[Strings.json.samples];
- var desiredFrameLength = 1000/60;
-
function findRegression(series, profile) {
var minIndex = Math.round(.025 * series.length);
var maxIndex = Math.round(.975 * (series.length - 1));
@@ -109,7 +107,7 @@
var complexityIndex = series.fieldMap[Strings.json.complexity];
var frameLengthIndex = series.fieldMap[Strings.json.frameLength];
- var regressionOptions = { desiredFrameLength: desiredFrameLength };
+ var regressionOptions = { desiredFrameLength: 1000/60 };
if (profile)
regressionOptions.preferredProfile = profile;
return {
@@ -124,9 +122,8 @@
};
}
- var complexitySamples;
// Convert these samples into SampleData objects if needed
- [Strings.json.complexity, Strings.json.complexityAverage, Strings.json.controller].forEach(function(seriesName) {
+ [Strings.json.complexity, Strings.json.controller].forEach(function(seriesName) {
var series = samples[seriesName];
if (series && !(series instanceof SampleData))
samples[seriesName] = new SampleData(series.fieldMap, series.data);
@@ -152,28 +149,20 @@
}
}
- [Strings.json.complexity, Strings.json.complexityAverage].forEach(function(seriesName) {
- if (!(seriesName in samples))
- return;
+ var regressionResult = findRegression(samples[Strings.json.complexity], predominantProfile);
+ var calculation = regressionResult.regression;
+ result[Strings.json.complexity] = {};
+ result[Strings.json.complexity][Strings.json.regressions.segment1] = [
+ [regressionResult.minComplexity, calculation.s1 + calculation.t1 * regressionResult.minComplexity],
+ [calculation.complexity, calculation.s1 + calculation.t1 * calculation.complexity]
+ ];
+ result[Strings.json.complexity][Strings.json.regressions.segment2] = [
+ [calculation.complexity, calculation.s2 + calculation.t2 * calculation.complexity],
+ [regressionResult.maxComplexity, calculation.s2 + calculation.t2 * regressionResult.maxComplexity]
+ ];
+ result[Strings.json.complexity][Strings.json.complexity] = calculation.complexity;
+ result[Strings.json.complexity][Strings.json.measurements.stdev] = Math.sqrt(calculation.error / samples[Strings.json.complexity].length);
- var regression = {};
- result[seriesName] = regression;
- var regressionResult = findRegression(samples[seriesName], predominantProfile);
- if (seriesName == Strings.json.complexity)
- complexitySamples = regressionResult.samples;
- var calculation = regressionResult.regression;
- regression[Strings.json.regressions.segment1] = [
- [regressionResult.minComplexity, calculation.s1 + calculation.t1 * regressionResult.minComplexity],
- [calculation.complexity, calculation.s1 + calculation.t1 * calculation.complexity]
- ];
- regression[Strings.json.regressions.segment2] = [
- [calculation.complexity, calculation.s2 + calculation.t2 * calculation.complexity],
- [regressionResult.maxComplexity, calculation.s2 + calculation.t2 * regressionResult.maxComplexity]
- ];
- regression[Strings.json.complexity] = calculation.complexity;
- regression[Strings.json.measurements.stdev] = Math.sqrt(calculation.error / samples[seriesName].length);
- });
-
if (isRampController) {
var timeComplexity = new Experiment;
data[Strings.json.controller].forEach(function(regression) {
@@ -188,15 +177,15 @@
experimentResult[Strings.json.measurements.percent] = timeComplexity.percentage();
const bootstrapIterations = 2500;
- var bootstrapResult = Regression.bootstrap(complexitySamples.data, bootstrapIterations, function(resampleData) {
- var complexityIndex = complexitySamples.fieldMap[Strings.json.complexity];
+ var bootstrapResult = Regression.bootstrap(regressionResult.samples.data, bootstrapIterations, function(resampleData) {
+ var complexityIndex = regressionResult.samples.fieldMap[Strings.json.complexity];
resampleData.sort(function(a, b) {
return a[complexityIndex] - b[complexityIndex];
});
- var resample = new SampleData(complexitySamples.fieldMap, resampleData);
- var regressionResult = findRegression(resample, predominantProfile);
- return regressionResult.regression.complexity;
+ var resample = new SampleData(regressionResult.samples.fieldMap, resampleData);
+ var bootstrapRegressionResult = findRegression(resample, predominantProfile);
+ return bootstrapRegressionResult.regression.complexity;
}, .8);
result[Strings.json.complexity][Strings.json.bootstrap] = bootstrapResult;
Modified: trunk/PerformanceTests/MotionMark/resources/strings.js (232560 => 232561)
--- trunk/PerformanceTests/MotionMark/resources/strings.js 2018-06-06 23:38:51 UTC (rev 232560)
+++ trunk/PerformanceTests/MotionMark/resources/strings.js 2018-06-06 23:46:43 UTC (rev 232561)
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2015-2017 Apple Inc. All rights reserved.
+ * Copyright (C) 2015-2018 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -37,7 +37,6 @@
controller: "controller",
time: "time",
complexity: "complexity",
- complexityAverage: "complexityAverage",
frameLength: "frameLength",
smoothedFrameLength: "smoothedFrameLength",
Modified: trunk/PerformanceTests/MotionMark/tests/resources/main.js (232560 => 232561)
--- trunk/PerformanceTests/MotionMark/tests/resources/main.js 2018-06-06 23:38:51 UTC (rev 232560)
+++ trunk/PerformanceTests/MotionMark/tests/resources/main.js 2018-06-06 23:46:43 UTC (rev 232561)
@@ -171,72 +171,31 @@
return this._sampler.processSamples();
},
- _processComplexitySamples: function(complexitySamples, complexityAverageSamples)
+ _processComplexitySamples: function(complexitySamples)
{
- complexityAverageSamples.addField(Strings.json.complexity, 0);
- complexityAverageSamples.addField(Strings.json.frameLength, 1);
- complexityAverageSamples.addField(Strings.json.measurements.stdev, 2);
-
complexitySamples.sort(function(a, b) {
return complexitySamples.getFieldInDatum(a, Strings.json.complexity) - complexitySamples.getFieldInDatum(b, Strings.json.complexity);
});
-
- // Samples averaged based on complexity
- var currentComplexity = -1;
- var experimentAtComplexity;
- function addSample() {
- var mean = experimentAtComplexity.mean();
- var stdev = experimentAtComplexity.standardDeviation();
-
- var averageSample = complexityAverageSamples.createDatum();
- complexityAverageSamples.push(averageSample);
- complexityAverageSamples.setFieldInDatum(averageSample, Strings.json.complexity, currentComplexity);
- complexityAverageSamples.setFieldInDatum(averageSample, Strings.json.frameLength, mean);
- complexityAverageSamples.setFieldInDatum(averageSample, Strings.json.measurements.stdev, stdev);
- }
- complexitySamples.forEach(function(sample) {
- var sampleComplexity = complexitySamples.getFieldInDatum(sample, Strings.json.complexity);
- if (sampleComplexity != currentComplexity) {
- if (currentComplexity > -1)
- addSample();
-
- currentComplexity = sampleComplexity;
- experimentAtComplexity = new Experiment;
- }
- experimentAtComplexity.sample(complexitySamples.getFieldInDatum(sample, Strings.json.frameLength));
- });
- // Finish off the last one
- addSample();
},
- processSamples: function(results)
+ _processMarks: function()
{
- var complexityExperiment = new Experiment;
- var smoothedFrameLengthExperiment = new Experiment;
-
- var samples = this._sampler.samples;
-
for (var markName in this._marks)
this._marks[markName].time -= this._startTimestamp;
- results[Strings.json.marks] = this._marks;
-
- results[Strings.json.samples] = {};
-
+ return this._marks;
+ },
+ _processControllerSamples: function()
+ {
var controllerSamples = new SampleData;
- results[Strings.json.samples][Strings.json.controller] = controllerSamples;
-
controllerSamples.addField(Strings.json.time, 0);
controllerSamples.addField(Strings.json.complexity, 1);
controllerSamples.addField(Strings.json.frameLength, 2);
controllerSamples.addField(Strings.json.smoothedFrameLength, 3);
- var complexitySamples = new SampleData(controllerSamples.fieldMap);
- results[Strings.json.samples][Strings.json.complexity] = complexitySamples;
-
+ var samples = this._sampler.samples;
samples[0].forEach(function(timestamp, i) {
var sample = controllerSamples.createDatum();
controllerSamples.push(sample);
- complexitySamples.push(sample);
// Represent time in milliseconds
controllerSamples.setFieldInDatum(sample, Strings.json.time, timestamp - this._startTimestamp);
@@ -251,9 +210,23 @@
controllerSamples.setFieldInDatum(sample, Strings.json.smoothedFrameLength, samples[2][i]);
}, this);
- var complexityAverageSamples = new SampleData;
- results[Strings.json.samples][Strings.json.complexityAverage] = complexityAverageSamples;
- this._processComplexitySamples(complexitySamples, complexityAverageSamples);
+ return controllerSamples;
+ },
+
+ processSamples: function(results)
+ {
+ results[Strings.json.marks] = this._processMarks();
+
+ var controllerSamples = this._processControllerSamples();
+ var complexitySamples = new SampleData(controllerSamples.fieldMap);
+
+ results[Strings.json.samples] = {};
+ results[Strings.json.samples][Strings.json.controller] = controllerSamples;
+ results[Strings.json.samples][Strings.json.complexity] = complexitySamples;
+ controllerSamples.forEach(function (sample) {
+ complexitySamples.push(sample);
+ });
+ this._processComplexitySamples(complexitySamples);
}
});
@@ -546,16 +519,17 @@
processSamples: function(results)
{
- Controller.prototype.processSamples.call(this, results);
-
+ results[Strings.json.marks] = this._processMarks();
// Have samplingTimeOffset represent time 0
var startTimestamp = this._marks[Strings.json.samplingStartTimeOffset].time;
-
for (var markName in results[Strings.json.marks]) {
results[Strings.json.marks][markName].time -= startTimestamp;
}
- var controllerSamples = results[Strings.json.samples][Strings.json.controller];
+ results[Strings.json.samples] = {};
+
+ var controllerSamples = this._processControllerSamples();
+ results[Strings.json.samples][Strings.json.controller] = controllerSamples;
controllerSamples.forEach(function(timeSample) {
controllerSamples.setFieldInDatum(timeSample, Strings.json.time, controllerSamples.getFieldInDatum(timeSample, Strings.json.time) - startTimestamp);
});
@@ -595,9 +569,7 @@
complexitySamples.push(controllerSamples.at(j));
});
- var complexityAverageSamples = new SampleData;
- results[Strings.json.samples][Strings.json.complexityAverage] = complexityAverageSamples;
- this._processComplexitySamples(complexitySamples, complexityAverageSamples);
+ this._processComplexitySamples(complexitySamples);
}
});
