Phedenskog has uploaded a new change for review.
https://gerrit.wikimedia.org/r/243027
Change subject: Added possibility to report metrics using more than statsv
......................................................................
Added possibility to report metrics using more than statsv
Reworked the structure so we can have different reporters,
reporting metrics. We now support statsv, json, csv and
Graphite. Lets add more later.
There's a new parameter --reporter that needs to be configured for
each run (fixed in the batch files) so this should work out of the
box in Jenkins.
Also documented the methods.
There are still some work that needs to be done:
* Better flow for testing full runs
* Choose what metrics to fetch
* A name
Bug: T114343
Change-Id: I7bd582a4a180b07b62d6c94552555957baf34ce6
---
A bin/index.js
M examples/batchExample.txt
M lib/cli.js
A lib/collectMetrics.js
M lib/index.js
A lib/reporter/csv.js
A lib/reporter/graphite.js
A lib/reporter/index.js
A lib/reporter/json.js
A lib/reporter/statsv.js
M lib/util.js
A lib/wpt.js
M package.json
M scripts/batch/desktop.txt
M scripts/batch/login-desktop.txt
M scripts/batch/login-mobile.txt
M scripts/batch/mobile-wpt-org.txt
M scripts/batch/mobile.txt
M scripts/batch/second-view-desktop.txt
M scripts/batch/second-view-mobile.txt
A test/batchTest.js
M test/cliTest.js
A test/collectMetricsTest.js
M test/files/batch.txt
A test/reporterTest.js
M test/utilTest.js
26 files changed, 816 insertions(+), 421 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/performance/WebPageTest
refs/changes/27/243027/1
diff --git a/bin/index.js b/bin/index.js
new file mode 100755
index 0000000..b5dd025
--- /dev/null
+++ b/bin/index.js
@@ -0,0 +1,32 @@
+#!/usr/bin/env node
+
+/**
+ * @fileoverview The bin file to run.
+ * @author Peter Hedenskog
+ * @copyright (c) 2015, Peter Hedenskog <peterwikimedia.org>.
+ * Released under the Apache 2.0 License.
+ */
+
+'use strict';
+var minimist = require('minimist');
+var cli = require('../lib/cli');
+var wpt = require('../lib/index');
+
+var argv = minimist(process.argv.slice(2), {
+ boolean: ['sendMetrics','verbose']
+});
+
+if (argv.help) {
+ cli.help();
+ process.exit(0);
+}
+
+if (!cli.validateArgs(argv)) {
+ process.exit(1);
+}
+
+if (argv.batch) {
+ wpt.runBatch(argv);
+} else {
+ wpt.runTest(argv);
+}
diff --git a/examples/batchExample.txt b/examples/batchExample.txt
index d593924..65df62f 100644
--- a/examples/batchExample.txt
+++ b/examples/batchExample.txt
@@ -19,4 +19,4 @@
--webPageTestKey <%WMF_WPT_KEY> --runs 15 --median SpeedIndex
https://en.wikipedia.org/wiki/Facebook
## And then test Barack 31 and use SpeedIndex as median
---webPageTestKey <%WMF_WPT_KEY> --runs 31 --median SpeedIndex
https://en.wikipedia.org/wiki/Barack_Obama
+--webPageTestKey <%WMF_WPT_KEY> --runs 31 --median SpeedIndex --reporter json
https://en.wikipedia.org/wiki/Barack_Obama
diff --git a/lib/cli.js b/lib/cli.js
index 4d107e8..9ac5563 100644
--- a/lib/cli.js
+++ b/lib/cli.js
@@ -1,31 +1,21 @@
-/*
-wptstatsv
-~~~~~~~
-A thin wrapper for the WebPageTest API that sends metrics to statsv.
-
-Copyright 2015 Peter Hedenskog <[email protected]>
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
+/**
+ * @fileoverview Command line helper methods.
+ * @author Peter Hedenskog
+ * @copyright (c) 2015, Peter Hedenskog <peterwikimedia.org>.
+ * Released under the Apache 2.0 License.
+ */
'use strict';
var util = require('./util');
+var reporters = require('./reporter');
var AVAILIBLE_USER_STATUS = ['anonymous', 'authenticated'];
module.exports = {
- // TODO add extra namespace (key?)
+ /**
+ * Print the options to the console.
+ */
help: function() {
console.log(' Thin wrapper for the WebPageTest API that sends metrics
to statsv.\n');
console.log(' Usage: ' + process.argv[1] + ' [options]
[URL/scriptFile]');
@@ -37,8 +27,6 @@
console.log(' --webPageTestKey The secret key for the
WebPageTest instance ' +
'[required]');
console.log(' --batch The path to a file containing
multiple URLs to test.');
- console.log(' --namespace The namespace of the key sent to
statsv.' +
- '[webpagetest]');
console.log(' --location The location and browser to use,
check ' +
' http://wpt.wmftest.org/getLocations.php [us-east-1:Chrome]');
console.log(' --connectivity Connectivity profile ' +
@@ -49,16 +37,25 @@
console.log(' --timeout The timeout in seconds until we
end the test [1200]');
console.log(' --userStatus Is the user logged in or not?
Used in the namespace ' +
'(anonymous|authenticated) [anonymous]');
- console.log(' --endpoint Where to send the statsv metrics
' +
- '[https://www.example.com]');
- console.log(' --sendMetrics Send metrics to statsv or not.
Set to send ' +
- 'metrics.');
+ console.log(' --reporter Choose how you want to report the
metrics ' +
+ '[csv|json|statsv] ');
console.log(' --customMetrics A file with custom WPT metrics. '
+
'https://sites.google.com/a/webpagetest.org/docs/using-webpagetest/custom-metrics
');
console.log(' --verbose Log the full JSON from
WebPageTest to the console\n');
+ console.log(' --namespace The namespace of the keys ' +
+ '[webpagetest]');
+ Object.keys(reporters.getReporters()).forEach(function(name) {
+ reporters.get(name).help();
+ });
+
console.log(' For a full list of options, check ' +
'https://github.com/marcelduran/webpagetest-api#test-works-for-runtest-method-only\n');
},
+ /**
+ * Validate the input arguments.
+ * @param {array} argv The input parameters for the run.
+ * @return {boolean} returns true if the argumenst are ok.
+ */
validateArgs: function(argv) {
if (argv.batch) {
// if it is batch job then test the parameters per line
@@ -80,8 +77,33 @@
'[' + AVAILIBLE_USER_STATUS + ']');
return false;
}
+ if (!argv.batch) {
+ if (!argv.reporter) {
+ console.error('Missing reporter. Needs to be one of [' +
+ Object.keys(reporters.getReporters()) + ']');
+ return false;
+ }
+
+ var reporter = reporters.get(argv.reporter);
+ if (!reporter) {
+ console.error('There\'s no matching reporter for ' +
argv.reporter);
+ return false;
+ }
+
+ var isOK = reporter.validate(argv);
+ if (!isOK) {
+ return isOK;
+ }
+ }
+
return true;
},
+ /**
+ * Get the file content or the URL for a run. The webpagetest api takes
either
+ * a URL or a file with URLs.
+ * @param {string} arg The argument to checkif it's a URL or file.
+ * @param {string} the URL or content of the file
+ */
getInputURLorFile: function(arg) {
// is it a file or URL we wanna test?
if (arg.indexOf('http') === -1) {
diff --git a/lib/collectMetrics.js b/lib/collectMetrics.js
new file mode 100644
index 0000000..8e4e983
--- /dev/null
+++ b/lib/collectMetrics.js
@@ -0,0 +1,99 @@
+/**
+ * @fileoverview Collect the metrics from the WebPageTest JSON.
+ * @author Peter Hedenskog
+ * @copyright (c) 2015, Peter Hedenskog <peterwikimedia.org>.
+ * Released under the Apache 2.0 License.
+ */
+
+'use strict';
+
+var NAMESPACE = 'webpagetest';
+var BROWSERS = {
+ 'Google Chrome': 'chrome',
+ Firefox: 'firefox',
+ 'Internet Explorer': 'ie',
+ Safari: 'safari'
+};
+var DEFAULT_USER_STATUS = 'anonymous';
+var DEFAULT_NAMESPACE = 'webpagetest';
+
+module.exports = {
+ // Here are the values we collect. Want to add more? Check the JSON that
is returned:
+ //
https://sites.google.com/a/webpagetest.org/docs/advanced-features/webpagetest-restful-apis
+ // #TOC-Sample
+ // Not 100% sure it's the latest though. Test by logging the output from
WebPageTest
+ // Note: It can differs depending on what agent that runs the tests.
+ // Note 2: Make sure we don't hit the statsv limit of maximum chars in one
request
+ METRICS: ['SpeedIndex', 'render', 'TTFB', 'fullyLoaded'],
+ ASSET_TYPES: ['html','js','css','image'],
+
+ /**
+ * Collect the metrics we want from the giant WebPageTest JSON. We always
+ * collect values from the median run.
+ * @param {object} wptJson The JSON returned from the WebPageTest API
+ * @param {array} argv The input parameters for the run.
+ */
+
+ collect: function(wptJson, argv) {
+ var self = this;
+ var metricsToSend = {
+ timings: {},
+ requests: {},
+ sizes: {}
+ };
+
+ var userStatus = argv.userStatus || DEFAULT_USER_STATUS;
+ var namespace = argv.namespace || DEFAULT_NAMESPACE;
+
+ var emulateMobile = argv.emulateMobile;
+ var firstViewOnly = argv.first;
+
+ var views = ['firstView'];
+ if (!firstViewOnly) {
+ views.push('repeatView');
+ }
+
+ views.forEach(function(view) {
+ // if we are missing browser info from WPT (happens when using MotoG
at least)
+ // use a normalized version of the location
+ // the browser/location can then look like
Dulles_MotoG_Motorola_G___Chrome
+ // but since there are no standard of naming it should be ok to just
use what we got
+ var browser = BROWSERS[wptJson.data.median[view].browser_name] ||
+ wptJson.data.location.replace(/[^A-Za-z_0-9]/g, '_');
+
+ if (emulateMobile) {
+ browser += '-emulateMobile';
+ }
+
+ // the actual location is the first part of the location string
+ // separated by either a : or _
+ var location = wptJson.data.location.split(/:|_/)[0];
+
+ var keyStart = namespace + '.' + location + '.' + userStatus + '.' +
browser +
+ '.' + view + '.';
+
+ self.METRICS.forEach(function(metric) {
+ metricsToSend.timings[keyStart + metric ] =
wptJson.data.median[view][metric];
+ });
+
+ if (wptJson.data.median[view].userTimes) {
+
Object.keys(wptJson.data.median[view].userTimes).forEach(function(userTiming) {
+ metricsToSend.timings[keyStart + userTiming ] =
+ wptJson.data.median[view].userTimes[userTiming];
+ });
+ }
+
+ // collect sizes & assets
+ self.ASSET_TYPES.forEach(function(assetType) {
+ metricsToSend.requests[keyStart +
+ assetType] = wptJson.data.median[view].breakdown[assetType].
+ requests;
+ metricsToSend.sizes[keyStart +
+ assetType] =
wptJson.data.median[view].breakdown[assetType].bytes;
+ });
+
+ });
+
+ return metricsToSend;
+ }
+};
diff --git a/lib/index.js b/lib/index.js
index 71c36ea..81683a2 100644
--- a/lib/index.js
+++ b/lib/index.js
@@ -1,132 +1,88 @@
-#!/usr/bin/env node
-
-/*
-wptstatsv
-~~~~~~~
-A thin wrapper for the WebPageTest API that sends metrics to statsv.
-
-Copyright 2015 Peter Hedenskog <[email protected]>
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
+/**
+ * @fileoverview Main file to test URLs at WebPageTest.
+ * @author Peter Hedenskog
+ * @copyright (c) 2015, Peter Hedenskog <peterwikimedia.org>.
+ * Released under the Apache 2.0 License.
+ */
'use strict';
-
-var WebPageTest = require('webpagetest');
-var minimist = require('minimist');
var util = require('./util');
var async = require('async');
var cli = require('./cli');
var eol = require('os').EOL;
+var wpt = require('./wpt');
+var collectMetrics = require('./collectMetrics');
-var DEFAULT_USER_STATUS = 'anonymous';
-var DEFAULT_NAMESPACE = 'webpagetest';
+module.exports = {
+ /**
+ * Test multiple URLs using a batch file
+ * @param {array} argv The arguments for the run
+ */
+ runBatch: function(argv, cb) {
+ var callback = cb || function() {};
+ var self = this;
+ var series = [];
+ var tests = util.readFile(argv.batch);
+ var lines = tests.split(eol);
+ lines.forEach(function(line) {
+ // only run tests where we have something on that line
+ if (line.indexOf('#') !== 0 && line.length > 1) {
-var argv = minimist(process.argv.slice(2), {
- boolean: ['sendMetrics','verbose']
-});
+ var myArgs = util.convertTextLineToMinimist(line);
+ if (!cli.validateArgs(myArgs)) {
+ process.exit(1);
+ }
+ series.push(function(callback) {
+ self.runTest(myArgs, callback);
+ });
+ }
+ });
+ // lets run the tests one by one after each other. in that way
+ // the wait time you configure (how long time you wait until a test is
finished)
+ // is more logical, meaning the configured time is per test and not
for a
+ // whole test suite.
+ async.series(series, // jshint unused:false
+ function(err, results) {
+ if (err) {
+ console.error('Couldn\'t execute all the runs' + err);
+ } else {
+ console.log('Succesfully run ' + series.length + '
tests.');
+ }
+ callback(err);
+ });
+ },
-function runBatch(argv) {
- var series = [];
- var tests = util.readFile(argv.batch);
- var lines = tests.split(eol);
- lines.forEach(function(line) {
- // only run tests where we have something on that line
- if (line.indexOf('#') !== 0 && line.length > 1) {
+ /**
+ * Test a single URL
+ * @param {array} argv The arguments for the run
+ * @param {function} cb The callback that will be called when the test
finished
+ */
+ runTest: function(argv, cb) {
+ var callback = cb || function() {};
+ var webPageTestHost = argv.webPageTestHost ||
'http://wpt.wmftest.org';
+ var arg = argv._[0];
- var myArgs = util.convertTextLineToMinimist(line);
- if (!cli.validateArgs(myArgs)) {
- process.exit(1);
- }
- series.push(function(callback) {
- runTest(myArgs, callback);
- });
- }
- });
- // lets run the tests one by one after each other. in that way
- // the wait time you configure (how long time you wait until a test is
finished)
- // is more logical, meaning the configured time is per test and not for a
- // whole test suite.
- async.series(series, // jshint unused:false
- function(err, results) {
- if (err) {
- console.error('Couldn\'t execute all the runs' + err);
- } else {
- console.log('Succesfully run ' + series.length + ' tests.');
- }
- });
-}
+ var input = cli.getInputURLorFile(arg);
-function runTest(argv, cb) {
- var callback = cb || function() {};
- var endpoint = argv.endpoint || 'https://www.example.com';
- var webPageTestHost = argv.webPageTestHost || 'http://wpt.wmftest.org';
- var wpt = new WebPageTest(webPageTestHost, argv.webPageTestKey);
- var userStatus = argv.userStatus || DEFAULT_USER_STATUS;
- var namespace = argv.namespace || DEFAULT_NAMESPACE;
-
- var arg = argv._[0];
-
- var input = cli.getInputURLorFile(arg);
-
- // Note: wptOptions are changed internally when you call runTest, so
- // pollResults and timeout are changed to milliseconds so if we set the
poll
- // to 10 it will be 10000, that's why we recreate the object per run :)
- var wptOptions = util.setupWPTOptions(argv);
- // read custom javascript metrics
- if (argv.customMetrics) {
- wptOptions.custom = util.readFile(argv.customMetrics);
- }
-
- console.log('Running WebPageTest [' + wptOptions.location + ' ' +
wptOptions.runs +
- ' time(s)] for ' + arg);
-
- wpt.runTest(input, wptOptions, function(err, data) {
-
- if (argv.verbose) {
- console.log(JSON.stringify(data));
+ // Note: wptOptions are changed internally when you call runTest, so
+ // pollResults and timeout are changed to milliseconds so if we set
the poll
+ // to 10 it will be 10000, that's why we recreate the object per run :)
+ var wptOptions = util.setupWPTOptions(argv);
+ // read custom javascript metrics
+ if (argv.customMetrics) {
+ wptOptions.custom = util.readFile(argv.customMetrics);
}
- if (err) {
- console.error('Couldn\'t fetch data from WebPageTest:' +
JSON.stringify(err));
- console.error('Configuration:' + JSON.stringify(wptOptions, null,
2));
- callback();
- return;
- } else {
- console.log('WebPageTest run: ' + data.data.summary);
- var collectedMetrics = util.collectMetrics(data, userStatus,
namespace,
- argv);
- console.log('Collected metrics: ' +
JSON.stringify(collectedMetrics, null, 2));
- if (argv.sendMetrics) {
- util.sendMetrics(collectedMetrics, endpoint);
+ console.log('Running WebPageTest [' + wptOptions.location + ' ' +
wptOptions.runs +
+ ' time(s)] for ' + arg);
+
+ wpt.run(webPageTestHost, argv.webPageTestKey, argv, input, wptOptions,
function(err, data) {
+ if (data) {
+ var collectedMetrics = collectMetrics.collect(data, argv);
+ var reporter = require('./reporter').get(argv.reporter);
+ reporter.report(collectedMetrics, argv);
}
callback();
- }
- });
-}
-// ---- Main
-
-if (argv.help) {
- cli.help();
- process.exit(0);
-}
-
-if (!cli.validateArgs(argv)) {
- process.exit(1);
-}
-
-if (argv.batch) {
- runBatch(argv);
-} else {
- runTest(argv);
-}
+ });
+ }
+};
diff --git a/lib/reporter/csv.js b/lib/reporter/csv.js
new file mode 100644
index 0000000..29dcafd
--- /dev/null
+++ b/lib/reporter/csv.js
@@ -0,0 +1,59 @@
+/**
+ * @fileoverview Report the metrics as CSV.
+ * @author Peter Hedenskog
+ * @copyright (c) 2015, Peter Hedenskog <peterwikimedia.org>.
+ * Released under the Apache 2.0 License.
+ */
+
+'use strict';
+
+var fs = require('fs');
+var eol = require('os').EOL;
+
+module.exports = {
+ /**
+ * Validate the input arguments.
+ * @param {array} argv The input parameters for the run.
+ * @return {boolean} returns true if the argumenst are ok.
+ */
+ validate: function(argv) {
+ return true;
+ },
+ /**
+ * Log help arguments to the console.
+ */
+ help: function() {
+ console.log(' --file The file and path to the file to
write the data. '
+ + 'If the file exists, the data will be appended');
+ },
+ /**
+ * Report the metrics by writing them as a CSV row.
+ * @param {object} collectedMetrics The metrics collected from the run.
+ * @param {array} argv The input parameters for the run.
+ */
+ report: function(collectedMetrics, argv) {
+
+ var keys = 'name,';
+ var values = argv._[0] + ',';
+ Object.keys(collectedMetrics).forEach(function(type) {
+ Object.keys(collectedMetrics[type]).forEach(function(metric) {
+ keys += metric + ',' ;
+ values += collectedMetrics[type][metric] + ',';
+ });
+ });
+ keys = keys.slice(0, -1);
+ values = values.slice(0, -1);
+
+ if (argv.file) {
+ try {
+ fs.statSync(argv.file);
+ fs.appendFileSync(argv.file, values + eol, 'utf8');
+ }
+ catch (error) {
+ // it's new file, lets add the keys as a header
+ fs.appendFileSync(argv.file, keys + eol + values + eol,
'utf8');
+ }
+ console.log('Wrote metrics to ' + argv.file);
+ }
+ }
+};
diff --git a/lib/reporter/graphite.js b/lib/reporter/graphite.js
new file mode 100644
index 0000000..19c83f3
--- /dev/null
+++ b/lib/reporter/graphite.js
@@ -0,0 +1,61 @@
+/**
+ * @fileoverview Report the metrics to Graphite.
+ * @author Peter Hedenskog
+ * @copyright (c) 2015, Peter Hedenskog <peterwikimedia.org>.
+ * Released under the Apache 2.0 License.
+ */
+
+'use strict';
+
+var net = require('net');
+
+module.exports = {
+ /**
+ * Validate the input arguments.
+ * @param {array} argv The input parameters for the run.
+ * @return {boolean} returns true if the argumenst are ok.
+ */
+ validate: function(argv) {
+ if (!argv.graphiteHost) {
+ console.error('Missing configuration for --graphiteHost');
+ }
+ return true;
+ },
+ /**
+ * Log help arguments to the console.
+ */
+ help: function() {
+ console.log(' --graphiteHost The Graphite hostname');
+ console.log(' --graphitePort The Graphite port [2003]');
+ },
+ /**
+ * Report the metrics by writing them as JSON.
+ * @param {object} collectedMetrics The metrics collected from the run.
+ * @param {array} argv The input parameters for the run.
+ */
+ report: function(metrics, argv) {
+ var port = argv.graphitePort || 2003;
+ var host = argv.graphiteHost;
+ var server = net.createConnection(port, host);
+ server.addListener('error', function(error) {
+ console.error('Could not send data to Graphite:' + error);
+ });
+
+ var timeStamp = ' ' + Math.round(new Date().getTime() / 1000) + '\n';
+ var data = '';
+ Object.keys(metrics).forEach(function(type) {
+ Object.keys(metrics[type]).forEach(function(metric) {
+ data += metric + ' ' + metrics[type][metric] + timeStamp;
+ });
+ });
+
+ if (argv.verbose) {
+ console.log(data);
+ }
+
+ server.on('connect', function() {
+ this.write(data);
+ this.end();
+ });
+ }
+};
diff --git a/lib/reporter/index.js b/lib/reporter/index.js
new file mode 100644
index 0000000..f5621d7
--- /dev/null
+++ b/lib/reporter/index.js
@@ -0,0 +1,43 @@
+/**
+ * @fileoverview Hold and know which reporters that exists.
+ * @author Peter Hedenskog
+ * @copyright (c) 2015, Peter Hedenskog <peterwikimedia.org>.
+ * Released under the Apache 2.0 License.
+ */
+
+'use strict';
+
+var csv = require('./csv');
+var json = require('./json');
+var statsv = require('./statsv');
+var graphite = require('./graphite');
+
+var reporters = {
+ csv: csv,
+ json: json,
+ graphite: graphite,
+ statsv: statsv
+};
+
+/**
+ * Get all reporters that exists
+ * @return {object} key/value for all reporters.
+*/
+module.exports.getReporters = function() {
+ return reporters;
+};
+
+/**
+ * Get a specifc reporter.
+ * @param {string} the name of the reporter
+ * @return {object} the reporter and null if the name doesn't matcj
+*/
+module.exports.get = function(name) {
+ var reporter = reporters[name];
+
+ if (!reporter) {
+ return null;
+ }
+
+ return reporter;
+};
diff --git a/lib/reporter/json.js b/lib/reporter/json.js
new file mode 100644
index 0000000..88a805a
--- /dev/null
+++ b/lib/reporter/json.js
@@ -0,0 +1,32 @@
+/**
+ * @fileoverview Report the metrics as JSON.
+ * @author Peter Hedenskog
+ * @copyright (c) 2015, Peter Hedenskog <peterwikimedia.org>.
+ * Released under the Apache 2.0 License.
+ */
+
+'use strict';
+module.exports = {
+ /**
+ * Validate the input arguments.
+ * @param {array} argv The input parameters for the run.
+ * @return {boolean} returns true if the argumenst are ok.
+ */
+ validate: function(argv) {
+ return true;
+ },
+ /**
+ * Log help arguments to the console.
+ */
+ help: function() {
+
+ },
+ /**
+ * Report the metrics by writing them as JSON.
+ * @param {object} collectedMetrics The metrics collected from the run.
+ * @param {array} argv The input parameters for the run.
+ */
+ report: function(metrics, argv) {
+ console.log(JSON.stringify(metrics,null, 2));
+ }
+};
diff --git a/lib/reporter/statsv.js b/lib/reporter/statsv.js
new file mode 100644
index 0000000..82df0a5
--- /dev/null
+++ b/lib/reporter/statsv.js
@@ -0,0 +1,70 @@
+/**
+ * @fileoverview Report the metrics to statsv.
+ * @author Peter Hedenskog
+ * @copyright (c) 2015, Peter Hedenskog <peterwikimedia.org>.
+ * Released under the Apache 2.0 License.
+ */
+
+'use strict';
+var request = require('request');
+
+module.exports = {
+ /**
+ * Validate the input arguments.
+ * @param {array} argv The input parameters for the run.
+ * @return {boolean} returns true if the argumenst are ok.
+ */
+ validate: function(argv) {
+ return true;
+ },
+ /**
+ * Log help arguments to the console.
+ */
+ help: function() {
+ console.log(' --endpoint Where to send the statsv metrics
' +
+ '[https://www.example.com]');
+ console.log(' --sendMetrics Send metrics to statsv or not.
Set to send ' +
+ 'metrics.');
+ },
+ /**
+ * Report the metrics by sending them to statsv.
+ * @param {object} collectedMetrics The metrics collected from the run.
+ * @param {array} argv The input parameters for the run.
+ */
+ report: function(metrics, argv) {
+ var endpoint = argv.endpoint || 'https://www.example.com';
+ var flatten = {};
+ // flatten the structure
+ Object.keys(metrics).forEach(function(type) {
+ Object.keys(metrics[type]).forEach(function(metric) {
+ flatten[metric] = metrics[type][metric] + ((type ===
'timings') ? 'ms' : 'g');
+ });
+ });
+
+
+ // Lets do something smarter in the future, now
+ // cut after 5 keys and send a new request
+ var MAX_KEYS_PER_REQUEST = 5;
+ var url = endpoint + '?';
+
+ var keys = Object.keys(flatten);
+ for (var i = 0; i < keys.length; i++) {
+
+ url += keys[i] + '=' + flatten[keys[i]] + '&';
+ // don't send first, and then for each MAX_KEYS_PER_REQUEST
+ // and the last time
+ if (i !== 0 && i % MAX_KEYS_PER_REQUEST === 0 || (i + 1 ===
flatten.length)) {
+ url = url.slice(0, -1);
+ console.log(url);
+ request(url, function(error, response, body) { // jshint
unused:false
+ if (!error) {
+ console.log('Succesfully sent metrics.');
+ } else {
+ console.error(error);
+ }
+ });
+ url = endpoint + '?';
+ }
+ }
+ }
+};
diff --git a/lib/util.js b/lib/util.js
index 2e5f77e..ed28396 100644
--- a/lib/util.js
+++ b/lib/util.js
@@ -1,74 +1,22 @@
-/*
-wptstatsv
-~~~~~~~
-A thin wrapper for the WebPageTest API that sends metrics to statsv.
-
-Copyright 2015 Peter Hedenskog <[email protected]>
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
+/**
+ * @fileoverview Utilities file holding help methods.
+ * @author Peter Hedenskog
+ * @copyright (c) 2015, Peter Hedenskog <peterwikimedia.org>.
+ * Released under the Apache 2.0 License.
+ */
'use strict';
-var request = require('request');
var fs = require('fs');
var path = require('path');
var minimist = require('minimist');
-var NAMESPACE = 'webpagetest';
-var BROWSERS = {
- 'Google Chrome': 'chrome',
- Firefox: 'firefox',
- 'Internet Explorer': 'ie',
- Safari: 'safari'
-};
-
-
module.exports = {
- // Here are the values we collect. Want to add more? Check the JSON that
is returned:
- //
https://sites.google.com/a/webpagetest.org/docs/advanced-features/webpagetest-restful-apis
- // #TOC-Sample
- // Not 100% sure it's the latest though. Test by logging the output from
WebPageTest
- // Note: It can differs depending on what agent that runs the tests.
- // Note 2: Make sure we don't hit the statsv limit of maximum chars in one
request
- METRICS: ['SpeedIndex', 'render', 'TTFB', 'fullyLoaded'],
- ASSET_TYPES: ['html','js','css','image'],
- sendMetrics: function(metrics, endpoint) {
-
- // Lets do something smarter in the future, now
- // cut after 5 keys and send a new request
- var MAX_KEYS_PER_REQUEST = 5;
- var url = endpoint + '?';
-
- var keys = Object.keys(metrics);
- for (var i = 0; i < keys.length; i++) {
-
- url += keys[i] + '=' + metrics[keys[i]] + '&';
- // don't send first, and then for each MAX_KEYS_PER_REQUEST
- // and the last time
- if (i !== 0 && i % MAX_KEYS_PER_REQUEST === 0 || (i + 1 ===
keys.length)) {
- url = url.slice(0, -1);
- request(url, function(error, response, body) { // jshint
unused:false
- if (!error) {
- console.log('Succesfully sent metrics.');
- } else {
- console.error(error);
- }
- });
- url = endpoint + '?';
- }
- }
- },
+ /**
+ * Create a WebPageTest options object from input arguments.
+ * @param {array} argv The arguments for the run
+ * @return {object} the WebPageTest options object
+ */
setupWPTOptions: function(argv) {
// some default options here
var wptOptions = {
@@ -89,6 +37,11 @@
return wptOptions;
},
+ /**
+ * Convert an input text line from a file to a minimist created argument
object
+ * @param {string} line The text line
+ * @return {object} a minimist argument object
+ */
convertTextLineToMinimist: function(line) {
// replace variables with env variables
line = this.replaceWithEnv(line);
@@ -113,11 +66,23 @@
}
return myArgs;
},
+ /**
+ * Read a text file (utf-8) and retunr the content.
+ * @param {string} filename The file and path to the file to read
+ * @return {string} the text file
+ */
readFile: function(filename) {
var fullPathToFile = (filename.charAt(0) === path.sep) ? filename :
path.join(process.cwd(),
path.sep, filename);
return fs.readFileSync(fullPathToFile, 'utf-8');
},
+ /**
+ * Replace all occurrences placeholders matching <%YOUR_KEY>
+ * with a matching named environment variable. Log error if we
+ * have placeholders but no matching variables.
+ * @param {string} text The text to replae
+ * @return {string} the text with replaced placeholders
+ */
replaceWithEnv: function(text) {
var matches = text.match(/<(.*?)>/g);
if (matches) {
@@ -135,63 +100,10 @@
}
return text;
},
- collectMetrics: function(wptJson, userStatus, namespace, argv) {
-
- var self = this;
- var metricsToSend = {};
-
- var emulateMobile = argv.emulateMobile;
- var firstViewOnly = argv.first;
-
- var views = ['firstView'];
- if (!firstViewOnly) {
- views.push('repeatView');
- }
-
- views.forEach(function(view) {
- // if we are missing browser info from WPT (happens when using
MotoG at least)
- // use a normalized version of the location
- // the browser/location can then look like
Dulles_MotoG_Motorola_G___Chrome
- // but since there are no standard of naming it should be ok to
just use what we got
- var browser = BROWSERS[wptJson.data.median[view].browser_name] ||
- wptJson.data.location.replace(/[^A-Za-z_0-9]/g, '_');
-
- if (emulateMobile) {
- browser += '-emulateMobile';
- }
-
- // the actual location is the first part of the location string
- // separated by either a : or _
- var location = wptJson.data.location.split(/:|_/)[0];
-
- var keyStart = namespace + '.' + location + '.' + userStatus + '.'
+ browser +
- '.' + view + '.';
-
- self.METRICS.forEach(function(metric) {
- metricsToSend[keyStart + metric ] =
wptJson.data.median[view][metric] + 'ms';
- });
-
- if (wptJson.data.median[view].userTimes) {
-
Object.keys(wptJson.data.median[view].userTimes).forEach(function(userTiming) {
- metricsToSend[keyStart + userTiming ] =
- wptJson.data.median[view].userTimes[userTiming] +
- 'ms';
- });
- }
-
- // collect sizes & assets
- self.ASSET_TYPES.forEach(function(assetType) {
- metricsToSend[keyStart +
- assetType + '.requests' ] =
wptJson.data.median[view].breakdown[assetType].
- requests + 'g';
- metricsToSend[keyStart +
- assetType + '.bytes' ] =
wptJson.data.median[view].breakdown[assetType].bytes + 'g';
- });
-
- });
-
- return metricsToSend;
- },
+ /**
+ * Get the current date and time in UTC.
+ * @return {string} the time with the format YYYY-MM-DD HH.MM
+ */
getTodaysDate: function() {
function addPadding(number) {
@@ -209,5 +121,5 @@
var minutes = addPadding(date.getUTCMinutes());
return date.getUTCFullYear() + '-' + month + '-' + day + ' ' + hours +
'.' + minutes;
- },
+ }
};
diff --git a/lib/wpt.js b/lib/wpt.js
new file mode 100644
index 0000000..75a22f7
--- /dev/null
+++ b/lib/wpt.js
@@ -0,0 +1,34 @@
+/**
+ * @fileoverview The functionallity to fetch data from WebPageTest.
+ * @author Peter Hedenskog
+ * @copyright (c) 2015, Peter Hedenskog <peterwikimedia.org>.
+ * Released under the Apache 2.0 License.
+ */
+
+'use strict';
+
+var WebPageTest = require('webpagetest');
+
+module.exports = {
+ run: function(host, key, argv, input, wptOptions, cb) {
+ var wpt = new WebPageTest(host, key);
+
+ wpt.runTest(input, wptOptions, function(err, data) {
+
+ if (argv.verbose) {
+ console.log(JSON.stringify(data, null, 1));
+ }
+
+ if (err) {
+ console.error('Couldn\'t fetch data from WebPageTest:' +
JSON.stringify(err));
+ console.error('Configuration:' + JSON.stringify(wptOptions,
null, 2));
+ cb(err);
+ return;
+ } else {
+ console.log('WebPageTest run: ' + data.data.summary);
+ cb(null, data);
+ }
+ });
+
+ }
+};
diff --git a/package.json b/package.json
index f2dcfb7..1c98411 100644
--- a/package.json
+++ b/package.json
@@ -1,29 +1,44 @@
{
- "name": "wptstatsv",
- "version": "0.0.1",
- "bin": "./lib/index.js",
- "description": "A thin wrapper for the WebPageTest API that sends metrics to
statsv",
- "license": "Apache-2.0",
- "repository": {
- "type": "git",
- "url": ""
- },
- "scripts": {
- "test": "mocha"
- },
- "engines": {
- "node": ">=0.10.7"
- },
- "devDependencies": {
- "mocha": "^2.2.5",
- "mocha-jscs": "^2.0.0",
- "mocha-jshint": "^2.2.3"
- },
- "main": "./lib/index.js",
- "dependencies": {
- "async": "1.4.2",
- "minimist": "1.1.3",
- "request": "2.61.0",
- "webpagetest": "0.3.3"
- }
-}
+ "name": "wpt",
+ "version": "0.0.1",
+ "bin": "./bin/index.js",
+ "description": "A thin wrapper for the WebPageTest API that report
specific metrics as csv/json or send them to Graphite/Statsv",
+ "keywords": [
+ "webpagetest",
+ "performance",
+ "reporter",
+ "graphite",
+ "csv",
+ "perfmatters",
+ "webperf"
+ ],
+ "homepage": "https://wikitech.wikimedia.org/wiki/WebPageTest";,
+ "license": "Apache-2.0",
+ "repository": {
+ "type": "git",
+ "url": "https://gerrit.wikimedia.org/r/p/performance/WebPageTest.git";
+ },
+ "files": [
+ "bin",
+ "lib"
+ ],
+ "scripts": {
+ "test": "mocha"
+ },
+ "engines": {
+ "node": ">=0.10.7"
+ },
+ "devDependencies": {
+ "mocha": "^2.2.5",
+ "mocha-jscs": "^2.0.0",
+ "mocha-jshint": "^2.2.3",
+ "mockery": "1.4.0"
+ },
+ "main": "./lib/index.js",
+ "dependencies": {
+ "async": "1.4.2",
+ "minimist": "1.1.3",
+ "request": "2.61.0",
+ "webpagetest": "0.3.3"
+ }
+}
\ No newline at end of file
diff --git a/scripts/batch/desktop.txt b/scripts/batch/desktop.txt
index d5b3ba1..9472f02 100644
--- a/scripts/batch/desktop.txt
+++ b/scripts/batch/desktop.txt
@@ -10,28 +10,28 @@
# $ WMF_WPT_KEY=SECRET_KEY STATSV_ENDPOINT=http://localhost WPT_RUNS=1 node
WMF_WPT_LOCATION=us-west-1 lib/index.js --batch scripts/batch/desktop.txt
# Collect metrics using Chrome
---webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Chrome --label chrome --runs <%WPT_RUNS> --endpoint
<%STATSV_ENDPOINT> --sendMetrics --namespace webpagetest.enwiki.Main_Page
https://en.wikipedia.org/wiki/Main_Page
+--webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Chrome --label chrome --runs <%WPT_RUNS> --endpoint
<%STATSV_ENDPOINT> --sendMetrics --namespace webpagetest.enwiki.Main_Page
--reporter statsv https://en.wikipedia.org/wiki/Main_Page
---webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Chrome --label chrome --runs <%WPT_RUNS> --endpoint
<%STATSV_ENDPOINT> --sendMetrics --namespace webpagetest.enwiki.Facebook
https://en.wikipedia.org/wiki/Facebook
+--webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Chrome --label chrome --runs <%WPT_RUNS> --endpoint
<%STATSV_ENDPOINT> --sendMetrics --namespace webpagetest.enwiki.Facebook
--reporter statsv https://en.wikipedia.org/wiki/Facebook
---webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Chrome --label chrome --runs <%WPT_RUNS> --endpoint
<%STATSV_ENDPOINT> --sendMetrics --namespace webpagetest.test2wiki.signup
https://test2.wikipedia.org/w/index.php?title=Special:UserLogin&type=signup
+--webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Chrome --label chrome --runs <%WPT_RUNS> --endpoint
<%STATSV_ENDPOINT> --sendMetrics --namespace webpagetest.test2wiki.signup
--reporter statsv
https://test2.wikipedia.org/w/index.php?title=Special:UserLogin&type=signup
---webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Chrome --label chrome-base --runs <%WPT_RUNS> --endpoint
<%STATSV_ENDPOINT> --sendMetrics --namespace webpagetest.enwiki.BlankPage
https://en.wikipedia.org/wiki/Special:BlankPage
+--webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Chrome --label chrome-base --runs <%WPT_RUNS> --endpoint
<%STATSV_ENDPOINT> --sendMetrics --namespace webpagetest.enwiki.BlankPage
--reporter statsv https://en.wikipedia.org/wiki/Special:BlankPage
# Collect metrics using Firefox
---webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Firefox --label ff --runs <%WPT_RUNS> --endpoint
<%STATSV_ENDPOINT> --sendMetrics --namespace webpagetest.enwiki.Main_Page
https://en.wikipedia.org/wiki/Main_Page
+--webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Firefox --label ff --runs <%WPT_RUNS> --endpoint
<%STATSV_ENDPOINT> --sendMetrics --namespace webpagetest.enwiki.Main_Page
--reporter statsv https://en.wikipedia.org/wiki/Main_Page
---webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Firefox --label ff --runs <%WPT_RUNS> --endpoint
<%STATSV_ENDPOINT> --sendMetrics --namespace webpagetest.enwiki.Facebook
https://en.wikipedia.org/wiki/Facebook
+--webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Firefox --label ff --runs <%WPT_RUNS> --endpoint
<%STATSV_ENDPOINT> --sendMetrics --namespace webpagetest.enwiki.Facebook
--reporter statsv https://en.wikipedia.org/wiki/Facebook
---webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Firefox --label ff --runs <%WPT_RUNS> --endpoint
<%STATSV_ENDPOINT> --sendMetrics --namespace webpagetest.test2wiki.signup
https://test2.wikipedia.org/w/index.php?title=Special:UserLogin&type=signup
+--webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Firefox --label ff --runs <%WPT_RUNS> --endpoint
<%STATSV_ENDPOINT> --sendMetrics --namespace webpagetest.test2wiki.signup
--reporter statsv
https://test2.wikipedia.org/w/index.php?title=Special:UserLogin&type=signup
---webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Firefox --label ff-base --runs <%WPT_RUNS> --endpoint
<%STATSV_ENDPOINT> --sendMetrics --namespace webpagetest.enwiki.BlankPage
https://en.wikipedia.org/wiki/Special:BlankPage
+--webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Firefox --label ff-base --runs <%WPT_RUNS> --endpoint
<%STATSV_ENDPOINT> --sendMetrics --namespace webpagetest.enwiki.BlankPage
--reporter statsv https://en.wikipedia.org/wiki/Special:BlankPage
# Collect metrics using IE <%WPT_RUNS> (running windows 7 = no SPDY)
---webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>_IE11 --label ie11 --runs <%WPT_RUNS> --endpoint
<%STATSV_ENDPOINT> --sendMetrics --namespace webpagetest.enwiki.Main_Page
https://en.wikipedia.org/wiki/Main_Page
+--webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>_IE11 --label ie11 --runs <%WPT_RUNS> --endpoint
<%STATSV_ENDPOINT> --sendMetrics --namespace webpagetest.enwiki.Main_Page
--reporter statsv https://en.wikipedia.org/wiki/Main_Page
---webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>_IE11 --label ie11 --runs <%WPT_RUNS> --endpoint
<%STATSV_ENDPOINT> --sendMetrics --namespace webpagetest.enwiki.Facebook
https://en.wikipedia.org/wiki/Facebook
+--webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>_IE11 --label ie11 --runs <%WPT_RUNS> --endpoint
<%STATSV_ENDPOINT> --sendMetrics --namespace webpagetest.enwiki.Facebook
--reporter statsv https://en.wikipedia.org/wiki/Facebook
---webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>_IE11 --label ie11 --runs <%WPT_RUNS> --endpoint
<%STATSV_ENDPOINT> --sendMetrics --namespace webpagetest.test2wiki.signup
https://test2.wikipedia.org/w/index.php?title=Special:UserLogin&type=signup
+--webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>_IE11 --label ie11 --runs <%WPT_RUNS> --endpoint
<%STATSV_ENDPOINT> --sendMetrics --namespace webpagetest.test2wiki.signup
--reporter statsv
https://test2.wikipedia.org/w/index.php?title=Special:UserLogin&type=signup
---webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>_IE11 --label ie11-base --runs <%WPT_RUNS> --endpoint
<%STATSV_ENDPOINT> --sendMetrics --namespace webpagetest.enwiki.BlankPage
https://en.wikipedia.org/wiki/Special:BlankPage
+--webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>_IE11 --label ie11-base --runs <%WPT_RUNS> --endpoint
<%STATSV_ENDPOINT> --sendMetrics --namespace webpagetest.enwiki.BlankPage
--reporter statsv https://en.wikipedia.org/wiki/Special:BlankPage
diff --git a/scripts/batch/login-desktop.txt b/scripts/batch/login-desktop.txt
index 03de767..eff7518 100644
--- a/scripts/batch/login-desktop.txt
+++ b/scripts/batch/login-desktop.txt
@@ -13,4 +13,4 @@
# Example (make sure to change WMF_WPT_KEY, and WPT_USER_PASSWORD)
# $ WMF_WPT_KEY=SECRET_KEY STATSV_ENDPOINT=http://localhost WPT_RUNS=1
WPT_USER=wptuser WPT_USER_PASSWORD=SECRET_PASSWORD WMF_WPT_LOCATION=us-west-1
node lib/index.js --batch ./scripts/batch/login-desktop.txt
---webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Chrome --label chrome-authenticated --runs <%WPT_RUNS>
--first true --userStatus authenticated --endpoint <%STATSV_ENDPOINT>
--sendMetrics --namespace webpagetest.enwiki.Facebook
./scripts/wptscripts/login-desktop-enwiki-facebook.txt
+--webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Chrome --label chrome-authenticated --runs <%WPT_RUNS>
--first true --userStatus authenticated --endpoint <%STATSV_ENDPOINT>
--sendMetrics --namespace webpagetest.enwiki.Facebook --reporter statsv
./scripts/wptscripts/login-desktop-enwiki-facebook.txt
diff --git a/scripts/batch/login-mobile.txt b/scripts/batch/login-mobile.txt
index b228367..6040ca4 100644
--- a/scripts/batch/login-mobile.txt
+++ b/scripts/batch/login-mobile.txt
@@ -13,4 +13,4 @@
# Example (make sure to change WMF_WPT_KEY, and WPT_USER_PASSWORD)
# $ WMF_WPT_KEY=SECRET_KEY STATSV_ENDPOINT=http://localhost WPT_RUNS=1
WPT_USER=wptuser WPT_USER_PASSWORD=SECRET_PASSWORD WMF_WPT_LOCATION=us-west-1
node lib/index.js --batch ./scripts/batch/login-mobile.txt
---webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Chrome --label chrome-mobile-authenticated --runs
<%WPT_MOBILE_RUNS> --first true --emulateMobile true --userStatus authenticated
--endpoint <%STATSV_ENDPOINT> --sendMetrics --connectivity 3GFast --namespace
webpagetest.enwiki-mobile.Facebook
./scripts/wptscripts/login-mobile-enwiki-facebook.txt
+--webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Chrome --label chrome-mobile-authenticated --runs
<%WPT_MOBILE_RUNS> --first true --emulateMobile true --userStatus authenticated
--endpoint <%STATSV_ENDPOINT> --sendMetrics --connectivity 3GFast --namespace
webpagetest.enwiki-mobile.Facebook --reporter statsv
./scripts/wptscripts/login-mobile-enwiki-facebook.txt
diff --git a/scripts/batch/mobile-wpt-org.txt b/scripts/batch/mobile-wpt-org.txt
index c233386..37d5d7c 100644
--- a/scripts/batch/mobile-wpt-org.txt
+++ b/scripts/batch/mobile-wpt-org.txt
@@ -11,4 +11,4 @@
# nodejs installed), just make sure to change the value of the WebPageTest key:
# $ WPT_ORG_WPT_KEY=SECRET_KEY STATSV_ENDPOINT=http://localhost
WPT_ORG_MOBILE_RUNS=1 node lib/index.js --batch
./scripts/batch/mobile-wpt-org.txt
---webPageTestKey <%WPT_ORG_WPT_KEY> --webPageTestHost www.webpagetest.org
--median SpeedIndex --location Dulles_MotoG:Motorola G - Chrome --label
chrome-m --runs <%WPT_ORG_MOBILE_RUNS> --endpoint <%STATSV_ENDPOINT>
--sendMetrics --connectivity 3GFast --namespace
webpagetest.enwiki-mobile.Facebook --timeout 2400
https://en.m.wikipedia.org/wiki/Facebook
+--webPageTestKey <%WPT_ORG_WPT_KEY> --webPageTestHost www.webpagetest.org
--median SpeedIndex --location Dulles_MotoG:Motorola G - Chrome --label
chrome-m --runs <%WPT_ORG_MOBILE_RUNS> --endpoint <%STATSV_ENDPOINT>
--sendMetrics --connectivity 3GFast --namespace
webpagetest.enwiki-mobile.Facebook --timeout 2400 --reporter statsv
https://en.m.wikipedia.org/wiki/Facebook
diff --git a/scripts/batch/mobile.txt b/scripts/batch/mobile.txt
index 5044039..6d2123c 100644
--- a/scripts/batch/mobile.txt
+++ b/scripts/batch/mobile.txt
@@ -11,10 +11,10 @@
# Example (make sure to change WMF_WPT_KEY)
# $ WMF_WPT_KEY=SECRET_KEY STATSV_ENDPOINT=http://localhost WPT_MOBILE_RUNS=1
WMF_WPT_LOCATION=us-west-1 node lib/index.js --batch ./scripts/batch/mobile.txt
---webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Chrome --label chrome-emulated-m --runs <%WPT_MOBILE_RUNS>
--endpoint <%STATSV_ENDPOINT> --sendMetrics --namespace
webpagetest.enwiki-mobile.San_Francisco --emulateMobile true --connectivity
3GFast https://en.m.wikipedia.org/wiki/San_Francisco
+--webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Chrome --label chrome-emulated-m --runs <%WPT_MOBILE_RUNS>
--endpoint <%STATSV_ENDPOINT> --sendMetrics --namespace
webpagetest.enwiki-mobile.San_Francisco --emulateMobile true --connectivity
3GFast --reporter statsv https://en.m.wikipedia.org/wiki/San_Francisco
---webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Chrome --label chrome-emulated-m --runs <%WPT_MOBILE_RUNS>
--endpoint <%STATSV_ENDPOINT> --sendMetrics --namespace
webpagetest.enwiki-mobile.Facebook --emulateMobile true --connectivity 3GFast
https://en.m.wikipedia.org/wiki/Facebook
+--webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Chrome --label chrome-emulated-m --runs <%WPT_MOBILE_RUNS>
--endpoint <%STATSV_ENDPOINT> --sendMetrics --namespace
webpagetest.enwiki-mobile.Facebook --emulateMobile true --connectivity 3GFast
--reporter statsv https://en.m.wikipedia.org/wiki/Facebook
---webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Chrome --label chrome-emulated-m --runs <%WPT_MOBILE_RUNS>
--endpoint <%STATSV_ENDPOINT> --sendMetrics --namespace
webpagetest.test2wiki-mobile.signup --emulateMobile true --connectivity 3GFast
https://test2.m.wikipedia.org/w/index.php?title=Special:UserLogin&type=signup
+--webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Chrome --label chrome-emulated-m --runs <%WPT_MOBILE_RUNS>
--endpoint <%STATSV_ENDPOINT> --sendMetrics --namespace
webpagetest.test2wiki-mobile.signup --emulateMobile true --connectivity 3GFast
--reporter statsv
https://test2.m.wikipedia.org/w/index.php?title=Special:UserLogin&type=signup
---webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Chrome --label chrome-emulated-m --runs <%WPT_MOBILE_RUNS>
--endpoint <%STATSV_ENDPOINT> --sendMetrics --namespace
webpagetest.enwiki-mobile.BlankPage --emulateMobile true --connectivity 3GFast
https://en.m.wikipedia.org/wiki/Special:BlankPage
+--webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Chrome --label chrome-emulated-m --runs <%WPT_MOBILE_RUNS>
--endpoint <%STATSV_ENDPOINT> --sendMetrics --namespace
webpagetest.enwiki-mobile.BlankPage --emulateMobile true --connectivity 3GFast
--reporter statsv https://en.m.wikipedia.org/wiki/Special:BlankPage
diff --git a/scripts/batch/second-view-desktop.txt
b/scripts/batch/second-view-desktop.txt
index 3822555..ad5766e 100644
--- a/scripts/batch/second-view-desktop.txt
+++ b/scripts/batch/second-view-desktop.txt
@@ -11,4 +11,4 @@
# Example (make sure to change WMF_WPT_KEY)
# $ WMF_WPT_KEY=SECRET_KEY STATSV_ENDPOINT=http://localhost WPT_RUNS=1
WMF_WPT_LOCATION=us-west-1 node lib/index.js --batch
./scripts/scripts/second-view-desktop.txt
---webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Chrome --label chrome-second --runs <%WPT_RUNS> --first
true --endpoint <%STATSV_ENDPOINT> --sendMetrics --namespace
webpagetest.enwiki.Facebook-second
./scripts/wptscripts/second-view-desktop-enwiki-facebook.txt
+--webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Chrome --label chrome-second --runs <%WPT_RUNS> --first
true --endpoint <%STATSV_ENDPOINT> --sendMetrics --namespace
webpagetest.enwiki.Facebook-second --reporter statsv
./scripts/wptscripts/second-view-desktop-enwiki-facebook.txt
diff --git a/scripts/batch/second-view-mobile.txt
b/scripts/batch/second-view-mobile.txt
index 10d7fe6..89d0726 100644
--- a/scripts/batch/second-view-mobile.txt
+++ b/scripts/batch/second-view-mobile.txt
@@ -9,4 +9,4 @@
# Example (make sure to change WMF_WPT_KEY)
# $ WMF_WPT_KEY=SECRET_KEY STATSV_ENDPOINT=http://localhost WPT_MOBILE_RUNS=1
WMF_WPT_LOCATION=us-west-1 node lib/index.js --batch
./scripts/batch/second-view-mobile.txt
---webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Chrome --label chrome-mobile-second --runs
<%WPT_MOBILE_RUNS> --first true --emulateMobile true --endpoint
<%STATSV_ENDPOINT> --sendMetrics --connectivity 3GFast --namespace
webpagetest.enwiki-mobile.Facebook-second
./scripts/wptscripts/second-view-mobile-enwiki-facebook.txt
+--webPageTestKey <%WMF_WPT_KEY> --median SpeedIndex --location
<%WMF_WPT_LOCATION>:Chrome --label chrome-mobile-second --runs
<%WPT_MOBILE_RUNS> --first true --emulateMobile true --endpoint
<%STATSV_ENDPOINT> --sendMetrics --connectivity 3GFast --namespace
webpagetest.enwiki-mobile.Facebook-second --reporter statsv
./scripts/wptscripts/second-view-mobile-enwiki-facebook.txt
diff --git a/test/batchTest.js b/test/batchTest.js
new file mode 100644
index 0000000..5a05e2a
--- /dev/null
+++ b/test/batchTest.js
@@ -0,0 +1,36 @@
+var underTest = require('../lib/');
+var mockery = require('mockery');
+var util = require('../lib/util');
+var desktopJson = JSON.parse(util.readFile('test/files/desktop_result.json'));
+
+var wptMock = {
+ run: function(host, key, argv, input, wptOptions, cb) {
+ cb(desktopJson);
+ }
+};
+
+//
+mockery.registerAllowable(underTest);
+mockery.registerMock('./wpt', wptMock);
+
+describe('Test the batch functionality', function() {
+ before(function() {
+ mockery.enable({
+ useCleanCache: true,
+ warnOnReplace: false,
+ warnOnUnregistered: false
+ });
+ });
+ it('A batch file should run through cleanly', function() {
+ var argv = {
+ batch: 'test/files/batch.txt'
+ };
+ var test = require('../lib/');
+ test.runBatch(argv, function() {});
+
+ });
+
+ after(function() {
+ mockery.disable();
+ });
+});
diff --git a/test/cliTest.js b/test/cliTest.js
index 0b172c6..4121884 100644
--- a/test/cliTest.js
+++ b/test/cliTest.js
@@ -20,8 +20,8 @@
'use strict';
-var cli = require('../lib/cli'),
-assert = require('assert');
+var cli = require('../lib/cli');
+var assert = require('assert');
// var apa = require("mocha-jscs")(["./lib"]);
@@ -52,7 +52,7 @@
});
it('Having both WebPageTestKey and a URL should be ok', function() {
- var argv = { webPageTestKey: 'thisIsMySuperSecretKey' };
+ var argv = { webPageTestKey: 'thisIsMySuperSecretKey', reporter:
'json' };
argv._ = ['https://www.wikipedia.org/'];
assert.strictEqual(cli.validateArgs(argv),true);
});
diff --git a/test/collectMetricsTest.js b/test/collectMetricsTest.js
new file mode 100644
index 0000000..1b3dc0a
--- /dev/null
+++ b/test/collectMetricsTest.js
@@ -0,0 +1,75 @@
+var cm = require('../lib/collectMetrics');
+var assert = require('assert');
+var util = require('../lib/util');
+var mobileJson = JSON.parse(util.readFile('test/files/mobile_result.json'));
+var desktopJson = JSON.parse(util.readFile('test/files/desktop_result.json'));
+
+describe('Test colllect metrics', function() {
+
+ it('We should be able to parse a JSON from WebPageTest collecting data
from desktop',
+ function() {
+ var userStatus = 'anonymous';
+ var namespace = 'webpagetest';
+ var metrics = cm.collect(desktopJson, []);
+ Object.keys(metrics).forEach(function(type) {
+ Object.keys(metrics[type]).forEach(function(key) {
+
assert.strictEqual(metrics[type][key].toString().indexOf('undefined'), -1,
+ 'We have an undefined value in ' + key);
+ });
+ });
+
+ // verify that we collect all the metrics that we want
+ cm.METRICS.forEach(function(definedMetric) {
+ var metricIncluded = false;
+ Object.keys(metrics).forEach(function(type) {
+ Object.keys(metrics[type]).forEach(function(key) {
+ if (key.indexOf(definedMetric) > -1) {
+ metricIncluded = true;
+ }
+ });
+ });
+ assert.strictEqual(metricIncluded, true, 'We are missing metric ' +
definedMetric);
+ });
+
+ // verify that we collect all the metrics that we want
+ cm.ASSET_TYPES.forEach(function(definedMetric) {
+ var metricIncluded = false;
+ Object.keys(metrics).forEach(function(type) {
+ Object.keys(metrics[type]).forEach(function(key) {
+ if (key.indexOf(definedMetric) > -1) {
+ metricIncluded = true;
+ }
+ });
+ });
+ assert.strictEqual(metricIncluded, true, 'We are missing asset type
' + definedMetric);
+ });
+ });
+
+ it('We should be able to parse a JSON from WebPageTest collecting data
from mobile',
+ function() {
+ var userStatus = 'anonymous';
+ var namespace = 'webpagetest';
+ var metrics = cm.collect(mobileJson, []);
+ Object.keys(metrics).forEach(function(type) {
+ Object.keys(metrics[type]).forEach(function(key) {
+ // verify that we aren't fetching any undefined values =
+ // values missing in the WPT file
+
assert.strictEqual(metrics[type][key].toString().indexOf('undefined'), -1,
+ 'We have an undefined value in ' + key);
+ });
+ });
+
+ // verify that we collect all the metrics that we want
+ cm.METRICS.forEach(function(definedMetric) {
+ var metricIncluded = false;
+ Object.keys(metrics).forEach(function(type) {
+ Object.keys(metrics[type]).forEach(function(key) {
+ if (key.indexOf(definedMetric) > -1) {
+ metricIncluded = true;
+ }
+ });
+ });
+ assert.strictEqual(metricIncluded, true, 'We are missing metric '
+ definedMetric);
+ });
+ });
+});
diff --git a/test/files/batch.txt b/test/files/batch.txt
index b705377..747bc50 100644
--- a/test/files/batch.txt
+++ b/test/files/batch.txt
@@ -1,3 +1,3 @@
---webPageTestKey <%WMF_WPT_KEY> --runs 15 --location Dulles:Chrome --median
SpeedIndex https://en.wikipedia.org/wiki/Facebook
---webPageTestKey <%WMF_WPT_KEY> --runs 31 --location Dulles_MotoG:Motorola G -
Chrome --median SpeedIndex https://en.wikipedia.org/wiki/Barack_Obama
---webPageTestKey <%WMF_WPT_KEY> --runs 31 --median SpeedIndex
https://en.wikipedia.org/wiki/Barack_Obama
+--webPageTestKey <%WMF_WPT_KEY> --runs 15 --location Dulles:Chrome --median
SpeedIndex --reporter json https://en.wikipedia.org/wiki/Facebook
+--webPageTestKey <%WMF_WPT_KEY> --runs 31 --location Dulles_MotoG:Motorola G -
Chrome --median SpeedIndex --reporter json
https://en.wikipedia.org/wiki/Barack_Obama
+--webPageTestKey <%WMF_WPT_KEY> --runs 31 --median SpeedIndex --reporter json
https://en.wikipedia.org/wiki/Barack_Obama
diff --git a/test/reporterTest.js b/test/reporterTest.js
new file mode 100644
index 0000000..9285b71
--- /dev/null
+++ b/test/reporterTest.js
@@ -0,0 +1,18 @@
+var reporter = require('../lib/reporter');
+var assert = require('assert');
+
+describe('Test reporter modules', function() {
+
+ it('All reporter modules should have the necessary methods', function() {
+
+ Object.keys(reporter.getReporters()).forEach(function(name) {
+ var mod = reporter.get(name);
+ assert.strictEqual(typeof mod.validate === 'function', true,
+ 'The reporter ' + name + ' is missing validate');
+ assert.strictEqual(typeof mod.help === 'function', true,
+ 'The reporter ' + name + ' is missing verify');
+ assert.strictEqual(typeof mod.report === 'function', true,
+ 'The reporter ' + name + ' is missing report');
+ });
+ });
+});
diff --git a/test/utilTest.js b/test/utilTest.js
index 07aaf21..5b77a24 100644
--- a/test/utilTest.js
+++ b/test/utilTest.js
@@ -20,8 +20,6 @@
'use strict';
var util = require('../lib/util');
var assert = require('assert');
-var mobileJson = JSON.parse(util.readFile('test/files/mobile_result.json'));
-var desktopJson = JSON.parse(util.readFile('test/files/desktop_result.json'));
var batchScript = util.readFile('test/files/batch.txt');
var minimist = require('minimist');
var eol = require('os').EOL;
@@ -29,15 +27,15 @@
describe('Test util', function() {
it('Location field should work with and without spaces and without a
location', function() {
- var validValues = ['Dulles:Chrome', 'Dulles_MotoG:Motorola G - Chrome',
undefined];
- var lines = batchScript.split(eol);
+ var validValues = ['Dulles:Chrome', 'Dulles_MotoG:Motorola G -
Chrome', undefined];
+ var lines = batchScript.split(eol);
- for (var i = 0; i < lines.length; i++) {
- if (lines[i].indexOf('#') !== 0 && lines[i].length > 1) {
- var myargs = util.convertTextLineToMinimist(lines[i]);
- assert.strictEqual(myargs.location, validValues[i]);
+ for (var i = 0; i < lines.length; i++) {
+ if (lines[i].indexOf('#') !== 0 && lines[i].length > 1) {
+ var myargs = util.convertTextLineToMinimist(lines[i]);
+ assert.strictEqual(myargs.location, validValues[i]);
+ }
}
- }
});
@@ -52,107 +50,41 @@
assert.deepEqual(wptOptions.location, 'ap-northeast-1_IE10');
assert.deepEqual(wptOptions.connectivity, '3G');
});
- });
- it('There should not be multiple spaces in the WebPageTest options',
function() {
- var lines = batchScript.split(eol);
- for (var i = 0; i < lines.length; i++) {
- if (lines[i].indexOf('#') !== 0 && lines[i].length > 1) {
- // we don't want double spaces
- assert.strictEqual(lines[i].indexOf(' '), -1);
- var myargs = util.convertTextLineToMinimist(lines[i]);
- // and make sure that the array is only having one
- // item (=url or script).
- assert.strictEqual(myargs._.length, 1);
- }
- }
- });
+
+ it('There should not be multiple spaces in the WebPageTest options',
function() {
+ var lines = batchScript.split(eol);
+ for (var i = 0; i < lines.length; i++) {
+ if (lines[i].indexOf('#') !== 0 && lines[i].length > 1) {
+ // we don't want double spaces
+ assert.strictEqual(lines[i].indexOf(' '), -1);
+ var myargs = util.convertTextLineToMinimist(lines[i]);
+ // and make sure that the array is only having one
+ // item (=url or script).
+ assert.strictEqual(myargs._.length, 1);
+ }
+ }
+});
it('Parameters specific for wptstatsv should be cleaned out from
WebPageTest options', function() {
- var keysToBeRemoved = ['webPageTestKey', 'webPageTestHost', '_',
'verbose', 'userStatus', 'sendMetrics', 'customMetrics', 'namespace'];
+ var keysToBeRemoved = ['webPageTestKey', 'webPageTestHost', '_',
'verbose',
+ 'userStatus', 'sendMetrics', 'customMetrics', 'namespace'];
var args = {
- webPageTestKey: 'aSupERSecrEtKey',
- webPageTestHost: 'http://www.example.org',
- _: ['all', 'extra', 'args'],
- verbose: true,
- userStatus: 'anonymous',
- customMetrics: 'javascript that collects custom metrics',
- namespace: 'super.special.namespace'
- };
+ webPageTestKey: 'aSupERSecrEtKey',
+ webPageTestHost: 'http://www.example.org',
+ _: ['all', 'extra', 'args'],
+ verbose: true,
+ userStatus: 'anonymous',
+ customMetrics: 'javascript that collects custom metrics',
+ namespace: 'super.special.namespace'
+ };
var wptOptions = util.setupWPTOptions(args);
keysToBeRemoved.forEach(function(key) {
- assert.strictEqual(wptOptions[key], undefined);
- });
- });
-
-
- it('We should be able to parse a JSON from WebPageTest collecting data from
mobile', function() {
- var userStatus = 'anonymous';
- var namespace = 'webpagetest';
- var metrics = util.collectMetrics(mobileJson, userStatus, namespace, []);
- Object.keys(metrics).forEach(function(key) {
- // verify that we aren't fetching any undefined values = values missing
in the WPT file
- assert.strictEqual(metrics[key].toString().indexOf('undefined'), -1, 'We
have an undefined value in ' + key);
- });
-
- // verify that we collect all the metrics that we want
- util.METRICS.forEach(function(definedMetric) {
- var metricIncluded = false;
- var keys = Object.keys(metrics);
- for (var i = 0; i < keys.length; i++) {
- if (keys[i].indexOf(definedMetric) > -1) {
- metricIncluded = true;
- }
- }
- assert.strictEqual(metricIncluded, true, 'We are missing metric ' +
definedMetric);
- });
- });
-
- it('We should be able to parse a JSON from WebPageTest collecting data
from desktop', function() {
- var userStatus = 'anonymous';
- var namespace = 'webpagetest';
- var metrics = util.collectMetrics(desktopJson, userStatus, namespace, []);
- Object.keys(metrics).forEach(function(metricKey) {
- assert.strictEqual(metrics[metricKey].toString().indexOf('undefined'),
-1, 'We have an undefined value in ' + metricKey);
- });
-
- // verify that we collect all the metrics that we want
- util.METRICS.forEach(function(definedMetric) {
- var included = false;
- Object.keys(metrics).forEach(function(metric) {
- if (metric.indexOf(definedMetric) > -1) {
- included = true;
- }
- });
-
- util.ASSET_TYPES.forEach(function(assetType) {
- Object.keys(metrics).forEach(function(type) {
- if (type.indexOf(assetType) > -1) {
- included = true;
- }
- var userStatus = 'anonymous';
- var namespace = 'webpagetest';
- var metrics = util.collectMetrics(desktopJson, userStatus, namespace,
[]);
- Object.keys(metrics).forEach(function(metric) {
- // verify that we aren't fetching any undefined values = values
missing in the WPT file
-
assert.strictEqual(metrics[metric].toString().indexOf('undefined'),-1,'We have
an undefined value in ' + metric);
- });
-
- // verify that we collect all the metrics that we want
- util.METRICS.forEach(function(definedMetric) {
- var metricIncluded = false;
- Object.keys(metrics).forEach(function(metric) {
- if (metrics[metric].toString().indexOf(definedMetric) > -1) {
- metricIncluded = true;
- }
- });
- assert.strictEqual(included, true, 'We are missing metric ' +
definedMetric);
- });
+ assert.strictEqual(wptOptions[key], undefined);
});
});
- });
it('We should be able to replace ENV variables', function() {
@@ -164,6 +96,5 @@
assert.strictEqual(replacedText.match(/VAR1/g).length, 2);
assert.strictEqual(replacedText.match(/VAR2/g).length, 1);
});
-
});
--
To view, visit https://gerrit.wikimedia.org/r/243027
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I7bd582a4a180b07b62d6c94552555957baf34ce6
Gerrit-PatchSet: 1
Gerrit-Project: performance/WebPageTest
Gerrit-Branch: master
Gerrit-Owner: Phedenskog <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
