Arlolra has uploaded a new change for review.
https://gerrit.wikimedia.org/r/281999
Change subject: Rename timer to stats, since we do counting too
......................................................................
Rename timer to stats, since we do counting too
* Also renames useDefaultPerformanceTimer to useDefaultStatsAggregator
Change-Id: Icfd6a39e93fd882d52cd06adf1b9e927e68aef21
---
M bin/server.js
M lib/api/ParsoidService.js
M lib/api/apiUtils.js
M lib/api/routes.js
M lib/config/ParsoidConfig.js
M lib/html2wt/SelectiveSerializer.js
M localsettings.js.example
M tests/mocha/apitest.localsettings.js
M tests/rttest.localsettings.js
9 files changed, 81 insertions(+), 85 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid
refs/changes/99/281999/1
diff --git a/bin/server.js b/bin/server.js
index a30d045..2a38f83 100755
--- a/bin/server.js
+++ b/bin/server.js
@@ -107,7 +107,7 @@
processLogger.log('fatal', 'uncaught exception', err);
});
-var timer = parsoidConfig.performanceTimer;
+var stats = parsoidConfig.stats;
// Workaround for https://github.com/nodejs/node/pull/3510
var fixClusterHandleLeak = function(worker) {
@@ -177,7 +177,7 @@
"Cpu timeout fetching: %s;
killing worker %s.",
msg.location, pid
));
- if (timer) {
timer.count('worker.exit.SIGKILL', ''); }
+ if (stats) {
stats.count('worker.exit.SIGKILL', ''); }
stopWorker(worker.id);
spawn();
}
@@ -197,7 +197,7 @@
if (!worker.suicide && !shuttingDown) {
var pid = worker.process.pid;
processLogger.log("warning", util.format("worker %s
died (%s), restarting.", pid, signal || code));
- if (timer) { timer.count('worker.exit.' + (signal ||
code), ''); }
+ if (stats) { stats.count('worker.exit.' + (signal ||
code), ''); }
spawn();
}
});
@@ -238,12 +238,12 @@
});
// Send heap usage statistics to Graphite at the requested sample rate
- if (timer && parsoidConfig.heapUsageSampleInterval) {
+ if (stats && parsoidConfig.heapUsageSampleInterval) {
setInterval(function() {
var heapUsage = process.memoryUsage();
- timer.timing('heap.rss', '', heapUsage.rss);
- timer.timing('heap.total', '', heapUsage.heapTotal);
- timer.timing('heap.used', '', heapUsage.heapUsed);
+ stats.timing('heap.rss', '', heapUsage.rss);
+ stats.timing('heap.total', '', heapUsage.heapTotal);
+ stats.timing('heap.used', '', heapUsage.heapUsed);
}, parsoidConfig.heapUsageSampleInterval);
}
diff --git a/lib/api/ParsoidService.js b/lib/api/ParsoidService.js
index ca5a079..f6cd8ec 100644
--- a/lib/api/ParsoidService.js
+++ b/lib/api/ParsoidService.js
@@ -152,8 +152,8 @@
// Count http error codes
app.use(function(req, res, next) {
- var timer = parsoidConfig.performanceTimer;
- if (timer) {
+ var stats = parsoidConfig.stats;
+ if (stats) {
var send;
var clear = function() {
res.removeListener('finish', send);
@@ -163,7 +163,7 @@
send = function() {
var code = String(res.statusCode || 'unknown');
if (code !== '200') {
- timer.count('http.status.' + code, '');
+ stats.count('http.status.' + code, '');
}
clear();
};
diff --git a/lib/api/apiUtils.js b/lib/api/apiUtils.js
index df62116..98653c0 100644
--- a/lib/api/apiUtils.js
+++ b/lib/api/apiUtils.js
@@ -252,10 +252,10 @@
env.log('info', 'started serializing');
// Performance Timing options
- var timer = env.conf.parsoid.performanceTimer;
+ var stats = env.conf.parsoid.stats;
var startTimers;
- if (timer) {
+ if (stats) {
startTimers = new Map();
startTimers.set('html2wt.init', Date.now());
startTimers.set('html2wt.total', Date.now());
@@ -267,11 +267,11 @@
// send domparse time, input size and init time to statsd/Graphite
// init time is the time elapsed before serialization
// init.domParse, a component of init time, is the time elapsed from
html string to DOM tree
- if (timer) {
- timer.timing('html2wt.init.domparse', '',
+ if (stats) {
+ stats.timing('html2wt.init.domparse', '',
Date.now() - startTimers.get('html2wt.init.domparse'));
- timer.timing('html2wt.size.input', '', html.length);
- timer.timing('html2wt.init', '',
+ stats.timing('html2wt.size.input', '', html.length);
+ stats.timing('html2wt.init', '',
Date.now() - startTimers.get('html2wt.init'));
}
@@ -285,7 +285,7 @@
apiUtils.endHtml2wt = function(ret) {
var env = ret.env;
- var timer = env.conf.parsoid.performanceTimer;
+ var stats = env.conf.parsoid.stats;
// var REQ_TIMEOUT = env.conf.parsoid.timeouts.request;
// As per https://www.mediawiki.org/wiki/Parsoid/API#v1_API_entry_points
@@ -297,10 +297,10 @@
return DU.serializeDOM(env, ret.doc.body, useSelser)
// .timeout(REQ_TIMEOUT)
.then(function(output) {
- if (timer) {
- timer.timing('html2wt.total', '',
+ if (stats) {
+ stats.timing('html2wt.total', '',
Date.now() -
ret.startTimers.get('html2wt.total'));
- timer.timing('html2wt.size.output', '', output.length);
+ stats.timing('html2wt.size.output', '', output.length);
}
apiUtils.logTime(env, ret.res, 'serializing');
return output;
@@ -345,10 +345,10 @@
}
// Performance Timing options
- var timer = env.conf.parsoid.performanceTimer;
+ var stats = env.conf.parsoid.stats;
var startTimers;
- if (timer) {
+ if (stats) {
startTimers = new Map();
// init refers to time elapsed before parsing begins
startTimers.set('wt2html.init', Date.now());
@@ -399,11 +399,11 @@
});
apiUtils.redirectToRevision = function(env, res, path, revid) {
- var timer = env.conf.parsoid.performanceTimer;
+ var stats = env.conf.parsoid.stats;
env.log('info', 'redirecting to revision', revid);
- if (timer) {
- timer.count('wt2html.redirectToOldid', '');
+ if (stats) {
+ stats.count('wt2html.redirectToOldid', '');
}
// Don't cache requests with no oldid
@@ -413,7 +413,7 @@
apiUtils.parsePageWithOldid = function(ret) {
var env = ret.env;
- var timer = env.conf.parsoid.performanceTimer;
+ var stats = env.conf.parsoid.stats;
var startTimers = ret.startTimers;
env.log('info', 'started parsing');
@@ -421,11 +421,11 @@
// ease of extraction in clients.
apiUtils.setHeader(ret.res, env, 'content-revision-id', ret.oldid);
- if (timer) {
- timer.timing('wt2html.pageWithOldid.init', '',
+ if (stats) {
+ stats.timing('wt2html.pageWithOldid.init', '',
Date.now() - startTimers.get('wt2html.init'));
startTimers.set('wt2html.pageWithOldid.parse', Date.now());
- timer.timing('wt2html.pageWithOldid.size.input', '',
env.page.src.length);
+ stats.timing('wt2html.pageWithOldid.size.input', '',
env.page.src.length);
}
var expansions = ret.reuse && ret.reuse.expansions;
@@ -449,7 +449,7 @@
apiUtils.parseWt = function(ret) {
var env = ret.env;
var res = ret.res;
- var timer = env.conf.parsoid.performanceTimer;
+ var stats = env.conf.parsoid.stats;
var startTimers = ret.startTimers;
env.log('info', 'started parsing');
@@ -459,11 +459,11 @@
// GET for wikitext parsing
apiUtils.setHeader(res, env, 'Cache-Control',
'private,no-cache,s-maxage=0');
- if (timer) {
- timer.timing('wt2html.wt.init', '',
+ if (stats) {
+ stats.timing('wt2html.wt.init', '',
Date.now() - startTimers.get('wt2html.init'));
startTimers.set('wt2html.wt.parse', Date.now());
- timer.timing('wt2html.wt.size.input', '', ret.wikitext.length);
+ stats.timing('wt2html.wt.size.input', '', ret.wikitext.length);
}
if (!res.locals.pageName) {
@@ -489,7 +489,7 @@
apiUtils.endWt2html = function(ret, doc, output) {
var env = ret.env;
var res = ret.res;
- var timer = env.conf.parsoid.performanceTimer;
+ var stats = env.conf.parsoid.stats;
var startTimers = ret.startTimers;
if (doc) {
@@ -501,17 +501,17 @@
apiUtils.sendResponse(res, env, output);
}
- if (timer) {
+ if (stats) {
if (startTimers.has('wt2html.wt.parse')) {
- timer.timing('wt2html.wt.parse', '',
+ stats.timing('wt2html.wt.parse', '',
Date.now() -
startTimers.get('wt2html.wt.parse'));
- timer.timing('wt2html.wt.size.output', '',
output.length);
+ stats.timing('wt2html.wt.size.output', '',
output.length);
} else if (startTimers.has('wt2html.pageWithOldid.parse')) {
- timer.timing('wt2html.pageWithOldid.parse', '',
+ stats.timing('wt2html.pageWithOldid.parse', '',
Date.now() -
startTimers.get('wt2html.pageWithOldid.parse'));
- timer.timing('wt2html.pageWithOldid.size.output', '',
output.length);
+ stats.timing('wt2html.pageWithOldid.size.output', '',
output.length);
}
- timer.timing('wt2html.total', '',
+ stats.timing('wt2html.total', '',
Date.now() - startTimers.get('wt2html.total'));
}
diff --git a/lib/api/routes.js b/lib/api/routes.js
index 0e81ab3..6a7ea2d 100644
--- a/lib/api/routes.js
+++ b/lib/api/routes.js
@@ -36,9 +36,9 @@
routes.v1Middle = function(req, res, next) {
res.locals.apiVersion = 1;
- var timer = parsoidConfig.performanceTimer;
- if (timer) {
- timer.count('api.version.' + res.locals.apiVersion, '');
+ var stats = parsoidConfig.stats;
+ if (stats) {
+ stats.count('api.version.' + res.locals.apiVersion, '');
}
res.locals.iwp = req.params[0] || parsoidConfig.defaultWiki ||
'';
res.locals.pageName = req.params[1] || 'Main_Page';
@@ -61,9 +61,9 @@
}
res.locals.apiVersion = version;
- var timer = parsoidConfig.performanceTimer;
- if (timer) {
- timer.count('api.version.' + res.locals.apiVersion, '');
+ var stats = parsoidConfig.stats;
+ if (stats) {
+ stats.count('api.version.' + res.locals.apiVersion, '');
}
res.locals.iwp = iwp;
diff --git a/lib/config/ParsoidConfig.js b/lib/config/ParsoidConfig.js
index 6a7a564..8f1cc45 100644
--- a/lib/config/ParsoidConfig.js
+++ b/lib/config/ParsoidConfig.js
@@ -157,20 +157,22 @@
// Make sure all critical required properties are present
this._sanitizeIt();
- // Timer that reports metrics to statsd
- if (this.useDefaultPerformanceTimer) {
+ // Initialize stats aggregator
+ if (this.useDefaultStatsAggregator ||
+ // FIXME: Remove this backwards compatibility some day.
+ this.useDefaultPerformanceTimer) {
var self = this;
Util.StatsD(this.txstatsdHost,
this.txstatsdPort).then(function(pt) {
- self.performanceTimer = pt;
+ self.stats = pt;
}).catch(function(err) {
// FIXME: This isn't going to the logger.
- console.log('Failed to initialize performance timer: '
+ err);
+ console.log('Failed to initialize : ' + err);
});
}
// ParsoidConfig is used across requests. Freeze it to avoid mutation.
var ignoreFields = {
- performanceTimer: true,
+ stats: true,
loggerBackend: true,
};
for (var prop in this) {
@@ -359,9 +361,9 @@
ParsoidConfig.prototype.batchConcurrency = 4;
/**
- * @property {null} Settings for Performance timer.
+ * @property {Object|null} Statistics aggregator, for counting and timing.
*/
-ParsoidConfig.prototype.performanceTimer = null;
+ParsoidConfig.prototype.stats = null;
/**
* @property {string} Default user agent used for making Mediawiki API requests
diff --git a/lib/html2wt/SelectiveSerializer.js
b/lib/html2wt/SelectiveSerializer.js
index 841f068..cf074a6 100644
--- a/lib/html2wt/SelectiveSerializer.js
+++ b/lib/html2wt/SelectiveSerializer.js
@@ -32,7 +32,7 @@
(this.env.conf.parsoid.traceFlags.indexOf("selser") !==
-1);
// Performance Timing option
- this.timer = this.env.conf.parsoid.performanceTimer;
+ this.stats = this.env.conf.parsoid.stats;
};
var SSP = SelectiveSerializer.prototype;
@@ -45,22 +45,22 @@
console.assert(DU.isBody(body), 'Expected a body node.');
var startTimers = new Map();
- var timer = this.timer;
+ var stats = this.stats;
if ((!this.env.page.dom && !this.env.page.domdiff) || this.env.page.src
=== null) {
- if (timer) {
+ if (stats) {
startTimers.set('html2wt.full.serialize', Date.now());
}
// If there's no old source, fall back to non-selective
serialization.
return this.wts.serializeDOM(body, false).tap(function() {
- if (timer) {
- timer.timing('html2wt.full.serialize', '',
+ if (stats) {
+ stats.timing('html2wt.full.serialize', '',
(Date.now() -
startTimers.get('html2wt.full.serialize')));
}
});
} else {
- if (timer) {
+ if (stats) {
startTimers.set('html2wt.selser.serialize', Date.now());
}
@@ -70,14 +70,14 @@
} else {
// Use provided diff-marked DOM (used during testing)
// or generate one (used in production)
- if (timer) {
+ if (stats) {
startTimers.set('html2wt.selser.domDiff',
Date.now());
}
diff = new DOMDiff(this.env).diff(body);
- if (timer) {
- timer.timing('html2wt.selser.domDiff', '',
+ if (stats) {
+ stats.timing('html2wt.selser.domDiff', '',
(Date.now() -
startTimers.get('html2wt.selser.domDiff')));
}
}
@@ -100,8 +100,8 @@
p = this.wts.serializeDOM(body, true);
}
return p.tap(function() {
- if (timer) {
- timer.timing('html2wt.selser.serialize', '',
+ if (stats) {
+ stats.timing('html2wt.selser.serialize', '',
(Date.now() -
startTimers.get('html2wt.selser.serialize')));
}
});
diff --git a/localsettings.js.example b/localsettings.js.example
index e79eb6f..6b39b3a 100644
--- a/localsettings.js.example
+++ b/localsettings.js.example
@@ -70,17 +70,17 @@
// restrict:
//parsoidConfig.allowCORS = 'some.domain.org';
- // Set to true for using the default performance metrics reporting to
statsd
+ // Set to true for using the default metrics reporting to statsd
// If true, provide the statsd host/port values
/*
- parsoidConfig.useDefaultPerformanceTimer = true;
+ parsoidConfig.useDefaultStatsAggregator = true;
parsoidConfig.txstatsdHost = 'statsd.domain.org';
parsoidConfig.txstatsdPort = 8125;
*/
- // Alternatively, define performanceTimer as follows:
+ // Alternatively, define `stats` as follows:
/*
- parsoidConfig.performanceTimer = {
+ parsoidConfig.stats = {
timing: function(metricName, time) { }, // do-something-with-it
count: function(metricName, value) { }, // do-something-with-it
};
@@ -89,7 +89,7 @@
// How often should we emit a heap sample? Time in ms.
// This setting is only relevant if you have enabled
// performance monitoring either via the default metrics
- // OR by defining your own performanceTimer properties
+ // OR by defining your own `stats` property.
//parsoidConfig.heapUsageSampleInterval = 5 * 60 * 1000;
// Allow override of port/interface:
diff --git a/tests/mocha/apitest.localsettings.js
b/tests/mocha/apitest.localsettings.js
index efc926b..6540409 100644
--- a/tests/mocha/apitest.localsettings.js
+++ b/tests/mocha/apitest.localsettings.js
@@ -8,8 +8,6 @@
'use strict';
exports.setup = function(parsoidConfig) {
- parsoidConfig.loadWMF = true;
-
// The URL of your MediaWiki API endpoint.
if (process.env.PARSOID_MOCKAPI_URL) {
parsoidConfig.setMwApi({
@@ -21,8 +19,8 @@
// We pre-define wikipedias as 'enwiki', 'dewiki' etc. Similarly
// for other projects: 'enwiktionary', 'enwikiquote', 'enwikibooks',
- // 'enwikivoyage' etc. (default true)
- // parsoidConfig.loadWMF = false;
+ // 'enwikivoyage' etc. (default false)
+ parsoidConfig.loadWMF = true;
// A default proxy to connect to the API endpoints. Default: undefined
// (no proxying). Overridden by per-wiki proxy config in setMwApi.
@@ -61,10 +59,9 @@
// changing api.php for load.php.
// parsoidConfig.modulesLoadURI = true;
- // Set to true to enable Performance timing
- parsoidConfig.useDefaultPerformanceTimer = false;
- // Peformance timing options for testing
- parsoidConfig.performanceTimer = {
+ // Add a metrics shim so these functions are called during testing.
+ parsoidConfig.useDefaultStatsAggregator = false;
+ parsoidConfig.stats = {
count: function() {},
timing: function() {},
};
diff --git a/tests/rttest.localsettings.js b/tests/rttest.localsettings.js
index 20d46b0..275ac2d 100644
--- a/tests/rttest.localsettings.js
+++ b/tests/rttest.localsettings.js
@@ -8,8 +8,6 @@
'use strict';
exports.setup = function(parsoidConfig) {
- parsoidConfig.loadWMF = true;
-
// The URL of your MediaWiki API endpoint.
if (process.env.PARSOID_MOCKAPI_URL) {
parsoidConfig.setMwApi({
@@ -24,8 +22,8 @@
// We pre-define wikipedias as 'enwiki', 'dewiki' etc. Similarly
// for other projects: 'enwiktionary', 'enwikiquote', 'enwikibooks',
- // 'enwikivoyage' etc. (default true)
- // parsoidConfig.loadWMF = false;
+ // 'enwikivoyage' etc. (default false)
+ parsoidConfig.loadWMF = true;
// A default proxy to connect to the API endpoints. Default: undefined
// (no proxying). Overridden by per-wiki proxy config in setMwApi.
@@ -70,10 +68,9 @@
// Enable bidi char stripping during serialization
parsoidConfig.scrubBidiChars = true;
- // Set to true to enable Performance timing
- parsoidConfig.useDefaultPerformanceTimer = false;
- // Peformance timing options for testing
- parsoidConfig.performanceTimer = {
+ // Add a metrics shim so these functions are called during testing.
+ parsoidConfig.useDefaultStatsAggregator = false;
+ parsoidConfig.stats = {
count: function() {},
timing: function() {},
};
--
To view, visit https://gerrit.wikimedia.org/r/281999
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Icfd6a39e93fd882d52cd06adf1b9e927e68aef21
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
