Mariapacana has uploaded a new change for review.
https://gerrit.wikimedia.org/r/117824
Change subject: WIP: Support structured logging and lazy pulling for backends.
......................................................................
WIP: Support structured logging and lazy pulling for backends.
Change-Id: I11aa47bf5ec460be35011a63b78cffe886bbfd28
---
M api/ParsoidService.js
A lib/LogData.js
M lib/Logger.js
3 files changed, 181 insertions(+), 97 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid
refs/changes/24/117824/1
diff --git a/api/ParsoidService.js b/api/ParsoidService.js
index ed7aeb9..f826cab 100644
--- a/api/ParsoidService.js
+++ b/api/ParsoidService.js
@@ -439,32 +439,12 @@
function parserEnvMw( req, res, next ) {
MWParserEnvironment.getParserEnv( parsoidConfig, null,
res.local('iwp'), res.local('pageName'), req.headers.cookie, function ( err,
env ) {
- function generateErrCBMessage (obj) {
- var messageString = "";
- if (obj.constructor.name === "Error") {
- messageString += obj.message;
- messageString += 'ERROR in ' +
res.local('iwp') + ':' + res.local('pageName');
- messageString += "\n" + obj.stack;
- } else {
- if (obj.msg) {
- messageString += obj.msg;
- }
- if (obj.location) {
- messageString += "\n" +
obj.location;
- }
- if (obj.stack) {
- messageString += "\n" +
obj.stack;
- }
- }
- return messageString;
- }
-
function errCB ( res, env, obj, callback ) {
try {
if (env.responseSent) {
return;
} else {
- var messageString =
generateErrCBMessage(obj);
+ var messageString = obj.fullMsg;
setHeader(res, env,
'Content-Type', 'text/plain; charset=UTF-8' );
sendResponse(res, env,
messageString, obj.code || 500);
res.on('finish', callback);
diff --git a/lib/LogData.js b/lib/LogData.js
new file mode 100644
index 0000000..207a7a9
--- /dev/null
+++ b/lib/LogData.js
@@ -0,0 +1,164 @@
+"use strict";
+require('./core-upgrade.js');
+
+var Util = require( './mediawiki.Util.js' ).Util,
+ DU = require( './mediawiki.DOMUtils.js').DOMUtils,
+ util = require('util'),
+ defines = require('./mediawiki.parser.defines.js'),
+ async = require('async');
+
+/**
+ *
+ * Extract properties and generate messages for logged objects.
+ *
+ * @class
+ * @constructor
+ * @param {MWParserEnvironment} env
+ * @param {string} logType
+ * @param {object} logObject
+ */
+
+var LogData = function (env, logType, logObject) {
+ this.env = env;
+ this.logType = logType;
+ this.logObject = logObject;
+
+ this.includeStackTrace =
/(^(error|fatal)|(^|\/)stacktrace)(\/|$)/.test(logType);
+ this.shouldAnnounceLocation = /^(error|warning)(\/|$)/.test(logType);
+
+ // Cache log information if previously constructed.
+ this.cachedStack = null;
+ this.cachedflatLO = null;
+ this.cachedMsg = null;
+ this.cachedLoc = null;
+ this.cachedFullMsg = null;
+
+ // Expose specific logData properties so backends can access them
+ // as needed; compute them on demand with defineProperty.
+ Object.defineProperty(this,
+
"flatLO",
+
{get: this.getOrReturnCachedFlatLO });
+
+ Object.defineProperty(this,
+
"stack",
+
{get: this.getOrReturnCachedStack});
+
+ Object.defineProperty(this,
+
"loc",
+
{get: this.getOrReturnCachedLoc });
+
+ Object.defineProperty(this,
+
"msg",
+
{get: this.getOrReturnCachedMsg });
+
+ Object.defineProperty(this,
+
"fullMsg",
+
{get: this.getOrReturnCachedFullMsg });
+
+};
+
+
+LogData.prototype.getOrReturnCachedFullMsg = function(){
+ if (this.cachedFullMsg) {
+ return this.cachedFullMsg;
+ } else {
+ var messageString = this.msg;
+ messageString += "\n" + this.loc;
+ messageString += "\n" + this.stack;
+ this.cachedFullMsg = messageString;
+ return this.cachedFullMsg;
+ }
+};
+
+LogData.prototype.getOrReturnCachedMsg = function() {
+ if (this.cachedMsg) {
+ return this.cachedMsg;
+ } else {
+ this.cachedMsg = this.flatLO.msg;
+ return this.cachedMsg;
+ }
+};
+
+LogData.prototype.getOrReturnCachedLoc = function() {
+ if (this.cachedLoc) {
+ return this.cachedLoc;
+ } else {
+ this.cachedLoc = this.announceLocation();
+ return this.cachedLoc;
+ }
+};
+
+LogData.prototype.getOrReturnCachedFlatLO = function() {
+ if (this.cachedFlatLO) {
+ return this.cachedFlatLO;
+ } else {
+ this.cachedFlatLO = this.flatten(this.logObject, 'top level');
+ return this.cachedFlatLO;
+ }
+};
+
+LogData.prototype.getOrReturnCachedStack = function(){
+ if (this.cachedStack) {
+ return this.cachedStack;
+ } else {
+ // This new error's stack doesn't seem very useful,
+ // since it is buried all the way in the middle of this.
+ if (this.includeStackTrace && !this.flatLO.stack ) {
+ this.cachedStack = new Error().stack;
+ } else if ( this.includeStackTrace && this.flatLO.stack ) {
+ this.cachedStack = this.flatLO.stack;
+ }
+ return this.cachedStack;
+ }
+};
+
+LogData.prototype.announceLocation = function () {
+ var location = this.logType + ' in ' + this.env.conf.wiki.iwp + ':' +
this.env.page.name;
+ if (this.env.page.revision && this.env.page.revision.revid) {
+ location += ' with oldid: ' + this.env.page.revision.revid;
+ }
+ return location;
+};
+
+LogData.prototype.flatten = function(o, topLevel) {
+ // returns an object with an arbitrary number of fields,
+ // including "msg" (for the message, if any)
+ // and "stack" (for the stack trace, if any)
+ var f, stack, msg, longMsg,
+ self = this;
+
+ if ( Array.isArray(o) && topLevel ) {
+ // flatten components, but no longer in a top-level context.
+ f = o.map(function(oo) { return self.flatten(oo); });
+ // join all the messages with spaces or newlines between them.
+ var tobool = function(x) { return !!x; };
+ msg = f.map(function(oo) { return oo.msg; }).filter(tobool).
+ join(' ');
+ longMsg = f.map(function(oo) { return oo.msg; }).filter(tobool).
+ join('\n');
+ // merge all custom fields
+ f = f.reduce(function(prev, oo) {
+ return Object.assign(prev, oo);
+ }, {});
+ return Object.assign(f, {
+ msg: msg,
+ longMsg: longMsg
+ });
+ } else if (o instanceof Error && o.code) {
+ return { msg: o.toString(), stack: o.stack, code: o.code };
+ } else if (o instanceof Error && !o.code) {
+ return { msg: o.toString(), stack: o.stack };
+ } else if (typeof(o)==='function') {
+ return self.flatten(o());
+ } else if (typeof (o) === 'object' && o.hasOwnProperty('msg')) {
+ return o;
+ } else if ( typeof(o) === 'string' ) {
+ return { msg: o };
+ } else {
+ return { msg: util.inspect(o)};
+ }
+};
+
+if (typeof module === "object") {
+ module.exports.LogData = LogData;
+}
\ No newline at end of file
diff --git a/lib/Logger.js b/lib/Logger.js
index 5eb8c9c..70cd999 100644
--- a/lib/Logger.js
+++ b/lib/Logger.js
@@ -1,7 +1,8 @@
"use strict";
require('./core-upgrade.js');
-var Util = require( './mediawiki.Util.js' ).Util,
+var LD = require('./LogData.js').LogData,
+ Util = require( './mediawiki.Util.js' ).Util,
DU = require( './mediawiki.DOMUtils.js').DOMUtils,
util = require('util'),
defines = require('./mediawiki.parser.defines.js'),
@@ -48,16 +49,9 @@
this.traceTestRegExp = this.traceTest();
};
-Logger.prototype.defaultBackend = function(obj) {
+Logger.prototype.defaultBackend = function(logData) {
try {
- var messageString = obj.msg;
- if (obj.location) {
- messageString += "\n" + obj.location;
- }
- if (obj.stack) {
- messageString += "\n" + obj.stack;
- }
- console.warn( messageString );
+ console.log(logData.fullMsg);
} catch (e) {
return;
}
@@ -77,41 +71,9 @@
}
};
-Logger.prototype.flatten = function(o, topLevel) {
- // returns an object with two fields, "msg" (for the message, if any)
- // and "stack" (for the stack trace, if any)
- var f, msg, stack,
- self = this;
-
- if ( Array.isArray(o) && topLevel ) {
- // flatten components, but no longer in a top-level context.
- f = o.map(function(oo) { return self.flatten(oo); });
- // join all the messages with spaces between them.
- msg = f.map(function(oo) { return oo.msg; }).join(' ');
- // use the stack of the first item in the array with a stack
- stack = f.reduce(function(prev, oo) {
- return prev || oo.stack;
- }, undefined);
- return { msg: msg, stack: stack };
- } else if (o instanceof Error && o.code) {
- return { msg: o.toString(), stack: o.stack, code:
o.code };
- } else if (o instanceof Error && !o.code) {
- return { msg: o.toString(), stack: o.stack };
- } else if (typeof(o)==='function') {
- f = this.flatten(o());
- return { msg: f.msg, stack: o.stack || f.stack };
- } else if (typeof(o)==='object' && o.hasOwnProperty('msg')) {
- f = this.flatten(o.msg);
- return { msg: f.msg, stack: o.stack || f.stack };
- } else if (typeof(o)==='string') {
- return { msg: o /* no stack */ };
- } else {
- return { msg: util.inspect(o) /* no stack */ };
- }
-};
-
-Logger.prototype.getApplicableBackends = function(logType, obj) {
+Logger.prototype.getApplicableBackends = function(logData) {
var applicableBackends = [];
+ var logType = logData.logType;
var self = this;
var error;
if (this.backendRegExp && this.backendRegExp.test(logType)) {
@@ -119,7 +81,7 @@
if (key === logType || key.test(logType)) {
applicableBackends.push(function(callback){
try {
- backends.get(key)(obj,
function(){
+ backends.get(key)(logData,
function(){
callback(null);
});
} catch (e) {
@@ -135,8 +97,8 @@
return applicableBackends;
};
-Logger.prototype.routeToBackends = function(logType, obj, cb) {
- var applicableBackends = this.getApplicableBackends(logType, obj);
+Logger.prototype.routeToBackends = function(logData, cb) {
+ var applicableBackends = this.getApplicableBackends(logData);
var self = this;
// If the logType is fatal, exits the process after logging
@@ -144,46 +106,24 @@
// Additionally runs processFatalCB, which looks for fatal
// events in the queue and logs them.
async.parallel(applicableBackends, function(){
- if (/^fatal$/.test(logType)) {
- self.defaultBackend(obj);
+ if (/^fatal$/.test(logData.logType)) {
+ self.defaultBackend(logData);
process.exit(1);
} else {
- self.defaultBackend(obj);
+ self.defaultBackend(logData);
}
cb();
});
};
-Logger.prototype.announceLocation = function (logType) {
- var location = logType + ' in ' + this.env.conf.wiki.iwp + ':' +
this.env.page.name;
- if (this.env.page.revision && this.env.page.revision.revid) {
- location += ' with oldid: ' + this.env.page.revision.revid;
- }
- return location;
-};
-
-Logger.prototype.emitMessage = function (logType, logObject, callback) {
- var includeStackTrace =
/(^(error|fatal)|(^|\/)stacktrace)(\/|$)/.test(logType);
- // XXX this should be configurable.
- var shouldAnnounceLocation = /^(error|warning)(\/|$)/.test(logType);
-
- // Gets everything that isn't logType
- var obj = this.flatten(logObject, 'top level');
- if (shouldAnnounceLocation) {
- obj.location = this.announceLocation(logType);
- }
- if (includeStackTrace && !obj.stack ) {
- obj.stack = new Error().stack;
- }
- this.routeToBackends(logType, obj, callback);
-};
-
Logger.prototype.log = function (logType) {
var self = this;
+ var traceTestRegExp = this.traceTest();
+ var logObject = Array.prototype.slice.call(arguments, 1);
+ var logData = new LD(this.env, logType, logObject);
// XXX this should be configurable.
if (this.traceTestRegExp.test(logType)) {
- var logObject = Array.prototype.slice.call(arguments, 1);
// If we are already processing a log request, but a log was
generated
// while processing the first request, processingLogRequest
will be true.
@@ -201,7 +141,7 @@
// process any fatal log events that we find on the queue.
} else {
self.processingLogRequest = true;
- this.emitMessage(logType, logObject, function(){
+ this.routeToBackends(logData, function(){
self.processingLogRequest = false;
if (self.logRequestQueue.length > 0) {
self.log.apply(self,
self.logRequestQueue.pop());
--
To view, visit https://gerrit.wikimedia.org/r/117824
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I11aa47bf5ec460be35011a63b78cffe886bbfd28
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Mariapacana <maria.pac...@gmail.com>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
