Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/226073
Change subject: Update to template v0.2.1 and add the monitoring spec
......................................................................
Update to template v0.2.1 and add the monitoring spec
We now have a monitoring utility able to monitor specific endpoints
exposed by the service at /?spec . This patch creates and exposes a
simple monitoring spec for Mathoid.
Also, some node modules have been updated and the Mathoid-specific route
has been moved over to routes/mathoid.js.
Bug: T105775
Change-Id: Id5e06852ccb4d1dc12b9376e690965c8e742ce18
---
M app.js
M config.dev.yaml
M lib/util.js
M package.json
A routes/mathoid.js
M routes/root.js
A spec.yaml
A test/features/app/spec.js
8 files changed, 556 insertions(+), 102 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/mathoid
refs/changes/73/226073/1
diff --git a/app.js b/app.js
index 76dec28..ea11449 100644
--- a/app.js
+++ b/app.js
@@ -15,6 +15,7 @@
var sUtil = require('./lib/util');
var packageInfo = require('./package.json');
var mjAPI = require("MathJax-node/lib/mj-single.js");
+var yaml = require('js-yaml');
/**
@@ -46,6 +47,40 @@
// set outgoing proxy
if(app.conf.proxy) {
process.env.HTTP_PROXY = app.conf.proxy;
+ // if there is a list of domains which should
+ // not be proxied, set it
+ if(app.conf.no_proxy_list) {
+ if(Array.isArray(app.conf.no_proxy_list)) {
+ process.env.NO_PROXY = app.conf.no_proxy_list.join(',');
+ } else {
+ process.env.NO_PROXY = app.conf.no_proxy_list;
+ }
+ }
+ }
+
+ // set up the spec
+ if(!app.conf.spec) {
+ app.conf.spec = __dirname + '/spec.yaml';
+ }
+ try {
+ app.conf.spec = yaml.safeLoad(fs.readFileSync(app.conf.spec));
+ } catch(e) {
+ app.logger.log('warn/spec', 'Could not load the spec: ' + e);
+ app.conf.spec = {};
+ }
+ if(!app.conf.spec.swagger) {
+ app.conf.spec.swagger = '2.0';
+ }
+ if(!app.conf.spec.info) {
+ app.conf.spec.info = {
+ version: app.info.version,
+ title: app.info.name,
+ description: app.info.description
+ };
+ }
+ app.conf.spec.info.version = app.info.version;
+ if(!app.conf.spec.paths) {
+ app.conf.spec.paths = {};
}
// set the CORS and CSP headers
diff --git a/config.dev.yaml b/config.dev.yaml
index f298895..cca23e7 100644
--- a/config.dev.yaml
+++ b/config.dev.yaml
@@ -33,6 +33,20 @@
port: 10042
# interface: localhost # uncomment to only listen on localhost
# more per-service config settings
+ # the location of the spec, defaults to spec.yaml if not specified
+ # spec: ./spec.template.yaml
+ # allow cross-domain requests to the API (default '*')
+ cors: '*'
+ # to disable use:
+ # cors: false
+ # to restrict to a particular domain, use:
+ # cors: restricted.domain.org
+ # URL of the outbound proxy to use (complete with protocol)
+ # proxy: http://my.proxy.org:8080
+ # the list of domains for which not to use the proxy defined above
+ # no_proxy_list:
+ # - domain1.com
+ # - domain2.org
svg: true
img: true
png: true #new feature
diff --git a/lib/util.js b/lib/util.js
index 3aef6da..5b6bbb6 100644
--- a/lib/util.js
+++ b/lib/util.js
@@ -4,7 +4,7 @@
var BBPromise = require('bluebird');
var util = require('util');
var express = require('express');
-var uuid = require('node-uuid');
+var uuid = require('cassandra-uuid');
var bunyan = require('bunyan');
@@ -86,11 +86,9 @@
*
* @return {String} the generated request ID
*/
-var reqIdBuff = new Buffer(16);
function generateRequestId() {
- uuid.v4(null, reqIdBuff);
- return reqIdBuff.toString('hex');
+ return uuid.TimeUuid.now().toString();
}
diff --git a/package.json b/package.json
index 94baa99..00d4680 100644
--- a/package.json
+++ b/package.json
@@ -36,29 +36,26 @@
},
"homepage": "https://github.com/wikimedia/mathoid";,
"dependencies": {
- "bluebird": "^2.9.24",
- "body-parser": "^1.12.3",
- "bunyan": "^1.3.5",
- "compression": "^1.4.3",
+ "bluebird": "~2.8.2",
+ "body-parser": "^1.13.2",
+ "bunyan": "^1.4.0",
+ "cassandra-uuid": "^0.0.2",
+ "compression": "^1.5.1",
"domino": "^1.0.18",
- "express": "^4.12.3",
- "js-yaml": "^3.2.7",
- "preq": "^0.3.13",
- "service-runner": "^0.1.8",
- "jsdom": "0.11.1",
- "node-uuid": "^1.4.3",
- "querystring": "0.x.x",
- "request": "2.x.x",
+ "express": "^4.13.1",
+ "js-yaml": "^3.3.1",
+ "preq": "^0.4.4",
+ "service-runner": "^0.2.1",
"texvcjs": "git+https://github.com/wikimedia/texvcjs";,
"MathJax-node":
"git+https://github.com/wikimedia/MathJax-node#mathoid-0-2-8";
},
"devDependencies": {
- "assert": "^1.3.0",
- "istanbul": "^0.3.13",
- "mocha": "^2.2.4",
- "mocha-jshint": "0.0.9",
- "mocha-lcov-reporter": "0.0.1",
- "coveralls": "2.11.2"
+ "coveralls": "2.11.2",
+ "istanbul": "^0.3.17",
+ "mocha": "^2.2.5",
+ "mocha-jshint": "^2.2.3",
+ "mocha-lcov-reporter": "^0.0.2",
+ "swagger-router": "^0.1.1"
},
"deploy": {
"target": "ubuntu",
diff --git a/routes/mathoid.js b/routes/mathoid.js
new file mode 100644
index 0000000..ab1e527
--- /dev/null
+++ b/routes/mathoid.js
@@ -0,0 +1,120 @@
+'use strict';
+
+
+var sUtil = require('../lib/util');
+var texvcjs = require('texvcjs');
+var HTTPError = sUtil.HTTPError;
+
+
+/**
+ * The main router object
+ */
+var router = sUtil.router();
+
+/**
+ * The main application object reported when this module is require()d
+ */
+var app;
+
+
+function handleRequest(res, q, type) {
+ var mml = true;
+ var sanitizedTex;
+ //Keep format variables constant
+ if (type === "tex") {
+ type = "TeX";
+ var sanitizationOutput = texvcjs.check(q);
+ // XXX properly handle errors here!
+ if (sanitizationOutput.status === '+') {
+ sanitizedTex = sanitizationOutput.output || '';
+ q = sanitizedTex;
+ } else {
+ throw new HTTPError
+ ({
+ status: 400,
+ log: sanitizationOutput.status + ': ' +
sanitizationOutput.details,
+ success: false
+ });
+ }
+ }
+ if (type === "mml" || type === "MathML") {
+ type = "MathML";
+ mml = false;
+ }
+ if (type === "ascii" || type === "asciimath") {
+ type = "AsciiMath";
+ }
+ app.mjAPI.typeset( {
+ math: q,
+ format: type,
+ svg: app.conf.svg,
+ img: app.conf.img,
+ mml: mml,
+ speakText: app.conf.speakText,
+ png: app.conf.png }, function (data) {
+ if (data.errors) {
+ data.success = false;
+ data.log = "Error:" + JSON.stringify(data.errors);
+ } else {
+ data.success = true;
+ data.log = "success";
+ }
+
+ // Strip some styling returned by MathJax
+ if (data.svg) {
+ data.svg = data.svg.replace(/style="([^"]+)"/, function(match,
style) {
+ return 'style="'
+ + style.replace(/(?:margin(?:-[a-z]+)?|position):[^;]+;
*/g, '')
+ + '"';
+ });
+ }
+
+ // Return the sanitized TeX to the client
+ if (sanitizedTex !== undefined) {
+ data.sanetex = sanitizedTex;
+ }
+
+ res.writeHead(200, {
+ 'Content-Type': 'application/json'
+ });
+ res.end(JSON.stringify(data));
+ });
+}
+
+
+/**
+ * POST /
+ * Performs the rendering request
+ */
+router.post(/^\/$/, function(req, res) {
+
+ // First some rudimentary input validation
+ if (!(req.body.q)) {
+ throw new HTTPError
+ ({
+ status: 400,
+ error: "q (query) post parameter is missing!"
+ });
+ }
+ var q = req.body.q;
+ var type = req.body.type;
+ if (!(req.body.type) ){
+ type = 'tex';
+ }
+ handleRequest(res, q, type);
+
+});
+
+
+module.exports = function(appObj) {
+
+ app = appObj;
+
+ return {
+ path: '/',
+ skip_domain: true,
+ router: router
+ };
+
+};
+
diff --git a/routes/root.js b/routes/root.js
index ce0027a..3b34d8e 100644
--- a/routes/root.js
+++ b/routes/root.js
@@ -8,13 +8,12 @@
* The main router object
*/
var router = sUtil.router();
-var texvcjs = require('texvcjs');
-var HTTPError = sUtil.HTTPError;
/**
* The main application object reported when this module is require()d
*/
var app;
+
/**
* GET /robots.txt
@@ -29,89 +28,19 @@
});
-function handleRequest(res, q, type) {
- var mml = true;
- var sanitizedTex;
- //Keep format variables constant
- if (type === "tex") {
- type = "TeX";
- var sanitizationOutput = texvcjs.check(q);
- // XXX properly handle errors here!
- if (sanitizationOutput.status === '+') {
- sanitizedTex = sanitizationOutput.output || '';
- q = sanitizedTex;
- } else {
- throw new HTTPError
- ({
- status: 400,
- log: sanitizationOutput.status + ': ' +
sanitizationOutput.details,
- success: false
- });
- }
- }
- if (type === "mml" || type === "MathML") {
- type = "MathML";
- mml = false;
- }
- if (type === "ascii" || type === "asciimath") {
- type = "AsciiMath";
- }
- app.mjAPI.typeset( {
- math: q,
- format: type,
- svg: app.conf.svg,
- img: app.conf.img,
- mml: mml,
- speakText: app.conf.speakText,
- png: app.conf.png }, function (data) {
- if (data.errors) {
- data.success = false;
- data.log = "Error:" + JSON.stringify(data.errors);
- } else {
- data.success = true;
- data.log = "success";
- }
- // Strip some styling returned by MathJax
- if (data.svg) {
- data.svg = data.svg.replace(/style="([^"]+)"/, function(match,
style) {
- return 'style="'
- + style.replace(/(?:margin(?:-[a-z]+)?|position):[^;]+;
*/g, '')
- + '"';
- });
- }
-
- // Return the sanitized TeX to the client
- if (sanitizedTex !== undefined) {
- data.sanetex = sanitizedTex;
- }
-
- res.writeHead(200, {
- 'Content-Type': 'application/json'
- });
- res.end(JSON.stringify(data));
- });
-}
/**
- * GET /robots.txt
- * Instructs robots no indexing should occur on this domain.
+ * GET /
+ * Main entry point. Currently it only responds if the spec query
+ * parameter is given, otherwise lets the next middleware handle it
*/
-router.post(/^\/$/, function(req, res) {
+router.get('/', function(req, res, next) {
- // First some rudimentary input validation
- if (!(req.body.q)) {
- throw new HTTPError
- ({
- status: 400,
- error: "q (query) post parameter is missing!"
- });
+ if(!(req.query || {}).hasOwnProperty('spec')) {
+ next();
+ } else {
+ res.json(app.conf.spec);
}
- var q = req.body.q;
- var type = req.body.type;
- if (!(req.body.type) ){
- type = 'tex';
- }
- handleRequest(res, q, type);
});
@@ -119,6 +48,7 @@
module.exports = function(appObj) {
app = appObj;
+
return {
path: '/',
skip_domain: true,
diff --git a/spec.yaml b/spec.yaml
new file mode 100644
index 0000000..e004c44
--- /dev/null
+++ b/spec.yaml
@@ -0,0 +1,75 @@
+swagger: '2.0'
+info:
+ version: 0.2.8
+ title: MAthoid
+ description: Renders tex to SVG and MathML using MathJax
+ termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use
+ contact:
+ name: the Wikimedia Services team
+ url: http://mediawiki.org/wiki/Services
+ license:
+ name: Apache2
+ url: http://www.apache.org/licenses/LICENSE-2.0
+paths:
+ # from routes/root.js
+ /robots.txt:
+ get:
+ tags:
+ - Root
+ - Robots
+ description: Gets robots.txt
+ x-monitor: false
+ /:
+ get:
+ tags:
+ - Root
+ description: The root service end-point
+ produces:
+ - application/json
+ x-amples:
+ - title: spec from root
+ request:
+ query:
+ spec: true
+ response:
+ status: 200
+ # from routes/info.js
+ /_info:
+ get:
+ tags:
+ - Service information
+ description: Gets information about the service
+ produces:
+ - application/json
+ x-amples:
+ - title: retrieve service info
+ request: {}
+ response:
+ status: 200
+ headers:
+ content-type: application/json
+ body:
+ name: /.+/
+ description: /.+/
+ version: /.+/
+ home: /.+/
+ # from routes/mathoid.js
+ /:
+ post:
+ tags:
+ - Render
+ description: Converts tex to MathML and SVG
+ produces:
+ - application/json
+ x-amples:
+ - title: Einstein equation
+ request:
+ body:
+ q: 'E=mc^2'
+ response:
+ status: 200
+ body:
+ log: success
+ success: true
+ svg: /.+/
+ mml: /.+/
diff --git a/test/features/app/spec.js b/test/features/app/spec.js
new file mode 100644
index 0000000..570fb40
--- /dev/null
+++ b/test/features/app/spec.js
@@ -0,0 +1,285 @@
+'use strict';
+
+
+var preq = require('preq');
+var assert = require('../../utils/assert.js');
+var server = require('../../utils/server.js');
+var URI = require('swagger-router').URI;
+var yaml = require('js-yaml');
+var fs = require('fs');
+
+
+function staticSpecLoad() {
+
+ var spec;
+ var myService =
server.config.conf.services[server.config.conf.services.length - 1].conf;
+ var specPath = __dirname + '/../../../' + (myService.spec ? myService.spec
: 'spec.yaml');
+
+ try {
+ spec = yaml.safeLoad(fs.readFileSync(specPath));
+ } catch(e) {
+ // this error will be detected later, so ignore it
+ spec = {paths: {}, 'x-default-params': {}};
+ }
+
+ return spec;
+
+}
+
+
+function validateExamples(pathStr, defParams, mSpec) {
+
+ var uri = new URI(pathStr, {}, true);
+
+ if(!mSpec) {
+ try {
+ uri.expand(defParams);
+ return true;
+ } catch(e) {
+ throw new Error('Missing parameter for route ' + pathStr + ' : ' +
e.message);
+ }
+ }
+
+ if(!Array.isArray(mSpec)) {
+ throw new Error('Route ' + pathStr + ' : x-amples must be an array!');
+ }
+
+ mSpec.forEach(function(ex, idx) {
+ if(!ex.title) {
+ throw new Error('Route ' + pathStr + ', example ' + idx + ': title
missing!');
+ }
+ ex.request = ex.request || {};
+ try {
+ uri.expand(Object.assign({}, defParams, ex.request.params || {}));
+ } catch(e) {
+ throw new Error('Route ' + pathStr + ', example ' + idx + ' (' +
ex.title + '): missing parameter: ' + e.message);
+ }
+ });
+
+ return true;
+
+}
+
+
+function constructTestCase(title, path, method, request, response) {
+
+ return {
+ title: title,
+ request: {
+ uri: server.config.uri + (path[0] === '/' ? path.substr(1) : path),
+ method: method,
+ headers: request.headers || {},
+ query: request.query,
+ body: request.body,
+ followRedirect: false
+ },
+ response: {
+ status: response.status || 200,
+ headers: response.headers || {},
+ body: response.body
+ }
+ };
+
+}
+
+
+function constructTests(paths, defParams) {
+
+ var ret = [];
+
+ Object.keys(paths).forEach(function(pathStr) {
+ Object.keys(paths[pathStr]).forEach(function(method) {
+ var p = paths[pathStr][method];
+ var uri;
+ if(p.hasOwnProperty('x-monitor') && !p['x-monitor']) {
+ return;
+ }
+ uri = new URI(pathStr, {}, true);
+ if(!p['x-amples']) {
+ ret.push(constructTestCase(
+ pathStr,
+ uri.toString({params: defParams}),
+ method,
+ {},
+ {}
+ ));
+ return;
+ }
+ p['x-amples'].forEach(function(ex) {
+ ex.request = ex.request || {};
+ ret.push(constructTestCase(
+ ex.title,
+ uri.toString({params: Object.assign({}, defParams,
ex.request.params || {})}),
+ method,
+ ex.request,
+ ex.response || {}
+ ));
+ });
+ });
+ });
+
+ return ret;
+
+}
+
+
+function cmp(result, expected, errMsg) {
+
+ if(expected === null || expected === undefined) {
+ // nothing to expect, so we can return
+ return true;
+ }
+ if(result === null || result === undefined) {
+ result = '';
+ }
+
+ if(expected.constructor === Object) {
+ Object.keys(expected).forEach(function(key) {
+ var val = expected[key];
+ assert.deepEqual(result.hasOwnProperty(key), true, 'Body field ' +
key + ' not found in response!');
+ cmp(result[key], val, key + ' body field mismatch!');
+ });
+ return true;
+ } else if(expected.constructor === Array) {
+ if(result.constructor !== Array) {
+ assert.deepEqual(result, expected, errMsg);
+ return true;
+ }
+ // only one item in expected - compare them all
+ if(expected.length === 1 && result.length > 1) {
+ result.forEach(function(item) {
+ cmp(item, expected[0], errMsg);
+ });
+ return true;
+ }
+ // more than one item expected, check them one by one
+ if(expected.length !== result.length) {
+ assert.deepEqual(result, expected, errMsg);
+ return true;
+ }
+ expected.forEach(function(item, idx) {
+ cmp(result[idx], item, errMsg);
+ });
+ return true;
+ }
+
+ if(expected.length > 1 && expected[0] === '/' && expected[expected.length
- 1] === '/') {
+ if((new RegExp(expected.slice(1, -1))).test(result)) {
+ return true;
+ }
+ } else if(expected.length === 0 && result.length === 0) {
+ return true;
+ } else if(result === expected || result.startsWith(expected)) {
+ return true;
+ }
+
+ assert.deepEqual(result, expected, errMsg);
+ return true;
+
+}
+
+
+function validateTestResponse(testCase, res) {
+
+ var expRes = testCase.response;
+
+ // check the status
+ assert.status(res, expRes.status);
+ // check the headers
+ Object.keys(expRes.headers).forEach(function(key) {
+ var val = expRes.headers[key];
+ assert.deepEqual(res.headers.hasOwnProperty(key), true, 'Header ' +
key + ' not found in response!');
+ cmp(res.headers[key], val, key + ' header mismatch!');
+ });
+ // check the body
+ if(!expRes.body) {
+ return true;
+ }
+ res.body = res.body || '';
+ if(Buffer.isBuffer(res.body)) { res.body = res.body.toString(); }
+ if(expRes.body.constructor !== res.body.constructor) {
+ if(expRes.body.constructor === String) {
+ res.body = JSON.stringify(res.body);
+ } else {
+ res.body = JSON.parse(res.body);
+ }
+ }
+ // check that the body type is the same
+ if(expRes.body.constructor !== res.body.constructor) {
+ throw new Error('Expected a body of type ' + expRes.body.constructor +
' but gotten ' + res.body.constructor);
+ }
+
+ // compare the bodies
+ cmp(res.body, expRes.body, 'Body mismatch!');
+
+ return true;
+
+}
+
+
+describe('Swagger spec', function() {
+
+ // the variable holding the spec
+ var spec = staticSpecLoad();
+ // default params, if given
+ var defParams = spec['x-default-params'] || {};
+
+ this.timeout(20000);
+
+ before(function () {
+ return server.start();
+ });
+
+ it('get the spec', function() {
+ return preq.get(server.config.uri + '?spec')
+ .then(function(res) {
+ assert.status(200);
+ assert.contentType(res, 'application/json');
+ assert.notDeepEqual(res.body, undefined, 'No body received!');
+ spec = res.body;
+ });
+ });
+
+ it('spec validation', function() {
+ if(spec['x-default-params']) {
+ defParams = spec['x-default-params'];
+ }
+ // check the high-level attributes
+ ['info', 'swagger', 'paths'].forEach(function(prop) {
+ assert.deepEqual(!!spec[prop], true, 'No ' + prop + ' field
present!');
+ });
+ // no paths - no love
+ assert.deepEqual(!!Object.keys(spec.paths), true, 'No paths given in
the spec!');
+ // now check each path
+ Object.keys(spec.paths).forEach(function(pathStr) {
+ var path;
+ assert.deepEqual(!!pathStr, true, 'A path cannot have a length of
zero!');
+ path = spec.paths[pathStr];
+ assert.deepEqual(!!Object.keys(path), true, 'No methods defined
for path: ' + pathStr);
+ Object.keys(path).forEach(function(method) {
+ var mSpec = path[method];
+ if(mSpec.hasOwnProperty('x-monitor') && !mSpec['x-monitor']) {
+ return;
+ }
+ validateExamples(pathStr, defParams, mSpec['x-amples']);
+ });
+ });
+ });
+
+ describe('routes', function() {
+
+ constructTests(spec.paths, defParams).forEach(function(testCase) {
+ it(testCase.title, function() {
+ return preq(testCase.request)
+ .then(function(res) {
+ validateTestResponse(testCase, res);
+ }, function(err) {
+ validateTestResponse(testCase, err);
+ });
+ });
+ });
+
+ });
+
+});
+
--
To view, visit https://gerrit.wikimedia.org/r/226073
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Id5e06852ccb4d1dc12b9376e690965c8e742ce18
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/mathoid
Gerrit-Branch: master
Gerrit-Owner: Mobrovac <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
