Yurik has submitted this change and it was merged. Change subject: Initial checkin, moved from graph extension repo ......................................................................
Initial checkin, moved from graph extension repo Change-Id: Iaff95aace6380375b05dd148a1e07831b15ad86e --- A .gitreview A graphoid-worker.js A graphoid.config.json A graphoid.js 4 files changed, 317 insertions(+), 0 deletions(-) Approvals: Yurik: Verified; Looks good to me, approved diff --git a/.gitreview b/.gitreview new file mode 100644 index 0000000..ae4946a --- /dev/null +++ b/.gitreview @@ -0,0 +1,7 @@ +[gerrit] +host=gerrit.wikimedia.org +port=29418 +project=mediawiki/services/graphoid.git +defaultbranch=master +defaultrebase=0 + diff --git a/graphoid-worker.js b/graphoid-worker.js new file mode 100644 index 0000000..476c12d --- /dev/null +++ b/graphoid-worker.js @@ -0,0 +1,226 @@ +"use strict"; + +var cluster = require('cluster'); + +/** + * The name of this instance. + * @property {string} + */ +var instanceName = cluster.isWorker ? 'worker(' + process.pid + ')' : 'master'; +console.log( ' - ' + instanceName + ' loading...' ); + +var express = require('express'), + fs = require('fs'), + child_process = require('child_process'), + request = require('request-promise'), + Promise = require('promise'), // https://www.npmjs.com/package/promise + http = require("http"), + url = require('url'), + querystring = require('querystring'), + vega = null; // Visualization grammar - https://github.com/trifacta/vega + + +try{ + vega = require("vega"); +} catch(err) { + console.log(err) +} + +var config; + +// Get the config +try { + config = JSON.parse(fs.readFileSync('./graphoid.config.json', 'utf8')); +} catch ( e ) { + console.error("Please set up your graphoid.config.json"); + process.exit(1); +} + +var serverRe = new RegExp('^([-a-z0-9]+\.)?(m\.|zero\.)?(' + config.domains.join('|') + ')$'); + +if (vega) { + vega.config.domainWhiteList = config.domains; + vega.config.safeMode = true; +} + +function merge() { + var result = {}; + for (var i = 0; i < arguments.length; i++) { + var obj = arguments[i]; + for (var key in obj) { + if (obj.hasOwnProperty(key)) { + result[key] = obj[key]; + } + } + } + return result; +} + +function getSpec(server, action, qs, id) { + + var url = 'http://' + server + '/w/api.php', + processResult, + callApiInt; + + processResult = function (response) { + var body = JSON.parse(response); + if ('error' in body) { + throw 'API result error: ' + data.error; + } + if ('warnings' in body) { + console.log('API warning: ' + JSON.stringify(body.warnings) + ' while getting ' + JSON.stringify({ + url: url, + opts: qs + })); + } + if ('query' in body && 'pages' in body.query) { + var obj = body.query.pages; + for (var k in obj) { + if (obj.hasOwnProperty(k)) { + var page = obj[k]; + if ('pageprops' in page && 'graph_specs' in page.pageprops) { + var gs = JSON.parse(page.pageprops.graph_specs); + if (id in gs) { + return gs[id]; + } + } + } + } + } + if ('continue' in body) { + return callApiInt(url, merge(qs, body.continue)); + } else { + return false; + } + }; + + callApiInt = function(url, options) { + var reqOpts = { + url: url, + qs: options, + headers: { + 'User-Agent': 'graph.ext backend (yurik at wikimedia)' + } + }; + return request(reqOpts) + .then(processResult) + .catch(function (reason) { + console.log(JSON.stringify(reqOpts)); + throw reason; // re-throw + }); + }; + + qs.action = action; + qs.format = 'json'; + return callApiInt(url, qs); +} + +function validateRequest(req) { + var query = url.parse(req.url, true).query; + if (!('revid' in query)) { + throw 'no revid'; + } + if (String(Math.abs(~~Number(query.revid))) !== query.revid) { + // must be a non-negative integer + throw 'bad revid param'; + } + // In case we switch to title, make sure to fail on query.title.indexOf('|') > -1 + + if (!('id' in query)) { + throw 'no id param'; + } + if (!('server' in query)) { + throw 'no server param'; + } + // Remove optional part #2 from host (makes m. links appear as desktop to optimize cache) + // 1 2 3 + // en.m.wikipedia.org + var srvParts = serverRe.exec(query.server); + if (!srvParts) { + throw 'bad server param'; + } + return { + server: (srvParts[1] || '') + srvParts[3], + action: 'query', + query: { + revids: query.revid, + prop: 'pageprops', + ppprop: 'graph_specs', + continue: '' + }, + id: query.id + }; +} + +function renderOnCanvas(spec, response) { + return new Promise(function (fulfill, reject){ + if (!vega) { + throw "Unable to load Vega npm module"; + } + vega.headless.render({spec: spec, renderer: "canvas"}, function (err, result) { + if (err) { + reject(err); + } else { + var stream = result.canvas.pngStream(); + response.writeHead(200, {"Content-Type": "image/png"}); + stream.on('data', function (chunk) { + response.write(chunk); + }); + stream.on('end', function () { + response.end(); + fulfill(); + }); + } + }); + }); +} + +// Adapted from https://www.promisejs.org/patterns/ +function delay(time) { + return new Promise(function (fulfill) { + setTimeout(fulfill, time); + }); +} +function timeout(promise, time) { + return Promise.race([promise, delay(time).then(function () { + throw 'Operation timed out'; + })]); +} + +var app = express(); // .createServer(); + +// robots.txt: no indexing. +app.get(/^\/robots.txt$/, function ( req, response ) { + response.end( "User-agent: *\nDisallow: /\n" ); +}); + + +app.get('/', function(req, response) { + + var params = false; + + var render = new Promise( + function (fulfill) { + // validate input params + params = validateRequest(req); + fulfill(params); + }).then(function (s) { + // get graph definition from the api + return getSpec(s.server, s.action, s.query, s.id); + }).then(function (spec) { + // render graph on canvas + // bug: timeout might happen right in the middle of streaming + return renderOnCanvas(spec, response); + }); + + // Limit request to 10 seconds, handle all errors + timeout(render, 10000) + .catch(function (reason) { + console.log(reason + (params ? '\n' + JSON.stringify(params) : '')); + response.writeHead(400); + response.end(JSON.stringify(reason)); + }); +}); + +console.log( ' - ' + instanceName + ' ready' ); +module.exports = app; diff --git a/graphoid.config.json b/graphoid.config.json new file mode 100644 index 0000000..7f1d795 --- /dev/null +++ b/graphoid.config.json @@ -0,0 +1,16 @@ +{ + "domains": [ + "mediawiki.org", + "wikibooks.org", + "wikidata.org", + "wikimedia.org", + "wikimediafoundation.org", + "wikinews.org", + "wikipedia.org", + "wikiquote.org", + "wikisource.org", + "wikiversity.org", + "wikivoyage.org", + "wiktionary.org" + ] +} diff --git a/graphoid.js b/graphoid.js new file mode 100644 index 0000000..1f22ac7 --- /dev/null +++ b/graphoid.js @@ -0,0 +1,68 @@ +#!/usr/bin/env node +/** + * A very basic cluster-based server runner. Restarts failed workers, but does + * not much else right now. + */ + +//var express = require('express'); +//var app = express.createServer(); +// +//var graphoidWorker = require('./graphoid-worker.js'); +//graphoidWorker.listen(port); +//return; + + +var cluster = require('cluster'), + // when running on appfog.com the listen port for the app + // is passed in an environment variable. Most users can ignore this! + port = process.env.GRAPHOID_PORT || 11042; + +if (cluster.isMaster) { + // Start a few more workers than there are cpus visible to the OS, so that we + // get some degree of parallelism even on single-core systems. A single + // long-running request would otherwise hold up all concurrent short requests. + var numCPUs = require('os').cpus().length + 3; + + + + + numCPUs = 1; + + + + + + // Fork workers. + for (var i = 0; i < numCPUs; i++) { + cluster.fork(); + } + + cluster.on('exit', function(worker) { + if (!worker.suicide) { + var exitCode = worker.process.exitCode; + console.log('worker', worker.process.pid, + 'died ('+exitCode+'), restarting.'); + cluster.fork(); + } + }); + + process.on('SIGTERM', function() { + console.log('master shutting down, killing workers'); + var workers = cluster.workers; + Object.keys(workers).forEach(function(id) { + console.log('Killing worker ' + id); + workers[id].destroy(); + }); + console.log('Done killing workers, bye'); + process.exit(1); + } ); + console.log('Starting Graphoid on port ' + port + + '\nPoint your browser to http://localhost:' + port + '/ for a test form\n'); +} else { + var graphoidWorker = require('./graphoid-worker.js'); + process.on('SIGTERM', function() { + console.log('Worker shutting down'); + process.exit(1); + }); + graphoidWorker.listen(port); +} -- To view, visit https://gerrit.wikimedia.org/r/191771 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Iaff95aace6380375b05dd148a1e07831b15ad86e Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/services/graphoid Gerrit-Branch: master Gerrit-Owner: Yurik <yu...@wikimedia.org> Gerrit-Reviewer: Yurik <yu...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits