Arlolra has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/391556 )
Change subject: Move out internal routes
......................................................................
Move out internal routes
Change-Id: Ib2bcaa1f56663e2cafe91c8c5d27ccf338f28cf8
---
M lib/api/ParsoidService.js
M lib/api/apiUtils.js
A lib/api/internal.js
M lib/api/routes.js
4 files changed, 229 insertions(+), 212 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid
refs/changes/56/391556/1
diff --git a/lib/api/ParsoidService.js b/lib/api/ParsoidService.js
index ffb218b..0515c97 100644
--- a/lib/api/ParsoidService.js
+++ b/lib/api/ParsoidService.js
@@ -75,9 +75,6 @@
var workers = workerFarm(farmOptions, require.resolve('../parse.js'));
var promiseWorkers = Promise.promisify(workers);
- // Load routes
- var routes = require('./routes')(parsoidConfig, processLogger,
parsoidOptions, promiseWorkers);
-
var app = express();
// Default express to production.
@@ -192,9 +189,10 @@
// Routes
+ var routes = require('./routes')(parsoidConfig, processLogger,
parsoidOptions, promiseWorkers);
+
var a = routes.acceptable;
var p = routes.parserEnvMw;
- var i = routes.internal;
var v3 = routes.v3Middle;
app.get('/', routes.home);
@@ -202,21 +200,23 @@
app.get('/version', routes.version);
app.get('/_version', routes.version); // for backwards compat.
- // private routes
- if (parsoidConfig.devAPI) {
- app.get('/_html/:prefix?/:title?', i, p, routes.html2wtForm);
- app.get('/_wikitext/:prefix?/:title?', i, p,
routes.wt2htmlForm);
- app.get('/_rt/:prefix?/:title?', i, p, routes.roundtripTesting);
- app.get('/_rtve/:prefix?/:title?', i, p,
routes.roundtripTestingNL);
- app.get('/_rtselser/:prefix?/:title?', i, p,
routes.roundtripSelser);
- app.get('/_rtform/:prefix?/:title?', i, p, routes.getRtForm);
- app.post('/_rtform/:prefix?/:title?', i, p, routes.postRtForm);
- }
-
// v3 API routes
app.get('/:domain/v3/page/:format/:title/:revision?', v3, p, a,
routes.v3Get);
app.post('/:domain/v3/transform/:from/to/:format/:title?/:revision?',
v3, p, a, routes.v3Post);
+ // private routes
+ if (parsoidConfig.devAPI) {
+ var internal = require('./internal')(parsoidConfig,
processLogger);
+ var i = internal.middle;
+ app.get('/_html/:prefix?/:title?', i, p, internal.html2wtForm);
+ app.get('/_wikitext/:prefix?/:title?', i, p,
internal.wt2htmlForm);
+ app.get('/_rt/:prefix?/:title?', i, p,
internal.roundtripTesting);
+ app.get('/_rtve/:prefix?/:title?', i, p,
internal.roundtripTestingNL);
+ app.get('/_rtselser/:prefix?/:title?', i, p,
internal.roundtripSelser);
+ app.get('/_rtform/:prefix?/:title?', i, p, internal.getRtForm);
+ app.post('/_rtform/:prefix?/:title?', i, p,
internal.postRtForm);
+ }
+
var server;
return new Promise(function(resolve, reject) {
app.on('error', function(err) {
diff --git a/lib/api/apiUtils.js b/lib/api/apiUtils.js
index a18557b..7f4f530 100644
--- a/lib/api/apiUtils.js
+++ b/lib/api/apiUtils.js
@@ -2,18 +2,15 @@
require('../../core-upgrade.js');
-var domino = require('domino');
var util = require('util');
var semver = require('semver');
var qs = require('querystring');
var cType = require('content-type');
-var Diff = require('../utils/Diff.js').Diff;
var DU = require('../utils/DOMUtils.js').DOMUtils;
var Util = require('../utils/Util.js').Util;
var PegTokenizer = require('../wt2html/tokenizer.js').PegTokenizer;
var PHPParseRequest = require('../mw/ApiRequest.js').PHPParseRequest;
-
/**
* @class apiUtils
@@ -172,47 +169,6 @@
env.log('info', util.format(
'completed %s in %s ms', str, Date.now() - res.locals.start
));
-};
-
-apiUtils.rtResponse = function(env, req, res, data) {
- apiUtils.renderResponse(res, 'roundtrip', data);
- apiUtils.logTime(env, res, 'parsing');
-};
-
-apiUtils.roundTripDiff = function(env, req, res, useSelser, doc) {
- // Re-parse the HTML to uncover foster-parenting issues
- doc = domino.createDocument(doc.outerHTML);
-
- var handler = env.getContentHandler();
- return handler.fromHTML(env, doc.body, useSelser).then(function(out) {
- // Strip selser trigger comment
- out = out.replace(/<!--rtSelserEditTestComment-->\n*$/, '');
-
- // Emit base href so all relative urls resolve properly
- var headNodes = "";
- for (var hNode = doc.head.firstChild; hNode; hNode =
hNode.nextSibling) {
- if (hNode.nodeName.toLowerCase() === 'base') {
- headNodes += DU.toXML(hNode);
- break;
- }
- }
-
- var bodyNodes = "";
- for (var bNode = doc.body.firstChild; bNode; bNode =
bNode.nextSibling) {
- bodyNodes += DU.toXML(bNode);
- }
-
- var htmlSpeChars = Util.escapeHtml(out);
- var patch =
Diff.convertChangesToXML(Diff.diffLines(env.page.src, out));
-
- return {
- headers: headNodes,
- bodyNodes: bodyNodes,
- htmlSpeChars: htmlSpeChars,
- patch: patch,
- reqUrl: req.url,
- };
- });
};
// To support the 'subst' API parameter, we need to prefix each
diff --git a/lib/api/internal.js b/lib/api/internal.js
new file mode 100644
index 0000000..2bfb57e
--- /dev/null
+++ b/lib/api/internal.js
@@ -0,0 +1,213 @@
+'use strict';
+
+require('../../core-upgrade.js');
+
+var domino = require('domino');
+
+var apiUtils = require('./apiUtils.js');
+var Diff = require('../utils/Diff.js').Diff;
+var DU = require('../utils/DOMUtils.js').DOMUtils;
+var Util = require('../utils/Util.js').Util;
+var TemplateRequest = require('../mw/ApiRequest.js').TemplateRequest;
+
+var roundTripDiff = function(env, req, res, useSelser, doc) {
+ // Re-parse the HTML to uncover foster-parenting issues
+ doc = domino.createDocument(doc.outerHTML);
+
+ var handler = env.getContentHandler();
+ return handler.fromHTML(env, doc.body, useSelser).then(function(out) {
+ // Strip selser trigger comment
+ out = out.replace(/<!--rtSelserEditTestComment-->\n*$/, '');
+
+ // Emit base href so all relative urls resolve properly
+ var headNodes = "";
+ for (var hNode = doc.head.firstChild; hNode; hNode =
hNode.nextSibling) {
+ if (hNode.nodeName.toLowerCase() === 'base') {
+ headNodes += DU.toXML(hNode);
+ break;
+ }
+ }
+
+ var bodyNodes = "";
+ for (var bNode = doc.body.firstChild; bNode; bNode =
bNode.nextSibling) {
+ bodyNodes += DU.toXML(bNode);
+ }
+
+ var htmlSpeChars = Util.escapeHtml(out);
+ var patch =
Diff.convertChangesToXML(Diff.diffLines(env.page.src, out));
+
+ return {
+ headers: headNodes,
+ bodyNodes: bodyNodes,
+ htmlSpeChars: htmlSpeChars,
+ patch: patch,
+ reqUrl: req.url,
+ };
+ });
+};
+
+var rtResponse = function(env, req, res, data) {
+ apiUtils.renderResponse(res, 'roundtrip', data);
+ apiUtils.logTime(env, res, 'parsing');
+};
+
+module.exports = function(parsoidConfig, processLogger) {
+ var internal = {};
+
+ // Middlewares
+
+ internal.middle = function(req, res, next) {
+ res.locals.errorEnc = 'plain';
+ var iwp = req.params.prefix || parsoidConfig.defaultWiki || '';
+ if (!parsoidConfig.mwApiMap.has(iwp)) {
+ var text = 'Invalid prefix: ' + iwp;
+ processLogger.log('fatal/request', new Error(text));
+ return apiUtils.errorResponse(res, text, 404);
+ }
+ res.locals.iwp = iwp;
+ res.locals.pageName = req.params.title || 'Main_Page';
+ res.locals.oldid = req.body.oldid || req.query.oldid || null;
+ // "body" flag to return just the body (instead of the entire
HTML doc)
+ res.locals.bodyOnly = !!(req.query.body || req.body.body);
+ // "subst" flag to perform {{subst:}} template expansion
+ res.locals.subst = !!(req.query.subst || req.body.subst);
+ res.locals.envOptions = {
+ prefix: res.locals.iwp,
+ pageName: res.locals.pageName,
+ };
+ next();
+ };
+
+ // Routes
+
+ // Form-based HTML DOM -> wikitext interface for manual testing.
+ internal.html2wtForm = function(req, res) {
+ var domain = parsoidConfig.mwApiMap.get(res.locals.iwp).domain;
+ var action = '/' + domain + '/v3/transform/html/to/wikitext/' +
res.locals.pageName;
+ if (req.query.hasOwnProperty('scrub_wikitext')) {
+ action += "?scrub_wikitext=" + req.query.scrub_wikitext;
+ }
+ apiUtils.renderResponse(res, 'form', {
+ title: 'Your HTML DOM:',
+ action: action,
+ name: 'html',
+ });
+ };
+
+ // Form-based wikitext -> HTML DOM interface for manual testing
+ internal.wt2htmlForm = function(req, res) {
+ var domain = parsoidConfig.mwApiMap.get(res.locals.iwp).domain;
+ apiUtils.renderResponse(res, 'form', {
+ title: 'Your wikitext:',
+ action: '/' + domain +
'/v3/transform/wikitext/to/html/' + res.locals.pageName,
+ name: 'wikitext',
+ });
+ };
+
+ // Round-trip article testing. Default to scrubbing wikitext here.
Can be
+ // overridden with qs param.
+ internal.roundtripTesting = function(req, res) {
+ var env = res.locals.env;
+ env.scrubWikitext = apiUtils.shouldScrub(req, true);
+
+ var target = env.normalizeAndResolvePageTitle();
+
+ var oldid = null;
+ if (req.query.oldid) {
+ oldid = req.query.oldid;
+ }
+
+ return TemplateRequest.setPageSrcInfo(env, target,
oldid).then(function() {
+ env.log('info', 'started parsing');
+ return env.getContentHandler().toHTML(env);
+ })
+ .then(roundTripDiff.bind(null, env, req, res, false))
+ .then(rtResponse.bind(null, env, req, res))
+ .catch(function(err) {
+ env.log('fatal/request', err);
+ });
+ };
+
+ // Round-trip article testing with newline stripping for editor-created
HTML
+ // simulation. Default to scrubbing wikitext here. Can be overridden
with qs
+ // param.
+ internal.roundtripTestingNL = function(req, res) {
+ var env = res.locals.env;
+ env.scrubWikitext = apiUtils.shouldScrub(req, true);
+
+ var target = env.normalizeAndResolvePageTitle();
+
+ var oldid = null;
+ if (req.query.oldid) {
+ oldid = req.query.oldid;
+ }
+
+ return TemplateRequest.setPageSrcInfo(env, target,
oldid).then(function() {
+ env.log('info', 'started parsing');
+ return env.getContentHandler().toHTML(env);
+ }).then(function(doc) {
+ // strip newlines from the html
+ var html = doc.innerHTML.replace(/[\r\n]/g, '');
+ return roundTripDiff(env, req, res, false,
DU.parseHTML(html));
+ })
+ .then(rtResponse.bind(null, env, req, res))
+ .catch(function(err) {
+ env.log('fatal/request', err);
+ });
+ };
+
+ // Round-trip article testing with selser over re-parsed HTML. Default
to
+ // scrubbing wikitext here. Can be overridden with qs param.
+ internal.roundtripSelser = function(req, res) {
+ var env = res.locals.env;
+ env.scrubWikitext = apiUtils.shouldScrub(req, true);
+
+ var target = env.normalizeAndResolvePageTitle();
+
+ var oldid = null;
+ if (req.query.oldid) {
+ oldid = req.query.oldid;
+ }
+
+ return TemplateRequest.setPageSrcInfo(env, target,
oldid).then(function() {
+ env.log('info', 'started parsing');
+ return env.getContentHandler().toHTML(env);
+ }).then(function(doc) {
+ doc = DU.parseHTML(DU.toXML(doc));
+ var comment =
doc.createComment('rtSelserEditTestComment');
+ doc.body.appendChild(comment);
+ return roundTripDiff(env, req, res, true, doc);
+ })
+ .then(rtResponse.bind(null, env, req, res))
+ .catch(function(err) {
+ env.log('fatal/request', err);
+ });
+ };
+
+ // Form-based round-tripping for manual testing
+ internal.getRtForm = function(req, res) {
+ apiUtils.renderResponse(res, 'form', {
+ title: 'Your wikitext:',
+ name: 'content',
+ });
+ };
+
+ // Form-based round-tripping for manual testing. Default to scrubbing
wikitext
+ // here. Can be overridden with qs param.
+ internal.postRtForm = function(req, res) {
+ var env = res.locals.env;
+ env.scrubWikitext = apiUtils.shouldScrub(req, true);
+
+ env.setPageSrcInfo(req.body.content);
+ env.log('info', 'started parsing');
+
+ return env.getContentHandler().toHTML(env)
+ .then(roundTripDiff.bind(null, env, req, res, false))
+ .then(rtResponse.bind(null, env, req, res))
+ .catch(function(err) {
+ env.log('fatal/request', err);
+ });
+ };
+
+ return internal;
+};
diff --git a/lib/api/routes.js b/lib/api/routes.js
index 7f8a186..00eea82 100644
--- a/lib/api/routes.js
+++ b/lib/api/routes.js
@@ -13,10 +13,7 @@
var MWParserEnv =
require('../config/MWParserEnvironment.js').MWParserEnvironment;
var Promise = require('../utils/promise.js');
var LogData = require('../logger/LogData.js').LogData;
-var ApiRequest = require('../mw/ApiRequest.js');
-
-var TemplateRequest = ApiRequest.TemplateRequest;
-
+var TemplateRequest = require('../mw/ApiRequest.js').TemplateRequest;
module.exports = function(parsoidConfig, processLogger, parsoidOptions,
promiseWorkers) {
var routes = {};
@@ -34,26 +31,6 @@
};
// Middlewares
-
- routes.internal = function(req, res, next) {
- res.locals.errorEnc = 'plain';
- var iwp = req.params.prefix || parsoidConfig.defaultWiki || '';
- if (!parsoidConfig.mwApiMap.has(iwp)) {
- return errOut(res, 'Invalid prefix: ' + iwp);
- }
- res.locals.iwp = iwp;
- res.locals.pageName = req.params.title || 'Main_Page';
- res.locals.oldid = req.body.oldid || req.query.oldid || null;
- // "body" flag to return just the body (instead of the entire
HTML doc)
- res.locals.bodyOnly = !!(req.query.body || req.body.body);
- // "subst" flag to perform {{subst:}} template expansion
- res.locals.subst = !!(req.query.subst || req.body.subst);
- res.locals.envOptions = {
- prefix: res.locals.iwp,
- pageName: res.locals.pageName,
- };
- next();
- };
var errorEncoding = new Map(Object.entries({
'pagebundle': 'json',
@@ -239,135 +216,6 @@
}
return versionCache.then(function(v) {
apiUtils.jsonResponse(res, v);
- });
- };
-
- // Form-based HTML DOM -> wikitext interface for manual testing.
- routes.html2wtForm = function(req, res) {
- var domain = parsoidConfig.mwApiMap.get(res.locals.iwp).domain;
- var action = '/' + domain + '/v3/transform/html/to/wikitext/' +
res.locals.pageName;
- if (req.query.hasOwnProperty('scrub_wikitext')) {
- action += "?scrub_wikitext=" + req.query.scrub_wikitext;
- }
- apiUtils.renderResponse(res, 'form', {
- title: 'Your HTML DOM:',
- action: action,
- name: 'html',
- });
- };
-
- // Form-based wikitext -> HTML DOM interface for manual testing
- routes.wt2htmlForm = function(req, res) {
- var domain = parsoidConfig.mwApiMap.get(res.locals.iwp).domain;
- apiUtils.renderResponse(res, 'form', {
- title: 'Your wikitext:',
- action: '/' + domain +
'/v3/transform/wikitext/to/html/' + res.locals.pageName,
- name: 'wikitext',
- });
- };
-
- // Round-trip article testing. Default to scrubbing wikitext here.
Can be
- // overridden with qs param.
- routes.roundtripTesting = function(req, res) {
- var env = res.locals.env;
- env.scrubWikitext = apiUtils.shouldScrub(req, true);
-
- var target = env.normalizeAndResolvePageTitle();
-
- var oldid = null;
- if (req.query.oldid) {
- oldid = req.query.oldid;
- }
-
- return TemplateRequest.setPageSrcInfo(env, target,
oldid).then(function() {
- env.log('info', 'started parsing');
- return env.getContentHandler().toHTML(env);
- })
- .then(apiUtils.roundTripDiff.bind(null, env, req, res, false))
- .then(apiUtils.rtResponse.bind(null, env, req, res))
- .catch(function(err) {
- env.log('fatal/request', err);
- });
- };
-
- // Round-trip article testing with newline stripping for editor-created
HTML
- // simulation. Default to scrubbing wikitext here. Can be overridden
with qs
- // param.
- routes.roundtripTestingNL = function(req, res) {
- var env = res.locals.env;
- env.scrubWikitext = apiUtils.shouldScrub(req, true);
-
- var target = env.normalizeAndResolvePageTitle();
-
- var oldid = null;
- if (req.query.oldid) {
- oldid = req.query.oldid;
- }
-
- return TemplateRequest.setPageSrcInfo(env, target,
oldid).then(function() {
- env.log('info', 'started parsing');
- return env.getContentHandler().toHTML(env);
- }).then(function(doc) {
- // strip newlines from the html
- var html = doc.innerHTML.replace(/[\r\n]/g, '');
- return apiUtils.roundTripDiff(env, req, res, false,
DU.parseHTML(html));
- })
- .then(apiUtils.rtResponse.bind(null, env, req, res))
- .catch(function(err) {
- env.log('fatal/request', err);
- });
- };
-
- // Round-trip article testing with selser over re-parsed HTML. Default
to
- // scrubbing wikitext here. Can be overridden with qs param.
- routes.roundtripSelser = function(req, res) {
- var env = res.locals.env;
- env.scrubWikitext = apiUtils.shouldScrub(req, true);
-
- var target = env.normalizeAndResolvePageTitle();
-
- var oldid = null;
- if (req.query.oldid) {
- oldid = req.query.oldid;
- }
-
- return TemplateRequest.setPageSrcInfo(env, target,
oldid).then(function() {
- env.log('info', 'started parsing');
- return env.getContentHandler().toHTML(env);
- }).then(function(doc) {
- doc = DU.parseHTML(DU.toXML(doc));
- var comment =
doc.createComment('rtSelserEditTestComment');
- doc.body.appendChild(comment);
- return apiUtils.roundTripDiff(env, req, res, true, doc);
- })
- .then(apiUtils.rtResponse.bind(null, env, req, res))
- .catch(function(err) {
- env.log('fatal/request', err);
- });
- };
-
- // Form-based round-tripping for manual testing
- routes.getRtForm = function(req, res) {
- apiUtils.renderResponse(res, 'form', {
- title: 'Your wikitext:',
- name: 'content',
- });
- };
-
- // Form-based round-tripping for manual testing. Default to scrubbing
wikitext
- // here. Can be overridden with qs param.
- routes.postRtForm = function(req, res) {
- var env = res.locals.env;
- env.scrubWikitext = apiUtils.shouldScrub(req, true);
-
- env.setPageSrcInfo(req.body.content);
- env.log('info', 'started parsing');
-
- return env.getContentHandler().toHTML(env)
- .then(apiUtils.roundTripDiff.bind(null, env, req, res, false))
- .then(apiUtils.rtResponse.bind(null, env, req, res))
- .catch(function(err) {
- env.log('fatal/request', err);
});
};
--
To view, visit https://gerrit.wikimedia.org/r/391556
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib2bcaa1f56663e2cafe91c8c5d27ccf338f28cf8
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
