Subramanya Sastry has uploaded a new change for review.
https://gerrit.wikimedia.org/r/225253
Change subject: [hackathon project] Convert github-flavored markdown to wikitext
......................................................................
[hackathon project] Convert github-flavored markdown to wikitext
* Added a commandline script and a private API endpoint for
doing the conversion. This accepts Github flavored markdown
right now.
Change-Id: I39185378b0e98b72e9909a533508fb8ec0668b13
---
M api/ParsoidService.js
M api/routes.js
M lib/mediawiki.DOMUtils.js
M package.json
A tests/md2wt.js
5 files changed, 131 insertions(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid
refs/changes/53/225253/1
diff --git a/api/ParsoidService.js b/api/ParsoidService.js
index c9a0573..4fdaf14 100644
--- a/api/ParsoidService.js
+++ b/api/ParsoidService.js
@@ -96,6 +96,10 @@
app.get( re('^/_rtform/(?:(' + mwApiRe + ')/(.*))?'), i, p,
routes.getRtForm );
app.post( re('^/_rtform/(?:(' + mwApiRe + ')/(.*))?'), i, p,
routes.postRtForm );
+ // markdown -> wt forms
+ app.get(re('^/_md2wt'), i, p, routes.md2wtForm);
+ app.post(re('^/_md2wt'), i, p, routes.postMarkdownForm);
+
// v1 API routes
app.get( re('^/(' + mwApiRe + ')/(.*)'), i, p, routes.v1Get );
app.post( re('^/(' + mwApiRe + ')/(.*)'), i, p, routes.v1Post );
diff --git a/api/routes.js b/api/routes.js
index f503e10..b945b69 100644
--- a/api/routes.js
+++ b/api/routes.js
@@ -303,6 +303,33 @@
});
};
+ // Form-based markdown -> wikitext interface
+ routes.md2wtForm = function( req, res ) {
+ var env = res.local('env');
+ var action = "/_md2wt";
+ apiUtils.renderResponse(res, env, "form", {
+ title: "Your Markdown:",
+ action: action,
+ name: "markdown"
+ });
+ };
+
+ // POST from markdown -> wikitext form
+ routes.postMarkdownForm = function(req, res) {
+ // Always scrub the wikitext
+ var env = res.local('env');
+ env.scrubWikitext = true;
+
+ // convert markdown -> html -> wikitext
+ var md = require('marked');
+ var dom = DU.parseHTML(md(req.body.markdown));
+ DU.massageMarkdownDOM(dom.body);
+ DU.serializeDOM(env, dom.body, false).then(function(out) {
+ apiUtils.setHeader(res, env, 'content-type',
apiUtils.WIKITEXT_CONTENT_TYPE);
+ apiUtils.endResponse(res, env, out);
+ });
+ };
+
// Form-based round-tripping for manual testing. Default to scrubbing
wikitext
// here. Can be overridden with qs param.
routes.postRtForm = function(req, res) {
diff --git a/lib/mediawiki.DOMUtils.js b/lib/mediawiki.DOMUtils.js
index 3c5c4eb..41b37b3 100644
--- a/lib/mediawiki.DOMUtils.js
+++ b/lib/mediawiki.DOMUtils.js
@@ -2621,6 +2621,35 @@
return out;
};
+DOMUtils.renameNode = function(node, newName) {
+ var newNode = node.ownerDocument.createElement(newName);
+ node.parentNode.insertBefore(newNode, node);
+ DU.migrateChildren(node, newNode);
+ DU.deleteNode(node);
+};
+
+// Make DOM changes to convert markdown HTML
+// to HTML that Parsoid can convert to core wikitext.
+// <strong> --> <b> and <em> --> <i>
+DOMUtils.massageMarkdownDOM = function(node) {
+ var next = node.nextSibling;
+
+ if (DU.isElt(node)) {
+ if (node.nodeName === 'STRONG') {
+ DU.renameNode(node, 'B');
+ } else if (node.nodeName === 'EM') {
+ DU.renameNode(node, 'I');
+ }
+
+ if (node.firstChild) {
+ DU.massageMarkdownDOM(node.firstChild);
+ }
+ }
+
+ if (next) {
+ DU.massageMarkdownDOM(next);
+ }
+};
if (typeof module === "object") {
module.exports.DOMUtils = DOMUtils;
diff --git a/package.json b/package.json
index 4a84551..7d253f1 100644
--- a/package.json
+++ b/package.json
@@ -22,7 +22,8 @@
"prfun": "~2.0.0",
"request": "~2.40.0",
"simplediff": "~0.1.1",
- "yargs": "~1.3.1"
+ "yargs": "~1.3.1",
+ "marked": "^0.3.3"
},
"devDependencies": {
"chai": "^3.0.0",
diff --git a/tests/md2wt.js b/tests/md2wt.js
new file mode 100644
index 0000000..368c185
--- /dev/null
+++ b/tests/md2wt.js
@@ -0,0 +1,69 @@
+'use strict';
+
+require('../lib/core-upgrade.js');
+var yargs = require('yargs');
+var md = require('marked');
+var DU = require('../lib/mediawiki.DOMUtils.js').DOMUtils;
+var Util = require('../lib/mediawiki.Util.js').Util;
+var ParserEnv =
require('../lib/mediawiki.parser.environment.js').MWParserEnvironment;
+var ParsoidConfig = require( '../lib/mediawiki.ParsoidConfig.js'
).ParsoidConfig;
+
+md.setOptions({
+ renderer: new md.Renderer(),
+ gfm: true,
+ tables: true,
+ breaks: false,
+ pedantic: false,
+ sanitize: true,
+ smartLists: true,
+ smartypants: false
+});
+
+var local = null;
+var opts = yargs.usage("Foobar", {});
+var argv = opts.argv;
+
+var setup = function(parsoidConfig) {
+ if (local && local.setup) {
+ local.setup(parsoidConfig);
+ }
+ Util.setTemplatingAndProcessingFlags(parsoidConfig, argv);
+ Util.setDebuggingFlags(parsoidConfig, argv);
+};
+
+var parsoidConfig = new ParsoidConfig(
+ { setup: setup },
+ { defaultWiki: 'enwiki' }
+);
+
+var p = new Promise(function(resolve) {
+ // collect input
+ var inputChunks = [];
+ var stdin = process.stdin;
+ stdin.resume();
+ stdin.setEncoding('utf8');
+ stdin.on('data', function(chunk) {
+ inputChunks.push(chunk);
+ });
+ stdin.on('end', function() {
+ resolve(inputChunks);
+ });
+}).then(function(inputChunks) {
+ var txt = inputChunks.join('');
+ // console.warn("text: " + txt);
+ var html = md(txt);
+ // console.warn("html: " + html);
+ var dom = DU.parseHTML(html);
+ DU.massageMarkdownDOM(dom.body);
+ // console.warn("dom: " + dom.body.outerHTML);
+ ParserEnv.getParserEnv(parsoidConfig, null, {
+ prefix: 'enwiki',
+ pageName: argv.page
+ }).then(function(env) {
+ // Always scrub the wikitext
+ env.scrubWikitext = true;
+ DU.serializeDOM(env, dom.body, false).then(function(out) {
+ console.log('wt:\n' + out);
+ });
+ });
+});
--
To view, visit https://gerrit.wikimedia.org/r/225253
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I39185378b0e98b72e9909a533508fb8ec0668b13
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Subramanya Sastry <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
