jenkins-bot has submitted this change and it was merged. Change subject: Generalized operation of TokenStreamPatcher a bit. ......................................................................
Generalized operation of TokenStreamPatcher a bit. * The token stream patcher is a workaround/hack to deal with limitations of not having a preprocessing tpl-expansion. So far, it had a special case for tackling re-parsing of table tags and nothing else. * This cleans up the argument handling a bit and adds a buffer, borrowed from change I54bed19d4a3ab1a096d8756fe62667b59fa0cdf4. * No change in behavior; no change in parser test results. Co-authored-by: C. Scott Ananian <[email protected]> Co-authored-by: Subramanya Sastry <[email protected]> Change-Id: Id974296f96ae6e222adad91b87dc3cbfd4e5fe30 --- M js/lib/ext.core.TokenStreamPatcher.js M js/lib/mediawiki.tokenizer.peg.js 2 files changed, 22 insertions(+), 9 deletions(-) Approvals: Subramanya Sastry: Looks good to me, approved jenkins-bot: Verified diff --git a/js/lib/ext.core.TokenStreamPatcher.js b/js/lib/ext.core.TokenStreamPatcher.js index e9fadeb..95ed483 100644 --- a/js/lib/ext.core.TokenStreamPatcher.js +++ b/js/lib/ext.core.TokenStreamPatcher.js @@ -13,6 +13,7 @@ var PegTokenizer = require('./mediawiki.tokenizer.peg.js').PegTokenizer, Util = require('./mediawiki.Util.js').Util, defines = require('./mediawiki.parser.defines.js'); + // define some constructor shortcuts var CommentTk = defines.CommentTk, TagTk = defines.TagTk, @@ -21,7 +22,7 @@ function TokenStreamPatcher( manager, options ) { this.manager = manager; - this.tokenizer = new PegTokenizer(); + this.tokenizer = new PegTokenizer(this.manager.env); manager.addTransform(this.onNewline.bind(this), "TokenStreamPatcher:onNewline", this.nlRank, 'newline'); @@ -37,20 +38,29 @@ TokenStreamPatcher.prototype.anyRank = 2.002; TokenStreamPatcher.prototype.endRank = 2.003; +TokenStreamPatcher.prototype.updateBuf = function() { + if (this.buf.length === 0) { + this.buf.sol = this.sol; + this.buf.srcOffset = this.srcOffset; + } +}; + TokenStreamPatcher.prototype.reset = function() { this.inNowiki = false; this.sol = true; - this.srcOffset = null; + this.srcOffset = 0; this.buf = []; + this.updateBuf(); }; -TokenStreamPatcher.prototype.onNewline = function(token, manager) { +TokenStreamPatcher.prototype.onNewline = function(token) { this.sol = true; this.srcOffset = (token.dataAttribs.tsr || [null,null])[1]; + this.updateBuf(); return {tokens: [token]}; }; -TokenStreamPatcher.prototype.onEnd = function(token, manager) { +TokenStreamPatcher.prototype.onEnd = function(token) { this.reset(); return {tokens: [token]}; }; @@ -61,7 +71,7 @@ this.sol = false; }; -TokenStreamPatcher.prototype.onAny = function(token, manager) { +TokenStreamPatcher.prototype.onAny = function(token) { // console.warn("T: " + JSON.stringify(token)); var tokens = [token]; switch (token.constructor) { diff --git a/js/lib/mediawiki.tokenizer.peg.js b/js/lib/mediawiki.tokenizer.peg.js index 5010642..6fb9c70 100644 --- a/js/lib/mediawiki.tokenizer.peg.js +++ b/js/lib/mediawiki.tokenizer.peg.js @@ -216,17 +216,20 @@ * Tokenize via a production passed in as an arg. * The text is tokenized synchronously in one shot. */ -PegTokenizer.prototype.tokenize = function( text, production ) { +PegTokenizer.prototype.tokenize = function( text, production, args ) { try { // Some productions use callbacks: start, tlb, toplevelblock. // All other productions return tokens directly. - var toks = [], - retToks = this.tokenizer.tokenize(text, production, { + var toks = []; + if (!args) { + args = { cb: function(r) { toks = toks.concat(r); }, pegTokenizer: this, srcOffset: this.offsets.startOffset || 0, env: this.env - }); + }; + } + var retToks = this.tokenizer.tokenize(text, production, args); if (retToks.constructor === Array && retToks.length > 0) { toks = toks.concat(retToks); -- To view, visit https://gerrit.wikimedia.org/r/76825 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Id974296f96ae6e222adad91b87dc3cbfd4e5fe30 Gerrit-PatchSet: 4 Gerrit-Project: mediawiki/extensions/Parsoid Gerrit-Branch: master Gerrit-Owner: Cscott <[email protected]> Gerrit-Reviewer: Subramanya Sastry <[email protected]> Gerrit-Reviewer: jenkins-bot _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
