jenkins-bot has submitted this change and it was merged.
Change subject: Don't strip trailing newlines and space on tokens from
sub-pipelines
......................................................................
Don't strip trailing newlines and space on tokens from sub-pipelines
The stripEOFTkfromTokens method used to strip trailing newlines and space-only
string tokens from sub-pipelines in addition to the expected EOFTk. This might
have been a work-around for missing paragraph suppression and possibly missing
stripping in other places, but does not seem to be needed any more.
It did create non-deterministic output when a trailing space-only token was
emitted from a sub-pipeline. The grouping of tokens returned from the
sub-pipeline depended on the order of template expansion, so the space would
only be stripped in some execution orders.
I have created a (currently unused) replacement method
stripTrailingNewlinesFromTokens that does the newline stripping in case that
still turns out to be necessary. We should remove it when we are satisfied
that the stripping is really not needed.
Change-Id: I83de5e87387d4d629eece946633e91b8e0f30854
---
M js/lib/mediawiki.TokenTransformManager.js
M js/lib/mediawiki.Util.js
2 files changed, 29 insertions(+), 20 deletions(-)
Approvals:
Subramanya Sastry: Looks good to me, approved
jenkins-bot: Verified
diff --git a/js/lib/mediawiki.TokenTransformManager.js
b/js/lib/mediawiki.TokenTransformManager.js
index fec8350..0eba1f7 100644
--- a/js/lib/mediawiki.TokenTransformManager.js
+++ b/js/lib/mediawiki.TokenTransformManager.js
@@ -327,7 +327,7 @@
* @param {Object} ret The chunk we're returning from the transform.
*/
AsyncTokenTransformManager.prototype.emitChunk = function( ret ) {
- this.env.dp( 'emitChunk', ret );
+ this.env.dp( 'AsyncTokenTransformManager.emitChunk', ret );
function checkForEOFTkErrors(ttm, ret, atEnd) {
if ( ttm.frame.depth === 0 &&
diff --git a/js/lib/mediawiki.Util.js b/js/lib/mediawiki.Util.js
index 845bd4a..77b7801 100644
--- a/js/lib/mediawiki.Util.js
+++ b/js/lib/mediawiki.Util.js
@@ -645,7 +645,7 @@
return tokens;
},
- // Strip 'end' tokens and trailing newlines
+ // Strip EOFTk token from token chunk
stripEOFTkfromTokens: function ( tokens ) {
// this.dp( 'stripping end or whitespace tokens' );
if ( tokens.constructor !== Array ) {
@@ -654,24 +654,33 @@
if ( ! tokens.length ) {
return tokens;
}
- // Strip 'end' tokens and trailing newlines
- var l = tokens[tokens.length - 1];
- if ( l &&
- ( l.constructor === pd.EOFTk ||
- l.constructor === pd.NlTk ||
- ( l.constructor === String && l.match( /^\s+$/
) ) ) ) {
- var origTokens = tokens;
- tokens = origTokens.slice();
- tokens.rank = origTokens.rank;
- while ( tokens.length &&
- (( l.constructor === pd.EOFTk ||
- l.constructor === pd.NlTk ) ||
- ( l.constructor === String && l.match( /^\s+$/
) ) ) )
- {
- // this.dp( 'stripping end or whitespace
tokens' );
- tokens.pop();
- l = tokens[tokens.length - 1];
- }
+ // Strip 'end' token
+ if ( tokens.length && tokens.last().constructor === pd.EOFTk ) {
+ var rank = tokens.rank;
+ tokens = tokens.slice(0,-1);
+ tokens.rank = rank;
+ }
+
+ return tokens;
+ },
+
+ // Strip NlTk and ws-only trailing text tokens. Used to be part of
+ // stripEOFTkfromTokens, but unclear if this is still needed.
+ // TODO: remove this if this is not needed any more!
+ stripTrailingNewlinesFromTokens: function (tokens) {
+ var token = tokens.last(),
+ lastMatches = function(toks) {
+ var lastTok = toks.last();
+ return lastTok && (
+ lastTok.constructor === pd.NlTk
||
+ lastTok.constructor === String
&& /^\s+$/.test(token));
+ };
+ if (lastMatches) {
+ tokens = tokens.slice();
+ }
+ while (lastMatches)
+ {
+ tokens.pop();
}
return tokens;
},
--
To view, visit https://gerrit.wikimedia.org/r/81711
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I83de5e87387d4d629eece946633e91b8e0f30854
Gerrit-PatchSet: 3
Gerrit-Project: mediawiki/extensions/Parsoid
Gerrit-Branch: master
Gerrit-Owner: GWicke <[email protected]>
Gerrit-Reviewer: Subramanya Sastry <[email protected]>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
