https://www.mediawiki.org/wiki/Special:Code/MediaWiki/113285
Revision: 113285
Author: gwicke
Date: 2012-03-07 20:06:54 +0000 (Wed, 07 Mar 2012)
Log Message:
-----------
Token representation clean-up. Now all tokens are differentiated using
constructors instead of type attributes.
Modified Paths:
--------------
trunk/extensions/VisualEditor/modules/parser/ext.Cite.js
trunk/extensions/VisualEditor/modules/parser/ext.core.LinkHandler.js
trunk/extensions/VisualEditor/modules/parser/ext.core.NoIncludeOnly.js
trunk/extensions/VisualEditor/modules/parser/ext.core.PostExpandParagraphHandler.js
trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js
trunk/extensions/VisualEditor/modules/parser/ext.util.TokenCollector.js
trunk/extensions/VisualEditor/modules/parser/mediawiki.HTML5TreeBuilder.node.js
trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformManager.js
trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.defines.js
trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.environment.js
trunk/extensions/VisualEditor/modules/parser/pegTokenizer.pegjs.txt
Modified: trunk/extensions/VisualEditor/modules/parser/ext.Cite.js
===================================================================
--- trunk/extensions/VisualEditor/modules/parser/ext.Cite.js 2012-03-07
20:03:59 UTC (rev 113284)
+++ trunk/extensions/VisualEditor/modules/parser/ext.Cite.js 2012-03-07
20:06:54 UTC (rev 113285)
@@ -125,7 +125,7 @@
return tokenCTX;
} else if ( this.isActive &&
// Also accept really broken ref close tags..
- ['TAG', 'ENDTAG', 'SELFCLOSINGTAG'].indexOf(token.type)
>= 0 &&
+ [TagTk, EndTagTk,
SelfclosingTagTk].indexOf(token.constructor) >= 0 &&
token.name.toLowerCase() === 'ref'
)
{
Modified: trunk/extensions/VisualEditor/modules/parser/ext.core.LinkHandler.js
===================================================================
--- trunk/extensions/VisualEditor/modules/parser/ext.core.LinkHandler.js
2012-03-07 20:03:59 UTC (rev 113284)
+++ trunk/extensions/VisualEditor/modules/parser/ext.core.LinkHandler.js
2012-03-07 20:06:54 UTC (rev 113285)
@@ -170,9 +170,9 @@
var token = tokens[i];
if ( token.constructor === String ) {
s += token;
- } else if ( token.type === 'NEWLINE' ) {
+ } else if ( token.constructor === NlTk ) {
s += '\n'; // XXX: preserve original newline
- } else if ( token.type === 'COMMENT' ) {
+ } else if ( token.constructor === CommentTk ) {
// strip it
} else {
var res = this.imageParser.processImageOptions( s,
'img_options' ),
Modified: trunk/extensions/VisualEditor/modules/parser/ext.core.NoIncludeOnly.js
===================================================================
--- trunk/extensions/VisualEditor/modules/parser/ext.core.NoIncludeOnly.js
2012-03-07 20:03:59 UTC (rev 113284)
+++ trunk/extensions/VisualEditor/modules/parser/ext.core.NoIncludeOnly.js
2012-03-07 20:06:54 UTC (rev 113285)
@@ -36,7 +36,7 @@
OnlyInclude.prototype.onAnyInclude = function ( token, manager ) {
//this.manager.env.dp( 'onAnyInclude', token, this );
- if ( token.type === 'END' ) {
+ if ( token.constructor === EOFTk ) {
this.inOnlyInclude = false;
if ( this.accum.length && ! this.foundOnlyInclude ) {
var res = this.accum;
@@ -92,7 +92,7 @@
} else {
tokens.shift();
if ( tokens.length &&
- tokens[tokens.length - 1].type
!== 'END' ) {
+ tokens[tokens.length -
1].constructor !== EOFTk ) {
tokens.pop();
}
return { tokens: tokens };
@@ -114,7 +114,7 @@
if ( isInclude ) {
tokens.shift();
if ( tokens.length &&
- tokens[tokens.length - 1].type
!== 'END' ) {
+ tokens[tokens.length -
1].constructor !== EOFTk ) {
tokens.pop();
}
return { tokens: tokens };
Modified:
trunk/extensions/VisualEditor/modules/parser/ext.core.PostExpandParagraphHandler.js
===================================================================
---
trunk/extensions/VisualEditor/modules/parser/ext.core.PostExpandParagraphHandler.js
2012-03-07 20:03:59 UTC (rev 113284)
+++
trunk/extensions/VisualEditor/modules/parser/ext.core.PostExpandParagraphHandler.js
2012-03-07 20:06:54 UTC (rev 113285)
@@ -73,7 +73,7 @@
PostExpandParagraphHandler.prototype.onAny = function ( token, frame, cb ) {
//console.warn( 'PostExpandParagraphHandler.onAny' );
this.tokens.push( token );
- if ( token.type === 'COMMENT' ||
+ if ( token.constructor === CommentTk ||
( token.constructor === String && token.match( /^[\t
]+$/ ) )
)
{
Modified:
trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js
===================================================================
--- trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js
2012-03-07 20:03:59 UTC (rev 113284)
+++ trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js
2012-03-07 20:06:54 UTC (rev 113285)
@@ -280,7 +280,7 @@
// Strip 'end' tokens and trailing newlines
var l = res[res.length - 1];
while ( res.length &&
- ( l.type === 'END' || l.constructor === NlTk )
+ ( l.constructor === EOFTk || l.constructor ===
NlTk )
)
{
this.manager.env.dp( 'TemplateHandler, stripping end or
whitespace tokens' );
Modified:
trunk/extensions/VisualEditor/modules/parser/ext.util.TokenCollector.js
===================================================================
--- trunk/extensions/VisualEditor/modules/parser/ext.util.TokenCollector.js
2012-03-07 20:03:59 UTC (rev 113284)
+++ trunk/extensions/VisualEditor/modules/parser/ext.util.TokenCollector.js
2012-03-07 20:06:54 UTC (rev 113285)
@@ -61,7 +61,7 @@
this.tokens.push ( token );
this.isActive = false;
this.manager.removeTransform( this.rank + this._anyDelta, 'any'
);
- if ( token.type !== 'END' || this.toEnd ) {
+ if ( token.constructor !== EOFTk || this.toEnd ) {
// end token
res = this.transformation ( this.tokens, this.cb,
this.manager );
this.tokens = [];
@@ -75,7 +75,7 @@
this.tokens = [];
return { tokens: res };
}
- } else if ( token.type !== 'END' ) {
+ } else if ( token.constructor !== EOFTk ) {
this.manager.env.dp( 'starting collection on ', token );
// start collection
this.tokens.push ( token );
Modified:
trunk/extensions/VisualEditor/modules/parser/mediawiki.HTML5TreeBuilder.node.js
===================================================================
---
trunk/extensions/VisualEditor/modules/parser/mediawiki.HTML5TreeBuilder.node.js
2012-03-07 20:03:59 UTC (rev 113284)
+++
trunk/extensions/VisualEditor/modules/parser/mediawiki.HTML5TreeBuilder.node.js
2012-03-07 20:06:54 UTC (rev 113285)
@@ -110,28 +110,25 @@
name: token.name,
data: this._att(token.attribs)});
break;
- default:
- switch (token.type) {
- case "COMMENT":
- this.emit('token', {type: 'Comment',
- data: token.value});
- break;
- case "END":
- this.emit('end');
- this.emit('token', { type: 'EOF' } );
- this.document = this.parser.document;
- if ( ! this.document.body ) {
- // HACK: This should not be
needed really.
- this.document.body =
this.parser.document.getElementsByTagName('body')[0];
- }
- // Emit the document to consumers
- //this.emit('document', this.document);
- break;
- default:
- console.warn("Unhandled token: " +
JSON.stringify(token));
- break;
+ case CommentTk:
+ this.emit('token', {type: 'Comment',
+ data: token.value});
+ break;
+ case EOFTk:
+ this.emit('end');
+ this.emit('token', { type: 'EOF' } );
+ this.document = this.parser.document;
+ if ( ! this.document.body ) {
+ // HACK: This should not be needed really.
+ this.document.body =
this.parser.document.getElementsByTagName('body')[0];
}
+ // Emit the document to consumers
+ //this.emit('document', this.document);
break;
+ default:
+ console.warn("Unhandled token: " +
JSON.stringify(token));
+ break;
+ break;
}
};
Modified:
trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformManager.js
===================================================================
---
trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformManager.js
2012-03-07 20:03:59 UTC (rev 113284)
+++
trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformManager.js
2012-03-07 20:06:54 UTC (rev 113285)
@@ -37,7 +37,7 @@
TokenTransformManager.prototype._construct = function () {
this.transformers = {
- tag: {}, // for TAG, ENDTAG, SELFCLOSINGTAG, keyed on name
+ tag: {}, // for TagTk, EndTagTk, SelfclosingTagTk, keyed on name
text: [],
newline: [],
comment: [],
@@ -452,19 +452,16 @@
case SelfclosingTagTk:
res = this._transformTagToken( token, cb );
break;
+ case CommentTk:
+ res = this._transformToken( token, ts.comment,
cb );
+ break;
+ case EOFTk:
+ res = this._transformToken( token, ts.end, cb );
+ break;
default:
- switch( token.type ) {
- case 'COMMENT':
- res = this._transformToken(
token, ts.comment, cb );
- break;
- case 'END':
- res = this._transformToken(
token, ts.end, cb );
- break;
- default:
- res = this._transformToken(
token, ts.martian, cb );
- break;
- }
+ res = this._transformToken( token, ts.martian,
cb );
break;
+ break;
}
if( res.tokens ) {
@@ -526,15 +523,9 @@
function ( tokens, notYetDone, allTokensProcessed ) {
//tokens = this._transformPhase2( this.frame, tokens, this.parentCB );
- //if ( tokens.length && tokens[tokens.length - 1].type === 'END' ) {
- // this.env.dp( 'AsyncTokenTransformManager, stripping end ' );
- // tokens.pop();
- //}
-
this.env.dp( 'AsyncTokenTransformManager._returnTokens, emitting chunk:
',
tokens );
-
if( !allTokensProcessed ) {
var res = this.transformTokens( tokens,
this._returnTokens.bind(this) );
this.emit( 'chunk', res.tokens );
@@ -559,13 +550,7 @@
this.emit( 'chunk', tokens );
if ( ! notYetDone ) {
-
//console.warn('AsyncTokenTransformManager._returnTokens done. tokens:' +
- // JSON.stringify( tokens, null, 2 ) + ',
listeners: ' +
- // JSON.stringify( this.listeners( 'chunk'
), null, 2 ) );
// signal our done-ness to consumers.
- //if ( this.atTopLevel ) {
- // this.emit( 'chunk', [{type: 'END'}]);
- //}
this.emit( 'end' );
// and reset internal state.
this._reset();
@@ -663,18 +648,15 @@
case SelfclosingTagTk:
res = this._transformTagToken( token,
this.prevToken );
break;
+ case CommentTk:
+ res = this._transformToken( token, ts.comment,
this.prevToken );
+ break;
+ case EOFTk:
+ res = this._transformToken( token, ts.end,
this.prevToken );
+ break;
default:
- switch( token.type ) {
- case 'COMMENT':
- res = this._transformToken(
token, ts.comment, this.prevToken );
- break;
- case 'END':
- res = this._transformToken(
token, ts.end, this.prevToken );
- break;
- default:
- res = this._transformToken(
token, ts.martian, this.prevToken );
- break;
- }
+ res = this._transformToken( token, ts.martian,
this.prevToken );
+ break;
}
if( res.tokens ) {
@@ -766,7 +748,7 @@
pipe.on( 'end',
this.onEnd.bind( this,
this._returnAttributeKey.bind( this, i ) )
);
- pipe.process( attributes[i].k.concat([{type:'END'}]) );
+ pipe.process( attributes[i].k.concat([ new EOFTk() ]) );
} else {
kv.key = cur.k;
}
@@ -785,7 +767,7 @@
this.onEnd.bind( this,
this._returnAttributeValue.bind( this, i ) )
);
//console.warn('starting attribute transform of ' +
JSON.stringify( attributes[i].v ) );
- pipe.process( cur.v.concat([{type:'END'}]) );
+ pipe.process( cur.v.concat([ new EOFTk() ]) );
} else {
kv.value = cur.v;
}
@@ -820,7 +802,7 @@
* Collect chunks returned from the pipeline
*/
AttributeTransformManager.prototype.onChunk = function ( cb, chunk ) {
- if ( chunk.length && chunk[chunk.length - 1].type === 'END' ) {
+ if ( chunk.length && chunk[chunk.length - 1].constructor === EOFTk ) {
chunk.pop();
}
cb( chunk, true );
Modified:
trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.defines.js
===================================================================
--- trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.defines.js
2012-03-07 20:03:59 UTC (rev 113284)
+++ trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.defines.js
2012-03-07 20:06:54 UTC (rev 113285)
@@ -18,7 +18,6 @@
TagTk.prototype.toString = toString;
function EndTagTk( name, attribs ) {
- //this.type = 'ENDTAG';
this.name = name;
this.attribs = attribs || [];
}
@@ -52,7 +51,6 @@
NlTk.prototype.toString = toString;
function CommentTk( value ) {
- this.type = 'COMMENT';
this.value = value;
}
CommentTk.prototype = new Object();
@@ -62,9 +60,7 @@
CommentTk.prototype.constructor = CommentTk;
CommentTk.prototype.toString = toString;
-function EOFTk( ) {
- this.type = 'END';
-}
+function EOFTk( ) { }
EOFTk.prototype = new Object();
EOFTk.prototype.toJSON = function () {
return $.extend( { type: 'EOFTk' }, this );
Modified:
trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.environment.js
===================================================================
---
trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.environment.js
2012-03-07 20:03:59 UTC (rev 113284)
+++
trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.environment.js
2012-03-07 20:06:54 UTC (rev 113285)
@@ -194,7 +194,7 @@
}
if ( token.constructor === String ) {
out.push( token );
- } else if ( token.type === 'COMMENT' || token.type ===
'NEWLINE' ) {
+ } else if ( token.constructor === CommentTk ||
token.constructor === NlTk ) {
// strip comments and newlines
} else {
if ( strict ) {
Modified: trunk/extensions/VisualEditor/modules/parser/pegTokenizer.pegjs.txt
===================================================================
--- trunk/extensions/VisualEditor/modules/parser/pegTokenizer.pegjs.txt
2012-03-07 20:03:59 UTC (rev 113284)
+++ trunk/extensions/VisualEditor/modules/parser/pegTokenizer.pegjs.txt
2012-03-07 20:06:54 UTC (rev 113285)
@@ -354,16 +354,6 @@
start
= e:toplevelblock* newline* {
// end is passed inline as a token, as well as a separate event for now.
-
- // this does not work yet.
- //console.warn('about to emit' + pp(self));
- //self._tokenizer.emit('chunk', [ { type: 'END' } ] );
- //self._tokenizer.emit('end');
- // Append the end (for obvious reasons this should not
- // be part of a stream, only when tokenizing complete
- // texts)
- //console.warn( pp( flatten ( e ) ) );
- cache = {};
__parseArgs[2]( [ new EOFTk( ) ] );
return []; //flatten(e);
}
@@ -550,7 +540,7 @@
comment
= '<!--' c:comment_chars* ('-->' / eof)
cs:(space* newline space* cn:comment { return cn })* {
- return [{ type: 'COMMENT', value: c.join('') }].concat(cs);
+ return [new CommentTk( c.join('') )].concat(cs);
}
comment_chars
_______________________________________________
MediaWiki-CVS mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-cvs
