https://www.mediawiki.org/wiki/Special:Code/MediaWiki/109653
Revision: 109653
Author: gwicke
Date: 2012-01-20 22:02:23 +0000 (Fri, 20 Jan 2012)
Log Message:
-----------
* NoInclude and IncludeOnly improvements
* Tokenizer support for templates and template args in template arguments and
titles
* Async attribute expansion fixes
Modified Paths:
--------------
trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js
trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformManager.js
trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.js
trunk/extensions/VisualEditor/modules/parser/pegTokenizer.pegjs.txt
Added Paths:
-----------
trunk/extensions/VisualEditor/modules/parser/ext.core.NoIncludeOnly.js
Removed Paths:
-------------
trunk/extensions/VisualEditor/modules/parser/ext.core.NoOnlyInclude.js
Copied: trunk/extensions/VisualEditor/modules/parser/ext.core.NoIncludeOnly.js
(from rev 109633,
trunk/extensions/VisualEditor/modules/parser/ext.core.NoOnlyInclude.js)
===================================================================
--- trunk/extensions/VisualEditor/modules/parser/ext.core.NoIncludeOnly.js
(rev 0)
+++ trunk/extensions/VisualEditor/modules/parser/ext.core.NoIncludeOnly.js
2012-01-20 22:02:23 UTC (rev 109653)
@@ -0,0 +1,59 @@
+/**
+ * Simple noinclude / onlyinclude implementation. Strips all tokens in
+ * noinclude sections.
+ *
+ * @author Gabriel Wicke <[email protected]>
+ */
+
+var TokenCollector = require( './ext.util.TokenCollector.js' ).TokenCollector;
+
+function NoInclude( manager, isInclude ) {
+ new TokenCollector(
+ manager,
+ function ( tokens ) {
+ if ( isInclude ) {
+ manager.env.dp( 'noinclude stripping',
tokens );
+ return {};
+ } else {
+ tokens.shift();
+ if ( tokens.length &&
+ tokens[tokens.length - 1].type
!== 'END' ) {
+ tokens.pop();
+ }
+ return { tokens: tokens };
+ }
+ }, // just strip it all..
+ true, // match the end-of-input if </noinclude> is
missing
+ 0.01, // very early in stage 1, to avoid any further
processing.
+ 'tag',
+ 'noinclude'
+ );
+}
+
+function IncludeOnly( manager, isInclude ) {
+ new TokenCollector(
+ manager,
+ function ( tokens ) {
+ if ( isInclude ) {
+ tokens.shift();
+ if ( tokens.length &&
+ tokens[tokens.length - 1].type
!== 'END' ) {
+ tokens.pop();
+ }
+ return { tokens: tokens };
+ } else {
+ return {};
+ }
+ },
+ true, // match the end-of-input if </noinclude> is
missing
+ 0.01, // very early in stage 1, to avoid any further
processing.
+ 'tag',
+ 'includeonly'
+ );
+}
+
+
+if (typeof module == "object") {
+ module.exports.NoInclude = NoInclude;
+ module.exports.IncludeOnly = IncludeOnly;
+}
Deleted: trunk/extensions/VisualEditor/modules/parser/ext.core.NoOnlyInclude.js
===================================================================
--- trunk/extensions/VisualEditor/modules/parser/ext.core.NoOnlyInclude.js
2012-01-20 21:55:15 UTC (rev 109652)
+++ trunk/extensions/VisualEditor/modules/parser/ext.core.NoOnlyInclude.js
2012-01-20 22:02:23 UTC (rev 109653)
@@ -1,39 +0,0 @@
-/**
- * Simple noinclude / onlyinclude implementation. Strips all tokens in
- * noinclude sections.
- *
- * @author Gabriel Wicke <[email protected]>
- */
-
-var TokenCollector = require( './ext.util.TokenCollector.js' ).TokenCollector;
-
-function NoInclude( manager ) {
- new TokenCollector(
- manager,
- function ( tokens ) {
- manager.env.dp( 'noinclude stripping', tokens );
- return {};
- }, // just strip it all..
- true, // match the end-of-input if </noinclude> is
missing
- 0.01, // very early in stage 1, to avoid any further
processing.
- 'tag',
- 'noinclude'
- );
-}
-
-function OnlyInclude( manager ) {
- new TokenCollector(
- manager,
- function ( ) { return {} }, // just strip it all..
- true, // match the end-of-input if </noinclude> is
missing
- 0.01, // very early in stage 1, to avoid any further
processing.
- 'tag',
- 'onlyinclude'
- );
-}
-
-
-if (typeof module == "object") {
- module.exports.NoInclude = NoInclude;
- module.exports.OnlyInclude = OnlyInclude;
-}
Modified:
trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js
===================================================================
--- trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js
2012-01-20 21:55:15 UTC (rev 109652)
+++ trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js
2012-01-20 22:02:23 UTC (rev 109653)
@@ -75,8 +75,8 @@
var attributes = [[[{ type: 'TEXT', value: '' }] , token.target ]]
.concat( this._nameArgs( token.orderedArgs ) );
- //console.log( 'before AttributeTransformManager: ' +
- // JSON.stringify( attributes,
null, 2 ) );
+ this.manager.env.dp( 'before AttributeTransformManager: ' +
+ JSON.stringify( attributes,
null, 2 ) );
new AttributeTransformManager(
this.manager,
this._returnAttributes.bind( this,
tplExpandData )
@@ -85,7 +85,10 @@
// Unblock finish
if ( ! tplExpandData.attribsAsync ) {
// Attributes were transformed synchronously
- this.manager.env.dp( 'sync attribs for ' + JSON.stringify(
token ));
+ this.manager.env.dp (
+ 'sync attribs for ' + JSON.stringify(
tplExpandData.target ),
+ tplExpandData.expandedArgs
+ );
// All attributes are fully expanded synchronously (no IO was
needed)
return this._expandTemplate ( tplExpandData );
} else {
@@ -110,7 +113,7 @@
out.push( orderedArgs[i] );
}
}
- //console.log( '_nameArgs: ' + JSON.stringify( out ) );
+ this.manager.env.dp( '_nameArgs: ' + JSON.stringify( out ) );
return out;
};
@@ -190,6 +193,7 @@
this.manager.env.KVtoHash(
tplExpandData.expandedArgs ),
tplExpandData.target
);
+ this.manager.env.dp( 'argHash:', this.manager.env.KVtoHash(
tplExpandData.expandedArgs ) );
// Hook up the inputPipeline output events to our handlers
inputPipeline.addListener( 'chunk', this._onChunk.bind ( this,
tplExpandData ) );
@@ -359,8 +363,8 @@
// ' vs. ' + JSON.stringify( this.manager.args )
);
res = this.manager.args[argName];
} else {
- //console.log( 'templateArg not found: ' + argName +
- // ' vs. ' + JSON.stringify( this.manager.args ) );
+ console.log( 'templateArg not found: ' + argName +
+ ' vs. ' + JSON.stringify( this.manager.args ) );
if ( token.attribs.length > 1 ) {
res = defaultValue;
} else {
Modified:
trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformManager.js
===================================================================
---
trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformManager.js
2012-01-20 21:55:15 UTC (rev 109652)
+++
trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformManager.js
2012-01-20 22:02:23 UTC (rev 109653)
@@ -664,7 +664,7 @@
}
}
this.env.dp( 'SyncTokenTransformManager.onChunk: emitting ' +
- JSON.stringify( localAccum ) );
+ JSON.stringify( localAccum, null, 2 ) );
this.emit( 'chunk', localAccum );
};
@@ -698,7 +698,7 @@
function AttributeTransformManager ( manager, callback ) {
this.manager = manager;
this.callback = callback;
- this.outstanding = 0;
+ this.outstanding = 1;
this.kvs = [];
//this.pipe = manager.getAttributePipeline( manager.args );
}
@@ -726,7 +726,7 @@
pipe.addListener( 'end',
this.onEnd.bind( this,
this._returnAttributeKey.bind( this, i ) )
);
- pipe.process( attributes[i][0] );
+ pipe.process( attributes[i][0].concat([{type:'END'}]) );
// transform the value
pipe = this.manager.getAttributePipeline( this.manager.args );
@@ -737,8 +737,12 @@
this.onEnd.bind( this,
this._returnAttributeValue.bind( this, i ) )
);
//console.log('starting attribute transform of ' +
JSON.stringify( attributes[i][1] ) );
- pipe.process( attributes[i][1] );
+ pipe.process( attributes[i][1].concat([{type:'END'}]) );
}
+ this.outstanding--;
+ if ( this.outstanding == 0 ) {
+ this._returnAttributes();
+ }
};
AttributeTransformManager.prototype._returnAttributes = function ( ) {
@@ -758,6 +762,9 @@
* Collect chunks returned from the pipeline
*/
AttributeTransformManager.prototype.onChunk = function ( cb, chunk ) {
+ if ( chunk.length && chunk[chunk.length - 1].type === 'END' ) {
+ chunk.pop();
+ }
cb( chunk, true );
};
@@ -773,7 +780,8 @@
* Callback for async argument value expansions
*/
AttributeTransformManager.prototype._returnAttributeValue = function ( ref,
tokens, notYetDone ) {
- //console.log( 'check _returnAttributeValue: ' + JSON.stringify( tokens
) );
+ //console.log( 'check _returnAttributeValue: ' + JSON.stringify( tokens
) +
+ // ' notYetDone:' + notYetDone );
this.kvs[ref].value = this.kvs[ref].value.concat( tokens );
if ( ! notYetDone ) {
this.outstanding--;
@@ -788,6 +796,8 @@
* Callback for async argument key expansions
*/
AttributeTransformManager.prototype._returnAttributeKey = function ( ref,
tokens, notYetDone ) {
+ //console.log( 'check _returnAttributeKey: ' + JSON.stringify( tokens )
+
+ // ' notYetDone:' + notYetDone );
this.kvs[ref].key = this.kvs[ref].key.concat( tokens );
if ( ! notYetDone ) {
this.outstanding--;
Modified: trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.js
===================================================================
--- trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.js
2012-01-20 21:55:15 UTC (rev 109652)
+++ trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.js
2012-01-20 22:02:23 UTC (rev 109653)
@@ -16,9 +16,9 @@
PegTokenizer =
require('./mediawiki.tokenizer.peg.js').PegTokenizer,
TokenTransformManager =
require('./mediawiki.TokenTransformManager.js'),
- NoOnlyInclude =
require('./ext.core.NoOnlyInclude.js'),
- OnlyInclude =
NoOnlyInclude.OnlyInclude,
- NoInclude =
NoOnlyInclude.NoInclude,
+ NoIncludeOnly =
require('./ext.core.NoIncludeOnly.js'),
+ IncludeOnly =
NoIncludeOnly.IncludeOnly,
+ NoInclude =
NoIncludeOnly.NoInclude,
QuoteTransformer =
require('./ext.core.QuoteTransformer.js').QuoteTransformer,
PostExpandParagraphHandler =
require('./ext.core.PostExpandParagraphHandler.js')
.PostExpandParagraphHandler,
@@ -65,7 +65,7 @@
};
// Create an input pipeline for the given input type.
- this.inputPipeline = this.makeInputPipeline ( inputType );
+ this.inputPipeline = this.makeInputPipeline ( inputType, {}, true );
this.inputPipeline.atTopLevel = true;
@@ -137,7 +137,7 @@
* accepts the wiki text this way. The last stage of the input pipeline is
* always an AsyncTokenTransformManager, which emits its output in events.
*/
-ParserPipeline.prototype.makeInputPipeline = function ( inputType, args ) {
+ParserPipeline.prototype.makeInputPipeline = function ( inputType, args,
isNoInclude ) {
switch ( inputType ) {
case 'text/wiki':
//console.log( 'makeInputPipeline ' + JSON.stringify(
args ) );
@@ -158,8 +158,9 @@
var tokenPreProcessor = new
TokenTransformManager.SyncTokenTransformManager ( this.env );
tokenPreProcessor.listenForTokensFrom (
wikiTokenizer );
+ new IncludeOnly( tokenPreProcessor, !
isNoInclude );
// Add noinclude transform for now
- new NoInclude( tokenPreProcessor );
+ new NoInclude( tokenPreProcessor, ! isNoInclude
);
var tokenExpander = new
TokenTransformManager.AsyncTokenTransformManager (
{
Modified: trunk/extensions/VisualEditor/modules/parser/pegTokenizer.pegjs.txt
===================================================================
--- trunk/extensions/VisualEditor/modules/parser/pegTokenizer.pegjs.txt
2012-01-20 21:55:15 UTC (rev 109652)
+++ trunk/extensions/VisualEditor/modules/parser/pegTokenizer.pegjs.txt
2012-01-20 22:02:23 UTC (rev 109653)
@@ -480,7 +480,11 @@
/ & { return syntaxFlags['extlink']; } "]" { return true; }
/ & { return syntaxFlags['linkdesc']; } link_end { return true; }
/ & { return syntaxFlags['h']; } '='+ space* newline { return true; }
- / & { return syntaxFlags['template']; } ('|' / '}}') { return true; }
+ / & { return syntaxFlags['template']; } ('|' / '}}' ) { return true; }
+ / & { return syntaxFlags['equal']; } '=' {
+ //console.log( 'equal stop!' );
+ return true;
+ }
inline
= c:(urltext / (! inline_breaks (inline_element / . )))+ {
@@ -539,6 +543,7 @@
= //& { dp('inline_element enter' + input.substr(pos, 10)); return true; }
& '<' ( comment / xmlish_tag )
/ & '{' ( & '{{{{{' template / tplarg / template )
+ /// & '{' ( tplarg / template )
// Eat three opening brackets as text.
/ '[[[' { return { type: 'TEXT', value: '[[[' } }
/ & '[' ( wikilink / extlink )
@@ -711,25 +716,26 @@
name: 'template',
attribs: [['data-target', JSON.stringify(target)]],
orderedArgs: params,
- args: {},
+ //args: {},
target: target
};
- if (params && params.length) {
- var position = 1;
- for ( var i = 0, l = params.length; i < l; i++ ) {
- var param = params[i];
- if ( param[0] === null ) {
- obj.args[position] = param[1];
- position++;
- } else {
- // Last value wins for duplicates.
- obj.args[param[0]] = param[1];
- }
- }
- // HACK: temporarily also push the args into an attribute
- // (just for debugging)
- obj.attribs.push(['data-json-args', JSON.stringify(obj.args)]);
- }
+ // XXX: this is kind of broken, as arg keys need to be expanded
+ //if (params && params.length) {
+ // var position = 1;
+ // for ( var i = 0, l = params.length; i < l; i++ ) {
+ // var param = params[i];
+ // if ( param[0] === null ) {
+ // obj.args[position] = param[1];
+ // position++;
+ // } else {
+ // // Last value wins for duplicates.
+ // obj.args[param[0]] = param[1];
+ // }
+ // }
+ // // HACK: temporarily also push the args into an attribute
+ // // (just for debugging)
+ // obj.attribs.push(['data-json-args', JSON.stringify(obj.args)]);
+ //}
// Should actually use a self-closing tag here, but the Node HTML5
// parser only recognizes known self-closing tags for now, so use an
// explicit end tag for now.
@@ -738,18 +744,14 @@
}
// XXX: support template and args in target!
-template_target
- = h:( !"}}" x:([^|\n]) { return x } )* { return { type: 'TEXT', value:
h.join('') } }
+//template_target
+// = h:( !"}}" x:([^|\n]) { return x } )* { return { type: 'TEXT', value:
h.join('') } }
-template_param
- = name:template_param_name space* "=" space* c:template_param_text {
- return [[{ type: 'TEXT', value: name }], flatten( c )];
- } / c:template_param_text {
- return [[], flatten( c ) ];
- }
-
tplarg
- = "{{{" name:template_param_text params:("|" p:template_param { return p })*
"}}}" {
+ = "{{{"
+ name:template_param_text
+ params:( newline? "|" newline? p:template_param { return p })*
+ "}}}" {
name = flatten( name );
var obj = {
type: 'SELFCLOSINGTAG',
@@ -764,21 +766,39 @@
obj.attribs.push(['data-json-args', JSON.stringify(params)]);
obj.defaultvalue = params[0][1];
}
- //console.log( 'tokenizer templatearg ' + JSON.stringify( obj ));
+ console.log( 'tokenizer tplarg ' + JSON.stringify( obj ));
return obj;
}
+template_param
+ = name:template_param_name space* "=" space* c:template_param_text {
+ //console.log( 'named template_param matched' + pp([name, flatten( c )])
);
+ return [name, flatten( c )];
+ } / c:template_param_text {
+ return [[], flatten( c ) ];
+ }
+
+
// FIXME: handle template args and templates in key! (or even parser
functions?)
template_param_name
- = h:( !"}}" x:([^=|\n]) { return x } )* { return h.join(''); }
+ = & { return setFlag( 'equal' ) }
+ tpt:template_param_text
+ & { clearFlag( 'equal' ); return true; }
+ {
+ //console.log( 'template param name matched: ' + pp( tpt ) );
+ return tpt;
+ }
+ / & { return clearFlag( 'equal' ) }
+ //= h:( !"}}" x:([^=|\n]) { return x } )* { return h.join(''); }
+
template_param_text
= & { return setFlag('template') }
- il:inline+ {
+ il:inline {
clearFlag('template');
return il;
}
- / & { clearFlag('template'); return false; }
+ / & { return clearFlag('template'); }
// TODO: handle link prefixes as in al[[Razi]]
wikilink
_______________________________________________
MediaWiki-CVS mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-cvs
