Arlolra has uploaded a new change for review.
https://gerrit.wikimedia.org/r/281076
Change subject: WIP: Tokenize html pre as an extension tag
......................................................................
WIP: Tokenize html pre as an extension tag
Change-Id: Icaffc8a73b0ae2281fa38223fb9f940075f2484f
---
M lib/config/ParsoidConfig.js
A lib/ext/Pre.js
M lib/wt2html/HTML5TreeBuilder.js
M lib/wt2html/pegTokenizer.pegjs.txt
M lib/wt2html/tokenizer.utils.js
5 files changed, 46 insertions(+), 123 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid
refs/changes/76/281076/1
diff --git a/lib/config/ParsoidConfig.js b/lib/config/ParsoidConfig.js
index 6a7a564..295589a 100644
--- a/lib/config/ParsoidConfig.js
+++ b/lib/config/ParsoidConfig.js
@@ -641,6 +641,7 @@
require('../ext/Cite.js').Cite,
require('../ext/LST.js'),
require('../ext/Translate.js'),
+ require('../ext/Pre.js'),
];
diff --git a/lib/ext/Pre.js b/lib/ext/Pre.js
new file mode 100644
index 0000000..20a25f1
--- /dev/null
+++ b/lib/ext/Pre.js
@@ -0,0 +1,38 @@
+'use strict';
+require('../../core-upgrade.js');
+
+var Util = require('../utils/Util.js').Util;
+var DU = require('../utils/DOMUtils.js').DOMUtils;
+var defines = require('../wt2html/parser.defines.js');
+
+// define some constructor shortcuts
+var KV = defines.KV;
+var TagTk = defines.TagTk;
+var EndTagTk = defines.EndTagTk;
+
+var tokenHandler = function(manager, pipelineOpts, extToken, cb) {
+ var argDict = Util.getArgInfo(extToken).dict;
+ var txt = argDict.body.extsrc;
+
+ if (!extToken.dataAttribs.tagWidths[1]) {
+ argDict.body = null; // Serialize to self-closing.
+ }
+
+ var start = new TagTk('pre', [
+ new KV('typeof', 'mw:Extension/' + argDict.name),
+ new KV('about', manager.env.newAboutId()),
+ new KV('data-mw', JSON.stringify(argDict)),
+ ], {
+ tsr: Util.clone(extToken.dataAttribs.tsr),
+ src: extToken.dataAttribs.src,
+ });
+
+ cb({ tokens: [start, txt, new EndTagTk('pre')] });
+};
+
+// Pre constructor
+module.exports = function() {
+ this.config = {
+ tags: [{ name: 'pre', tokenHandler: tokenHandler }],
+ };
+};
\ No newline at end of file
diff --git a/lib/wt2html/HTML5TreeBuilder.js b/lib/wt2html/HTML5TreeBuilder.js
index 46c7e0f..26b7516 100644
--- a/lib/wt2html/HTML5TreeBuilder.js
+++ b/lib/wt2html/HTML5TreeBuilder.js
@@ -160,7 +160,7 @@
return JSON.stringify(token);
});
- var tName, attrs, tProperty, data;
+ var tName, attrs, data;
switch (token.constructor) {
case String:
case NlTk:
@@ -226,35 +226,6 @@
// Re-expand an empty-line meta-token into its
constituent comment + WS tokens
if (Util.isEmptyLineMetaToken(token)) {
this.onChunk(dataAttribs.tokens);
- break;
- }
-
- tProperty = token.getAttribute("property");
- if (tName === "pre" && tProperty &&
tProperty.match(/^mw:html$/)) {
- // Unpack pre tags.
- var toks;
- attribs = attribs.filter(function(attr) {
- if (attr.k === "content") {
- toks = attr.v;
- return false;
- } else {
- return attr.k !== "property";
- }
- });
- var endpos = dataAttribs.endpos;
- dataAttribs.endpos = undefined;
- var tsr = dataAttribs.tsr;
- if (tsr) {
- dataAttribs.tsr = [ tsr[0], endpos ];
- }
- dataAttribs.stx = 'html';
- toks.unshift(new TagTk('pre', attribs,
dataAttribs));
- dataAttribs = { stx: 'html'};
- if (tsr) {
- dataAttribs.tsr = [ tsr[1] - 6, tsr[1]
];
- }
- toks.push(new EndTagTk('pre', [], dataAttribs));
- this.onChunk(toks);
break;
}
diff --git a/lib/wt2html/pegTokenizer.pegjs.txt
b/lib/wt2html/pegTokenizer.pegjs.txt
index bc12ec2..c55cde4 100644
--- a/lib/wt2html/pegTokenizer.pegjs.txt
+++ b/lib/wt2html/pegTokenizer.pegjs.txt
@@ -179,8 +179,7 @@
block
= &sof r:redirect {return [r];} // has to be first alternative; otherwise
gets parsed as a <ol>
/ block_lines
- / & '<' rs:( pre // tag variant can start anywhere
- / c:comment &eolf { return c; }
+ / & '<' rs:( c:comment &eolf { return c; }
/ nowiki
// avoid a paragraph if we know that the line starts with a block
tag
/ bt:block_tag
@@ -249,7 +248,6 @@
return st.concat(r);
}
/ ! { return stops.counters.nopre; } pi:pre_indent { return pi; }
- / pre
/ // Horizontal rules
"----" d:"-"*
// Check if a newline or content follows
@@ -283,12 +281,9 @@
inline_breaks
= & { return inlineBreaks(input, endOffset(), stops); }
-pre_start = "<" pre_tag_name [^>]* ">"
-
inlineline
= c:(urltext
/ !inline_breaks
- !pre_start
r:(inline_element / [^\r\n]) { return r; })+ {
return tu.flattenStringlist(c);
}
@@ -812,7 +807,6 @@
= c:((sol full_table_in_link_caption)
/ urltext
/ (!inline_breaks
- !pre_start
r:( inline_element / '[' text_char+ ']' / . ) { return r; }
)
)+ {
@@ -873,11 +867,7 @@
// Indented pre blocks differ from their non-indented (purely tag-based)
// cousins by having their contents parsed.
pre_indent
- =
- // FIXME: Disabled for now. This is T108216.
- // pre_indent_in_tags
- // /
- l:pre_indent_line
+ = l:pre_indent_line
// keep consuming indented lines unless they start a table
ls:(s:sol
!(space* "{|")
@@ -889,82 +879,10 @@
return l.concat(ls);
}
-pre_tag_name =
- tag:"pre"i !tag_name_chars {
- return tag;
- }
-
-// An indented pre block that is surrounded with pre tags. The pre tags are
-// used directly.
-// XXX gwicke: check if the first line is not indented, and round-trip spaces;
-// possibly merge with the regular 'pre' rule.
-// FIXME: fix tag end position
-pre_indent_in_tags
- = & { return stops.inc('pre'); }
- s:spaces // XXX: capture space for round-tripping
- "<" pre_tag_name
- attribs:generic_newline_attributes
- ">"
- l:nested_block_line
- ls:(sol pre_indent_line)*
- "</" pre_tag_name ">"
- {
- stops.dec('pre');
- var ret = [ new TagTk('pre', attribs, { tsr: tsrOffsets('start') }) ];
- // ls will always be an array
- return ret.concat(l, tu.flattenIfArray(ls), [ new EndTagTk('pre') ]);
- }
- / & { return stops.dec('pre'); }
-
// Don't recognize tabs
pre_indent_line = " " l:nested_block_line {
return [' '].concat(l);
}
-
-/*
- * Pre blocks defined using non-indented HTML tags only parse nowiki tags and
- * html entities inside them, and convert other content to verbatim text.
- * Nowiki inside pre is not functionally needed, but supported for backwards
- * compatibility.
- *
- * TODO: add entity support!
- */
-pre
- = & { return stops.inc('pre'); }
- "<" pre_tag_name
- attribs:generic_newline_attributes
- space*
- endpos:(">" { return endOffset(); })
- // MediaWiki <pre> is special in that it converts all pre content to plain
- // text.
- ts:( newlineToken
- / (htmlentity / [^&<]+)+
- / nowiki
- / !("</" pre_tag_name ">") t2:(htmlentity / .) { return t2; })*
- ("</" pre_tag_name ">" / eof) {
- stops.dec('pre');
- // return nowiki tags as well?
-
- // Emit as SelfclosingTag in order to avoid the nested pre problem in
- // the PreHandler.
- attribs.push(new KV('property', 'mw:html'));
- attribs.push(new KV('content', tu.flattenStringlist(ts)));
- return [
- new SelfclosingTagTk('pre', attribs, {
- tsr: tsrOffsets(),
- endpos: endpos,
- }),
- ];
-
- }
- / "</" pre_tag_name ">" { stops.dec('pre'); return "</pre>"; }
- // if this is still preish, emit as a string
- // necessary to work with the pre_start lookaheads
- / p:('<' pre_tag_name) {
- stops.dec('pre');
- return tu.flattenStringlist(p);
- }
- / & { return stops.dec('pre'); }
/* -----------------------------------------------------------------------
* Extension tags should be parsed with higher priority than anything else.
@@ -999,7 +917,7 @@
var skipLen = 0;
// EndTagTk
- if (t.constructor === EndTagTk || isHtmlTag) {
+ if (t.constructor === EndTagTk || (isHtmlTag && !isInstalledExt)) {
return t;
// SelfclosingTagTk
@@ -1023,10 +941,7 @@
dp.src = input.substring(dp.tsr[0], dp.tsr[1]);
dp.tagWidths = [dp.tsr[1] - dp.tsr[0], 0];
- // We accept unclosed references tags,
- // as does the PHP parser. They will normalize
- // to self-closed in a round trip.
- if (tagName !== 'references' || !isInstalledExt) {
+ if (!isInstalledExt) {
return t;
}
@@ -1140,8 +1055,7 @@
}
// nowiki fallback: source-based round-tripping
// of unbalanced nowiki tags that are treated as text.
- / ! { return stops.counters.pre > 0; }
- nw0:("" { return endOffset(); })
+ / nw0:("" { return endOffset(); })
"<" "/"? nowiki_tag_name space* "/"? space* ">" {
var nowiki = input.substring(nw0, endOffset());
return Util.placeholder(nowiki, {
diff --git a/lib/wt2html/tokenizer.utils.js b/lib/wt2html/tokenizer.utils.js
index 6a446b7..5d6d966 100644
--- a/lib/wt2html/tokenizer.utils.js
+++ b/lib/wt2html/tokenizer.utils.js
@@ -257,8 +257,7 @@
return stops.onStack('extlink') ||
(stops.onStack('linkdesc') && input[pos
+ 1] === ']');
case '<':
- return (counters.pre && input.substr(pos, 6)
=== '<pre>') ||
- (counters.noinclude &&
input.substr(pos, 12) === '</noinclude>') ||
+ return (counters.noinclude && input.substr(pos,
12) === '</noinclude>') ||
(counters.includeonly &&
input.substr(pos, 14) === '</includeonly>') ||
(counters.onlyinclude &&
input.substr(pos, 14) === '</onlyinclude>');
default:
--
To view, visit https://gerrit.wikimedia.org/r/281076
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Icaffc8a73b0ae2281fa38223fb9f940075f2484f
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
