[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Consolidate bailing out from ext/wiki links

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/406126 )

Change subject: Consolidate bailing out from ext/wiki links
..

Consolidate bailing out from ext/wiki links

So that extlinks get the same expanded attribute treatment.

Fixes the regression in,
http://localhost:8000/ar.wikipedia.org/v3/page/html/%D8%B0%D8%B1%D8%A9_%D8%A7%D9%84%D9%87%D9%8A%D8%AF%D8%B1%D9%88%D8%AC%D9%8A%D9%86/25945469

Follows up to 217c68a by adding typeof "mw:ExpandedAttrs" to the nodes
where xmlish attributes contain extension tags for expanding.

Change-Id: I188f1c903619320052b0502ba13c7b3be7040e6f
---
M lib/wt2html/tt/AttributeExpander.js
M lib/wt2html/tt/LinkHandler.js
M tests/parserTests.txt
3 files changed, 75 insertions(+), 102 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/26/406126/1

diff --git a/lib/wt2html/tt/AttributeExpander.js 
b/lib/wt2html/tt/AttributeExpander.js
index 59d88e8..bb7a751 100644
--- a/lib/wt2html/tt/AttributeExpander.js
+++ b/lib/wt2html/tt/AttributeExpander.js
@@ -124,7 +124,6 @@
  * -- */
 function stripMetaTags(env, tokens, wrapTemplates) {
var buf = [];
-   var isPushed = false;
var hasGeneratedContent = false;
 
for (var i = 0, l = tokens.length; i < l; i++) {
@@ -141,10 +140,12 @@
return prev + next.textContent;
}, '');
buf.push(str);
+   hasGeneratedContent = true;
+   continue;
// TODO: Maybe cleanup the remaining about 
sibbling wrappers
// but the sanitizer will drop them anyways
}
-   isPushed = false;
+
if (wrapTemplates) {
// Strip all meta tags.
var type = t.getAttribute("typeof");
@@ -154,12 +155,12 @@
hasGeneratedContent = true;
}
} else {
-   isPushed = true;
buf.push(t);
+   continue;
}
}
 
-   if (!isPushed && t.name !== "meta") {
+   if (t.name !== "meta") {
// Dont strip token if it is not a meta-tag
buf.push(t);
}
diff --git a/lib/wt2html/tt/LinkHandler.js b/lib/wt2html/tt/LinkHandler.js
index 7397b3c..a9aa56c 100644
--- a/lib/wt2html/tt/LinkHandler.js
+++ b/lib/wt2html/tt/LinkHandler.js
@@ -210,13 +210,71 @@
});
 };
 
+var bailTokens = function(env, token, isExtLink) {
+   var count = isExtLink ? 1 : 2;
+   var tokens = ["[".repeat(count)];
+
+   if (/mw:ExpandedAttrs/.test(token.getAttribute("typeof"))) {
+   var dataMW = JSON.parse(token.getAttribute("data-mw")).attribs;
+   var html;
+   for (var i = 0; i < dataMW.length; i++) {
+   if (dataMW[i][0].txt === "href") {
+   html = dataMW[i][1].html;
+   break;
+   }
+   }
+
+   // Since we are splicing off '['s and ']'s from the incoming 
token,
+   // adjust TSR of the DOM-fragment by `count` each on both end.
+   var tsr = token.dataAttribs && token.dataAttribs.tsr;
+   if (tsr && typeof (tsr[0]) === 'number' && typeof (tsr[1]) === 
'number') {
+   tsr = [tsr[0] + count, tsr[1] - count];
+   } else {
+   tsr = null;
+   }
+
+   var body = DU.ppToDOM(html);
+   var dft = DU.buildDOMFragmentTokens(env, token, body, null, {
+   noPWrapping: true,
+   tsr: tsr,
+   });
+
+   tokens = tokens.concat(dft);
+   } else {
+   // FIXME: Duplicate work
+   tokens = tokens.concat(token.getAttribute("href"));
+   }
+
+   // Append rest of the attributes
+   if (isExtLink) {
+   var content = Util.lookup(token.attribs, 'mw:content');
+
+   // FIXME: Use this attribute in regular extline
+   // cases to rt spaces correctly maybe?  Unsure
+   // it is worth it.
+   var spaces = token.getAttribute('spaces') || '';
+
+   tokens = tokens.concat(spaces, content);
+   } else {
+   token.attribs.forEach(function(a) {
+   if (a.k === "mw:maybeContent") {
+   

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Strip everything before the first pf token

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/406057 )

Change subject: Strip everything before the first pf token
..

Strip everything before the first pf token

In this isolate case,
{{safesubst:#if:{{{1}
only "subst:" is part of the same token as the prefix so
`normalizeTarget` doesn't take care of remove the "safesubst:"

The fix here is from noting that we wouldn't be at this place in the
code if the target didn't normalize a canonical function name, so the
contents before the prefix can be safely discarded.

This is another patch along the line of 5608e83 and bcd5572

Bug: T185643
Change-Id: I7002804fcf5305ffb6c88d93c201d9e972f9037f
---
M lib/wt2html/tt/TemplateHandler.js
1 file changed, 1 insertion(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/57/406057/1

diff --git a/lib/wt2html/tt/TemplateHandler.js 
b/lib/wt2html/tt/TemplateHandler.js
index bd00531..33968e4 100644
--- a/lib/wt2html/tt/TemplateHandler.js
+++ b/lib/wt2html/tt/TemplateHandler.js
@@ -473,7 +473,7 @@
if (canonicalFunctionName !== undefined) {
// Extract toks that make up pfArg
var pfArgToks;
-   var re = new RegExp(prefix, 'i');
+   var re = new RegExp('^(.*?)' + prefix, 'i');
 
// Because of the lenient stringifying above, we need to find 
the prefix
var i = targetToks.findIndex(function(t) {

-- 
To view, visit https://gerrit.wikimedia.org/r/406057
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I7002804fcf5305ffb6c88d93c201d9e972f9037f
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Stop clearing tplInfo if don't find a table for fostered con...

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/406049 )

Change subject: Stop clearing tplInfo if don't find a table for fostered content
..

Stop clearing tplInfo if don't find a table for fostered content

Also, remove some commented out code that should have gone with dd4bda7

Fixes the following and others,
http://localhost:8000/en.wikipedia.org/v3/page/html/Phulwari_(Vidhan_Sabha_constituency)/793701085
http://localhost:8000/en.wikipedia.org/v3/page/html/User%3ADylanrocks96%2Fsandbox/776441874

Bug: T185346
Change-Id: Ib3c635715ed8936e514e4e8f2f725ab345921f76
---
M lib/wt2html/pp/processors/linter.js
1 file changed, 5 insertions(+), 15 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/49/406049/1

diff --git a/lib/wt2html/pp/processors/linter.js 
b/lib/wt2html/pp/processors/linter.js
index 7d093e4..55206a7 100644
--- a/lib/wt2html/pp/processors/linter.js
+++ b/lib/wt2html/pp/processors/linter.js
@@ -377,15 +377,19 @@
  */
 function logFosteredContent(env, node, dp, tplInfo) {
var maybeTable = node.nextSibling;
+   var clear = false;
+
while (maybeTable && !DU.hasNodeName(maybeTable, 'table')) {
if (tplInfo && maybeTable === tplInfo.last) {
-   tplInfo.clear = true;
+   clear = true;
}
maybeTable = maybeTable.nextSibling;
}
 
if (!maybeTable) {
return null;
+   } else if (clear && tplInfo) {
+   tplInfo.clear = true;
}
 
// In pathological cases, we might walk past fostered nodes
@@ -832,21 +836,7 @@
 function logWikitextFixups(node, env, tplInfo) {
var dp = DU.getDataParsoid(node);
 
-   /*
-* Turn this off since this is wasted processing
-* that is not really actionable at this time.
-*
-   if (tplInfo && tplInfo.first === node) {
-   // Log transclusions with more than one part
-   logTransclusions(env, node, dp, tplInfo);
-   }
-   */
-
logTreeBuilderFixup(env, node, dp, tplInfo);
-   // Turning this off for now since
-   // (a) this needs fixing (b) not exposed in linter extension output yet
-   // (c) this is source of error logspam in kibana
-   // logIgnoredTableAttr(env, node, dp, tplInfo);
logDeletableTables(env, node, dp, tplInfo); // For T161341
logBadPWrapping(env, node, dp, tplInfo);// For T161306
logObsoleteHTMLTags(env, node, dp, tplInfo);

-- 
To view, visit https://gerrit.wikimedia.org/r/406049
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib3c635715ed8936e514e4e8f2f725ab345921f76
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Fix crasher from rendering self-closed html pre tags

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/405823 )

Change subject: Fix crasher from rendering self-closed html pre tags
..

Fix crasher from rendering self-closed html pre tags

For example,
http://localhost:8000/fr.wikipedia.org/v3/page/html/Utilisateur%3ADepil/125767760

Bug: T185385
Change-Id: I6c2211c0cffb9f899d4efb390649e72729e0f705
---
M lib/ext/Pre/index.js
M tests/parserTests.txt
2 files changed, 12 insertions(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/23/405823/1

diff --git a/lib/ext/Pre/index.js b/lib/ext/Pre/index.js
index 1e34ad8..50b8b84 100644
--- a/lib/ext/Pre/index.js
+++ b/lib/ext/Pre/index.js
@@ -45,7 +45,7 @@
stx: 'html',
});
 
-   var txt = argDict.body.extsrc;
+   var txt = argDict.body && argDict.body.extsrc || '';
 
// Support nowikis in pre.  Do this before stripping newlines, see test,
// " with  inside (compatibility with 1.6 and earlier)"
diff --git a/tests/parserTests.txt b/tests/parserTests.txt
index c1b893c..fd05393 100644
--- a/tests/parserTests.txt
+++ b/tests/parserTests.txt
@@ -2723,6 +2723,17 @@
 !! end
 
 !! test
+Self-closed pre
+!! wikitext
+
+!! html/php
+
+
+!! html/parsoid
+
+!! end
+
+!! test
 Parsoid: Don't paragraph-wrap fosterable content even if table syntax is 
unbalanced
 !! options
 parsoid=wt2html

-- 
To view, visit https://gerrit.wikimedia.org/r/405823
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I6c2211c0cffb9f899d4efb390649e72729e0f705
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Fix a bug linting content from templated extension tags with...

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/405820 )

Change subject: Fix a bug linting content from templated extension tags with 
lint handlers
..

Fix a bug linting content from templated extension tags with lint handlers

The lint handler was only called if the extension tag was the first node
of the template.

Change-Id: I3bac02671130ea0cf61b827247081954768181cc
---
M lib/wt2html/pp/processors/linter.js
M tests/mocha/linter.js
2 files changed, 3 insertions(+), 4 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/20/405820/1

diff --git a/lib/wt2html/pp/processors/linter.js 
b/lib/wt2html/pp/processors/linter.js
index 8339b4c..7d093e4 100644
--- a/lib/wt2html/pp/processors/linter.js
+++ b/lib/wt2html/pp/processors/linter.js
@@ -878,12 +878,11 @@
continue;
}
 
-   var nodeTypeOf = null;
+   var nodeTypeOf = node.getAttribute('typeof');
 
// !tplInfo check is to protect against templated content in
// extensions which might in turn be nested in templated 
content.
if (!tplInfo && DU.isFirstEncapsulationWrapperNode(node)) {
-   nodeTypeOf = node.getAttribute('typeof');
tplInfo = {
first: node,
last: 
JSUtils.lastItem(DU.getAboutSiblings(node, node.getAttribute("about"))),
diff --git a/tests/mocha/linter.js b/tests/mocha/linter.js
index 8889a11..5b3b82f 100644
--- a/tests/mocha/linter.js
+++ b/tests/mocha/linter.js
@@ -910,7 +910,7 @@
});
});
it('should attribute linter issues to the ref tag even when ref 
and references are both templated', function() {
-   return parseWT('a x b 
{{1x|x}} c {{1x|y}} 
{{1x|}}').then(function(result) {
+   return parseWT('a x b 
{{1x|x}} {{1x|c y}} 
{{1x|}}').then(function(result) {
result.should.have.length(3);
result[0].should.have.a.property("type", 
"missing-end-tag");
result[0].dsr.should.deep.equal([ 7, 11, 3, 0 
]);
@@ -923,7 +923,7 @@

result[1].should.have.a.property("templateInfo");

result[1].templateInfo.should.have.a.property("name", "Template:1x");
result[2].should.have.a.property("type", 
"missing-end-tag");
-   result[2].dsr.should.deep.equal([ 45, 67, null, 
null]);
+   result[2].dsr.should.deep.equal([ 43, 67, null, 
null]);
result[2].should.have.a.property("params");
result[2].params.should.have.a.property("name", 
"b");

result[2].should.have.a.property("templateInfo");

-- 
To view, visit https://gerrit.wikimedia.org/r/405820
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I3bac02671130ea0cf61b827247081954768181cc
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Fix infinite recursion from linting named ref cycles

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/405821 )

Change subject: Fix infinite recursion from linting named ref cycles
..

Fix infinite recursion from linting named ref cycles

On pages like,
http://localhost:8000/fr.wikipedia.org/v3/page/html/Krzysztof_Charamsa/142052535
http://localhost:8000/es.wikipedia.org/v3/page/html/Taxonom%C3%ADa/104913080

Bug: T185267
Change-Id: I156cce384873609a65b659360667323b8f0a497b
---
M lib/ext/Cite/index.js
M lib/utils/DOMUtils.js
M tests/mocha/linter.js
3 files changed, 25 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/21/405821/1

diff --git a/lib/ext/Cite/index.js b/lib/ext/Cite/index.js
index d2b27e4..25319e2 100644
--- a/lib/ext/Cite/index.js
+++ b/lib/ext/Cite/index.js
@@ -182,6 +182,10 @@
 };
 
 Ref.prototype.lintHandler = function(ref, env, tplInfo, domLinter) {
+   // Don't lint the content of ref in ref, since it can lead to cycles
+   // using named refs
+   if (DU.fromExtensionContent(ref, 'references')) { return ref.nextNode; }
+
var linkBackId = ref.firstChild.getAttribute('href').replace(/[^#]*#/, 
'');
var refNode = ref.ownerDocument.getElementById(linkBackId);
if (refNode) {
diff --git a/lib/utils/DOMUtils.js b/lib/utils/DOMUtils.js
index abb1eb2..3b90142 100644
--- a/lib/utils/DOMUtils.js
+++ b/lib/utils/DOMUtils.js
@@ -2891,6 +2891,21 @@
}
 };
 
+/**
+ * Is the node from extension content?
+ */
+DOMUtils.fromExtensionContent = function(node, extType) {
+   var parentNode = node.parentNode;
+   var extReg = new RegExp('\\bmw:Extension\\/' + extType + '\\b');
+   while (parentNode && !DU.atTheTop(parentNode)) {
+   if (extReg.test(parentNode.getAttribute('typeof'))) {
+   return true;
+   }
+   parentNode = parentNode.parentNode;
+   }
+   return false;
+};
+
 if (typeof module === "object") {
module.exports.DOMUtils = DOMUtils;
 }
diff --git a/tests/mocha/linter.js b/tests/mocha/linter.js
index 5b3b82f..4195f2a 100644
--- a/tests/mocha/linter.js
+++ b/tests/mocha/linter.js
@@ -945,5 +945,11 @@

result[1].templateInfo.should.have.a.property("name", "Template:1x");
});
});
+   it('should not get into a cycle trying to lint ref in ref', 
function() {
+   return parseWT("{{#tag:ref||name='x'}}{{#tag:ref||name='y'}}")
+   .then(function() {
+   return parseWT("{{#tag:ref||name=x}}");
+   });
+   });
});
 });

-- 
To view, visit https://gerrit.wikimedia.org/r/405821
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I156cce384873609a65b659360667323b8f0a497b
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...deploy[master]: Hardcode num_workers since remote_vars takes precedent

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/405060 )

Change subject: Hardcode num_workers since remote_vars takes precedent
..

Hardcode num_workers since remote_vars takes precedent

As described on,
https://wikitech.wikimedia.org/wiki/Scap3#Config_Deployments

Change-Id: I53f90fb97d7720d38450dcfba34bdb38817192b3
---
M scap/templates/config.yaml.j2
M scap/vars.yaml
2 files changed, 2 insertions(+), 4 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid/deploy 
refs/changes/60/405060/1

diff --git a/scap/templates/config.yaml.j2 b/scap/templates/config.yaml.j2
index e8029cf..7a450d1 100644
--- a/scap/templates/config.yaml.j2
+++ b/scap/templates/config.yaml.j2
@@ -1,7 +1,7 @@
 # Number of worker processes to spawn.
 # Set to 0 to run everything in a single process without clustering.
 # Use 'ncpu' to run as many workers as there are CPU units
-num_workers: <%= num_workers %>
+num_workers: 1
 
 # Log error messages and gracefully restart a worker if v8 reports that it
 # uses more heap (note: not RSS) than this many mb.
@@ -51,7 +51,7 @@
   linter:
 sendAPI: <%= linter_send_api %>
 
-  useWorker: <%= use_worker %>
+  useWorker: true
 
   mwApiServer: <%= mwapi_server if mwapi_server else '' %>
   defaultAPIProxyURI: <%= mwapi_proxy if mwapi_proxy else '' %>
diff --git a/scap/vars.yaml b/scap/vars.yaml
index 273ee8c..49bf590 100644
--- a/scap/vars.yaml
+++ b/scap/vars.yaml
@@ -16,5 +16,3 @@
 linter_send_api: true
 mwapi_server:
 mwapi_proxy:
-num_workers: 1
-use_worker: true

-- 
To view, visit https://gerrit.wikimedia.org/r/405060
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I53f90fb97d7720d38450dcfba34bdb38817192b3
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid/deploy
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...deploy[master]: Bump src/ to af06386 for deploy

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/405036 )

Change subject: Bump src/ to af06386 for deploy
..

Bump src/ to af06386 for deploy

Change-Id: Iaf099daf7e0b90d5a4740679065831acf7e32bab
---
M src
1 file changed, 1 insertion(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid/deploy 
refs/changes/36/405036/1

diff --git a/src b/src
index 231bfff..af06386 16
--- a/src
+++ b/src
@@ -1 +1 @@
-Subproject commit 231bfff1f41a8c3e1cd32b030a4792247e65d681
+Subproject commit af06386635ff44101ac534cdc5d3ae6df6ced96a

-- 
To view, visit https://gerrit.wikimedia.org/r/405036
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Iaf099daf7e0b90d5a4740679065831acf7e32bab
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid/deploy
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Fix crasher from tokenizing headings

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/404871 )

Change subject: Fix crasher from tokenizing headings
..

Fix crasher from tokenizing headings

Introduced in c167933

Fixes the render of many pages in logstash, including,
http://localhost:8000/en.wikipedia.org/v3/page/html/User%3A2011arvind/816572769

Change-Id: I54e6078b09202461a5921712d13b800d8d526592
---
M lib/wt2html/pegTokenizer.pegjs
M tests/parserTests.txt
2 files changed, 11 insertions(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/71/404871/1

diff --git a/lib/wt2html/pegTokenizer.pegjs b/lib/wt2html/pegTokenizer.pegjs
index 44974cd..17c6300 100644
--- a/lib/wt2html/pegTokenizer.pegjs
+++ b/lib/wt2html/pegTokenizer.pegjs
@@ -517,7 +517,7 @@
 // split up equal signs into two equal parts, with at least
 // one character in the middle.
 level = Math.floor((s.length - 1) / 2);
-c = '='.repeat(s.length - 2 * level);
+c = ['='.repeat(s.length - 2 * level)];
 s = e = '='.repeat(level);
 }
 level = Math.min(6, level);
diff --git a/tests/parserTests.txt b/tests/parserTests.txt
index c01e750..2cf6aca 100644
--- a/tests/parserTests.txt
+++ b/tests/parserTests.txt
@@ -25350,6 +25350,16 @@
 
 !! end
 
+!! test
+Headings: Used as horizontal rule
+!! options
+parsoid=wt2html
+!! wikitext
+===
+!! html/parsoid
+===
+!! end
+
  --- Lists ---
  0. Outside nests (*foo, etc.)
  1. Nested inside html *foo

-- 
To view, visit https://gerrit.wikimedia.org/r/404871
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I54e6078b09202461a5921712d13b800d8d526592
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...deploy[master]: Enable useWorker in production

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/404803 )

Change subject: Enable useWorker in production
..

Enable useWorker in production

Change-Id: I154d1c4b255715ca9945e182da0950dd65792bd1
---
M scap/environments/beta/vars.yaml
M scap/vars.yaml
2 files changed, 2 insertions(+), 4 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid/deploy 
refs/changes/03/404803/1

diff --git a/scap/environments/beta/vars.yaml b/scap/environments/beta/vars.yaml
index ba583db..33a5a26 100644
--- a/scap/environments/beta/vars.yaml
+++ b/scap/environments/beta/vars.yaml
@@ -1,4 +1,2 @@
 localsettings: conf/wmf/betalabs.localsettings.js
 load_wmf: false
-num_workers: 1
-use_worker: true
diff --git a/scap/vars.yaml b/scap/vars.yaml
index 6b99950..273ee8c 100644
--- a/scap/vars.yaml
+++ b/scap/vars.yaml
@@ -16,5 +16,5 @@
 linter_send_api: true
 mwapi_server:
 mwapi_proxy:
-num_workers: ncpu
-use_worker: false
+num_workers: 1
+use_worker: true

-- 
To view, visit https://gerrit.wikimedia.org/r/404803
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I154d1c4b255715ca9945e182da0950dd65792bd1
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid/deploy
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Deep clone expression result before modifying

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/404738 )

Change subject: Deep clone expression result before modifying
..

Deep clone expression result before modifying

Necessary for the follow up which tickled the bug.

Change-Id: I380bb7a9bb16026954eb39a24c5915f5992153a9
---
M lib/wt2html/pegTokenizer.pegjs
1 file changed, 5 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/38/404738/1

diff --git a/lib/wt2html/pegTokenizer.pegjs b/lib/wt2html/pegTokenizer.pegjs
index 1a0f97c..b8afe0b 100644
--- a/lib/wt2html/pegTokenizer.pegjs
+++ b/lib/wt2html/pegTokenizer.pegjs
@@ -1057,6 +1057,11 @@
   }
   var lvsrc = input.substring(lv0, lv1);
   var attribs = [];
+
+  // Do a deep clone since we may be destructively modifying
+  // (the `t[fld] = name;` below) the result of a cached expression.
+  ts = Util.clone(ts, true);
+
   ts.forEach(function(t) {
 // move token strings into KV attributes so that they are
 // properly expanded by early stages of the token pipeline

-- 
To view, visit https://gerrit.wikimedia.org/r/404738
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I380bb7a9bb16026954eb39a24c5915f5992153a9
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Stop service-runner so that cli invoked rt test exits

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/404697 )

Change subject: Stop service-runner so that cli invoked rt test exits
..

Stop service-runner so that cli invoked rt test exits

Change-Id: Ida82d19f3fdd30201b0507d15e1ec5abcaa1c0a4
---
M bin/roundtrip-test.js
1 file changed, 1 insertion(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/97/404697/1

diff --git a/bin/roundtrip-test.js b/bin/roundtrip-test.js
index 5fb2da8..8ebcc5c 100755
--- a/bin/roundtrip-test.js
+++ b/bin/roundtrip-test.js
@@ -765,6 +765,7 @@
xmlFormat : plainFormat;
var r = yield runTests(title, argv, formatter);
console.log(r.output);
+   yield ret.runner.stop();
if (argv.check) {
process.exit(r.exitCode);
}

-- 
To view, visit https://gerrit.wikimedia.org/r/404697
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ida82d19f3fdd30201b0507d15e1ec5abcaa1c0a4
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: Fix typo in parsoid-rt.config.yaml.erb

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/404691 )

Change subject: Fix typo in parsoid-rt.config.yaml.erb
..

Fix typo in parsoid-rt.config.yaml.erb

Change-Id: I2f5983624e3f4696213b19578526aa31bb427ddd
---
M modules/testreduce/templates/parsoid-rt.config.yaml.erb
1 file changed, 1 insertion(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/91/404691/1

diff --git a/modules/testreduce/templates/parsoid-rt.config.yaml.erb 
b/modules/testreduce/templates/parsoid-rt.config.yaml.erb
index 5651d18..9c9b435 100644
--- a/modules/testreduce/templates/parsoid-rt.config.yaml.erb
+++ b/modules/testreduce/templates/parsoid-rt.config.yaml.erb
@@ -11,7 +11,7 @@
 
 defaultAPIProxyURI: <%= @default_api_proxy_uri %>
 
-linting: true;
+linting: true
 linter:
   apiSampling: 1000
   sendAPI: false

-- 
To view, visit https://gerrit.wikimedia.org/r/404691
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I2f5983624e3f4696213b19578526aa31bb427ddd
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...deploy[master]: Bump src/ to 231bfff for deploy

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/404503 )

Change subject: Bump src/ to 231bfff for deploy
..

Bump src/ to 231bfff for deploy

Change-Id: I42941e20a5866584abe48e8c39f2849b8a9a3dc6
---
M src
1 file changed, 1 insertion(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid/deploy 
refs/changes/03/404503/1

diff --git a/src b/src
index e133312..231bfff 16
--- a/src
+++ b/src
@@ -1 +1 @@
-Subproject commit e133312621f5aba3dd7d3cf011f6f5a8dde042dc
+Subproject commit 231bfff1f41a8c3e1cd32b030a4792247e65d681

-- 
To view, visit https://gerrit.wikimedia.org/r/404503
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I42941e20a5866584abe48e8c39f2849b8a9a3dc6
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid/deploy
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...VisualEditor[master]: Remove autoGenerated hack

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/403751 )

Change subject: Remove autoGenerated hack
..

Remove autoGenerated hack

Reverts some of 471e40e

Depends-On: I2b5c7ff552b3322be74f79a81936c41d58fecabc
Change-Id: Ibfcd4ced6c58c1cd0d362af3bdb43ce4e6879941
---
M modules/ve-mw/ce/nodes/ve.ce.MWTransclusionNode.js
1 file changed, 1 insertion(+), 5 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/VisualEditor 
refs/changes/51/403751/1

diff --git a/modules/ve-mw/ce/nodes/ve.ce.MWTransclusionNode.js 
b/modules/ve-mw/ce/nodes/ve.ce.MWTransclusionNode.js
index 5c1f70c..8216cd8 100644
--- a/modules/ve-mw/ce/nodes/ve.ce.MWTransclusionNode.js
+++ b/modules/ve-mw/ce/nodes/ve.ce.MWTransclusionNode.js
@@ -115,11 +115,7 @@
 
// Filter out auto-generated items, e.g. reference lists
contentNodes = contentNodes.filter( function ( node ) {
-   // HACK: Also check first-child as auto-generated is applied 
inside ref wrapper (T181230)
-   if ( isAutoGenerated( node ) || isAutoGenerated( 
node.childNodes[ 0 ] ) ) {
-   return false;
-   }
-   return true;
+   return !isAutoGenerated( node );
} );
 
function isWhitespaceNode( node ) {

-- 
To view, visit https://gerrit.wikimedia.org/r/403751
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ibfcd4ced6c58c1cd0d362af3bdb43ce4e6879941
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/VisualEditor
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...deploy[master]: Remove logger sampling for obsolete logs

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/403468 )

Change subject: Remove logger sampling for obsolete logs
..

Remove logger sampling for obsolete logs

Empty element warnings were removed in cfff63a of the Parsoid repo.

Change-Id: Id626ec64b8411713f68dd9f8ed416f42ea571776
---
M conf/wmf/betalabs.localsettings.js
D conf/wmf/localsettings.js
M scap/templates/config.yaml.j2
M scap/vars.yaml
4 files changed, 6 insertions(+), 21 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid/deploy 
refs/changes/68/403468/1

diff --git a/conf/wmf/betalabs.localsettings.js 
b/conf/wmf/betalabs.localsettings.js
index 60ef935..f4f843d 100644
--- a/conf/wmf/betalabs.localsettings.js
+++ b/conf/wmf/betalabs.localsettings.js
@@ -20,12 +20,4 @@
 
// The production enwiki: a work-around to be able to use the labs 
parsoid instance from RESTBase
parsoidConfig.setMwApi({ prefix: 'enwikiprod', uri: 
'https://en.wikipedia.org/w/api.php' });
-
-   // Sample verbose logs
-   parsoidConfig.loggerSampling = [
-   ['warn/dsr/inconsistent', 5],
-   ['warn/empty/li', 10],
-   ['warn/empty/tr', 10],
-   [/^warn\/empty\//, 10],
-   ];
 };
diff --git a/conf/wmf/localsettings.js b/conf/wmf/localsettings.js
deleted file mode 100644
index efb4cb2..000
--- a/conf/wmf/localsettings.js
+++ /dev/null
@@ -1,12 +0,0 @@
-"use strict";
-exports.setup = function(parsoidConfig) {
-   // Sample these verbose logs to prevent overwhelm
-   // 1% and 2% for empty/tr and empty/li is based on
-   // seeing the volume in rt-testing.
-   parsoidConfig.loggerSampling = [
-   ['warn/dsr/inconsistent', 5],
-   ['warn/empty/li', 1],
-   ['warn/empty/tr', 0],
-   [/^warn\/empty\//, 5],
-   ];
-};
diff --git a/scap/templates/config.yaml.j2 b/scap/templates/config.yaml.j2
index e93ba32..e8029cf 100644
--- a/scap/templates/config.yaml.j2
+++ b/scap/templates/config.yaml.j2
@@ -70,3 +70,8 @@
 wt2html:
   maxTableCells: 4
   maxListItems: 4
+
+  # Sample verbose logs
+  loggerSampling:
+- - 'warn/dsr/inconsistent'
+  - 5
diff --git a/scap/vars.yaml b/scap/vars.yaml
index 6fbf827..6b99950 100644
--- a/scap/vars.yaml
+++ b/scap/vars.yaml
@@ -10,7 +10,7 @@
 # service
 name: parsoid
 port: 8000
-localsettings: conf/wmf/localsettings.js
+localsettings:
 load_wmf: true
 linting: true
 linter_send_api: true

-- 
To view, visit https://gerrit.wikimedia.org/r/403468
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Id626ec64b8411713f68dd9f8ed416f42ea571776
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid/deploy
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] operations/puppet[production]: Switch to YAML configuration for Parsoid on ruthenium

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/403464 )

Change subject: Switch to YAML configuration for Parsoid on ruthenium
..

Switch to YAML configuration for Parsoid on ruthenium

This also enables cpu_workers with useWorker.

The removed empty warnings are from cfff63a in the Parsoid repo.

Change-Id: I0aab1a81f34e717ee1fade5f7b2765d49eb2773b
---
M modules/parsoid/manifests/init.pp
M modules/role/manifests/parsoid/rt_client.pp
M modules/role/manifests/parsoid/testing.pp
D modules/testreduce/templates/parsoid-rt-client.rttest.localsettings.js.erb
A modules/testreduce/templates/parsoid-rt.config.yaml.erb
5 files changed, 31 insertions(+), 74 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/operations/puppet 
refs/changes/64/403464/1

diff --git a/modules/parsoid/manifests/init.pp 
b/modules/parsoid/manifests/init.pp
index 4a0b7ea..3741c8f 100644
--- a/modules/parsoid/manifests/init.pp
+++ b/modules/parsoid/manifests/init.pp
@@ -8,10 +8,9 @@
 # [*port*]
 #   Port to run the Parsoid service on. Default: 8000
 #
-# [*settings_file*]
-#   Location of the old-format Parsoid configuration and settings file. Note
-#   that Parsoid still draws part of its configuration from there when it is
-#   specified. Default: 'conf/wmf/localsettings.js'
+# [*conf*]
+#   Hash or YAML-formatted string that gets merged into the service's
+#   configuration.  Only applicable for non-scap3 deployments.
 #
 # [*logging_name*]
 #   The logging name to send to logstash. Default: 'parsoid'
@@ -33,12 +32,13 @@
 #   'http://api.svc.eqiad.wmnet'
 #
 # [*discovery*]
-# If defined, will use that discovery key to discover if the current 
datacenter is active
-# for the MediaWiki API, and use HTTP or HTTPS to connect the host 
${discovery}.discovery.wmnet
+#   If defined, will use that discovery key to discover if the current 
datacenter is active
+#   for the MediaWiki API, and use HTTP or HTTPS to connect the host 
${discovery}.discovery.wmnet
 #
 class parsoid(
 $port  = 8000,
-$settings_file = 'conf/wmf/localsettings.js',
+$conf  = undef,
+$no_workers= 'ncpu',
 $logging_name  = 'parsoid',
 $statsd_prefix = 'parsoid',
 $deployment= 'scap3',
@@ -92,6 +92,8 @@
 } else {
 service::node::config { 'parsoid':
 port   => $port,
+config => $conf,
+no_workers => $no_workers,
 starter_module => 'src/lib/index.js',
 entrypoint => 'apiServiceWorker',
 logging_name   => $logging_name,
@@ -99,9 +101,6 @@
 heartbeat_to   => 18,
 statsd_prefix  => $statsd_prefix,
 auto_refresh   => false,
-config => {
-localsettings => $settings_file,
-},
 }
 }
 }
diff --git a/modules/role/manifests/parsoid/rt_client.pp 
b/modules/role/manifests/parsoid/rt_client.pp
index 9b2a52a..028a0c1 100644
--- a/modules/role/manifests/parsoid/rt_client.pp
+++ b/modules/role/manifests/parsoid/rt_client.pp
@@ -2,14 +2,6 @@
 class role::parsoid::rt_client {
 include ::testreduce
 
-file { 
'/srv/deployment/parsoid/deploy/src/tests/testreduce/parsoid-rt-client.rttest.localsettings.js':
-content => 
template('testreduce/parsoid-rt-client.rttest.localsettings.js.erb'),
-owner   => 'root',
-group   => 'root',
-mode=> '0444',
-notify  => Service['parsoid-rt-client'],
-}
-
 testreduce::client { 'parsoid-rt-client':
 instance_name => 'parsoid-rt-client',
 parsoid_port  => hiera('parsoid::testing::parsoid_port'),
diff --git a/modules/role/manifests/parsoid/testing.pp 
b/modules/role/manifests/parsoid/testing.pp
index ec686c4..a2beed1 100644
--- a/modules/role/manifests/parsoid/testing.pp
+++ b/modules/role/manifests/parsoid/testing.pp
@@ -9,8 +9,9 @@
 
 class { '::parsoid':
 port  => $parsoid_port,
-settings_file => '/srv/deployment/parsoid/deploy/src/localsettings.js',
 deployment=> 'git',
+no_workers=> 1,
+conf  => template('testreduce/parsoid-rt.config.yaml.erb'),
 }
 
 file { '/usr/local/bin/update_parsoid.sh':
@@ -18,15 +19,6 @@
 owner  => 'root',
 group  => 'root',
 mode   => '0555',
-}
-
-# Use this parsoid instance for parsoid rt-testing
-file { '/srv/deployment/parsoid/deploy/src/localsettings.js':
-content => 
template('testreduce/parsoid-rt-client.rttest.localsettings.js.erb'),
-owner   => 'root',
-group   => 'wikidev',
-mode=> '0444',
-before  => Service['parsoid'],
 }
 
 # mysql client and configuration to provide command line access to
diff --git 
a/modules/testreduce/templates/parsoid-rt-client.rttest.localsettings.js.erb 

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: WIP: Permit extension tags in xmlish attribute values

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/403319 )

Change subject: WIP: Permit extension tags in xmlish attribute values
..

WIP: Permit extension tags in xmlish attribute values

Bug: T183515
Authored-by: Arlo Breault 
Authored-by: Shannon Bailey 
Change-Id: I311d0fbbd2cdcf2a0c0a3dbf698912e6df5f3356
---
M lib/wt2html/pegTokenizer.pegjs
M lib/wt2html/tt/AttributeExpander.js
M tests/mockAPI.js
M tests/parserTests.txt
4 files changed, 41 insertions(+), 9 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/19/403319/1

diff --git a/lib/wt2html/pegTokenizer.pegjs b/lib/wt2html/pegTokenizer.pegjs
index 1a0f97c..65d10f9 100644
--- a/lib/wt2html/pegTokenizer.pegjs
+++ b/lib/wt2html/pegTokenizer.pegjs
@@ -1412,6 +1412,12 @@
 tag_name_chars = [^\t\n\v />\0]
 tag_name = $([A-Za-z] tag_name_chars*)
 
+extension_tag =
+  &{ return !stops.onStack('extTag'); }
+  extToken:xmlish_tag
+  &{ return extToken.name === 'extension'; }
+  { return extToken; }
+
 xmlish_tag
   = & {
   // By the time we get to `doTableStuff` in the php parser, we've already
@@ -2258,7 +2264,7 @@
   = r:( $[^{}&<\-|/ \t\n\r\x0c>]+
   / !inline_breaks
 !'/>'
-s:( directive / [{}&<\-|/] ) { return s; }
+s:( directive / extension_tag / [{}&<\-|/] ) { return s; }
   )+ {
 return tu.flattenString(r);
   }
@@ -2268,7 +2274,7 @@
   = r:( $[^{}&<\-|/'>]+
   / !inline_breaks
 !'/>'
-s:( directive / [{}&<\-|/] ) { return s; }
+s:( directive / extension_tag / [{}&<\-|/] ) { return s; }
   )* {
 return tu.flattenString(r);
   }
@@ -2278,7 +2284,7 @@
   = r:( $[^{}&<\-|/">]+
   / !inline_breaks
 !'/>'
-s:( directive / [{}&<\-|/] ) { return s; }
+s:( directive / extension_tag / [{}&<\-|/] ) { return s; }
   )* {
 return tu.flattenString(r);
   }
@@ -2294,7 +2300,7 @@
 // The stop set is space_or_newline and | which matches table_att_value.
 table_attribute_preprocessor_text
   = r:( $[^{}&<\-!\[ \t\n\r\x0c|]+
-  / !inline_breaks s:( directive / [{}&<\-!\[] ) { return s; }
+  / !inline_breaks s:( directive / extension_tag / [{}&<\-!\[] ) { return s; }
   )+ {
 return tu.flattenString(r);
   }
@@ -2302,7 +2308,7 @@
 // The stop set is '\r\n| which matches table_att_value.
 table_attribute_preprocessor_text_single
   = r:( $[^{}&<\-!\['\r\n|]+
-  / !inline_breaks s:( directive / [{}&<\-!\[] ) { return s; }
+  / !inline_breaks s:( directive / extension_tag / [{}&<\-!\[] ) { return s; }
   )* {
 return tu.flattenString(r);
   }
@@ -2310,7 +2316,7 @@
 // The stop set is "\r\n| which matches table_att_value.
 table_attribute_preprocessor_text_double
   = r:( $[^{}&<\-!\["\r\n|]+
-  / !inline_breaks s:( directive / [{}&<\-!\[] ) { return s; }
+  / !inline_breaks s:( directive / extension_tag / [{}&<\-!\[] ) { return s; }
   )* {
 return tu.flattenString(r);
   }
diff --git a/lib/wt2html/tt/AttributeExpander.js 
b/lib/wt2html/tt/AttributeExpander.js
index 9a76502..59d88e8 100644
--- a/lib/wt2html/tt/AttributeExpander.js
+++ b/lib/wt2html/tt/AttributeExpander.js
@@ -122,7 +122,7 @@
  * This helper method strips all meta tags introduced by
  * transclusions, etc. and returns the content.
  * -- */
-function stripMetaTags(tokens, wrapTemplates) {
+function stripMetaTags(env, tokens, wrapTemplates) {
var buf = [];
var isPushed = false;
var hasGeneratedContent = false;
@@ -130,6 +130,20 @@
for (var i = 0, l = tokens.length; i < l; i++) {
var t = tokens[i];
if ([TagTk, SelfclosingTagTk].indexOf(t.constructor) !== -1) {
+   // Reinsert expanded extension content that's been 
parsed to DOM
+   // as a string.  This should match what the php parser 
does since
+   // extension content is html being placed in an 
attribute context.
+   if (t.getAttribute('typeof') === 'mw:DOMFragment') {
+   var nodes = 
env.fragmentMap.get(t.dataAttribs.html);
+   var str = nodes.reduce(function(prev, next) {
+   // We strip tags since the sanitizer 
would normally drop
+   // tokens but we're already at html
+   return prev + next.textContent;
+   }, '');
+   buf.push(str);
+   // TODO: Maybe cleanup the remaining about 
sibbling wrappers
+   // but the sanitizer will drop them anyways
+   }
isPushed = false;
if (wrapTemplates) {
// Strip all meta tags.
@@ -348,7 

[MediaWiki-commits] [Gerrit] mediawiki...deploy[master]: Enable cpu workers on the beta cluster

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/403075 )

Change subject: Enable cpu workers on the beta cluster
..

Enable cpu workers on the beta cluster

Change-Id: Ifefe6560c7c4d44d786cfff3ba49e823625addd5
---
M scap/environments/beta/vars.yaml
M scap/templates/config.yaml.j2
M scap/vars.yaml
3 files changed, 7 insertions(+), 2 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid/deploy 
refs/changes/75/403075/1

diff --git a/scap/environments/beta/vars.yaml b/scap/environments/beta/vars.yaml
index 33a5a26..ba583db 100644
--- a/scap/environments/beta/vars.yaml
+++ b/scap/environments/beta/vars.yaml
@@ -1,2 +1,4 @@
 localsettings: conf/wmf/betalabs.localsettings.js
 load_wmf: false
+num_workers: 1
+use_worker: true
diff --git a/scap/templates/config.yaml.j2 b/scap/templates/config.yaml.j2
index 9e61161..e93ba32 100644
--- a/scap/templates/config.yaml.j2
+++ b/scap/templates/config.yaml.j2
@@ -1,7 +1,7 @@
 # Number of worker processes to spawn.
 # Set to 0 to run everything in a single process without clustering.
 # Use 'ncpu' to run as many workers as there are CPU units
-num_workers: ncpu
+num_workers: <%= num_workers %>
 
 # Log error messages and gracefully restart a worker if v8 reports that it
 # uses more heap (note: not RSS) than this many mb.
@@ -51,6 +51,8 @@
   linter:
 sendAPI: <%= linter_send_api %>
 
+  useWorker: <%= use_worker %>
+
   mwApiServer: <%= mwapi_server if mwapi_server else '' %>
   defaultAPIProxyURI: <%= mwapi_proxy if mwapi_proxy else '' %>
 
diff --git a/scap/vars.yaml b/scap/vars.yaml
index a152df6..6fbf827 100644
--- a/scap/vars.yaml
+++ b/scap/vars.yaml
@@ -16,4 +16,5 @@
 linter_send_api: true
 mwapi_server:
 mwapi_proxy:
-
+num_workers: ncpu
+use_worker: false

-- 
To view, visit https://gerrit.wikimedia.org/r/403075
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ifefe6560c7c4d44d786cfff3ba49e823625addd5
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid/deploy
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Give some time for async loggers to deliver fatal messages

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/403070 )

Change subject: Give some time for async loggers to deliver fatal messages
..

Give some time for async loggers to deliver fatal messages

Fatals in logstash are only reporting "fatal/request" where the process
isn't exited, however these logs are showing up in the file based
logging.

Change-Id: Ib9b271007213a79c2444a40debe2945d6e1435ac
---
M lib/logger/Logger.js
1 file changed, 2 insertions(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/70/403070/1

diff --git a/lib/logger/Logger.js b/lib/logger/Logger.js
index 8b8b6d6..0171663 100644
--- a/lib/logger/Logger.js
+++ b/lib/logger/Logger.js
@@ -286,7 +286,8 @@
// events in the queue and logs them.
async.parallel(applicableBackends, function() {
if (/^fatal$/.test(logData.logType)) {
-   process.exit(1);
+   // Give some time for async loggers to deliver the 
message
+   setTimeout(function() { process.exit(1); }, 100);
}
cb();
});

-- 
To view, visit https://gerrit.wikimedia.org/r/403070
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib9b271007213a79c2444a40debe2945d6e1435ac
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...deploy[master]: Bump src/ to e133312 for deploy

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/402869 )

Change subject: Bump src/ to e133312 for deploy
..

Bump src/ to e133312 for deploy

Change-Id: Ib7b90fd922628aa07b5783abdf10bab267861404
---
M src
1 file changed, 1 insertion(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid/deploy 
refs/changes/69/402869/1

diff --git a/src b/src
index 5abd780..e133312 16
--- a/src
+++ b/src
@@ -1 +1 @@
-Subproject commit 5abd780336c54c0fcd8fd953588085fe505bf455
+Subproject commit e133312621f5aba3dd7d3cf011f6f5a8dde042dc

-- 
To view, visit https://gerrit.wikimedia.org/r/402869
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib7b90fd922628aa07b5783abdf10bab267861404
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid/deploy
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Add a function to create the references node uniformly

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/402454 )

Change subject: Add a function to create the references node uniformly
..

Add a function to create the references node uniformly

Change-Id: I58fd7ea7c692b1a38dab160083d3d9a529527fb0
---
M lib/ext/Cite/index.js
M tests/citeParserTests-blacklist.js
2 files changed, 43 insertions(+), 38 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/54/402454/1

diff --git a/lib/ext/Cite/index.js b/lib/ext/Cite/index.js
index 4679ce8..0c32c92 100644
--- a/lib/ext/Cite/index.js
+++ b/lib/ext/Cite/index.js
@@ -332,6 +332,28 @@
this.cite = cite;
 }
 
+var createReferences = function(doc, about, refsOpts, modifyDp) {
+   var ol = doc.createElement('ol');
+   DU.addAttributes(ol, {
+   class: 'mw-references references',
+   typeof: 'mw:Extension/references',
+   about: about,
+   });
+   var dp = DU.getDataParsoid(ol);
+   if (refsOpts.group) {  // No group for the empty string either
+   dp.group = refsOpts.group;
+   ol.setAttribute('data-mw-group', refsOpts.group);
+   }
+   if (refsOpts.responsive !== null) {
+   // Pass along the `responsive` parameter
+   dp.tmp.responsive = refsOpts.responsive;
+   }
+   if (typeof modifyDp === 'function') {
+   modifyDp(dp);
+   }
+   return ol;
+};
+
 var dummyDoc = domino.createDocument();
 
 /**
@@ -339,11 +361,13 @@
  */
 References.prototype.tokenHandler = function(manager, pipelineOpts, refsTok, 
cb) {
var env = manager.env;
+   var kvs = refsTok.getAttribute('options');
+   var opts = Util.kvToHash(kvs, true);
 
-   // group is the only recognized option?
var refsOpts = Object.assign({
group: null,
-   }, Util.kvToHash(refsTok.getAttribute("options"), true));
+   responsive: null,
+   }, opts);
 
// Assign an about id and intialize the nested refs html
var referencesId = env.newAboutId();
@@ -352,37 +376,23 @@
// token so that the dom post processor can generate
// and emit references at this point in the DOM.
var emitReferencesFragment = function(toks, body) {
-   var ol;
-   if (body) {
-   ol = body.ownerDocument.createElement('ol');
-   DU.migrateChildren(body, ol);
-   } else {
-   ol = dummyDoc.createElement('ol');
+   var doc = body ? body.ownerDocument : dummyDoc;
+   var ol = createReferences(doc, referencesId, refsOpts, 
function(dp) {
+   dp.src = refsTok.getAttribute('source');
+   });
+   if (!body) {
body = dummyDoc.createElement('body');
+   } else {
+   DU.migrateChildren(body, ol);
}
body.appendChild(ol);
-   DU.addAttributes(ol, {
-   'class': 'mw-references references',
-   typeof: 'mw:Extension/references',
-   about: referencesId,
-   });
-   var olProcessor = function(ol) {
-   var dp = DU.getDataParsoid(ol);
-   dp.src = refsTok.getAttribute('source');
-   if (refsOpts.group) {
-   dp.group = refsOpts.group;
-   ol.setAttribute('data-mw-group', 
refsOpts.group);
-   }
-   // Pass along the `responsive` parameter
-   dp.tmp.responsive = refsOpts.responsive;
-   };
cb({
async: false,
tokens: DU.buildDOMFragmentTokens(
manager.env,
refsTok,
body,
-   olProcessor,
+   null,
// The  HTML above is wrapper HTML added on 
and doesn't
// have any DSR on it. We want DSR added to it.
{ aboutId: referencesId, setDSR: true, 
isForeignContent: true }
@@ -612,25 +622,20 @@
var self = this;
 
refsData.refGroups.forEach(function(refsValue, refsGroup) {
-   var ol = doc.createElement('ol');
-   var dp = DU.getDataParsoid(ol);
-   DU.addAttributes(ol, {
-   'class': 'mw-references references',
-   typeof: 'mw:Extension/references',
-   about: env.newAboutId(),
+   var ol = createReferences(doc, env.newAboutId(), {
+   group: refsGroup,
+ 

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Rethink responsive references wrappers

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/402455 )

Change subject: Rethink responsive references wrappers
..

Rethink responsive references wrappers

This needs testing with VE and a better description.

Bug: T181408
Change-Id: I2b5c7ff552b3322be74f79a81936c41d58fecabc
---
M lib/ext/Cite/index.js
M lib/wt2html/pp/processors/linter.js
M tests/citeParserTests.txt
3 files changed, 70 insertions(+), 69 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/55/402455/1

diff --git a/lib/ext/Cite/index.js b/lib/ext/Cite/index.js
index 0c32c92..12185c2 100644
--- a/lib/ext/Cite/index.js
+++ b/lib/ext/Cite/index.js
@@ -332,17 +332,40 @@
this.cite = cite;
 }
 
-var createReferences = function(doc, about, refsOpts, modifyDp) {
-   var ol = doc.createElement('ol');
-   DU.addAttributes(ol, {
-   class: 'mw-references references',
+var dummyDoc = domino.createDocument();
+
+var createReferences = function(env, body, about, refsOpts, modifyDp) {
+   var doc = body ? body.ownerDocument : dummyDoc;
+
+   var frag = doc.createElement('ol');
+   frag.classList.add('mw-references');
+   frag.classList.add('references');
+
+   if (body) {
+   DU.migrateChildren(body, frag);
+   }
+
+   // Support the `responsive` parameter
+   var rrOpts = env.conf.wiki.responsiveReferences;
+   var responsiveWrap = rrOpts.enabled;
+   if (refsOpts.responsive !== null) {
+   responsiveWrap = refsOpts.responsive !== '0';
+   }
+   if (responsiveWrap) {
+   var div = doc.createElement('div');
+   div.classList.add('mw-references-wrap');
+   div.appendChild(frag);
+   frag = div;
+   }
+
+   DU.addAttributes(frag, {
typeof: 'mw:Extension/references',
about: about,
});
-   var dp = DU.getDataParsoid(ol);
+   var dp = DU.getDataParsoid(frag);
if (refsOpts.group) {  // No group for the empty string either
dp.group = refsOpts.group;
-   ol.setAttribute('data-mw-group', refsOpts.group);
+   frag.setAttribute('data-mw-group', refsOpts.group);
}
if (refsOpts.responsive !== null) {
// Pass along the `responsive` parameter
@@ -351,10 +374,9 @@
if (typeof modifyDp === 'function') {
modifyDp(dp);
}
-   return ol;
-};
 
-var dummyDoc = domino.createDocument();
+   return frag;
+};
 
 /**
  * Sanitize the references tag and convert it into a meta-token
@@ -376,27 +398,20 @@
// token so that the dom post processor can generate
// and emit references at this point in the DOM.
var emitReferencesFragment = function(toks, body) {
-   var doc = body ? body.ownerDocument : dummyDoc;
-   var ol = createReferences(doc, referencesId, refsOpts, 
function(dp) {
+   if (!body) { body = dummyDoc.createElement('body'); }
+   var frag = createReferences(env, body, referencesId, refsOpts, 
function(dp) {
dp.src = refsTok.getAttribute('source');
});
-   if (!body) {
-   body = dummyDoc.createElement('body');
-   } else {
-   DU.migrateChildren(body, ol);
-   }
-   body.appendChild(ol);
+   body.appendChild(frag);
cb({
async: false,
-   tokens: DU.buildDOMFragmentTokens(
-   manager.env,
-   refsTok,
-   body,
-   null,
+   tokens: DU.buildDOMFragmentTokens(env, refsTok, body, 
null, {
+   aboutId: referencesId,
+   isForeignContent: true,
// The  HTML above is wrapper HTML added on 
and doesn't
// have any DSR on it. We want DSR added to it.
-   { aboutId: referencesId, setDSR: true, 
isForeignContent: true }
-   ),
+   setDSR: true,
+   }),
});
};
 
@@ -569,6 +584,17 @@
DU.setDataMw(refsNode, dataMw);
}
 
+   var refGroup = refsData.getRefGroup(group);
+
+   // Deal with responsive warpper
+   if (refsNode.classList.contains('mw-references-wrap')) {
+   var rrOpts = env.conf.wiki.responsiveReferences;
+   if (refGroup && refGroup.refs.length > rrOpts.threshold) {
+   refsNode.classList.add('mw-references-columns');
+   }
+   refsNode = refsNode.firstChild;
+   }
+
 

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Avoid expanding multiline sol transparent template ranges un...

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/402107 )

Change subject: Avoid expanding multiline sol transparent template ranges 
unnecessarily
..

Avoid expanding multiline sol transparent template ranges unnecessarily

Follow up to 4d7a25b, where reaching the eol with unmatched meta markers
was unaccounted for.

Fixes all the regressions at,
https://parsoid-rt-tests.wikimedia.org/regressions/between/28d7734fa2385a8fa599e3ce1bfbcd8582c05bc4/5abd780336c54c0fcd8fd953588085fe505bf455

Change-Id: Iaf63202349380c7e1b4c2e6d0a768c49301b1869
---
M lib/wt2html/tt/ParagraphWrapper.js
M tests/parserTests.txt
2 files changed, 27 insertions(+), 2 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/07/402107/1

diff --git a/lib/wt2html/tt/ParagraphWrapper.js 
b/lib/wt2html/tt/ParagraphWrapper.js
index a0483ed..6f84534 100644
--- a/lib/wt2html/tt/ParagraphWrapper.js
+++ b/lib/wt2html/tt/ParagraphWrapper.js
@@ -236,10 +236,10 @@
// Not a transclusion meta; Check for 
nl/sol-transparent tokens
// and leave them out of the p-wrapping.
if (!Util.isSolTransparent(this.env, t) && 
t.constructor !== NlTk) {
-   if (tplStartIndex > -1) { i = tplStartIndex; }
break;
}
}
+   if (tplStartIndex > -1) { i = tplStartIndex; }
out.splice(i, 0, new TagTk('p'));
this.hasOpenPTag = true;
}
@@ -270,10 +270,10 @@
// Not a transclusion meta; Check for 
nl/sol-transparent tokens
// and leave them out of the p-wrapping.
if (!Util.isSolTransparent(this.env, t) && 
t.constructor !== NlTk) {
-   if (tplEndIndex > -1) { i = tplEndIndex; }
break;
}
}
+   if (tplEndIndex > -1) { i = tplEndIndex; }
out.splice(i + 1, 0, new EndTagTk('p'));
this.hasOpenPTag = false;
}
diff --git a/tests/parserTests.txt b/tests/parserTests.txt
index 05bc36b..afbd6d6 100644
--- a/tests/parserTests.txt
+++ b/tests/parserTests.txt
@@ -2149,6 +2149,31 @@
 
 !! end
 
+!! test
+Avoid expanding multiline sol transparent template ranges unnecessarily
+!! wikitext
+hi
+
+
+{{echo|
+}}
+
+[[Category:Ho]]
+!! html/php
+hi
+
+
+
+!! html/parsoid
+hi
+
+
+
+
+
+
+!! end
+
 ###
 ### Preformatted text
 ###

-- 
To view, visit https://gerrit.wikimedia.org/r/402107
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Iaf63202349380c7e1b4c2e6d0a768c49301b1869
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Update blacklist after 7c6e86d

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/402106 )

Change subject: Update blacklist after 7c6e86d
..

Update blacklist after 7c6e86d

Change-Id: I5c374d2240a80c99f54b27786c3f1b2c14969e04
---
M tests/citeParserTests-blacklist.js
M tests/parserTests-blacklist.js
2 files changed, 23 insertions(+), 23 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/06/402106/1

diff --git a/tests/citeParserTests-blacklist.js 
b/tests/citeParserTests-blacklist.js
index 756199d..9a3491a 100644
--- a/tests/citeParserTests-blacklist.js
+++ b/tests/citeParserTests-blacklist.js
@@ -40,37 +40,37 @@
 
 // Blacklist for wt2html
 add("wt2html", "Simple , no ", "Wikipedia rocks![1]\n↑  Proceeds of Rockology, vol. XXI");
-add("wt2html", "Simple , with ", "Wikipedia rocks![1]\n\n↑  Proceeds of Rockology, vol. XXI");
-add("wt2html", " with a simple template", "Templating[1]\n\n↑  A
 simple template.");
-add("wt2html", " with a ", "Templating[1]\n\n↑  {{simple template}}");
+add("wt2html", "Simple , with ", "Wikipedia rocks![1]\n\n↑  Proceeds of Rockology, vol. XXI");
+add("wt2html", " with a simple template", "Templating[1]\n\n↑  A
 simple template.");
+add("wt2html", " with a ", "Templating[1]\n\n↑  {{simple template}}");
 add("wt2html", " in a ", "Templatingref>{{simple 
template}}/ref>\n\n");
 add("wt2html", " in a ", "Templating\n\n");
-add("wt2html", " in a  (bug 5384)", "Templating[1]\n\n↑  Text");
-add("wt2html", " after  (bug 6164)", "[1]\n\n\n\n\n\n↑  one");
+add("wt2html", " in a  (bug 5384)", "Templating[1]\n\n↑  Text");
+add("wt2html", " after  (bug 6164)", "[1]\n\n\n\n\n\n↑  one");
 add("wt2html", "{{REVISIONID}} on page with  (bug 6299)", "Parser
 function implementation for pf_revisionid missing in Parsoid.[1]\n↑  elite");
 add("wt2html", "{{REVISIONID}} on page without  (bug 6299 sanity check)", 
"Parser
 function implementation for pf_revisionid missing in Parsoid.");
-add("wt2html", "Ref with content followed by blank ref", "[1]\n\n[1]\n\n1 2 
 content");
-add("wt2html", "Blank ref followed by ref with content", "[1]\n\n[1]\n\n1 2 
 content");
-add("wt2html", "Regression: non-blank ref \"0\" followed by ref with content", 
"[1]\n\n[1]\n\n1 2 
 0");
-add("wt2html", "Regression sanity check: non-blank ref \"1\" followed by ref 
with content", "[1]\n\n[1]\n\n1 2 
 1");
-add("wt2html", "Ref names containing a number", "[1]\n[2]\n[3]\n\n↑  One↑  Two↑  Three");
-add("wt2html", "Erroneous refs", "[1]\n\n[2]\n\n[3]\n\n[4]\n\n[5]\n\n↑  Zero↑  Also zero, but differently! (Normal ref)↑  ↑  ↑  \n\n");
-add("wt2html", "Simple , with  in group", "Wikipedia rocks![1]\nWikipedia rocks![note 
1]\n\n↑  Proceeds of Rockology, vol. XXI\n↑  Proceeds of Rockology, vol. XXI");
+add("wt2html", "Ref with content followed by blank ref", "[1]\n\n[1]\n\n1 2 
 content");
+add("wt2html", "Blank ref followed by ref with content", "[1]\n\n[1]\n\n1 2 
 content");
+add("wt2html", "Regression: non-blank ref \"0\" followed by ref with content", 
"[1]\n\n[1]\n\n1 2 
 0");
+add("wt2html", "Regression sanity check: non-blank ref \"1\" followed by ref 
with content", "[1]\n\n[1]\n\n1 2 
 1");
+add("wt2html", "Ref names containing a number", "[1]\n[2]\n[3]\n\n↑  One↑  Two↑  Three");
+add("wt2html", "Erroneous refs", "[1]\n\n[2]\n\n[3]\n\n[4]\n\n[5]\n\n↑  Zero↑  Also zero, but differently! (Normal ref)↑  ↑  ↑  \n\n");
+add("wt2html", "Simple , with  in group", "Wikipedia rocks![1]\nWikipedia rocks![note 
1]\n\n↑  Proceeds of Rockology, vol. XXI\n↑  Proceeds of Rockology, vol. XXI");
 add("wt2html", "Simple , with  in group, with groupname in 
Chinese", "AAA[参 
1]BBB[注 
1]CCC[参 
2]\n\nrefs\n↑ 
 ref a↑ 
 ref c\nnotes\n↑ 
 note b");
 add("wt2html", " defined in ", "[1]\n\n↑  BAR");
 add("wt2html", " defined in  called with #tag", "[1]\n\n\n[1]\n\n1 2 
 BAR");
-add("wt2html", " defined in  error conditions", "[2 
1]\n\n↑ 
  BAR BAR BAR\n bad group");
-add("wt2html", " with custom group link with number names in Klingon", 
"Wikipedia rocks![klingon 
1]\n\n↑  Proceeds of Rockology, vol. XXI");
-add("wt2html", "Bug 31374 regression check: nested strip items", "note[1]\n\n↑  reference");
+add("wt2html", " defined in  error conditions", "[2 
1]\n\n↑ 
  BAR BAR BAR\n bad group");
+add("wt2html", " with custom group link with number names in Klingon", 
"Wikipedia rocks![klingon 
1]\n\n↑  Proceeds of Rockology, vol. XXI");
+add("wt2html", "Bug 31374 regression check: nested strip items", "note[1]\n\n↑  reference");
 add("wt2html", "Bug 13073 regression check: wrapped ", "[1]\n↑  foo\n");
 add("wt2html", " with no name and no content.", "Bla.[1]\n↑  ");
 add("wt2html", " with an empty-string name parameter and no content.", 
"Bla.[1]\n↑  ");
 add("wt2html", " with a non-empty name parameter and no content.", "Bla.[1]\n↑  ");
-add("wt2html", "s with the follow parameter", "Page one.[1]\n\nPage two.[2]\n\n References \n↑  

[MediaWiki-commits] [Gerrit] mediawiki...ProofreadPage[master]: Preserve SOL context when serializing page content as wikitext

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/401816 )

Change subject: Preserve SOL context when serializing page content as wikitext
..

Preserve SOL context when serializing page content as wikitext

When fetching the parser output, the body is put on a newline, which
allows SOL sensitive syntax to render as expected.  However, when
serializing for fetching wikitext from the action API, that context
isn't preserved.

This results in broken renders as in,
https://es.wikisource.org/api/rest_v1/page/html/P%C3%A1gina%3AFiguras_i_descripciones_de_aves_chilenas.djvu%2F44

where the returned wikitext to Parsoid begins,

{{c|28}}
{|style="font-size:90%;margin:0 auto;" width=70%
| || {{menor|Pulg.}} || {{menor|lín.}}
|-|Lonjitud del dedo mediano ||—|| 10½
|-
|{{ditto|Lonjitud}} del dedo exterior ||—|| 7½
|-
|{{ditto|Lonjitud}} del dedo interior ||—|| 7
|-
|{{ditto|Lonjitud}} del dedo posterior ||—|| 8
|-
|{{ditto|Lonjitud}} de su uña sólo ||—|| 5
|}
...

Until Parsoid has native rendering the proofread-page contentmodel, this
is resulting in fostered content, as reported,
https://www.mediawiki.org/wiki/Topic:U4yizxwac6fnqzch

A native extension for Parsoid is in,
https://gerrit.wikimedia.org/r/#/c/331799/

Bug: T184021
Change-Id: I342590860d91e08260904e35d0e7ebef8d139540
---
M includes/Page/PageContent.php
M includes/Page/PageContentHandler.php
2 files changed, 4 insertions(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/ProofreadPage 
refs/changes/16/401816/1

diff --git a/includes/Page/PageContent.php b/includes/Page/PageContent.php
index 01a85c0..4850687 100644
--- a/includes/Page/PageContent.php
+++ b/includes/Page/PageContent.php
@@ -269,7 +269,9 @@
 
// create content
$wikitextContent = new WikitextContent(
-   $this->header->getNativeData() . "\n\n" . 
$this->body->getNativeData() .
+   $this->header->getNativeData() .
+   "\n\n" .  // Matches 
PageContentHandler::serializeContentInWikitext
+   $this->body->getNativeData() .
$this->footer->getNativeData()
);
$parserOutput = $wikitextContent->getParserOutput( $title, 
$revId, $options, $generateHtml );
diff --git a/includes/Page/PageContentHandler.php 
b/includes/Page/PageContentHandler.php
index 99b8b0b..9441d46 100644
--- a/includes/Page/PageContentHandler.php
+++ b/includes/Page/PageContentHandler.php
@@ -85,6 +85,7 @@
'' .
$content->getHeader()->serialize() .
'' .
+   "\n\n" .  // Matches PageContent::getParserOutput
$content->getBody()->serialize() .
'' .
$content->getFooter()->serialize() .

-- 
To view, visit https://gerrit.wikimedia.org/r/401816
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I342590860d91e08260904e35d0e7ebef8d139540
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/ProofreadPage
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...deploy[master]: Bump src/ to 5abd780 + add pn

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/401809 )

Change subject: Bump src/ to 5abd780 + add pn
..

Bump src/ to 5abd780 + add pn

Change-Id: I136ce84512a512bf0abc9656853b579a6468b777
---
A node_modules/pn/.travis.yml
A node_modules/pn/README.md
A node_modules/pn/_promise.js
A node_modules/pn/_promisify.js
A node_modules/pn/assert.js
A node_modules/pn/async_hooks.js
A node_modules/pn/buffer.js
A node_modules/pn/child_process.js
A node_modules/pn/cluster.js
A node_modules/pn/console.js
A node_modules/pn/crypto.js
A node_modules/pn/dgram.js
A node_modules/pn/dns.js
A node_modules/pn/domain.js
A node_modules/pn/events.js
A node_modules/pn/fs.js
A node_modules/pn/http.js
A node_modules/pn/http2.js
A node_modules/pn/https.js
A node_modules/pn/inspector.js
A node_modules/pn/net.js
A node_modules/pn/os.js
A node_modules/pn/package.json
A node_modules/pn/path.js
A node_modules/pn/perf_hooks.js
A node_modules/pn/process.js
A node_modules/pn/punycode.js
A node_modules/pn/querystring.js
A node_modules/pn/readline.js
A node_modules/pn/repl.js
A node_modules/pn/scripts/generate.js
A node_modules/pn/stream.js
A node_modules/pn/string_decoder.js
A node_modules/pn/test/cp.js
A node_modules/pn/test/custom.js
A node_modules/pn/test/fs.js
A node_modules/pn/test/load.js
A node_modules/pn/test/timers.js
A node_modules/pn/timers.js
A node_modules/pn/tls.js
A node_modules/pn/tty.js
A node_modules/pn/url.js
A node_modules/pn/util.js
A node_modules/pn/v8.js
A node_modules/pn/vm.js
A node_modules/pn/zlib.js
M src
47 files changed, 1,251 insertions(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid/deploy 
refs/changes/09/401809/1

diff --git a/node_modules/pn/.travis.yml b/node_modules/pn/.travis.yml
new file mode 100644
index 000..e1519e4
--- /dev/null
+++ b/node_modules/pn/.travis.yml
@@ -0,0 +1,13 @@
+language: node_js
+node_js:
+  - "9.3.0"
+  - "8.9.4"
+  - "7.10.1"
+  - "6.12.3"
+  - "5.12.0"
+  - "4.8.7"
+  - "3.3.1"
+  - "2.5.0"
+  - "0.12"
+  - "0.11"
+  - "0.10"
diff --git a/node_modules/pn/README.md b/node_modules/pn/README.md
new file mode 100644
index 000..29ca995
--- /dev/null
+++ b/node_modules/pn/README.md
@@ -0,0 +1,148 @@
+# pn
+[![NPM][NPM1]][NPM2]
+
+[![Build Status][1]][2]
+
+The `pn` library gives you the Promise-using node standard library
+you've always dreamed of.  Think "Promised Node" when saying it.
+
+Just about every node standard library method that takes a callback will now
+return a `Promise` iff no callback is supplied.  But existing code
+that passes callbacks will still work fine --- and fast:  No unnecessary
+`Promise`s are created if you supply a callback.
+
+The wrappers are generated automatically by a script, with a table to
+handle exceptions and odd cases.  See below for more details.
+
+## Installation
+```
+npm install pn
+```
+## Usage
+```
+var fs = require('pn/fs');
+fs.writeFile('foo', 'bar', 'utf-8').then(function() { console.log('done'); });
+// But you can use callbacks, too -- existing code won't break.
+fs.writeFile('foo', 'bat', 'utf-8', function(err) { console.log('yay'); });
+```
+
+This library uses node native promises (ie `global.Promise`) by
+default, and thus works with node >= 0.11.
+
+You can use [`core-js`](https://www.npmjs.org/package/core-js) or
+[`es6-shim`](https://www.npmjs.org/package/es6-shim)
+to add ES6 Promises to earlier versions of node, for example:
+```
+require('es6-shim');
+var fs = require('pn/fs');
+```
+Just be sure that `es6-shim` is loaded before the `pn` package in that
+case.
+
+You might also want to look into packages like
+[`prfun`](https://www.npmjs.org/package/prfun)
+which add some helpers to make working with native promises much more
+fun.
+
+In particular, the `Promise#done` method is very useful when
+debugging, at least until v8's native Promise debugging
+capabilities are completed.
+
+## Custom Promise types
+
+You can also specify a custom `Promise` type to use, as follows:
+```
+var MyPromise = require('prfun');  // Use prfun's Promises, for example.
+require('pn/_promise')(MyPromise); // This only needs to be done once.
+```
+
+## Exceptions and odd cases
+
+The wrappers are automatically generated by `scripts/generate.js`;
+there is a table in that file which specifies all the odd cases.
+
+In general: if the node API has a callback which takes multiple
+value arguments, the `Promise` returned will be an object with
+named fields corresponding to the different values. If the node
+API takes a callback *and* returns a value, `pn` will return
+the original value with a nonenumerable field named `promise`
+corresponding to the callback.  Combining these two cases:
+```
+var child_process = require('pn/child_process');
+var cp = child_process.execFile('true');
+console.log('pid', cp.pid);
+cp.promise.then(function(result) {
+  console.log('stdout: ', result.stdout);
+  

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Assert supported options for the parser pipeline cache key

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/401619 )

Change subject: Assert supported options for the parser pipeline cache key
..

Assert supported options for the parser pipeline cache key

Removing "token" in several places, since that's only an option for
getDOMFragmentToken but was refactored in 6624260

Passing a key to `pipeline.process` is removed since it has no effect.

The cacheKey option itself is removed since it goes unused.

Change-Id: I41753d5c4e311b970bdbd00fd9b5903197e33aac
---
M lib/utils/DOMUtils.js
M lib/utils/Util.js
M lib/wt2html/TokenTransformManager.js
M lib/wt2html/parser.js
M lib/wt2html/tt/ExtensionHandler.js
M lib/wt2html/tt/LanguageVariantHandler.js
M lib/wt2html/tt/LinkHandler.js
M lib/wt2html/tt/TemplateHandler.js
8 files changed, 48 insertions(+), 41 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/19/401619/1

diff --git a/lib/utils/DOMUtils.js b/lib/utils/DOMUtils.js
index aad5b46..abb1eb2 100644
--- a/lib/utils/DOMUtils.js
+++ b/lib/utils/DOMUtils.js
@@ -1662,12 +1662,13 @@
// XXX gwicke: use 
proper key that is not
// source-based? This 
also needs to work for
// transclusion output.
-   key = dp.cacheKey;
+   key = null;
}
 
if (key) {
expAccum[key] = 
DU.makeExpansion(env, nodes);
}
+
node = lastItem(nodes);
} else {

doExtractExpansions(node.firstChild);
diff --git a/lib/utils/Util.js b/lib/utils/Util.js
index 9d3e18d..cd998cb 100644
--- a/lib/utils/Util.js
+++ b/lib/utils/Util.js
@@ -1195,7 +1195,7 @@
}
 
// Off the starting block ... ready, set, go!
-   pipeline.process(content, opts.tplArgs ? opts.tplArgs.cacheKey 
: undefined);
+   pipeline.process(content);
},
 
/**
diff --git a/lib/wt2html/TokenTransformManager.js 
b/lib/wt2html/TokenTransformManager.js
index 67ddbc0..286f43b 100644
--- a/lib/wt2html/TokenTransformManager.js
+++ b/lib/wt2html/TokenTransformManager.js
@@ -1456,7 +1456,6 @@
 
opts.tplArgs = {
name: null,
-   cacheKey: this.title,
};
 
var content;
diff --git a/lib/wt2html/parser.js b/lib/wt2html/parser.js
index 3d63eb2..b2d7bc8 100644
--- a/lib/wt2html/parser.js
+++ b/lib/wt2html/parser.js
@@ -175,12 +175,44 @@
],
 };
 
-// SSS FIXME: maybe there is some built-in method for this already?
+var supportedOptions = new Set([
+   // If true, templates found in content will be encapsulated with data
+   // (target, arguments, etc.) and its contents expanded
+   'wrapTemplates',
+
+   // If true, indicates pipeline is processing the expanded content of a
+   // template or its arguments
+   'inTemplate',
+
+   // If true, indicates that we are in a  context
+   // (in current usage, isInclude === inTemplate)
+   'isInclude',
+
+   // The extension tag that is being processed (Ex: ref, references)
+   // (in current usage, only used for native tag implementation)
+   'extTag',
+
+   // Any id assigned to the extension tag (ex: )
+   'extTagId',
+
+   // Suppress paragraph-wrapping for content in this pipeline
+   'noPWrapping',
+
+   // Suppress indent-pre processing for content in this pipeline
+   'noPre',
+
+   // Are we processing content of attributes?
+   // (in current usage, used for transcluded attr. keys/values)
+   'attrExpansion',
+]);
+
 // Default options processing
-ParserPipelineFactory.prototype.defaultOptions = function(options) {
-   if (!options) {
-   options = {};
-   }
+var defaultOptions = function(options) {
+   if (!options) { options = {}; }
+
+   Object.keys(options).forEach(function(k) {
+   console.assert(supportedOptions.has(k), 'Invalid cacheKey 
option: ' + k);
+   });
 
// default: not an include context
if (options.isInclude === undefined) {
@@ -192,10 +224,6 @@
options.wrapTemplates = true;
}
 
-   if (options.cacheKey === undefined) {
-   options.cacheKey = null;
-   }
-
return options;
 };
 
@@ -204,7 +232,7 @@
  */
 ParserPipelineFactory.prototype.makePipeline = function(type, options) {
// SSS 

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Sync parserTests with core

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/401531 )

Change subject: Sync parserTests with core
..

Sync parserTests with core

Change-Id: Idefc00c87a7ab413a608d52c00fe2086ffd30b24
---
M tests/parserTests-blacklist.js
M tests/parserTests.json
M tests/parserTests.txt
3 files changed, 46 insertions(+), 46 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/31/401531/1

diff --git a/tests/parserTests-blacklist.js b/tests/parserTests-blacklist.js
index cc7c8a0..4d33cfe 100644
--- a/tests/parserTests-blacklist.js
+++ b/tests/parserTests-blacklist.js
@@ -403,12 +403,12 @@
 add("html2html", "Disabled subpages", "[/index.php?title=/subpageaction=editredlink=1
 /subpage]\n");
 add("html2html", "T2561: {{/Subpage}}", "[/index.php?title=Page/Subpageaction=editredlink=1
 Page/Subpage]\n");
 add("html2html", "Link to category", "Category:MediaWiki User's 
Guide\n");
-add("html2html", "Section headings with TOC", "\n Contents \n\n\n 1 Headline 
1\n\n 1.1 Subheadline 
1\n\n 1.1.1 Skipping a 
level\n\n 1.1.1.1 Skipping a 
level\n 2 Headline 
2\n\n 2.1 Another 
headline\n\n\n\n Headline 1 \n\n Subheadline 1 
\n\n 
Skipping a level \n\n Skipping a level \n\n Headline 2 \nSome text\n\n Another 
headline \n");
-add("html2html", "TOC regression (T11764)", "\n Contents \n\n\n 1 title 
1\n\n 1.1 title 
1.1\n\n 1.1.1 title 
1.1.1\n 1.2 title 
1.2\n 2 title 
2\n\n 2.1 title 
2.1\n\n\n\n title 1 \n\n title 1.1 \n\n title 1.1.1 
\n\n title 1.2 
\n\n title 2 
\n\n title 2.1 
\n");
-add("html2html", "TOC with wgMaxTocLevel=3 (T8204)", "\n Contents 
\n\n\n 1 title 
1\n\n 1.1 title 
1.1\n 1.2 title 
1.2\n 2 title 
2\n\n 2.1 title 
2.1\n\n\n\n title 1 \n\n title 1.1 \n\n title 1.1.1 
\n\n title 1.2 
\n\n title 2 
\n\n title 2.1 
\n");
-add("html2html", "TOC with wgMaxTocLevel=3 and two level four headings 
(T8204)", "\n Contents \n\n\n 1 Section 
1\n\n 1.1 Section 
1.1\n 2 Section 
2\n\n\n\n Section 1 \n\n Section 1.1 
\n\n 
Section 1.1.1 \n\n Section 1.1.1.1 \n\n Section 2 \n");
+add("html2html", "Section headings with TOC", "\n Contents \n\n\n 1 Headline 
1\n\n 1.1 Subheadline 
1\n\n 1.1.1 Skipping a 
level\n\n 1.1.1.1 Skipping a 
level\n 2 Headline 
2\n\n 2.1 Another 
headline\n\n\n\n Headline 1 \n\n Subheadline 1 
\n\n 
Skipping a level \n\n Skipping a level \n\n Headline 2 \nSome text\n\n Another 
headline \n");
+add("html2html", "TOC regression (T11764)", "\n Contents \n\n\n 1 title 
1\n\n 1.1 title 
1.1\n\n 1.1.1 title 
1.1.1\n 1.2 title 
1.2\n 2 title 
2\n\n 2.1 title 
2.1\n\n\n\n title 1 \n\n title 1.1 \n\n title 1.1.1 
\n\n title 1.2 
\n\n title 2 
\n\n title 2.1 
\n");
+add("html2html", "TOC with wgMaxTocLevel=3 (T8204)", "\n Contents \n\n\n 1 title 
1\n\n 1.1 title 
1.1\n 1.2 title 
1.2\n 2 title 
2\n\n 2.1 title 
2.1\n\n\n\n title 1 \n\n title 1.1 \n\n title 1.1.1 
\n\n title 1.2 
\n\n title 2 
\n\n title 2.1 
\n");
+add("html2html", "TOC with wgMaxTocLevel=3 and two level four headings 
(T8204)", "\n Contents \n\n\n 1 Section 
1\n\n 1.1 Section 
1.1\n 2 Section 
2\n\n\n\n Section 1 \n\n Section 1.1 
\n\n 
Section 1.1.1 \n\n Section 1.1.1.1 \n\n Section 2 \n");
 add("html2html", "Link inside a section heading", " Section 
with a link in it \n");
-add("html2html", "TOC regression (T14077)", "\n Contents \n\n\n 1 title 
1\n\n 1.1 title 
1.1\n 2 title 
2\n\n\n\n title 1 \n\n title 1.1 \n\n title 2 \n");
+add("html2html", "TOC regression (T14077)", "\n Contents \n\n\n 1 title 
1\n\n 1.1 title 
1.1\n 2 title 
2\n\n\n\n title 1 \n\n title 1.1 \n\n title 2 \n");
 add("html2html", "Character reference normalization in link text (T3938)", "thisthat\n");
 add("html2html", "(T21451) Links should refer to the normalized form.", "אַ\nאַ\nאַ\nאַ\nאַ\n");
 add("html2html", "T4095: link with pipe and three closing brackets, version 
2", "[http://example.com/]");
@@ -437,7 +437,7 @@
 add("html2html", "Parents of subpages, two levels up", "[/index.php?title=Subpage_test/L1action=editredlink=1
 L1]2\n\n[/index.php?title=Subpage_test/L1action=editredlink=1
 L1l]\n");
 add("html2html", "Parents of subpages, two levels up, with lots of extra 
trailing slashes.", "[/index.php?title=Subpage_test/L1action=editredlink=1
 Subpage test/L1]\n");
 add("html2html", "RAW magic word", "[/index.php?title=Template:QUERTYaction=editredlink=1
 Template:QUERTY]\n");
-add("html2html", "Out-of-order TOC heading levels", "\n Contents 
\n\n\n 1 2\n\n 1.1 6\n 1.2 3\n 2 1\n\n 2.1 5\n 2.2 2\n\n\n\n 2 \n\n 6 \n\n 3 \n\n 1 \n\n 5 \n\n 2 \n");
+add("html2html", "Out-of-order TOC heading levels", "\n Contents \n\n\n 1 2\n\n 1.1 6\n 1.2 3\n 2 1\n\n 2.1 5\n 2.2 2\n\n\n\n 2 \n\n 6 \n\n 3 \n\n 1 \n\n 5 \n\n 2 \n");
 add("html2html", "Free external link invading image caption", "http://example.com/images/thumb/3/3a/Foobar.jpg/180px-Foobar.jpg\; 
alt=\"180px-Foobar.jpg\" rel=\"mw:externalImage\" 

[MediaWiki-commits] [Gerrit] mediawiki/core[master]: Sync up with Parsoid parserTests.txt

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/401522 )

Change subject: Sync up with Parsoid parserTests.txt
..

Sync up with Parsoid parserTests.txt

This now aligns with Parsoid commit 28d7734fa2385a8fa599e3ce1bfbcd8582c05bc4

Change-Id: Iee7f3f0022c9c770c1a02b401699df0915ea7c86
---
M tests/parser/parserTests.txt
1 file changed, 35 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core 
refs/changes/22/401522/1

diff --git a/tests/parser/parserTests.txt b/tests/parser/parserTests.txt
index 840c74d..aa03c0e 100644
--- a/tests/parser/parserTests.txt
+++ b/tests/parser/parserTests.txt
@@ -12633,6 +12633,19 @@
 
 !! end
 
+!! test
+Preprocessor precedence 17: template w/o target shouldn't prevent closing
+!! options
+parsoid=wt2html
+!! wikitext
+{{echo|hi {{
+!! html/php
+hi {{}}
+
+!! html/parsoid
+hi {{}}
+!! end
+
 ###
 ### Token Stream Patcher tests
 ###
@@ -30608,3 +30621,25 @@
  2 
 b
 !! end
+
+!! test
+Pseudo-sections emitted by templates should have id -2 
+!! options
+parsoid={
+  "wrapSections": true
+}
+!! wikitext
+foo
+{{echo|
+== a ==
+== b ==
+
+}}
+!! html/parsoid
+foo
+
+ a 
+ b 
+
+
+!! end

-- 
To view, visit https://gerrit.wikimedia.org/r/401522
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Iee7f3f0022c9c770c1a02b401699df0915ea7c86
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/core
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Account for SOL transparent templates in p-wrapping

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/399766 )

Change subject: Account for SOL transparent templates in p-wrapping
..

Account for SOL transparent templates in p-wrapping

Fixes the gratuitous template wrapping on,
/en.wikipedia.org/v3/page/html/Wikimedia_Foundation/803552163

Bug: T182349
Change-Id: I5093a3af83db2352309c2a39e95412c441cf68aa
---
M lib/wt2html/tt/ParagraphWrapper.js
M tests/parserTests-blacklist.js
M tests/parserTests.txt
3 files changed, 33 insertions(+), 8 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/66/399766/1

diff --git a/lib/wt2html/tt/ParagraphWrapper.js 
b/lib/wt2html/tt/ParagraphWrapper.js
index af24c3b..f1badfc 100644
--- a/lib/wt2html/tt/ParagraphWrapper.js
+++ b/lib/wt2html/tt/ParagraphWrapper.js
@@ -215,6 +215,7 @@
 ParagraphWrapper.prototype.openPTag = function(out) {
if (!this.hasOpenPTag) {
var i = 0;
+   var enteredTransclusion = -1;
// Be careful not to expand template ranges unnecessarily.
// Look for open markers before starting a p-tag.
for (; i < out.length; i++) {
@@ -222,18 +223,20 @@
if (t.name === "meta") {
var typeOf = t.getAttribute("typeof");
if (/^mw:Transclusion$/.test(typeOf)) {
+   enteredTransclusion = i;
// We hit a start tag and everything 
before it is sol-transparent.
-   break;
+   continue;
} else if (/^mw:Transclusion/.test(typeOf)) {
// End tag. All tokens before this are 
sol-transparent.
// Let leave them all out of the 
p-wrapping.
-   i += 1;
-   break;
+   enteredTransclusion = -1;
+   continue;
}
}
// Not a transclusion meta; Check for 
nl/sol-transparent tokens
// and leave them out of the p-wrapping.
if (!Util.isSolTransparent(this.env, t) && 
t.constructor !== NlTk) {
+   if (enteredTransclusion > -1) { i = 
enteredTransclusion; }
break;
}
}
@@ -245,6 +248,7 @@
 ParagraphWrapper.prototype.closeOpenPTag = function(out) {
if (this.hasOpenPTag) {
var i = out.length - 1;
+   var enteredTransclusion = -1;
// Be careful not to expand template ranges unnecessarily.
// Look for open markers before closing.
for (; i > -1; i--) {
@@ -254,18 +258,20 @@
if (/^mw:Transclusion$/.test(typeOf)) {
// We hit a start tag and everything 
after it is sol-transparent.
// Don't include the sol-transparent 
tags OR the start tag.
-   i -= 1;
-   break;
+   enteredTransclusion = -1;
+   continue;
} else if (/^mw:Transclusion/.test(typeOf)) {
// End tag. We're done.
// The rest of the tags past this are 
sol-transparent.
// Let us leave them all out of the 
p-wrapping.
-   break;
+   enteredTransclusion = i;
+   continue;
}
}
// Not a transclusion meta; Check for 
nl/sol-transparent tokens
// and leave them out of the p-wrapping.
if (!Util.isSolTransparent(this.env, t) && 
t.constructor !== NlTk) {
+   if (enteredTransclusion > -1) { i = 
enteredTransclusion; }
break;
}
}
diff --git a/tests/parserTests-blacklist.js b/tests/parserTests-blacklist.js
index cc7c8a0..6699748 100644
--- a/tests/parserTests-blacklist.js
+++ b/tests/parserTests-blacklist.js
@@ -123,7 +123,7 @@
 add("wt2html", "Templates: Lists: Multi-line list-items via templates", "a 
{{nonexistent|\nunused}}\nb 
{{nonexistent|\nunused}}");
 add("wt2html", "Templates: Ugly nesting: 1. Quotes opened/closed across 
templates (echo)", "abcde");

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: [WIP] Don't insert new paragraph start before end tags

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/399471 )

Change subject: [WIP] Don't insert new paragraph start before end tags
..

[WIP] Don't insert new paragraph start before end tags

Fixes the gratuitous template wrapping on,
/en.wikipedia.org/v3/page/html/Wikimedia_Foundation/803552163

Bug: T182349
Change-Id: I6b8df14d989750d66219b5234085dd636ca29a83
---
M lib/wt2html/tt/ParagraphWrapper.js
1 file changed, 2 insertions(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/71/399471/1

diff --git a/lib/wt2html/tt/ParagraphWrapper.js 
b/lib/wt2html/tt/ParagraphWrapper.js
index af24c3b..eb4c020 100644
--- a/lib/wt2html/tt/ParagraphWrapper.js
+++ b/lib/wt2html/tt/ParagraphWrapper.js
@@ -292,7 +292,8 @@
blockTagCount--;
}
this.closeOpenPTag(newToks);
-   } else if (blockTagCount === 0 && 
!Util.isSolTransparent(this.env, t) && !this.hasOpenPTag) {
+   } else if (blockTagCount === 0 && 
!Util.isSolTransparent(this.env, t)
+   && !this.hasOpenPTag && t.constructor 
!== EndTagTk) {
// SSS FIXME: This check below is strictly not 
necessary since
// removeUselessPWrappers will take care of it. 
But, doing this
// here will eliminate useless array copying. 
Not sure if this

-- 
To view, visit https://gerrit.wikimedia.org/r/399471
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I6b8df14d989750d66219b5234085dd636ca29a83
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...deploy[master]: Bump src/ to 439abf7 + add worker-farm package

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/399323 )

Change subject: Bump src/ to 439abf7 + add worker-farm package
..

Bump src/ to 439abf7 + add worker-farm package

Change-Id: I33e3160f216eaad6b14a22ece0848b9d4f0b2eb1
---
A node_modules/.bin/errno
A node_modules/errno/.jshintrc
A node_modules/errno/.travis.yml
A node_modules/errno/README.md
A node_modules/errno/build.js
A node_modules/errno/cli.js
A node_modules/errno/custom.js
A node_modules/errno/errno.js
A node_modules/errno/package.json
A node_modules/errno/test.js
A node_modules/prr/.jshintrc
A node_modules/prr/.npmignore
A node_modules/prr/.travis.yml
A node_modules/prr/LICENSE.md
A node_modules/prr/README.md
A node_modules/prr/package.json
A node_modules/prr/prr.js
A node_modules/prr/test.js
M node_modules/sshpk/package.json
A node_modules/worker-farm/.travis.yml
A node_modules/worker-farm/LICENSE.md
A node_modules/worker-farm/README.md
A node_modules/worker-farm/examples/basic/child.js
A node_modules/worker-farm/examples/basic/index.js
A node_modules/worker-farm/examples/pi/calc.js
A node_modules/worker-farm/examples/pi/index.js
A node_modules/worker-farm/index.d.ts
A node_modules/worker-farm/lib/child/index.js
A node_modules/worker-farm/lib/farm.js
A node_modules/worker-farm/lib/fork.js
A node_modules/worker-farm/lib/index.js
A node_modules/worker-farm/package.json
A node_modules/worker-farm/tests/child.js
A node_modules/worker-farm/tests/debug.js
A node_modules/worker-farm/tests/index.js
M src
36 files changed, 2,590 insertions(+), 4 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid/deploy 
refs/changes/23/399323/1

diff --git a/node_modules/.bin/errno b/node_modules/.bin/errno
new file mode 12
index 000..5a98e53
--- /dev/null
+++ b/node_modules/.bin/errno
@@ -0,0 +1 @@
+../errno/cli.js
\ No newline at end of file
diff --git a/node_modules/errno/.jshintrc b/node_modules/errno/.jshintrc
new file mode 100644
index 000..c8ef3ca
--- /dev/null
+++ b/node_modules/errno/.jshintrc
@@ -0,0 +1,59 @@
+{
+"predef": [ ]
+  , "bitwise": false
+  , "camelcase": false
+  , "curly": false
+  , "eqeqeq": false
+  , "forin": false
+  , "immed": false
+  , "latedef": false
+  , "noarg": true
+  , "noempty": true
+  , "nonew": true
+  , "plusplus": false
+  , "quotmark": true
+  , "regexp": false
+  , "undef": true
+  , "unused": true
+  , "strict": false
+  , "trailing": true
+  , "maxlen": 120
+  , "asi": true
+  , "boss": true
+  , "debug": true
+  , "eqnull": true
+  , "esnext": true
+  , "evil": true
+  , "expr": true
+  , "funcscope": false
+  , "globalstrict": false
+  , "iterator": false
+  , "lastsemic": true
+  , "laxbreak": true
+  , "laxcomma": true
+  , "loopfunc": true
+  , "multistr": false
+  , "onecase": false
+  , "proto": false
+  , "regexdash": false
+  , "scripturl": true
+  , "smarttabs": false
+  , "shadow": false
+  , "sub": true
+  , "supernew": false
+  , "validthis": true
+  , "browser": true
+  , "couch": false
+  , "devel": false
+  , "dojo": false
+  , "mootools": false
+  , "node": true
+  , "nonstandard": true
+  , "prototypejs": false
+  , "rhino": false
+  , "worker": true
+  , "wsh": false
+  , "nomen": false
+  , "onevar": false
+  , "passfail": false
+}
\ No newline at end of file
diff --git a/node_modules/errno/.travis.yml b/node_modules/errno/.travis.yml
new file mode 100644
index 000..f996821
--- /dev/null
+++ b/node_modules/errno/.travis.yml
@@ -0,0 +1,11 @@
+sudo: false
+
+language: node_js
+
+node_js:
+  - 9
+  - 8
+  - 7
+  - 6
+  - 5
+  - 4
diff --git a/node_modules/errno/README.md b/node_modules/errno/README.md
new file mode 100644
index 000..a4d0fb5
--- /dev/null
+++ b/node_modules/errno/README.md
@@ -0,0 +1,145 @@
+# node-errno
+
+> Better 
[libuv](https://github.com/libuv/libuv)/[Node.js](https://nodejs.org)/[io.js](https://iojs.org)
 error handling & reporting. Available in npm as *errno*.
+
+[![npm](https://img.shields.io/npm/v/errno.svg)](https://www.npmjs.com/package/errno)
+[![Build 
Status](https://secure.travis-ci.org/rvagg/node-errno.png)](http://travis-ci.org/rvagg/node-errno)
+[![npm](https://img.shields.io/npm/dm/errno.svg)](https://www.npmjs.com/package/errno)
+
+* [errno exposed](#errnoexposed)
+* [Custom errors](#customerrors)
+
+
+## errno exposed
+
+Ever find yourself needing more details about Node.js errors? Me too, so 
*node-errno* contains the errno mappings direct from libuv so you can use them 
in your code.
+
+**By errno:**
+
+```js
+require('errno').errno[3]
+// → {
+// "errno": 3,
+// "code": "EACCES",
+// "description": "permission denied"
+//   }
+```
+
+**By code:**
+
+```js
+require('errno').code.ENOTEMPTY
+// → {
+// "errno": 53,
+// "code": "ENOTEMPTY",
+// "description": "directory not empty"
+//   }
+```
+
+**Make your errors more descriptive:**
+
+```js
+var errno = require('errno')
+
+function 

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Only use max depth detection since we aren't doing template ...

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/399245 )

Change subject: Only use max depth detection since we aren't doing template 
expansion
..

Only use max depth detection since we aren't doing template expansion

Testing this in parserTests runs afoul since we're still using native
template expansion there and we hit loop detection first.

Locally, once can try a Template:Gallery,


File:Test.png|123 {{Gallery}}


which renders similarly to the php parser, but with 2x instances.

References are a special case though, which we'll see in the follow-up.

Change-Id: I1a026191c4246180c210ad385a624fe9a6b2d2f6
---
M lib/wt2html/TokenTransformManager.js
M lib/wt2html/pegTokenizer.pegjs
M lib/wt2html/tt/TemplateHandler.js
M tests/parserTests.txt
4 files changed, 16 insertions(+), 13 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/45/399245/1

diff --git a/lib/wt2html/TokenTransformManager.js 
b/lib/wt2html/TokenTransformManager.js
index eb4e899..67ddbc0 100644
--- a/lib/wt2html/TokenTransformManager.js
+++ b/lib/wt2html/TokenTransformManager.js
@@ -1499,12 +1499,12 @@
  *
  * @param {string} title
  */
-Frame.prototype.loopAndDepthCheck = function(title, maxDepth) {
-   // XXX: set limit really low for testing!
+Frame.prototype.loopAndDepthCheck = function(title, maxDepth, ignoreLoop) {
if (this.depth > maxDepth) {
-   // too deep
+   // Too deep
return 'Error: Expansion depth limit exceeded at ';
}
+   if (ignoreLoop) { return false; }
var elem = this;
do {
if (elem.title === title) {
diff --git a/lib/wt2html/pegTokenizer.pegjs b/lib/wt2html/pegTokenizer.pegjs
index 17495b2..1a0f97c 100644
--- a/lib/wt2html/pegTokenizer.pegjs
+++ b/lib/wt2html/pegTokenizer.pegjs
@@ -801,12 +801,12 @@
  */
 tplarg_or_template
   = &'{{' &{
-  // Refuse to recurse beyond 40 levels. Default in the PHP parser
+  // Refuse to recurse beyond `maxDepth` levels. Default in the PHP parser
   // is $wgMaxTemplateDepth = 40; This is to prevent crashing from
   // buggy wikitext with lots of unclosed template calls, as in
   // eswiki/Usuario:C%C3%A1rdenas/PRUEBAS?oldid=651094
   if (stops.onCount('templatedepth') === undefined ||
-  stops.onCount('templatedepth') < 40) {
+  stops.onCount('templatedepth') < env.conf.parsoid.maxDepth) {
 return true;
   } else {
 return false;
diff --git a/lib/wt2html/tt/TemplateHandler.js 
b/lib/wt2html/tt/TemplateHandler.js
index bd19d9b..72aaf6c 100644
--- a/lib/wt2html/tt/TemplateHandler.js
+++ b/lib/wt2html/tt/TemplateHandler.js
@@ -149,11 +149,10 @@
var templateName = tgt.target;
var attribs = [];
 
-   // We still need to check for loops and depth limit 
violations
-   // because of the higher precedence of extension tags, 
which can
-   // result in nested templates even while using the php 
preprocessor
-   // for expansion.
-   var checkRes = this.checkRes(templateName);
+   // We still need to check for limit violations because 
of the
+   // higher precedence of extension tags, which can 
result in nested
+   // templates even while using the php preprocessor for 
expansion.
+   var checkRes = this.checkRes(templateName, true);
if (Array.isArray(checkRes)) {
cb({ tokens: checkRes });
return;
@@ -632,8 +631,8 @@
 /**
  * checkRes
  */
-TemplateHandler.prototype.checkRes = function(target) {
-   var checkRes = this.manager.frame.loopAndDepthCheck(target, 
this.env.conf.parsoid.maxDepth);
+TemplateHandler.prototype.checkRes = function(target, ignoreLoop) {
+   var checkRes = this.manager.frame.loopAndDepthCheck(target, 
this.env.conf.parsoid.maxDepth, ignoreLoop);
if (checkRes) {
// Loop detected or depth limit exceeded, abort!
var res = [
@@ -708,7 +707,7 @@
return;
}
 
-   var checkRes = this.checkRes(target);
+   var checkRes = this.checkRes(target, false);
if (Array.isArray(checkRes)) {
checkRes.rank = this.manager.phaseEndRank;
cb({ tokens: checkRes });
diff --git a/tests/parserTests.txt b/tests/parserTests.txt
index 0e234de..8adcaf2 100644
--- a/tests/parserTests.txt
+++ b/tests/parserTests.txt
@@ -294,12 +294,16 @@
 ###
 ### Basic tests
 ###
+
 !! test
 Blank input
 !! wikitext
 !! html
 !! end
 
+## This isn't a great test in Parsoid since native template expansion in
+## parserTests takes us to the path of expansion loop detection, which isn't
+## present 

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Restrict nested refs not coming from templates

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/399246 )

Change subject: Restrict nested refs not coming from templates
..

Restrict nested refs not coming from templates

This better matches the behaviour of the php extension.  An instructive
example is,

123 {{#tag:ref|321}}

{{#tag:ref|456 654}}

The php implementation only parses the ref contents when producing the
output for reference tags, at which point, nested refs will be ignored,
as in the former case.  The "321" is dropped.

The latter is special, since the extension tag takes precedence over the
parser function, the inner ref will already have been processed when the
outer is added to the stack, and hence the nesting is permitted.  This
is why the inner ref precedes the outer in the references list (has a
lower number).  Unfortunately, Parsoid doesn't yet get that ordering
right.

Change-Id: Ieb0e418cca634605c2a9f1487139b15095f38d81
---
M lib/ext/Cite/index.js
M tests/citeParserTests.txt
M tests/parserTests.txt
3 files changed, 49 insertions(+), 21 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/46/399246/1

diff --git a/lib/ext/Cite/index.js b/lib/ext/Cite/index.js
index ced0b51..66a8580 100644
--- a/lib/ext/Cite/index.js
+++ b/lib/ext/Cite/index.js
@@ -81,12 +81,25 @@
cb({ tokens: toks, async: false });
};
 
+   // Drop nested refs entirely, unless we've explicitly allowed them
+   if (pipelineOpts.extTag === 'ref' && !pipelineOpts.allowNestedRef) {
+   cb({ tokens: [], async: false });
+   return;
+   }
+
+   // The one supported case for nested refs is from the {{#tag:ref}} 
parser
+   // function.  However, we're overly permissive here since we can't
+   // distinguish when that's nested in another template.  The php
+   // preprocessor did our expansion.
+   var allowNestedRef = pipelineOpts.inTemplate && pipelineOpts.extTag !== 
'ref';
+
Util.processExtSource(manager, refTok, {
// Full pipeline for processing ref-content
pipelineType: 'text/x-mediawiki/full',
pipelineOpts: {
extTag: "ref",
inTemplate: pipelineOpts.inTemplate,
+   allowNestedRef: allowNestedRef,
noPre: true,
noPWrapping: true,
},
diff --git a/tests/citeParserTests.txt b/tests/citeParserTests.txt
index 5e29443..87d17d3 100644
--- a/tests/citeParserTests.txt
+++ b/tests/citeParserTests.txt
@@ -1265,9 +1265,18 @@
 !! wikitext
 test hi {{refinref}}
 
+!! html/php
+test 1
+
+
+↑ hi 
+
+
+Cite error: 
ref tag defined in references has no 
name attribute.
+
 !! html/parsoid
-test [1]
-↑  hi [2]↑  ho
+test [1]
+↑  hi 
 !! end
 
 ## Parsoid responsive references tests
@@ -1415,3 +1424,28 @@
 !! end
 
 ## End Parsoid responsive references tests
+
+!! article
+Template:CircularRef
+!! text
+Hi {{CircularRef}}
+!! endarticle
+
+!! test
+CircularRef
+!! wikitext
+Hi {{CircularRef}}
+
+!! html/php
+1
+
+
+↑ Hi 
+
+
+Cite error: 
ref tag defined in references has no 
name attribute.
+
+!! html/parsoid
+[1]
+↑  Hi 
+!! end
diff --git a/tests/parserTests.txt b/tests/parserTests.txt
index 8adcaf2..aedb460 100644
--- a/tests/parserTests.txt
+++ b/tests/parserTests.txt
@@ -280,12 +280,6 @@
 !!endarticle
 
 !! article
-Template:CircularRef
-!! text
-{{CircularRef}}
-!! endarticle
-
-!! article
 Template:With: Colon
 !! text
 Template with colon
@@ -299,19 +293,6 @@
 Blank input
 !! wikitext
 !! html
-!! end
-
-## This isn't a great test in Parsoid since native template expansion in
-## parserTests takes us to the path of expansion loop detection, which isn't
-## present in production, since we don't do our own expansion.  See T64424
-!! test
-CircularRef
-!! wikitext
-{{CircularRef}}
-
-!! html/parsoid
-[1]
-↑  Error: Expansion loop detected at Template:CircularRef
 !! end
 
 !! test

-- 
To view, visit https://gerrit.wikimedia.org/r/399246
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ieb0e418cca634605c2a9f1487139b15095f38d81
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: [WIP] Emit lint markers in the token stream

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/398909 )

Change subject: [WIP] Emit lint markers in the token stream
..

[WIP] Emit lint markers in the token stream

Unfortunately, this complicates template wrapping.

Change-Id: I61dbbf422ecc0e3d0ab1103ccf2b308cbd2aad44
---
M lib/wt2html/pp/handlers/linter.js
M lib/wt2html/tt/LinkHandler.js
M tests/mocha/linter.js
3 files changed, 38 insertions(+), 21 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/09/398909/1

diff --git a/lib/wt2html/pp/handlers/linter.js 
b/lib/wt2html/pp/handlers/linter.js
index ffa1079..58d6a44 100644
--- a/lib/wt2html/pp/handlers/linter.js
+++ b/lib/wt2html/pp/handlers/linter.js
@@ -813,6 +813,23 @@
env.log('lint/multiline-html-table-in-list', lintObj);
 }
 
+/**
+ * Log lints for linting meta markers that were inserted in the token stream
+ */
+var logLintMetas = function(env, node, dp, tplInfo) {
+   console.log('here', node.nodeName, node.getAttribute('typeof'))
+   if (!DU.isMarkerMeta(node, 'mw:Lint/lint/Marker')) {
+   return false;
+   }
+   var lintObj = dp.lintObj;
+   var templateInfo = findEnclosingTemplateName(env, tplInfo);
+   lintObj.templateInfo = templateInfo;
+   lintObj.dsr = findLintDSR(templateInfo, tplInfo, lintObj.dsr);
+   env.log('lint/' + dp.type, lintObj);
+   DU.deleteNode(node);
+   return true;
+};
+
 function logWikitextFixups(node, env, atTopLevel, tplInfo) {
// For now, don't run linter in subpipelines.
// Only on the final DOM for the top-level page.
@@ -827,6 +844,11 @@
 
var dp = DU.getDataParsoid(node);
 
+   var next = node.nextSibling;
+   if (logLintMetas(env, node, dp, tplInfo)) {
+   return next;
+   }
+
/*
 * Turn this off since this is wasted processing
 * that is not really actionable at this time.
diff --git a/lib/wt2html/tt/LinkHandler.js b/lib/wt2html/tt/LinkHandler.js
index 5aea514..5c4dc0b 100644
--- a/lib/wt2html/tt/LinkHandler.js
+++ b/lib/wt2html/tt/LinkHandler.js
@@ -72,7 +72,7 @@
  *
  * @return {Object} The target info
  */
-WikiLinkHandler.prototype.getWikiLinkTargetInfo = function(token, hrefKV) {
+WikiLinkHandler.prototype.getWikiLinkTargetInfo = function(token, hrefKV, cb) {
var env = this.manager.env;
 
var info = {
@@ -87,23 +87,19 @@
}
if (/^:/.test(info.href)) {
if (env.conf.parsoid.linting) {
-   var lint = {
-   dsr: token.dataAttribs.tsr,
-   params: { href: ':' + info.href },
-   templateInfo: undefined,
-   };
-   if (this.options.inTemplate) {
-   // `frame.title` is already the result of 
calling
-   // `getPrefixedDBKey`, but for the sake of 
consistency with
-   // `findEnclosingTemplateName`, we do a little 
more work to
-   // match `env.makeLink`.
-   var name = 
Util.sanitizeTitleURI(env.page.relativeLinkPrefix +
-   
this.manager.frame.title).replace(/^\.\//, '');
-   lint.templateInfo = { name: name };
-   // TODO(arlolra): Pass tsr info to the frame
-   lint.dsr = [0, 0];
-   }
-   env.log('lint/multi-colon-escape', lint);
+   var tsr = token.dataAttribs.tsr;
+   var meta = new SelfclosingTagTk('meta', [
+   new KV('typeof', 'mw:Lint/lint/Marker'),
+   ], {
+   tsr: [tsr[0], tsr[0]],
+   type: 'multi-colon-escape',
+   lintObj: {
+   dsr: tsr,
+   params: { href: ':' + info.href },
+   templateInfo: undefined,
+   }
+   });
+   cb({ tokens: [meta] });
}
// This will get caught by the caller, and mark the target as 
invalid
throw new Error('Multiple colons prefixing href.');
@@ -130,7 +126,7 @@
// Recurse!
hrefKV = new KV('href', (/:/.test(info.href) ? 
':' : '') + info.href);
hrefKV.vsrc = info.hrefSrc;
-   info = this.getWikiLinkTargetInfo(token, 
hrefKV);
+   info = this.getWikiLinkTargetInfo(token, 
hrefKV, cb);

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Cleanup and refactor so mw api warnings are always logged

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/398337 )

Change subject: Cleanup and refactor so mw api warnings are always logged
..

Cleanup and refactor so mw api warnings are always logged

Change-Id: I3688168f3826e4eba704f3ee6dfd79bd97b98249
---
M lib/mw/ApiRequest.js
1 file changed, 34 insertions(+), 53 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/37/398337/1

diff --git a/lib/mw/ApiRequest.js b/lib/mw/ApiRequest.js
index de717e9..c7ef4fb 100644
--- a/lib/mw/ApiRequest.js
+++ b/lib/mw/ApiRequest.js
@@ -273,7 +273,7 @@
 var httpAgent = null;
 var httpsAgent = null;
 
-ApiRequest.prototype.request = function(requestOpts, callback) {
+ApiRequest.prototype.request = function(requestOpts) {
var env = this.env;
var proxy = this.proxy;
var mwApiServer = this.mwApiServer;
@@ -334,7 +334,7 @@
}
this.trace("Starting HTTP request: ", options);
 
-   return request(options, callback);
+   return request(options, this._requestCB.bind(this));
 };
 
 /**
@@ -410,7 +410,7 @@
this.retries--;
// retry
this.requestOptions.timeout *= 3 + Math.random();
-   this.request(this.requestOptions, 
this._requestCB.bind(this));
+   this.request(this.requestOptions);
return;
} else {
var dnee = new Error(this.reqType + ' failure for '
@@ -438,6 +438,11 @@
delete this.env.requestQueue[this.queueKey];
 };
 
+ApiRequest.prototype._logWarningsAndHandleJSON = function(error, data) {
+   logAPIWarnings(this, data);
+   this._handleJSON(error, data);
+};
+
 /**
  * Default body handler: parse to JSON and call _handleJSON.
  *
@@ -448,7 +453,7 @@
  */
 ApiRequest.prototype._handleBody = function(error, body) {
if (error) {
-   this._handleJSON(error, {});
+   this._logWarningsAndHandleJSON(error, {});
return;
}
var data;
@@ -472,7 +477,7 @@
this.reqType);
}
}
-   this._handleJSON(error, data);
+   this._logWarningsAndHandleJSON(error, data);
 };
 
 ApiRequest.prototype.trace = function() {
@@ -529,12 +534,20 @@
timeout: env.conf.parsoid.timeouts.mwApi.srcFetch,
};
 
-   // Start the request
-   this.request(this.requestOptions, this._requestCB.bind(this));
+   this.request(this.requestOptions);
 }
 
-// Inherit from ApiRequest
 util.inherits(TemplateRequest, ApiRequest);
+
+// Function which returns a promise for the result of a template request.
+TemplateRequest.promise = promiseFor(TemplateRequest);
+
+// Function which returns a promise to set page src info.
+TemplateRequest.setPageSrcInfo = function(env, target, oldid, opts) {
+   return TemplateRequest.promise(env, target, oldid, 
opts).then(function(src) {
+   env.setPageSrcInfo(src);
+   });
+};
 
 /**
  * @method _handleJSON
@@ -544,8 +557,6 @@
  * @param {Object} data The response from the server - parsed JSON object
  */
 TemplateRequest.prototype._handleJSON = function(error, data) {
-   logAPIWarnings(this, data);
-
if (!error && !data.query) {
error = this._errorObj(data, '', 'Missing data.query');
}
@@ -618,16 +629,6 @@
this._processListeners(null, metadata);
 };
 
-// Function which returns a promise for the result of a template request.
-TemplateRequest.promise = promiseFor(TemplateRequest);
-
-// Function which returns a promise to set page src info.
-TemplateRequest.setPageSrcInfo = function(env, target, oldid, opts) {
-   return TemplateRequest.promise(env, target, oldid, 
opts).then(function(src) {
-   env.setPageSrcInfo(src);
-   });
-};
-
 /**
  * @class
  * @extends ApiRequest
@@ -677,17 +678,12 @@
timeout: env.conf.parsoid.timeouts.mwApi.preprocessor,
};
 
-   // Start the request
-   this.request(this.requestOptions, this._requestCB.bind(this));
+   this.request(this.requestOptions);
 }
 
-
-// Inherit from ApiRequest
 util.inherits(PreprocessorRequest, ApiRequest);
 
 PreprocessorRequest.prototype._handleJSON = function(error, data) {
-   logAPIWarnings(this, data);
-
if (!error && !(data && data.expandtemplates)) {
error = this._errorObj(data, this.text, 'Missing 
data.expandtemplates.');
}
@@ -757,19 +753,15 @@
timeout: env.conf.parsoid.timeouts.mwApi.extParse,
};
 
-   // Start the request
-   this.request(this.requestOptions, this._requestCB.bind(this));
+   this.request(this.requestOptions);
 }
 
-// Inherit from ApiRequest
 util.inherits(PHPParseRequest, ApiRequest);
 
 // Function which returns a promise for the 

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Log api warnings from batch requests

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/398297 )

Change subject: Log api warnings from batch requests
..

Log api warnings from batch requests

Change-Id: I95cbaa5100db2bcbdeb1c395daf57659eabd033b
---
M lib/mw/ApiRequest.js
1 file changed, 3 insertions(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/97/398297/1

diff --git a/lib/mw/ApiRequest.js b/lib/mw/ApiRequest.js
index 1f8812c..992eed7 100644
--- a/lib/mw/ApiRequest.js
+++ b/lib/mw/ApiRequest.js
@@ -86,7 +86,7 @@
// split up warnings by API module
Object.keys(data.warnings).forEach(function(apiModule) {
var re = req.env.conf.parsoid.suppressMwApiWarnings;
-   var msg = data.warnings[apiModule]['*'];
+   var msg = data.warnings[apiModule]['warnings'] || 
data.warnings[apiModule]['*'];
if (re instanceof RegExp && re.test(msg)) {
return; // suppress this message
}
@@ -827,6 +827,8 @@
 util.inherits(BatchRequest, ApiRequest);
 
 BatchRequest.prototype._handleJSON = function(error, data) {
+   logAPIWarnings(this, data);
+
if (!error && !(data && data['parsoid-batch'] && 
Array.isArray(data['parsoid-batch']))) {
error = this._errorObj(data, this.batchText, 'Missing/invalid 
data.parsoid-batch');
}

-- 
To view, visit https://gerrit.wikimedia.org/r/398297
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I95cbaa5100db2bcbdeb1c395daf57659eabd033b
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Pass the page name when parsing extension content

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/398298 )

Change subject: Pass the page name when parsing extension content
..

Pass the page name when parsing extension content

This looks like a refactoring failure when moving to mediawiki-title,
since the "key" property was correct for the old Title constructor when
the code was added.

Both with and without batching, the revid was considered canonical so
this likely had no effect other than producing a warning.

Change-Id: Icc6fc6b812a6aae3a2d836ed28bc829fb2cd4935
---
M lib/wt2html/tt/ExtensionHandler.js
1 file changed, 1 insertion(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/98/398298/1

diff --git a/lib/wt2html/tt/ExtensionHandler.js 
b/lib/wt2html/tt/ExtensionHandler.js
index e90a7d3..22c3c6b 100644
--- a/lib/wt2html/tt/ExtensionHandler.js
+++ b/lib/wt2html/tt/ExtensionHandler.js
@@ -84,7 +84,7 @@
env.log('debug', 'Note: trying to expand ', text);
parentCB({ async: true });
// Pass the page title to the API.
-   var title = env.page && env.page.title && env.page.title.key || 'API';
+   var title = env.page.name || '';
env.batcher.parse(title, text).nodify(cb);
 };
 

-- 
To view, visit https://gerrit.wikimedia.org/r/398298
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Icc6fc6b812a6aae3a2d836ed28bc829fb2cd4935
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Start the bin/rt launched Parsoid logging at "info"

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/398203 )

Change subject: Start the bin/rt launched Parsoid logging at "info"
..

Start the bin/rt launched Parsoid logging at "info"

On ruthenium we pass at parsoidURL so that remains unaffected.  This is
just for command line use.

Change-Id: Ic9fc002b5391801a72ff5dba67b47a307b049518
---
M bin/roundtrip-test.js
M tests/serviceWrapper.js
2 files changed, 4 insertions(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/03/398203/1

diff --git a/bin/roundtrip-test.js b/bin/roundtrip-test.js
index 76a544b..f544359 100755
--- a/bin/roundtrip-test.js
+++ b/bin/roundtrip-test.js
@@ -760,7 +760,9 @@
if (argv.parsoidURL) { return; }
// Start our own Parsoid server
var serviceWrapper = 
require('../tests/serviceWrapper.js');
-   var serverOpts = {};
+   var serverOpts = {
+   logging: { level: 'info' },
+   };
if (argv.apiURL) {
serverOpts.mockURL = argv.apiURL;
argv.domain = 'customwiki';
diff --git a/tests/serviceWrapper.js b/tests/serviceWrapper.js
index bf4e087..feb6283 100644
--- a/tests/serviceWrapper.js
+++ b/tests/serviceWrapper.js
@@ -100,6 +100,7 @@
num_workers: 1,
worker_heartbeat_timeout: 2 * 60 * 1000,
services: services,
+   logging: options.logging,
})
.then(function() {
return ret;

-- 
To view, visit https://gerrit.wikimedia.org/r/398203
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ic9fc002b5391801a72ff5dba67b47a307b049518
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Give the rt client a user agent

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/398204 )

Change subject: Give the rt client a user agent
..

Give the rt client a user agent

Change-Id: I12c8c1a965ae978e4c0a541eb0c636869800ebeb
---
M bin/roundtrip-test.js
1 file changed, 10 insertions(+), 6 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/04/398204/1

diff --git a/bin/roundtrip-test.js b/bin/roundtrip-test.js
index f544359..2743392 100755
--- a/bin/roundtrip-test.js
+++ b/bin/roundtrip-test.js
@@ -465,10 +465,15 @@
return results;
 };
 
+var UA = 'Roundtrip-Test';
+
 function parsoidPost(profile, options) {
var httpOptions = {
method: 'POST',
body: options.data,
+   headers: {
+   'User-Agent': UA,
+   },
};
 
var uri = options.uri + 'transform/';
@@ -480,17 +485,13 @@
httpOptions.body.scrub_wikitext = true;
// We want to encode the request but *not* decode the response.
httpOptions.body = JSON.stringify(httpOptions.body);
-   httpOptions.headers = {
-   'Content-Type': 'application/json',
-   };
+   httpOptions.headers['Content-Type'] = 'application/json';
} else {  // wt2html
uri += 'wikitext/to/pagebundle/' + options.title;
if (options.oldid) {
uri += '/' + options.oldid;
}
-   httpOptions.headers = {
-   Accept: apiUtils.pagebundleContentType(null, 
options.contentVersion),
-   };
+   httpOptions.headers.Accept = 
apiUtils.pagebundleContentType(null, options.contentVersion);
// setting json here encodes the request *and* decodes the 
response.
httpOptions.json = true;
}
@@ -608,6 +609,9 @@
return Util.retryingHTTPRequest(10, {
method: 'GET',
uri: uri2,
+   headers: {
+   'User-Agent': UA,
+   },
});
}).spread(function(res, body) {
profile.start = Date.now();

-- 
To view, visit https://gerrit.wikimedia.org/r/398204
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I12c8c1a965ae978e4c0a541eb0c636869800ebeb
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Template syntax w/ empty link target shouldn't be considered...

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/398189 )

Change subject: Template syntax w/ empty link target shouldn't be considered 
broken
..

Template syntax w/ empty link target shouldn't be considered broken

The spec at https://www.mediawiki.org/wiki/Preprocessor_ABNF says the
link target can be zero or more xml characters.

This improves both the render and parse time for enwiki/User:Pdebee

Bug: T143378
Change-Id: Iaf7221571b79dfbe5d1f32be9476ff0b2d8ac821
---
M lib/wt2html/pegTokenizer.pegjs
M tests/parserTests.txt
2 files changed, 14 insertions(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/89/398189/1

diff --git a/lib/wt2html/pegTokenizer.pegjs b/lib/wt2html/pegTokenizer.pegjs
index e1bd3f9..17495b2 100644
--- a/lib/wt2html/pegTokenizer.pegjs
+++ b/lib/wt2html/pegTokenizer.pegjs
@@ -890,7 +890,7 @@
   params.unshift(new KV(tu.flattenIfArray(target.tokens), '', 
target.srcOffsets));
   var obj = new SelfclosingTagTk('template', params, { tsr: tsrOffsets(), 
src: text() });
   return obj;
-} / $('{{' space_or_newline+ '}}')
+} / $('{{' space_or_newline* '}}')
 
 tplarg
   = stopLen:("" { return stops.push('preproc', /* {{ */'}}'); })
diff --git a/tests/parserTests.txt b/tests/parserTests.txt
index c729776..0e234de 100644
--- a/tests/parserTests.txt
+++ b/tests/parserTests.txt
@@ -12633,6 +12633,19 @@
 
 !! end
 
+!! test
+Preprocessor precedence 17: template w/o target shouldn't prevent closing
+!! options
+parsoid=wt2html
+!! wikitext
+{{echo|hi {{
+!! html/php
+hi {{}}
+
+!! html/parsoid
+hi {{}}
+!! end
+
 ###
 ### Token Stream Patcher tests
 ###

-- 
To view, visit https://gerrit.wikimedia.org/r/398189
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Iaf7221571b79dfbe5d1f32be9476ff0b2d8ac821
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Remove long obsolete comment from 29362cc

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/398170 )

Change subject: Remove long obsolete comment from 29362cc
..

Remove long obsolete comment from 29362cc

Change-Id: Ib5d10f75e1c07ab2c16621e9599973a9b707819c
---
M lib/wt2html/parser.js
1 file changed, 0 insertions(+), 2 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/70/398170/1

diff --git a/lib/wt2html/parser.js b/lib/wt2html/parser.js
index caf40ce..3d63eb2 100644
--- a/lib/wt2html/parser.js
+++ b/lib/wt2html/parser.js
@@ -13,8 +13,6 @@
 
 var Promise = require('../utils/promise.js');
 
-// make this global for now
-// XXX: figure out a way to get away without a global for PEG actions!
 var PegTokenizer = require('./tokenizer.js').PegTokenizer;
 var TokenTransformManager = require('./TokenTransformManager.js');
 var ExtensionHandler = require('./tt/ExtensionHandler.js').ExtensionHandler;

-- 
To view, visit https://gerrit.wikimedia.org/r/398170
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib5d10f75e1c07ab2c16621e9599973a9b707819c
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: [WIP] Tokenize noincludeonly content asynchronously

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/398171 )

Change subject: [WIP] Tokenize noincludeonly content asynchronously
..

[WIP] Tokenize noincludeonly content asynchronously

Bug: T143378
Change-Id: I465a618eba89bbcf8d4afe8e01610278d2453c75
---
M lib/wt2html/parser.js
M lib/wt2html/pegTokenizer.pegjs
M lib/wt2html/tokenizer.js
M lib/wt2html/tt/NoIncludeOnly.js
4 files changed, 64 insertions(+), 19 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/71/398171/1

diff --git a/lib/wt2html/parser.js b/lib/wt2html/parser.js
index 3d63eb2..82e9bc2 100644
--- a/lib/wt2html/parser.js
+++ b/lib/wt2html/parser.js
@@ -38,9 +38,9 @@
 var IncludeOnly = NoIncludeOnly.IncludeOnly;
 var NoInclude = NoIncludeOnly.NoInclude;
 var OnlyInclude = NoIncludeOnly.OnlyInclude;
+var TokenizeInclude = NoIncludeOnly.TokenizeInclude;
 var WikiLinkHandler = LinkHandler.WikiLinkHandler;
 var ExternalLinkHandler = LinkHandler.ExternalLinkHandler;
-
 
 var ParserPipeline; // forward declaration
 var globalPipelineId = 0;
@@ -81,6 +81,13 @@
// a fully expanded token stream ready for consumption by the
// tokens/expanded pipeline.
'tokens/x-mediawiki': [
+   [
+   AsyncTokenTransformManager,
+   [ 0, 'tokens/x-mediawiki' ],
+   [
+   TokenizeInclude,  // 0.001
+   ],
+   ],
// Synchronous in-order per input
[
SyncTokenTransformManager,
diff --git a/lib/wt2html/pegTokenizer.pegjs b/lib/wt2html/pegTokenizer.pegjs
index e1bd3f9..09a916a 100644
--- a/lib/wt2html/pegTokenizer.pegjs
+++ b/lib/wt2html/pegTokenizer.pegjs
@@ -15,7 +15,6 @@
 var DU = pegIncludes.DOMUtils;
 var Util = pegIncludes.Util;
 var JSUtils = pegIncludes.JSUtils;
-var PegTokenizer = pegIncludes.PegTokenizer;
 var defines = pegIncludes.defines;
 var constants = pegIncludes.constants;
 var tu = pegIncludes.tu;
@@ -239,13 +238,15 @@
 ], dp);
 } else if (isIncludeTag) {
 // Parse ext-content, strip eof, and shift tsr
-var extContent = dp.src.substring(dp.tagWidths[0], dp.src.length - 
dp.tagWidths[1]);
-var extContentToks = (new 
PegTokenizer(env)).tokenizeSync(extContent);
-if (dp.tagWidths[1] > 0) {
-extContentToks = Util.stripEOFTkfromTokens(extContentToks);
-}
-Util.shiftTokenTSR(extContentToks, dp.tsr[0] + dp.tagWidths[0]);
-return [t].concat(extContentToks);
+dp.tmp = (dp.tmp || {});
+dp.tmp.toTokenize = {
+content: dp.src.substring(dp.tagWidths[0], dp.src.length - 
dp.tagWidths[1]),
+shift: dp.tsr[0] + dp.tagWidths[0],
+stripEOF: dp.tagWidths[1] > 0,
+name: tagName,
+};
+t.name = 'tokenizeinclude';
+return t;
 } else {
 console.assert(false, 'Should not be reachable.');
 }
@@ -2182,16 +2183,17 @@
 }
 
 // Tokenize include content in a new tokenizer
-var inclContentToks = (new PegTokenizer(env)).tokenizeSync(inclContent);
-inclContentToks = Util.stripEOFTkfromTokens(inclContentToks);
-
-// Shift tsr
-Util.shiftTokenTSR(inclContentToks, endOffset());
+dp.toTokenize = {
+content: inclContent,
+shift: endOffset(),
+stripEOF: true,
+name: name,
+};
 
 // Skip past content
 peg$currPos += inclContent.length;
 
-return [new TagTk(name, [], dp)].concat(inclContentToks);
+return new TagTk('tokenizeinclude', [], dp);
   }) & { return !!il; } { return il; }
 
 // Start of file
diff --git a/lib/wt2html/tokenizer.js b/lib/wt2html/tokenizer.js
index 4318ed9..8eacedd 100644
--- a/lib/wt2html/tokenizer.js
+++ b/lib/wt2html/tokenizer.js
@@ -36,8 +36,6 @@
DOMUtils: require('../utils/DOMUtils.js').DOMUtils,
JSUtils: JSUtils,
Util: require('../utils/Util.js').Util,
-   // defined below to satisfy JSHint
-   PegTokenizer: null,
 };
 
 /**
@@ -56,8 +54,6 @@
this.options = options || {};
this.offsets = {};
 }
-
-pegIncludes.PegTokenizer = PegTokenizer;
 
 // Inherit from EventEmitter
 util.inherits(PegTokenizer, events.EventEmitter);
diff --git a/lib/wt2html/tt/NoIncludeOnly.js b/lib/wt2html/tt/NoIncludeOnly.js
index 66bce5c..de7db72 100644
--- a/lib/wt2html/tt/NoIncludeOnly.js
+++ b/lib/wt2html/tt/NoIncludeOnly.js
@@ -8,6 +8,8 @@
 var coreutil = require('util');
 var TokenHandler = require('./TokenHandler.js');
 var TokenCollector = require('./TokenCollector.js').TokenCollector;
+var PegTokenizer = require('../tokenizer.js').PegTokenizer;
+var Util = require('../../utils/Util.js').Util;
 var defines = 

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Make the log levels configurable for the initial logger

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/398117 )

Change subject: Make the log levels configurable for the initial logger
..

Make the log levels configurable for the initial logger

To avoid having to reconstruct and set a logger at a reduced level, as
in parserTests.

Set the default for bin/parse to "warn".  The necessitates bumping the
level for linting (when not sending to the MW API) so that --linting
reports the errors w/o needing the newly added --verbose, which restores
the "info" level.

Change-Id: I2d04edc15912b0ce73285ab7d9c4fc7cf5f9b457
---
M bin/parse.js
M bin/parserTests.js
M lib/config/MWParserEnvironment.js
M lib/logger/LintLogger.js
4 files changed, 31 insertions(+), 21 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/17/398117/1

diff --git a/bin/parse.js b/bin/parse.js
index e9ddc7a..b0a7c58 100755
--- a/bin/parse.js
+++ b/bin/parse.js
@@ -110,6 +110,11 @@
'boolean': true,
'default': false,
},
+   'verbose': {
+   description: 'Log at level "info" as well',
+   'boolean': true,
+   'default': false,
+   },
 
// These are ParsoidConfig properties
 
@@ -290,6 +295,11 @@
}
}
 
+   var logLevels;
+   if (!argv.verbose) {
+   logLevels = ["fatal", "error", "warn"];
+   }
+
var envOptions = {
domain: domain,
prefix: prefix,
@@ -298,6 +308,7 @@
nativeGallery: argv.nativeGallery,
pageBundle: argv.pageBundle || argv.pboutfile,
wrapSections: argv.wrapSections,
+   logLevels: logLevels,
};
 
return Promise.resolve()
diff --git a/bin/parserTests.js b/bin/parserTests.js
index 1a4bf04..9fe93e6 100755
--- a/bin/parserTests.js
+++ b/bin/parserTests.js
@@ -1049,16 +1049,18 @@
 
var pc = new ParsoidConfig({ setup: setup }, parsoidOptions);
 
+   var logLevels;
+   if (Util.booleanOption(options.quiet)) {
+   logLevels = ["fatal", "error"];
+   }
+
// Create a new parser environment
-   return MWParserEnvironment.getParserEnv(pc, { prefix: 'enwiki' })
+   return MWParserEnvironment.getParserEnv(pc, {
+   prefix: 'enwiki',
+   logLevels: logLevels,
+   })
.then(function(env) {
this.env = env;
-
-   if (Util.booleanOption(options.quiet)) {
-   var logger = new ParsoidLogger(env);
-   logger.registerLoggingBackends(["fatal", "error"], pc);
-   env.setLogger(logger);
-   }
 
// Save default logger so we can be reset it after temporarily
// switching to the suppressLogger to suppress expected error
diff --git a/lib/config/MWParserEnvironment.js 
b/lib/config/MWParserEnvironment.js
index 3fac953..d5dc934 100644
--- a/lib/config/MWParserEnvironment.js
+++ b/lib/config/MWParserEnvironment.js
@@ -145,7 +145,6 @@
console.assert(parsoidConfig.mwApiMap.has(options.prefix));
this.conf.wiki = new WikiConfig(parsoidConfig, null, options.prefix);
 
-   this.lintLogger = new LintLogger(this);
this.configureLogging();
 
this.initializeForPageName(options.pageName || this.conf.wiki.mainpage);
@@ -285,16 +284,19 @@
console.warn('-'.repeat(55));
 };
 
-MWParserEnvironment.prototype.configureLogging = function() {
-   var logger = new ParsoidLogger(this);
-   this.setLogger(logger);
+MWParserEnvironment.prototype.setLogger = function(logger) {
+   this.logger = logger;
+   this.log = this.logger.log.bind(this.logger);
+};
 
-   var defaultLogLevels = [
+MWParserEnvironment.prototype.configureLogging = function() {
+   this.lintLogger = new LintLogger(this);
+   var logger = new ParsoidLogger(this);
+   var logLevels = this.logLevels || [
"fatal", "error", "warn", "info",
];
-
-   // Configure backends
-   logger.registerLoggingBackends(defaultLogLevels, this.conf.parsoid, 
this.lintLogger);
+   logger.registerLoggingBackends(logLevels, this.conf.parsoid, 
this.lintLogger);
+   this.setLogger(logger);
 };
 
 // Cache for wiki configurations, shared between requests.
@@ -414,11 +416,6 @@
this.page.pagelanguage = metadata.pagelanguage;
this.page.pagelanguagedir = metadata.pagelanguagedir;
this.page.src = (metadata.revision && metadata.revision['*']) || '';
-};
-
-MWParserEnvironment.prototype.setLogger = function(logger) {
-   this.logger = logger;
-   this.log = this.logger.log.bind(this.logger);
 };
 
 /**
diff --git a/lib/logger/LintLogger.js b/lib/logger/LintLogger.js
index 89f1c6e..edd9d16 100644
--- a/lib/logger/LintLogger.js
+++ b/lib/logger/LintLogger.js
@@ 

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Restore location data to start/complete parse logs

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/398112 )

Change subject: Restore location data to start/complete parse logs
..

Restore location data to start/complete parse logs

This was an oversight from e7997dd.

Change-Id: I9804f55c9446b911bf12ac3ed0696b725ee0aa80
---
M bin/parse.js
M bin/parserTests.js
M lib/api/apiUtils.js
M lib/api/routes.js
M lib/config/MWParserEnvironment.js
M lib/logger/LintLogger.js
M lib/parse.js
7 files changed, 34 insertions(+), 24 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/12/398112/1

diff --git a/bin/parse.js b/bin/parse.js
index e9ddc7a..85cf96f 100755
--- a/bin/parse.js
+++ b/bin/parse.js
@@ -110,6 +110,11 @@
'boolean': true,
'default': false,
},
+   'verbose': {
+   description: 'Log at level "info" as well',
+   'boolean': true,
+   'default': false,
+   },
 
// These are ParsoidConfig properties
 
@@ -290,6 +295,11 @@
}
}
 
+   var defaultLogLevels;
+   if (!argv.verbose) {
+   defaultLogLevels = ["fatal", "error", "warn"];
+   }
+
var envOptions = {
domain: domain,
prefix: prefix,
@@ -298,6 +308,7 @@
nativeGallery: argv.nativeGallery,
pageBundle: argv.pageBundle || argv.pboutfile,
wrapSections: argv.wrapSections,
+   defaultLogLevels: defaultLogLevels,
};
 
return Promise.resolve()
diff --git a/bin/parserTests.js b/bin/parserTests.js
index 1a4bf04..0b8d8dd 100755
--- a/bin/parserTests.js
+++ b/bin/parserTests.js
@@ -1049,16 +1049,18 @@
 
var pc = new ParsoidConfig({ setup: setup }, parsoidOptions);
 
+   var defaultLogLevels;
+   if (Util.booleanOption(options.quiet)) {
+   defaultLogLevels = ["fatal", "error"];
+   }
+
// Create a new parser environment
-   return MWParserEnvironment.getParserEnv(pc, { prefix: 'enwiki' })
+   return MWParserEnvironment.getParserEnv(pc, {
+   prefix: 'enwiki',
+   defaultLogLevels: defaultLogLevels,
+   })
.then(function(env) {
this.env = env;
-
-   if (Util.booleanOption(options.quiet)) {
-   var logger = new ParsoidLogger(env);
-   logger.registerLoggingBackends(["fatal", "error"], pc);
-   env.setLogger(logger);
-   }
 
// Save default logger so we can be reset it after temporarily
// switching to the suppressLogger to suppress expected error
diff --git a/lib/api/apiUtils.js b/lib/api/apiUtils.js
index 2b855e3..e3dc5bc 100644
--- a/lib/api/apiUtils.js
+++ b/lib/api/apiUtils.js
@@ -2,7 +2,6 @@
 
 require('../../core-upgrade.js');
 
-var util = require('util');
 var semver = require('semver');
 var qs = require('querystring');
 var cType = require('content-type');
@@ -164,12 +163,6 @@
err.suppressLoggingStack = true;
}
env.log('fatal/request', err);
-};
-
-apiUtils.logTime = function(processLogger, res, str) {
-   processLogger.log('info', util.format(
-   'completed %s in %sms', str, Date.now() - res.locals.start
-   ));
 };
 
 // To support the 'subst' API parameter, we need to prefix each
diff --git a/lib/api/routes.js b/lib/api/routes.js
index a2a5f56..cc2e8ea 100644
--- a/lib/api/routes.js
+++ b/lib/api/routes.js
@@ -313,8 +313,6 @@
}
}
 
-   processLogger.log('info', 'started parsing');
-
var envOptions = Object.assign({
pageBundle: pageBundle,
// Set data-parsoid to be discarded, so that 
the subst'ed
@@ -395,7 +393,6 @@
}
metrics.endTiming('wt2html.total', 
startTimers.get('wt2html.total'));
}
-   apiUtils.logTime(processLogger, res, 'parsing');
});
});
});
@@ -410,8 +407,6 @@
 
// Performance Timing options
var startTimers = new Map();
-
-   processLogger.log('info', 'started serializing');
 
if (metrics) {
startTimers.set('html2wt.init', Date.now());
@@ -535,7 +530,6 @@
startTimers.get('html2wt.total'));
metrics.timing('html2wt.size.output', 
out.wt.length);
}
-   apiUtils.logTime(processLogger, res, 'serializing');
apiUtils.plainResponse(res, out.wt, undefined, 

[MediaWiki-commits] [Gerrit] mediawiki...deploy[master]: Bump src/ to 741fc5d for deploy

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/397880 )

Change subject: Bump src/ to 741fc5d for deploy
..

Bump src/ to 741fc5d for deploy

Change-Id: Id15caa45752eb942d19a68c0c4b146015035feae
---
M src
1 file changed, 1 insertion(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid/deploy 
refs/changes/80/397880/1

diff --git a/src b/src
index 01c1fc3..741fc5d 16
--- a/src
+++ b/src
@@ -1 +1 @@
-Subproject commit 01c1fc33c15cec32871b5440d7d6b7a57a53a225
+Subproject commit 741fc5dc0f683df8d81bc09843734dad966138bd

-- 
To view, visit https://gerrit.wikimedia.org/r/397880
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Id15caa45752eb942d19a68c0c4b146015035feae
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid/deploy
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: [WIP] Handle table end following templates in attribute posi...

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/397721 )

Change subject: [WIP] Handle table end following templates in attribute position
..

[WIP] Handle table end following templates in attribute position

Bug: T126834
Change-Id: I1b83ef52efdafcd96aeac9a06a951235bbb69f5b
---
M lib/wt2html/tt/AttributeExpander.js
1 file changed, 4 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/21/397721/1

diff --git a/lib/wt2html/tt/AttributeExpander.js 
b/lib/wt2html/tt/AttributeExpander.js
index 9a76502..f34cd24 100644
--- a/lib/wt2html/tt/AttributeExpander.js
+++ b/lib/wt2html/tt/AttributeExpander.js
@@ -14,6 +14,7 @@
 // define some constructor shortcuts
 var NlTk = defines.NlTk;
 var TagTk = defines.TagTk;
+var EndTagTk = defines.EndTagTk;
 var SelfclosingTagTk = defines.SelfclosingTagTk;
 
 
@@ -346,6 +347,9 @@
expandedK = updatedK.preNLBuf;
postNLToks = updatedK.postNLBuf;
metaTokens = 
updatedK.metaTokens;
+   if 
(postNLToks[postNLToks.length - 1] === '|}') {
+   
postNLToks.splice(postNLToks.length - 1, 1, new EndTagTk('table'));
+   }
} else {
// Scenario 2 from the 
documentation comment above.
updatedK = 
stripMetaTags(expandedK, wrapTemplates);

-- 
To view, visit https://gerrit.wikimedia.org/r/397721
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I1b83ef52efdafcd96aeac9a06a951235bbb69f5b
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Add some metrics to the http api to count the original version

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/397647 )

Change subject: Add some metrics to the http api to count the original version
..

Add some metrics to the http api to count the original version

Change-Id: I63316e525d8a8f1569868283f981391b831b7f94
---
M lib/api/routes.js
1 file changed, 7 insertions(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/47/397647/1

diff --git a/lib/api/routes.js b/lib/api/routes.js
index a2a5f56..d36b083 100644
--- a/lib/api/routes.js
+++ b/lib/api/routes.js
@@ -443,9 +443,12 @@
} else if (original && original.html) {
env.originalVersion = 
apiUtils.versionFromType(original.html);
if (env.originalVersion === null) {
-   // Or, should we check for a meta in the 
original html?
return apiUtils.fatalRequest(env, 'Content-type 
of original html is missing.', 400);
}
+   }
+   if (metrics) {
+   var ver = env.hasOwnProperty('originalVersion') ? 
env.originalVersion : 'default';
+   metrics.increment('html2wt.original.version.' + ver);
}
 
var pb, origPb;
@@ -553,6 +556,9 @@
if (env.originalVersion === null) {
return apiUtils.fatalRequest(env, 'Content-type of 
revision html is missing.', 400);
}
+   if (metrics) {
+   metrics.increment('pb2pb.original.version.' + 
env.originalVersion);
+   }
 
var contentmodel = (revision && revision.contentmodel);
 

-- 
To view, visit https://gerrit.wikimedia.org/r/397647
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I63316e525d8a8f1569868283f981391b831b7f94
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Stop emitting isIW in dp

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/397648 )

Change subject: Stop emitting isIW in dp
..

Stop emitting isIW in dp

Change-Id: I22fb1d8bdf1cceea6855d67feacf158f127243c6
---
M lib/html2wt/LinkHandler.js
M lib/wt2html/tt/LinkHandler.js
M tests/parserTests-blacklist.js
M tests/parserTests.txt
4 files changed, 22 insertions(+), 19 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/48/397648/1

diff --git a/lib/html2wt/LinkHandler.js b/lib/html2wt/LinkHandler.js
index b9f4f06..c5ba776 100644
--- a/lib/html2wt/LinkHandler.js
+++ b/lib/html2wt/LinkHandler.js
@@ -89,16 +89,21 @@
tail: dp.tail || '',
prefix: dp.prefix || '',
content: {},  // string or tokens
+   isInterwiki: false,
};
+
+   var isIW = false;
 
// Figure out the type of the link
var rel = node.getAttribute('rel');
if (rel) {
-   // Parsoid only emits and recognizes ExtLink, WikiLink, and 
PageProp rel values.
-   // Everything else defaults to ExtLink during serialization 
(unless it is
-   // serializable to a wikilink)
+   // Parsoid only emits and recognizes ExtLink, WikiLink, Media, 
and
+   // PageProp rel values.  Everything else defaults to ExtLink 
during
+   // serialization (unless it is serializable to a wikilink).
var typeMatch = 
rel.match(/\b(mw:(WikiLink|ExtLink|Media|PageProp)[^\s]*)\b/);
if (typeMatch) {
+   // Note this before stripping the subtype
+   isIW = (typeMatch[0] === 'mw:WikiLink/Interwiki');
rtData.type = typeMatch[1];
// Strip link subtype info
if (/^mw:(Wiki|Ext)Link\//.test(rtData.type)) {
@@ -193,9 +198,8 @@
&& !/^#|\?./.test(interWikiMatch[1])
// ExtLinks should have content to convert.
&& (rtData.type !== 'mw:ExtLink' || 
rtData.content.string || rtData.contentNode)
-   && (dp.isIW || target.modified || 
rtData.contentModified)) {
+   && (isIW || target.modified || rtData.contentModified)) 
{
// External link that is really an interwiki link. 
Convert it.
-   // TODO: Leaving this for backwards compatibility, 
remove when 1.5 is no longer bound
if (rtData.type === 'mw:ExtLink') {
rtData.type = 'mw:WikiLink';
}
diff --git a/lib/wt2html/tt/LinkHandler.js b/lib/wt2html/tt/LinkHandler.js
index 5aea514..9faf0ae 100644
--- a/lib/wt2html/tt/LinkHandler.js
+++ b/lib/wt2html/tt/LinkHandler.js
@@ -633,8 +633,7 @@
 
// Change the rel to be mw:ExtLink
Util.lookupKV(newTk.attribs, 'rel').v = 'mw:WikiLink/Interwiki';
-   // Remember that this was using wikitext syntax though
-   newTk.dataAttribs.isIW = true;
+
// Add title unless it's just a fragment (and trim off fragment)
// (The normalization here is similar to what Title#getPrefixedDBKey() 
does.)
if (target.href[0] !== "#") {
diff --git a/tests/parserTests-blacklist.js b/tests/parserTests-blacklist.js
index f10fdc9..1a37b1f 100644
--- a/tests/parserTests-blacklist.js
+++ b/tests/parserTests-blacklist.js
@@ -59,7 +59,7 @@
 add("wt2html", "Self-link to numeric title", "0");
 add("wt2html", " inside a link", "Main Page the main page [it's not very good]");
 add("wt2html", "Interlanguage link with spacing", "Blah blah blah\nhttp://zh.wikipedia.org/wiki/%20%20%20%20Chinese%20%20%20%20%20\; 
data-parsoid='{\"stx\":\"simple\",\"a\":{\"href\":\"http://zh.wikipedia.org/wiki/%20%20%20%20Chinese%20%20%20%20%20\"},\"sa\":{\"href\":\;
   zh  :Chinese \"},\"dsr\":[15,43,null,null]}'/>");
-add("wt2html", "Escaping of interlanguage links (T129218, T156308)", "Blah blah blah\nhttp://es.wikipedia.org/wiki/Spanish\; 
title=\"es:Spanish\" 
data-parsoid='{\"stx\":\"simple\",\"a\":{\"href\":\"http://es.wikipedia.org/wiki/Spanish\"},\"sa\":{\"href\":\":es:Spanish\"},\"isIW\":true,\"dsr\":[15,30,2,2]}'>es:Spanish\n : zh : Chinese ");
+add("wt2html", "Escaping of interlanguage links (T129218, T156308)", "Blah blah blah\nhttp://es.wikipedia.org/wiki/Spanish\; 
title=\"es:Spanish\" 
data-parsoid='{\"stx\":\"simple\",\"a\":{\"href\":\"http://es.wikipedia.org/wiki/Spanish\"},\"sa\":{\"href\":\":es:Spanish\"},\"dsr\":[15,30,2,2]}'>es:Spanish\n : zh : Chinese ");
 add("wt2html", "Space and question mark encoding in interlanguage links 
(T95473)", "Blah blah blah\nhttp://es.wikipedia.org/wiki/Foo%20bar?\; 
data-parsoid='{\"stx\":\"simple\",\"a\":{\"href\":\"http://es.wikipedia.org/wiki/Foo%20bar?\"},\"sa\":{\"href\":\"es:Foo
 

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: [WIP] Start removing some backwards compatibility code

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/396539 )

Change subject: [WIP] Start removing some backwards compatibility code
..

[WIP] Start removing some backwards compatibility code

Should we enable `strictAcceptCheck`?

Change-Id: I3d464d631af2377f656a3ff2d91c3c9bbd49c5ff
---
M lib/api/apiUtils.js
M lib/api/routes.js
M lib/html2wt/WikitextSerializer.js
M tests/mocha/api.js
M tests/mocha/templatedata.js
5 files changed, 7 insertions(+), 139 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/39/396539/1

diff --git a/lib/api/apiUtils.js b/lib/api/apiUtils.js
index bdc85d2..94cbd31 100644
--- a/lib/api/apiUtils.js
+++ b/lib/api/apiUtils.js
@@ -281,7 +281,7 @@
var t = cType.parse(ct);
var profile = t.parameters && t.parameters.profile;
if (profile) {
-   var p = apiUtils.parseProfile(profile, 'html');
+   var p = apiUtils.parseProfile(profile);
return p && p.version;
} else {
return null;
@@ -294,7 +294,6 @@
}
 };
 
-var oldSpec = /^mediawiki.org\/specs\/(html)\/(\d+\.\d+\.\d+)$/;
 var newSpec = 
/^https:\/\/www.mediawiki.org\/wiki\/Specs\/(HTML|pagebundle)\/(\d+\.\d+\.\d+)$/;
 
 /**
@@ -302,18 +301,10 @@
  *
  * @method
  * @param {String} profile
- * @param {String} format
- *   Just used for backwards compatibility w/ <= 1.2.0
- *   where the pagebundle didn't have a spec.
  * @return {Object|null}
  */
 apiUtils.parseProfile = function(profile, format) {
var match = newSpec.exec(profile);
-   // TODO(arlolra): Remove when this version is no longer supported.
-   if (!match) {
-   match = oldSpec.exec(profile);
-   if (match) { match[1] = format; }
-   }
if (match) {
return {
format: match[1].toLowerCase(),
@@ -341,11 +332,9 @@
return !acceptableTypes.length || acceptableTypes.some(function(t) {
var profile = t.parameters && t.parameters.profile;
if ((opts.format === 'html' && t.type === 'text/html') ||
-   (opts.format === 'pagebundle' && t.type === 
'application/json') ||
-   // 'pagebundle' is sending 'text/html' in older 
versions
-   oldSpec.exec(profile)) {
+   (opts.format === 'pagebundle' && t.type === 
'application/json')) {
if (profile) {
-   var p = apiUtils.parseProfile(profile, 
opts.format);
+   var p = apiUtils.parseProfile(profile);
if (p && (opts.format === p.format)) {
var contentVersion = 
env.resolveContentVersion(p.version);
if (contentVersion !== null) {
diff --git a/lib/api/routes.js b/lib/api/routes.js
index 7540504..f5e0c2f 100644
--- a/lib/api/routes.js
+++ b/lib/api/routes.js
@@ -482,12 +482,6 @@
}
apiUtils.validatePageBundle(pb, 
env.originalVersion);
DU.applyPageBundle(doc, pb);
-
-   // TODO(arlolra): data-parsoid is no longer 
versioned
-   // independently, but we leave this for 
backwards compatibility
-   // until content version <= 1.2.0 is 
deprecated.  Anything new
-   // should only depend on `env.originalVersion`.
-   envOptions.dpContentType = 
(original['data-parsoid'].headers || {})['content-type'];
}
 
// If we got original src, set it
diff --git a/lib/html2wt/WikitextSerializer.js 
b/lib/html2wt/WikitextSerializer.js
index bdaba5e..473a1f4 100644
--- a/lib/html2wt/WikitextSerializer.js
+++ b/lib/html2wt/WikitextSerializer.js
@@ -498,13 +498,6 @@
// Parse custom format specification, if present.
var defaultBlockSpc  = '{{_\n| _ = _\n}}'; // "block"
var defaultInlineSpc = '{{_|_=_}}'; // "inline"
-   // FIXME: Do a full regexp test maybe?
-   if (/.*data-parsoid\/0.0.1"$/.test(this.env.dpContentType)) {
-   // For previous versions of data-parsoid,
-   // wt2html pipeline used "|foo = bar" style args
-   // as the default.
-   defaultInlineSpc = "{{_|_ = _}}";
-   }
 
var format = tplData && tplData.format ? tplData.format.toLowerCase() : 
null;
if (format === 'block') { format = defaultBlockSpc; }
diff --git a/tests/mocha/api.js b/tests/mocha/api.js
index 7965320..7d0bceb 100644
--- 

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: T182338: Abort on invalid chars in interwiki links

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/396163 )

Change subject: T182338: Abort on invalid chars in interwiki links
..

T182338: Abort on invalid chars in interwiki links

Change-Id: I4b6907ffeb6383ed02fd84f7af1c8f5c0761da15
---
M lib/wt2html/tt/LinkHandler.js
M tests/parserTests-blacklist.js
M tests/parserTests.txt
3 files changed, 9 insertions(+), 15 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/63/396163/1

diff --git a/lib/wt2html/tt/LinkHandler.js b/lib/wt2html/tt/LinkHandler.js
index f1de3a2..85cde86 100644
--- a/lib/wt2html/tt/LinkHandler.js
+++ b/lib/wt2html/tt/LinkHandler.js
@@ -137,6 +137,8 @@
}
} else if (interwikiInfo && interwikiInfo.url) {
info.href = hrefBits[2];
+   // Ensure a valid title, even though we're discarding 
the result
+   env.makeTitleFromURLDecodedStr(title);
// Interwiki or language link? If no language info, or 
if it starts
// with an explicit ':' (like [[:en:Foo]]), it's not a 
language link.
if (info.fromColonEscapedText ||
@@ -254,7 +256,7 @@
tokens = tokens.concat(dft);
} else {
// FIXME: Duplicate work
-   tokens[0] += 
Util.tokensToString(token.getAttribute("href"));
+   tokens = tokens.concat(token.getAttribute("href"));
}
 
// Append rest of the attributes
diff --git a/tests/parserTests-blacklist.js b/tests/parserTests-blacklist.js
index 751e385..d9f5409 100644
--- a/tests/parserTests-blacklist.js
+++ b/tests/parserTests-blacklist.js
@@ -151,7 +151,6 @@
 add("wt2html", "CSS line continuation 2", "");
 add("wt2html", "Sanitizer: Closing of closed but not open table tags", "Table 
not started");
 add("wt2html", "Sanitizer: Validating that  and  work, but only 
for Microdata", "\n\tmeta itemprop=\"hello\" 
content=\"world\">\n\tmeta http-equiv=\"refresh\" 
content=\"5\">\n\tmeta itemprop=\"hello\" http-equiv=\"refresh\" 
content=\"5\">\n\tlink itemprop=\"hello\" href=\"{{SERVER}}\">\n\tlink 
rel=\"stylesheet\" href=\"{{SERVER}}\">\n\tlink rel=\"stylesheet\" 
itemprop=\"hello\" href=\"{{SERVER}}\">\n");
-add("wt2html", "Sanitizer: angle brackets are invalid, even in interwiki links 
(T182338)", "http://www.usemod.com/cgi-bin/mb.pl?FooBar\; 
title=\"meatball:FooBar\" 
data-parsoid='{\"stx\":\"simple\",\"a\":{\"href\":\"http://www.usemod.com/cgi-bin/mb.pl?FooBar\"},\"sa\":{\"href\":\"meatball:FooBar\"},\"isIW\":true,\"dsr\":[0,20,2,2]}'>meatball:FooBar\nhttp://www.usemod.com/cgi-bin/mb.pl?Foo>Bar\" title=\"meatball:Foo>Bar\" 
data-parsoid='{\"stx\":\"simple\",\"a\":{\"href\":\"http://www.usemod.com/cgi-bin/mb.pl?Foo>Bar\"},\"sa\":{\"href\":\"meatball:Foo>Bar\"},\"isIW\":true,\"dsr\":[21,41,2,2]}'>meatball:Foo>Bar\nhttp://www.usemod.com/cgi-bin/mb.pl?Foobar\; 
title=\"meatball:Foobar\" 
data-parsoid='{\"stx\":\"simple\",\"a\":{\"href\":\"http://www.usemod.com/cgi-bin/mb.pl?Foobar\"},\"sa\":{\"href\":\"meatball:Foolt;bar\"},\"isIW\":true,\"dsr\":[42,65,2,2]}'>meatball:Foobar\nhttp://www.usemod.com/cgi-bin/mb.pl?Foo>bar\" title=\"meatball:Foo>bar\" 
data-parsoid='{\"stx\":\"simple\",\"a\":{\"href\":\"http://www.usemod.com/cgi-bin/mb.pl?Foo>bar\"},\"sa\":{\"href\":\"meatball:Foogt;bar\"},\"isIW\":true,\"dsr\":[66,89,2,2]}'>meatball:Foo>bar");
 add("wt2html", "Language converter glossary rules inside attributes 
(T119158)", "\n\n");
 add("wt2html", "Inline HTML vs wiki block nesting", "Bold
 paragraph\n\nNew wiki paragraph");
 add("wt2html", "Special page transclusion", "{{Special:Prefixindex/Xyzzyx}}");
@@ -418,7 +417,6 @@
 add("html2html", "CSS line continuation 2", "\n");
 add("html2html", "Parser hook: nested tags", "/tag>");
 add("html2html", "Sanitizer: Validating that  and  work, but only 
for Microdata", "\n\tmeta itemprop=\"hello\" 
content=\"world\" />\n\tmeta http-equiv=\"refresh\" 
content=\"5\">\n\tmeta itemprop=\"hello\" content=\"5\" />\n\thttp://example.org\; 
data-parsoid='{\"targetOff\":162,\"contentOffsets\":[162,162],\"dsr\":[143,163,19,1]}'>\n\tlink
 rel=\"stylesheet\" href=\"http://example.org\;>\n\thttp://example.org\; 
data-parsoid='{\"targetOff\":235,\"contentOffsets\":[235,235],\"dsr\":[216,236,19,1]}'>\n\n\n");
-add("html2html", "Sanitizer: angle brackets are invalid, even in interwiki 
links (T182338)", "http://www.usemod.com/cgi-bin/mb.pl?FooBar\; 
title=\"meatball:FooBar\" 
data-parsoid='{\"stx\":\"simple\",\"a\":{\"href\":\"http://www.usemod.com/cgi-bin/mb.pl?FooBar\"},\"sa\":{\"href\":\"meatball:FooBar\"},\"isIW\":true,\"dsr\":[0,20,2,2]}'>meatball:FooBar\nhttp://www.usemod.com/cgi-bin/mb.pl?Foo>Bar\" title=\"meatball:Foo>Bar\" 

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Further reduce mocha noise by suppressing unnecessary stacks

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/396099 )

Change subject: Further reduce mocha noise by suppressing unnecessary stacks
..

Further reduce mocha noise by suppressing unnecessary stacks

Change-Id: I2dff85ad1ab9e93ab1aeead352688553ad0e0612
---
M lib/api/routes.js
M lib/html2wt/DOMHandlers.js
2 files changed, 3 insertions(+), 5 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/99/396099/1

diff --git a/lib/api/routes.js b/lib/api/routes.js
index f7969db..6917969 100644
--- a/lib/api/routes.js
+++ b/lib/api/routes.js
@@ -25,11 +25,8 @@
// is setup.  The logger doesn't emit the expected location info.
// You probably want `apiUtils.fatalRequest` instead.
var errOut = function(res, text, httpStatus) {
-   var err = new Error(text);
-   err.httpStatus = httpStatus || 404;
-   err.suppressLoggingStack = true;
-   processLogger.log('fatal/request', err);
-   apiUtils.errorResponse(res, text, err.httpStatus);
+   processLogger.log('fatal/request', text);
+   apiUtils.errorResponse(res, text, httpStatus || 404);
};
 
// Middlewares
diff --git a/lib/html2wt/DOMHandlers.js b/lib/html2wt/DOMHandlers.js
index dcf51a2..d6984a4 100644
--- a/lib/html2wt/DOMHandlers.js
+++ b/lib/html2wt/DOMHandlers.js
@@ -1436,6 +1436,7 @@
this.name = 'Bad Request';
this.message = message || 'Bad Request';
this.httpStatus = 400;
+   this.suppressLoggingStack = true;
 }
 ClientError.prototype = Error.prototype;
 

-- 
To view, visit https://gerrit.wikimedia.org/r/396099
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I2dff85ad1ab9e93ab1aeead352688553ad0e0612
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: T65370: Use a map for namespaceIds

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/396096 )

Change subject: T65370: Use a map for namespaceIds
..

T65370: Use a map for namespaceIds

Change-Id: I8e1681bc3a2f9e56f60113385ea5544a826b619a
---
M lib/config/WikiConfig.js
M lib/html2wt/LinkHandler.js
M lib/wt2html/tt/LinkHandler.js
M lib/wt2html/tt/ParserFunctions.js
M tests/parserTests.utils.js
5 files changed, 11 insertions(+), 12 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/96/396096/1

diff --git a/lib/config/WikiConfig.js b/lib/config/WikiConfig.js
index 318d2d8..a689117 100644
--- a/lib/config/WikiConfig.js
+++ b/lib/config/WikiConfig.js
@@ -128,7 +128,7 @@
};
 
// Seed localized namespace name to id mapping with canonicalNamespaces
-   this.namespaceIds = Object.create(this.canonicalNamespaces);
+   this.namespaceIds = JSUtils.mapObject(this.canonicalNamespaces);
 
// The interwiki prefix
this.iwp = prefix || "";
@@ -180,7 +180,7 @@
if (nsid === "14" && name['*'] !== "Category") {
this._categoryRegexpSource += "|" + name['*'];
}
-   this.namespaceIds[Util.normalizeNamespaceName(name['*'])] = 
Number(nsid);
+   this.namespaceIds.set(Util.normalizeNamespaceName(name['*']), 
Number(nsid));
if (name.canonical) {
// XXX: is this b/c?

this.canonicalNamespaces[Util.normalizeNamespaceName(name.canonical)] =
@@ -198,7 +198,7 @@
 
var aliases = resultConf.namespacealiases;
for (var ax = 0; ax < aliases.length; ax++) {
-   
this.namespaceIds[Util.normalizeNamespaceName(aliases[ax]['*'])] = 
aliases[ax].id;
+   
this.namespaceIds.set(Util.normalizeNamespaceName(aliases[ax]['*']), 
aliases[ax].id);
if (aliases[ax].id === -1) {
this._specialAliases.push(aliases[ax]['*']);
}
diff --git a/lib/html2wt/LinkHandler.js b/lib/html2wt/LinkHandler.js
index 8dfccac..b9f4f06 100644
--- a/lib/html2wt/LinkHandler.js
+++ b/lib/html2wt/LinkHandler.js
@@ -172,7 +172,7 @@
if (dp.sa && dp.sa.namespace) {
mediaPrefix = dp.sa.namespace;
} else {
-   mediaPrefix = 
wiki.namespaceNames[wiki.namespaceIds.media];
+   mediaPrefix = 
wiki.namespaceNames[wiki.namespaceIds.get('media')];
}
 
// Unless edited, get original shadowed info for 
'fileName'
@@ -460,7 +460,7 @@
// it needs a ':' on front to distingish from a 
category link.
var categoryMatch = 
linkTarget.match(/^([^:]+)[:]/);
if (categoryMatch) {
-   var ns = 
wiki.namespaceIds[Util.normalizeNamespaceName(categoryMatch[1])];
+   var ns = 
wiki.namespaceIds.get(Util.normalizeNamespaceName(categoryMatch[1]));
if (ns === 
wiki.canonicalNamespaces.category) {
// Check that the next node 
isn't a category link,
// in which case we don't want 
the ':'.
diff --git a/lib/wt2html/tt/LinkHandler.js b/lib/wt2html/tt/LinkHandler.js
index f1de3a2..ff3ef7f 100644
--- a/lib/wt2html/tt/LinkHandler.js
+++ b/lib/wt2html/tt/LinkHandler.js
@@ -117,8 +117,7 @@
var nnn = Util.normalizeNamespaceName(nsPrefix.trim());
var interwikiInfo = env.conf.wiki.interwikiMap.get(nnn);
// check for interwiki / language links
-   // SSS FIXME: Use a map for namespaceIds!
-   var ns = env.conf.wiki.namespaceIds.hasOwnProperty(nnn) ? 
env.conf.wiki.namespaceIds[nnn] : undefined;
+   var ns = env.conf.wiki.namespaceIds.get(nnn);
// also check for url to protect against [[constructor:foo]]
if (ns !== undefined) {
info.title = env.makeTitleFromURLDecodedStr(title);
diff --git a/lib/wt2html/tt/ParserFunctions.js 
b/lib/wt2html/tt/ParserFunctions.js
index c72d53d..522784f 100644
--- a/lib/wt2html/tt/ParserFunctions.js
+++ b/lib/wt2html/tt/ParserFunctions.js
@@ -816,8 +816,8 @@
var env = this.env;
var normalizedTarget = target.toLowerCase().replace(' ', '_');
 
-   if (env.conf.wiki.namespaceIds[normalizedTarget]) {
-   nsid = env.conf.wiki.namespaceIds[normalizedTarget];
+   if (env.conf.wiki.namespaceIds.has(normalizedTarget)) {
+   nsid = env.conf.wiki.namespaceIds.get(normalizedTarget);
} else if (env.conf.wiki.canonicalNamespaces[normalizedTarget]) {
 

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: T66003: Make strictSSL configurable per wiki as well

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/396032 )

Change subject: T66003: Make strictSSL configurable per wiki as well
..

T66003: Make strictSSL configurable per wiki as well

Change-Id: Ifac871d49b88f85d4f65273e20a86204fb49264e
---
M lib/mw/ApiRequest.js
1 file changed, 1 insertion(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/32/396032/1

diff --git a/lib/mw/ApiRequest.js b/lib/mw/ApiRequest.js
index 129f545..a4515a4 100644
--- a/lib/mw/ApiRequest.js
+++ b/lib/mw/ApiRequest.js
@@ -298,7 +298,7 @@
// Set default options, forward cookie if set.
options.headers['User-Agent'] = env.conf.parsoid.userAgent;
options.headers.Connection = 'close';
-   options.strictSSL = env.conf.parsoid.strictSSL;
+   options.strictSSL = env.conf.wiki.strictSSL || 
env.conf.parsoid.strictSSL;
if (env.cookie) {
options.headers.Cookie = env.cookie;
}

-- 
To view, visit https://gerrit.wikimedia.org/r/396032
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ifac871d49b88f85d4f65273e20a86204fb49264e
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Fix accepting a posted original title

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/395879 )

Change subject: Fix accepting a posted original title
..

Fix accepting a posted original title

 * This was broken in 37f5c20

Change-Id: Ie492b4c15eb5ba2c90c873f58e49e65cbced3f70
---
M lib/api/routes.js
M tests/mocha/api.js
2 files changed, 24 insertions(+), 8 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/79/395879/1

diff --git a/lib/api/routes.js b/lib/api/routes.js
index 4253180..607eecd 100644
--- a/lib/api/routes.js
+++ b/lib/api/routes.js
@@ -91,14 +91,6 @@
}
res.locals.iwp = iwp;
 
-   res.locals.envOptions = {
-   prefix: res.locals.iwp,
-   pageName: res.locals.pageName,
-   cookie: req.headers.cookie,
-   reqId: req.headers['x-request-id'],
-   userAgent: req.headers['user-agent'],
-   };
-
// "subst" flag to perform {{subst:}} template expansion
res.locals.subst = !!(req.query.subst || req.body.subst);
// This is only supported for the html format
@@ -117,6 +109,14 @@
}
}
 
+   res.locals.envOptions = {
+   prefix: res.locals.iwp,
+   pageName: res.locals.pageName,
+   cookie: req.headers.cookie,
+   reqId: req.headers['x-request-id'],
+   userAgent: req.headers['user-agent'],
+   };
+
res.locals.opts = opts;
next();
};
diff --git a/tests/mocha/api.js b/tests/mocha/api.js
index 6a1c2d7..7965320 100644
--- a/tests/mocha/api.js
+++ b/tests/mocha/api.js
@@ -672,6 +672,22 @@
.end(done);
});
 
+   it('should accept an original title, other than main', 
function(done) {
+   request(api)
+   .post(mockDomain + '/v3/transform/wikitext/to/html/')
+   .send({
+   original: {
+   title: 'Lint Page',
+   },
+   })
+   .expect(302)  // no revid or wikitext source provided
+   .expect(function(res) {
+   res.headers.should.have.property('location');
+   res.headers.location.should.equal('/' + 
mockDomain + '/v3/page/html/Lint_Page/102');
+   })
+   .end(done);
+   });
+
it('should not require a title when empty wikitext is provided 
(html)', function(done) {
request(api)
.post(mockDomain + '/v3/transform/wikitext/to/html/')

-- 
To view, visit https://gerrit.wikimedia.org/r/395879
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ie492b4c15eb5ba2c90c873f58e49e65cbced3f70
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: T169006: Preserve request method when redirecting to oldid

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/395880 )

Change subject: T169006: Preserve request method when redirecting to oldid
..

T169006: Preserve request method when redirecting to oldid

Change-Id: Ifab6e8d003a5e2afa11ca4077d2beafb684ca1a5
---
M lib/api/apiUtils.js
M tests/mocha/api.js
2 files changed, 42 insertions(+), 33 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/80/395880/1

diff --git a/lib/api/apiUtils.js b/lib/api/apiUtils.js
index bdc85d2..d2a5b13 100644
--- a/lib/api/apiUtils.js
+++ b/lib/api/apiUtils.js
@@ -369,7 +369,7 @@
});
 };
 
-apiUtils._redirect = function(req, res, target, processRedirect) {
+apiUtils._redirect = function(req, res, target, httpStatus, processRedirect) {
var locals = res.locals;
var path = processRedirect([
'',
@@ -382,7 +382,7 @@
 
// Don't cache redirect requests
apiUtils.setHeader(res, 'Cache-Control', 'private,no-cache,s-maxage=0');
-   apiUtils.relativeRedirect(res, path);
+   apiUtils.relativeRedirect(res, path, httpStatus);
 };
 
 /**
@@ -392,26 +392,24 @@
  */
 apiUtils.redirectToOldid = function(req, res) {
var env = res.locals.env;
-   return this._redirect(
-   req,
-   res,
-   env.normalizeAndResolvePageTitle(),
-   function(redirPath) {
-   var revid = env.page.meta.revision.revid;
-   redirPath += '/' + revid;
-   if (Object.keys(req.query).length > 0) {
-   redirPath += '?' + qs.stringify(req.query);
-   }
-   var format = res.locals.opts.format;
-   env.log('info', 'redirecting to revision', revid, 
'for', format);
-   var metrics = env.conf.parsoid.metrics;
-   if (metrics) {
-   metrics.increment('redirectToOldid.' + 
format.toLowerCase());
-   }
-   return redirPath;
+   var target = env.normalizeAndResolvePageTitle();
+   // Preserve the request method since we currently don't allow GETing the
+   // "lint" format.  See T169006
+   var httpStatus = (req.method === 'GET') ? 302 : 307;
+   return this._redirect(req, res, target, httpStatus, function(redirPath) 
{
+   var revid = env.page.meta.revision.revid;
+   redirPath += '/' + revid;
+   if (Object.keys(req.query).length > 0) {
+   redirPath += '?' + qs.stringify(req.query);
}
-   );
-
+   var format = res.locals.opts.format;
+   env.log('info', 'redirecting to revision', revid, 'for', 
format);
+   var metrics = env.conf.parsoid.metrics;
+   if (metrics) {
+   metrics.increment('redirectToOldid.' + 
format.toLowerCase());
+   }
+   return redirPath;
+   });
 };
 
 /**
@@ -421,15 +419,10 @@
  * @param {Response} res
  */
 apiUtils._redirectToPage = function(title, req, res) {
-   return this._redirect(
-   req,
-   res,
-   title,
-   function(path) {
-   res.locals.env.log('info', 'redirecting to ', path);
-   return path;
-   }
-   );
+   return this._redirect(req, res, title, undefined, function(path) {
+   res.locals.env.log('info', 'redirecting to ', path);
+   return path;
+   });
 };
 
 /**
diff --git a/tests/mocha/api.js b/tests/mocha/api.js
index 7965320..46a8df0 100644
--- a/tests/mocha/api.js
+++ b/tests/mocha/api.js
@@ -466,6 +466,22 @@
.end(done);
});
 
+   it('should redirect title to latest revision (lint)', 
function(done) {
+   request(api)
+   .post(mockDomain + '/v3/transform/wikitext/to/lint/')
+   .send({
+   original: {
+   title: 'Lint_Page',
+   },
+   })
+   .expect(307)  // no revid or wikitext source provided
+   .expect(function(res) {
+   res.headers.should.have.property('location');
+   res.headers.location.should.equal('/' + 
mockDomain + '/v3/page/lint/Lint_Page/102');
+   })
+   .end(done);
+   });
+
});
 
describe("wt2html", function() {
@@ -648,7 +664,7 @@
title: 'Main_Page',
},
})
-   .expect(302)  // no 

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Prefer wiki.mainpage to defaultPageName

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/395830 )

Change subject: Prefer wiki.mainpage to defaultPageName
..

Prefer wiki.mainpage to defaultPageName

Change-Id: I24610c34f7b2d0d8769fba3a3d774efcb3ca3510
---
M bin/parse.js
M bin/parserTests.js
M lib/api/internal.js
M lib/api/routes.js
M lib/config/MWParserEnvironment.js
M tools/sync-baseconfig.js
6 files changed, 7 insertions(+), 12 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/30/395830/1

diff --git a/bin/parse.js b/bin/parse.js
index 5d97a24..e9ddc7a 100755
--- a/bin/parse.js
+++ b/bin/parse.js
@@ -156,7 +156,7 @@
'pageName': {
description: 'The page name, returned for {{PAGENAME}}. If no 
input is given (ie. empty/stdin closed), it downloads and parses the page. This 
should be the actual title of the article (that is, not including any 
URL-encoding that might be necessary in wikitext).',
'boolean': false,
-   'default': ParserEnvProto.defaultPageName,
+   'default': '',
},
'pageBundle': {
description: 'Output pagebundle JSON',
diff --git a/bin/parserTests.js b/bin/parserTests.js
index cd03108..1a4bf04 100755
--- a/bin/parserTests.js
+++ b/bin/parserTests.js
@@ -683,7 +683,7 @@
 
// Since we are reusing the 'env' object, set it to the default
// so that relative link prefix is back to "./"
-   this.env.initializeForPageName(this.env.defaultPageName);
+   this.env.initializeForPageName(this.env.conf.wiki.mainpage);
 
if (item.options.subpage !== undefined) {
this.env.conf.wiki.namespacesWithSubpages[0] = true;
@@ -693,7 +693,7 @@
!Array.isArray(item.options.title)) {
// This sets the page name as well as the relative link 
prefix
// for the rest of the parse.  Do this redundantly with 
the above
-   // so that we start from the defaultPageName when 
resolving
+   // so that we start from the wiki.mainpage when 
resolving
// absolute subpages.
this.env.initializeForPageName(item.options.title);
}
diff --git a/lib/api/internal.js b/lib/api/internal.js
index 0823b83..cef8744 100644
--- a/lib/api/internal.js
+++ b/lib/api/internal.js
@@ -65,7 +65,7 @@
return apiUtils.errorResponse(res, text, 404);
}
res.locals.iwp = iwp;
-   res.locals.pageName = req.params.title || 'Main_Page';
+   res.locals.pageName = req.params.title || '';
res.locals.oldid = req.body.oldid || req.query.oldid || null;
// "body" flag to return just the body (instead of the entire 
HTML doc)
res.locals.bodyOnly = !!(req.query.body || req.body.body);
diff --git a/lib/api/routes.js b/lib/api/routes.js
index f7969db..4253180 100644
--- a/lib/api/routes.js
+++ b/lib/api/routes.js
@@ -54,7 +54,7 @@
 
routes.v3Middle = function(req, res, next) {
res.locals.titleMissing = !req.params.title;
-   res.locals.pageName = req.params.title || 'Main_Page';
+   res.locals.pageName = req.params.title || '';
res.locals.oldid = req.params.revision || null;
 
// "body_only" flag to return just the body (instead of the 
entire HTML doc)
diff --git a/lib/config/MWParserEnvironment.js 
b/lib/config/MWParserEnvironment.js
index d448823..3fac953 100644
--- a/lib/config/MWParserEnvironment.js
+++ b/lib/config/MWParserEnvironment.js
@@ -31,7 +31,6 @@
  */
 var MWParserEnvironment = function(parsoidConfig, options) {
options = options || {};
-   var self = this;
 
// page information
this.page = (function() {
@@ -42,7 +41,7 @@
/**
 * @property {String} name
 */
-   Page.prototype.name = self.defaultPageName;
+   Page.prototype.name = '';
 
/**
 * Any leading ..?/ strings that will be necessary for building 
links.
@@ -149,7 +148,7 @@
this.lintLogger = new LintLogger(this);
this.configureLogging();
 
-   this.initializeForPageName(options.pageName || this.defaultPageName);
+   this.initializeForPageName(options.pageName || this.conf.wiki.mainpage);
 
this.pipelineFactory = new ParserPipelineFactory(this);
 
@@ -297,9 +296,6 @@
// Configure backends
logger.registerLoggingBackends(defaultLogLevels, this.conf.parsoid, 
this.lintLogger);
 };
-
-// The default page name (true name, without wikitext url encoding)
-MWParserEnvironment.prototype.defaultPageName = "Main Page";
 
 // Cache for wiki configurations, 

[MediaWiki-commits] [Gerrit] mediawiki...deploy[master]: Bump src/ to deploy-2017-12-06 branch for deploy

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/395780 )

Change subject: Bump src/ to deploy-2017-12-06 branch for deploy
..

Bump src/ to deploy-2017-12-06 branch for deploy

That branch was produced by doing:

$ git checkout -b deploy-2017-12-06 f087b5d
$ git rebase e71937d -i

And choosing:

pick 3a91a182 T178253: Permit table rows from templates in figure captions
pick 3048db62 Stop rewriting html escaping everywhere
pick 4d66ad79 Sync parserTests with core
pick 4aebdbe8 Move expansion reuse to the env so cpu workers can partake
pick 27975cec T61840: Remove top level dependence on jsDiff
drop 1584dd79 T114072: Add  wrappers to Parsoid output
drop 0723e5c4 Use a Map instead of an array in order to avoid quadratic running 
time
drop db2edc33 Sync parserTests and citeParserTests with core/Cite.
drop c9f40476 T118520: Use figure-inline instead of span for inline media
drop 44259698 T181229: Stop adding packing wrappers for last nodes
drop 58c60632 T114072: Always emit a lead  tag
drop cba26a92 Add lead section before resolving template-section conflicts
drop 451e37f7 roundtrip-test.js: Strip section tags before comparing DOMs
drop bedc8201 Fix crasher in getDSR in section wrapping code
drop c1679332 Properly handle short headings
drop bd724104 Move wrapSections option from ParsoidConfig to MWParserEnvironment
drop e6a6f261 T181226: Don't emit  wrappers in body only mode
drop a4ba7e0a Fix crasher in section <-> template conflict resolution code
drop fe216e8e Update Parsoid to generate modern HTML5 IDs w/ legacy fallback
drop 73686df2 Fix crasher with lonely `=` and `==`
drop 36f84a3d Fix RT issues with duplicate heading ids
pick 5506240d Strip [[ ]] from parser tests options in the parser
drop 02bae936 Bump Parsoid HTML version to 1.6.0
pick 37f5c209 Use lib/parse.js in api/routes.js
pick 7058f64a Move out internal routes
pick 3f26d448 lib/parse.js: Move env initializations to relevant parsing paths
pick b667451e lib/parse.js: Make wt2html and html2wt methods do what they say
drop dc046be4 T97093: Use mw:WikiLink/InterWiki for interwiki links
pick f33bfc0c T180930: Selser shouldn't reuse orig sep for autoinserted tags
pick 7eecd545 Preserve original transclusion's parameter order
pick f087b5d0 Improve parameter aliases handling when ordering template 
parameters

Change-Id: Ib9f347b7054a563fd637d86a0a7af50089e09f05
---
M src
1 file changed, 1 insertion(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid/deploy 
refs/changes/80/395780/1

diff --git a/src b/src
index c9f4047..01c1fc3 16
--- a/src
+++ b/src
@@ -1 +1 @@
-Subproject commit c9f404761cd288e7b58b89623ac459bbb2901a7d
+Subproject commit 01c1fc33c15cec32871b5440d7d6b7a57a53a225

-- 
To view, visit https://gerrit.wikimedia.org/r/395780
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib9f347b7054a563fd637d86a0a7af50089e09f05
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid/deploy
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Reduce noise in mocha testing by bumping the logger level to...

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/395678 )

Change subject: Reduce noise in mocha testing by bumping the logger level to 
warn
..

Reduce noise in mocha testing by bumping the logger level to warn

Change-Id: Iee25de77fe6118786c31fa683ccfdffcf5b0a055
---
M tests/serviceWrapper.js
1 file changed, 0 insertions(+), 3 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/78/395678/1

diff --git a/tests/serviceWrapper.js b/tests/serviceWrapper.js
index 43076db..bf4e087 100644
--- a/tests/serviceWrapper.js
+++ b/tests/serviceWrapper.js
@@ -99,9 +99,6 @@
// service-runner doesn't actually call close on the 
worker.
num_workers: 1,
worker_heartbeat_timeout: 2 * 60 * 1000,
-   logging: {
-   level: 'info',  // Default is 'warn'
-   },
services: services,
})
.then(function() {

-- 
To view, visit https://gerrit.wikimedia.org/r/395678
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Iee25de77fe6118786c31fa683ccfdffcf5b0a055
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Title is not optional in v3 of the http api

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/395677 )

Change subject: Title is not optional in v3 of the http api
..

Title is not optional in v3 of the http api

Change-Id: I0df7d6204f01c0f73070ae370b9d66a879ef08bb
---
M lib/api/routes.js
1 file changed, 1 insertion(+), 6 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/77/395677/1

diff --git a/lib/api/routes.js b/lib/api/routes.js
index 7ca157f..b3c4754 100644
--- a/lib/api/routes.js
+++ b/lib/api/routes.js
@@ -54,7 +54,7 @@
 
routes.v3Middle = function(req, res, next) {
res.locals.titleMissing = !req.params.title;
-   res.locals.pageName = req.params.title || 'Main_Page';
+   res.locals.pageName = req.params.title;
res.locals.oldid = req.params.revision || null;
 
// "body_only" flag to return just the body (instead of the 
entire HTML doc)
@@ -343,11 +343,6 @@

startTimers.get('wt2html.init'));
startTimers.set('wt2html.wt.parse', 
Date.now());
metrics.timing('wt2html.wt.size.input', 
wikitext.length);
-   }
-
-   if (!res.locals.pageName) {
-   // clear default page name
-   env.page.name = '';
}
} else if (oldid) {
envOptions.pageWithOldid = true;

-- 
To view, visit https://gerrit.wikimedia.org/r/395677
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I0df7d6204f01c0f73070ae370b9d66a879ef08bb
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Reject lint requests if linting isn't enabled

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/395676 )

Change subject: Reject lint requests if linting isn't enabled
..

Reject lint requests if linting isn't enabled

Change-Id: I41cf86d45466db75b07fb09ceffcacc72bc3f620
---
M lib/api/routes.js
1 file changed, 4 insertions(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/76/395676/1

diff --git a/lib/api/routes.js b/lib/api/routes.js
index d830061..7ca157f 100644
--- a/lib/api/routes.js
+++ b/lib/api/routes.js
@@ -43,8 +43,11 @@
 
var validGets = new Set(['wikitext', 'html', 'pagebundle']);
 
+   var wikitextTransforms = ['html', 'pagebundle'];
+   if (parsoidConfig.linting) { wikitextTransforms.push('lint'); }
+
var validTransforms = new Map(Object.entries({
-   'wikitext': ['html', 'pagebundle', 'lint'],
+   'wikitext': wikitextTransforms,
'html': ['wikitext'],
'pagebundle': ['wikitext', 'pagebundle'],
}));

-- 
To view, visit https://gerrit.wikimedia.org/r/395676
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I41cf86d45466db75b07fb09ceffcacc72bc3f620
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Alias map

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/395665 )

Change subject: Alias map
..

Alias map

Change-Id: Ica3ea544cec7183b3474b5cdf7489e3f04c91eef
---
M lib/html2wt/WikitextSerializer.js
1 file changed, 12 insertions(+), 3 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/65/395665/1

diff --git a/lib/html2wt/WikitextSerializer.js 
b/lib/html2wt/WikitextSerializer.js
index 895de1c..c54987e 100644
--- a/lib/html2wt/WikitextSerializer.js
+++ b/lib/html2wt/WikitextSerializer.js
@@ -423,16 +423,19 @@
}));
// Record order of parameters in templatedata (if present)
var tplDataOrder = new Map();
+   var aliasMap = new Map();
var keys = [];
if (tplData && Array.isArray(tplData.paramOrder)) {
var params = tplData.params;
tplData.paramOrder.forEach(function(k, i) {
-   tplDataOrder.set(k, { order: keys.length });
+   tplDataOrder.set(k, { order: i });
+   aliasMap.set(k, { key: k, order: -1 });
keys.push(k);
// Aliases have the same sort order as the main name.
var aliases = params && params[k] && params[k].aliases;
-   (aliases || []).forEach(function(a) {
-   tplDataOrder.set(a, { order: keys.length });
+   (aliases || []).forEach(function(a, j) {
+   tplDataOrder.set(a, { order: i });
+   aliasMap.set(a, { key: k, order: j });
keys.push(a);
});
});
@@ -467,6 +470,12 @@
var aOrder = defaultGet(nearestOrder, a);
var bOrder = defaultGet(nearestOrder, b);
if (aOrder !== bOrder) { return aOrder - bOrder; }
+   // canon
+   var aCanon = aliasMap.get(a);
+   var bCanon = aliasMap.get(b);
+   if (aCanon && bCanon && aCanon.key === bCanon.key) {
+   return aCanon.order - bCanon.order;
+   }
// secondary key is templatedata order
aOrder = defaultGet(tplDataOrder, a);
bOrder = defaultGet(tplDataOrder, b);

-- 
To view, visit https://gerrit.wikimedia.org/r/395665
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ica3ea544cec7183b3474b5cdf7489e3f04c91eef
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Slowly start to wean us off using parser env in http api routes

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/395583 )

Change subject: Slowly start to wean us off using parser env in http api routes
..

Slowly start to wean us off using parser env in http api routes

 * The pageName change is because title is not optional in v3 so it had
   no effect.

 * Also, reduce noise in mocha testing by bumping the logger level to
   warn.

Change-Id: I55cb78bb09198cf419409c4802f8492adac25aea
---
M lib/api/apiUtils.js
M lib/api/internal.js
M lib/api/routes.js
M lib/parse.js
M tests/serviceWrapper.js
5 files changed, 22 insertions(+), 32 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/83/395583/1

diff --git a/lib/api/apiUtils.js b/lib/api/apiUtils.js
index c919a76..bdc85d2 100644
--- a/lib/api/apiUtils.js
+++ b/lib/api/apiUtils.js
@@ -166,9 +166,9 @@
env.log('fatal/request', err);
 };
 
-apiUtils.logTime = function(env, res, str) {
-   env.log('info', util.format(
-   'completed %s in %s ms', str, Date.now() - res.locals.start
+apiUtils.logTime = function(processLogger, res, str) {
+   processLogger.log('info', util.format(
+   'completed %s in %sms', str, Date.now() - res.locals.start
));
 };
 
diff --git a/lib/api/internal.js b/lib/api/internal.js
index 2bfb57e..0823b83 100644
--- a/lib/api/internal.js
+++ b/lib/api/internal.js
@@ -48,7 +48,7 @@
 
 var rtResponse = function(env, req, res, data) {
apiUtils.renderResponse(res, 'roundtrip', data);
-   apiUtils.logTime(env, res, 'parsing');
+   env.log('info', 'completed in ' + (Date.now() - res.locals.start) + 
'ms');
 };
 
 module.exports = function(parsoidConfig, processLogger) {
diff --git a/lib/api/routes.js b/lib/api/routes.js
index 260185f..78a9776 100644
--- a/lib/api/routes.js
+++ b/lib/api/routes.js
@@ -51,7 +51,7 @@
 
routes.v3Middle = function(req, res, next) {
res.locals.titleMissing = !req.params.title;
-   res.locals.pageName = req.params.title || 'Main_Page';
+   res.locals.pageName = req.params.title;
res.locals.oldid = req.params.revision || null;
 
// "body_only" flag to return just the body (instead of the 
entire HTML doc)
@@ -198,7 +198,7 @@
 
// Validate and set the content version
if (!apiUtils.validateAndSetContentVersion(res, 
acceptableTypes)) {
-   if (env.conf.parsoid.strictAcceptCheck) {
+   if (parsoidConfig.strictAcceptCheck) {
var text = 
env.availableVersions.reduce(function(prev, curr) {
return prev + apiUtils[opts.format + 
'ContentType'](env, curr) + '\n';
}, 'Not acceptable.\n');
@@ -283,15 +283,9 @@
});
}
 
-   p = p.tap(function() {
-   env.bumpParserResourceUse('wikitextSize',
-   (typeof wt !== 'string' ? env.page.src : 
wt).length);
-   });
-
var doSubst = (typeof wt === 'string' && res.locals.subst);
if (doSubst) {
p = p.then(function(wikitext) {
-   // FIXME: reset limits after subst'ing
return apiUtils.substTopLevelTemplates(env, 
target, wikitext);
});
}
@@ -306,7 +300,7 @@
}
 
// Follow redirects if asked
-   if (env.conf.parsoid.devAPI && 
req.query.follow_redirects) {
+   if (parsoidConfig.devAPI && req.query.follow_redirects) 
{
// Get localized redirect matching regexp
var reSrc = 
env.conf.wiki.getMagicWordMatcher('redirect').source;
reSrc = '^[ \\t\\n\\r\\0\\x0b]*' +
@@ -321,7 +315,7 @@
}
}
 
-   env.log('info', 'started parsing');
+   processLogger.log('info', 'started parsing');
 
var envOptions = Object.assign({
pageBundle: pageBundle,
@@ -346,11 +340,6 @@

startTimers.get('wt2html.init'));
startTimers.set('wt2html.wt.parse', 
Date.now());
metrics.timing('wt2html.wt.size.input', 
wikitext.length);
-   }
-
-   if (!res.locals.pageName) {
-   // clear default page name
-   env.page.name = '';
}
} else if (oldid) 

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Use lib/parse.js in api/routes.js

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/394601 )

Change subject: Use lib/parse.js in api/routes.js
..

Use lib/parse.js in api/routes.js

Change-Id: I20f6cc3078853bbc462e5d565269d97bf097f880
---
M lib/api/ParsoidService.js
M lib/api/apiUtils.js
M lib/api/routes.js
M lib/html2wt/WikitextSerializer.js
M lib/index.js
5 files changed, 98 insertions(+), 99 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/01/394601/1

diff --git a/lib/api/ParsoidService.js b/lib/api/ParsoidService.js
index b4d52ae..4953b67 100644
--- a/lib/api/ParsoidService.js
+++ b/lib/api/ParsoidService.js
@@ -18,6 +18,8 @@
 var util = require('util');
 
 var Promise = require('../utils/promise.js');
+var ParsoidConfig = require('../config/ParsoidConfig.js').ParsoidConfig;
+var parseJsPath = require.resolve('../parse.js');
 
 /**
  * ParsoidService
@@ -33,15 +35,17 @@
  * Instantiates an [express](http://expressjs.com/) server
  * to handle HTTP requests and begins listening on the configured port.
  *
- * @param {ParsoidConfig} parsoidConfig
+ * @param {Object} parsoidOptions
  * @param {Object} processLogger
  *   WARNING: `processLogger` is not necessarily an instance of `Logger`.
  *   The interface is merely that exposed by service-runner, `log(level, 
info)`.
  *   Don't expect it to exit after you've logged "fatal" and other such things.
  * @return {Promise} server
  */
-ParsoidService.init = Promise.method(function(parsoidConfig, processLogger) {
+ParsoidService.init = Promise.method(function(parsoidOptions, processLogger) {
processLogger.log('info', 'loading ...');
+
+   var parsoidConfig = new ParsoidConfig(null, parsoidOptions);
 
// Get host and port from the environment, if available
// note: in production, the port is exposed via the 'port' config 
stanza and
@@ -55,8 +59,10 @@
// and 'serverInterface' is the legacy option name
var host = parsoidConfig.interface || parsoidConfig.serverInterface || 
process.env.INTERFACE;
 
+   var parse = require(parseJsPath);
+
// Load routes
-   var routes = require('./routes')(parsoidConfig, processLogger);
+   var routes = require('./routes')(parsoidConfig, processLogger, 
parsoidOptions, parse);
 
var app = express();
 
@@ -205,7 +211,8 @@
reject(err);
});
server = app.listen(port, host, resolve);
-   }).then(function() {
+   })
+   .then(function() {
port = server.address().port;
processLogger.log('info',
util.format('ready on %s:%s', host || '', port));
diff --git a/lib/api/apiUtils.js b/lib/api/apiUtils.js
index d6a1d35..8172d0f 100644
--- a/lib/api/apiUtils.js
+++ b/lib/api/apiUtils.js
@@ -236,13 +236,7 @@
}
// Now pass it to the MediaWiki API with onlypst set so that it
// subst's the templates.
-   return PHPParseRequest.promise(env, target, wt, 
true).then(function(wikitext) {
-   // Set data-parsoid to be discarded, so that the subst'ed
-   // content is considered new when it comes back.
-   env.discardDataParsoid = true;
-   // Use the returned wikitext as the page source.
-   return wikitext;
-   });
+   return PHPParseRequest.promise(env, target, wt, true);
 };
 
 apiUtils.wikitextContentType = function(env) {
diff --git a/lib/api/routes.js b/lib/api/routes.js
index d09d554..c6ecb22 100644
--- a/lib/api/routes.js
+++ b/lib/api/routes.js
@@ -18,8 +18,7 @@
 
 var TemplateRequest = ApiRequest.TemplateRequest;
 
-
-module.exports = function(parsoidConfig, processLogger) {
+module.exports = function(parsoidConfig, processLogger, parsoidOptions, parse) 
{
var routes = {};
var metrics = parsoidConfig.metrics;
var REQ_TIMEOUT = parsoidConfig.timeouts.request;
@@ -50,6 +49,10 @@
res.locals.bodyOnly = !!(req.query.body || req.body.body);
// "subst" flag to perform {{subst:}} template expansion
res.locals.subst = !!(req.query.subst || req.body.subst);
+   res.locals.envOptions = {
+   prefix: res.locals.iwp,
+   pageName: res.locals.pageName,
+   };
next();
};
 
@@ -106,6 +109,14 @@
return errOut(res, 'Invalid domain: ' + 
req.params.domain);
}
res.locals.iwp = iwp;
+
+   res.locals.envOptions = {
+   prefix: res.locals.iwp,
+   pageName: res.locals.pageName,
+   cookie: req.headers.cookie,
+   reqId: req.headers['x-request-id'],
+   userAgent: req.headers['user-agent'],
+   };
 
// "subst" flag 

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: T180930: Selser shouldn't reuse orig sep for autoinserted ends

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/394412 )

Change subject: T180930: Selser shouldn't reuse orig sep for autoinserted ends
..

T180930: Selser shouldn't reuse orig sep for autoinserted ends

 * This might be more generally applicable than to just tables.

Change-Id: I76418f44bb078c5f70cf2ef6b7982dff58448491
---
M lib/html2wt/DOMHandlers.js
M tests/parserTests-blacklist.js
2 files changed, 16 insertions(+), 31 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/12/394412/1

diff --git a/lib/html2wt/DOMHandlers.js b/lib/html2wt/DOMHandlers.js
index 0720450..c1d626b 100644
--- a/lib/html2wt/DOMHandlers.js
+++ b/lib/html2wt/DOMHandlers.js
@@ -695,7 +695,14 @@
// is never computed and set here.
state.sep.constraints = { a: {}, b: {}, 
min: 1, max: 2 };
}
+
+   if (state.selserMode && dp.autoInsertedEnd) {
+   // Set modified flag so that original 
separators aren't
+   // reused for the end tag since this is 
new content.
+   state.inModifiedContent = true;
+   }
WTSUtils.emitEndTag(dp.endTagSrc || "|}", node, 
state);
+
if (indentTable) {
state.singleLineContext.pop();
}
diff --git a/tests/parserTests-blacklist.js b/tests/parserTests-blacklist.js
index fda090d..b5d6104 100644
--- a/tests/parserTests-blacklist.js
+++ b/tests/parserTests-blacklist.js
@@ -1645,39 +1645,17 @@
 add("selser", "HTML bullet list, closed tags (T7497) [[4,[2],3,0,0]]", 
"1ll4gfgwnpb5dOne\nTwo\n");
 add("selser", "HTML nested ordered list, open tags (T7497) [[2,3,4,[2,1],3]]", 
"1ph637d\nl97dbh1lg6gmxTwo:\n\nSub-one\nSub-two\n\n");
 add("selser", "Fuzz testing: Parser13 [2]", "1s3ei1s\n{| \n| http://a|");
-add("selser", "Fuzz testing: Parser13 [[0,[[1", "{| \n| 
data-foobar=\"nmk8w6\" http: / a ||}");
-add("selser", "Fuzz testing: Parser13 [1]", "{| data-foobar=\"13wllqe\" \n| 
http://a||}");
-add("selser", "Fuzz testing: Parser13 [[2,2]]", "{| 
\n| http://a||}");
-add("selser", "Fuzz testing: Parser13 [[4,2]]", 
"{|\n| http://a||}");
-add("selser", "Fuzz testing: Parser13 [[0,1]]", "{| \n| http://a||}");
-add("selser", "Fuzz testing: Parser13 [[0,2]]", "{| \n| 
http://a||}");
-add("selser", "Fuzz testing: Parser13 [[0,[1]]]", "{| \n| http://a||}");
-add("selser", "Fuzz testing: Parser13 [[2,[[1", "{| \n| 
data-foobar=\"cmzmng\" http: / a ||}");
-add("selser", "Fuzz testing: Parser13 [[4,[1]]]", "{|\n| 
http://a||}");
-add("selser", "Fuzz testing: Parser13 [[4,0]]", "{|\n| 
http://a||}");
-add("selser", "Fuzz testing: Parser13 [[2,0]]", "{| \n| 
http://a||}");
+add("selser", "Fuzz testing: Parser13 [1]", "{| data-foobar=\"13wllqe\" \n| 
http://a|\n|}");
+add("selser", "Fuzz testing: Parser13 [[2,2]]", "{| 
\n| http://a|\n|}");
+add("selser", "Fuzz testing: Parser13 [[4,2]]", 
"{|\n| http://a|\n|}");
+add("selser", "Fuzz testing: Parser13 [[0,1]]", "{| \n| http://a|\n|}");
+add("selser", "Fuzz testing: Parser13 [[0,2]]", "{| \n| 
http://a|\n|}");
+add("selser", "Fuzz testing: Parser13 [[0,[1]]]", "{| \n| http://a|\n|}");
+add("selser", "Fuzz testing: Parser13 [[4,[1]]]", "{|\n| 
http://a|\n|}");
+add("selser", "Fuzz testing: Parser13 [[4,0]]", "{|\n| 
http://a|\n|}");
+add("selser", "Fuzz testing: Parser13 [[2,0]]", "{| \n| 
http://a|\n|}");
 add("selser", "Fuzz testing: Parser16 [2]", "1jeb7ja\n{|\n!https://||");
-add("selser", "Fuzz testing: Parser16 [[0,[[0,2,0,0", 
"{|\n!https://\n!7nk0ln|||}");
-add("selser", "Fuzz testing: Parser16 [[0,[[3,4,0,0", 
"{|\n!18o2jid|}");
-add("selser", "Fuzz testing: Parser16 [[3,2]]", 
"{|\n!https://|||}");
-add("selser", "Fuzz testing: Parser16 [[0,1]]", "{|\n!https://|||}");
-add("selser", "Fuzz testing: Parser16 [[3,[2]]]", 
"{|\n!https://|||}");
-add("selser", "Fuzz testing: Parser16 [[0,[[1,1,1,0", "{|\n! 
data-foobar=\"1ypnx5s\" |https://|| data-foobar=\"lc27jn\" ||| 
data-foobar=\"14qvamm\" }");
-add("selser", "Fuzz testing: Parser16 [1]", "{| 
data-foobar=\"1j1pbvp\"\n!https://|||}");
-add("selser", "Fuzz testing: Parser16 [[2,[1]]]", 
"{|\n!https://|||}");
-add("selser", "Fuzz testing: Parser21 [[2,2]]", 
"{|\n! irc://{{ftp://a\; onmouseover=\"alert('hello 
world');\"\n||}");
-add("selser", "Fuzz testing: Parser21 [1]", "{| data-foobar=\"14162hx\"\n! 
irc://{{ftp://a\; onmouseover=\"alert('hello world');\"\n||}");
-add("selser", "Fuzz testing: Parser21 [[4,[2]]]", 
"{|\n! irc://{{ftp://a\; onmouseover=\"alert('hello 
world');\"\n||}");
-add("selser", "Fuzz testing: Parser21 

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: T181229: Stop adding packing wrappers for last nodes

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/393634 )

Change subject: T181229: Stop adding packing wrappers for last nodes
..

T181229: Stop adding packing wrappers for last nodes

 * This unnecessarily complicates unpacking content nested in templates.

   If the content to be packed only expands to one node, no last node
   wrapper is added, which puts us in the position of trying to decide
   if a sibling is just an added wrapper or potentially some other
   template generated content.

   This is unfortunately resulting in dropped content as in the added
   test case, since the latter possibility is ignored.

 * Since the wrapper type is already determined by inspecting all nodes
   for block content and currently used for both edges, it seems
   unnecessary to have a wrapper for the last node, which is probably a
   relic from before that logic was added, as indicated by the analysis
   in the remove comment.

Change-Id: Id08ae1679fb43c50285fbc14cb11c3f2c5cd08f3
---
M lib/utils/DOMUtils.js
M lib/wt2html/pp/handlers/unpackDOMFragments.js
M tests/parserTests.txt
3 files changed, 9 insertions(+), 26 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/34/393634/1

diff --git a/lib/utils/DOMUtils.js b/lib/utils/DOMUtils.js
index 773d196..c4864ae 100644
--- a/lib/utils/DOMUtils.js
+++ b/lib/utils/DOMUtils.js
@@ -1804,8 +1804,7 @@
// content. It could potentially introduce minor rendering 
differences when
// compared to PHP parser output, but we'll swallow it for now.
var firstNode = nodes[0];
-   var lastNode = nodes.length > 1 ? lastItem(nodes) : null;
-   var wrapperType;
+   var wrapperType = 'INLINE';
if (opts.noPWrapping) {
// If the DOM fragment is being processed in the 
context where P wrapping
// has been suppressed, we represent the DOM fragment 
with inline-tokens.
@@ -1814,12 +1813,7 @@
// is correct in scenarios where link-content or 
image-captions are being
// processed in a sub-pipeline and we don't want a 
 in the link-caption
// to cause the .. to get split apart.
-   //
-   wrapperType = 'INLINE';
-   } else if (lastNode && DU.isBlockNode(lastNode)) {
-   wrapperType = 'BLOCK';
} else {
-   wrapperType = 'INLINE';
for (var i = 0; i < nodes.length; i++) {
if (DU.isBlockNode(nodes[i]) || 
DU.hasBlockElementDescendant(nodes[i])) {
wrapperType = 'BLOCK';
@@ -1828,18 +1822,7 @@
}
}
 
-   // Get two tokens each representing the start and end elements.
-   //
-   // The assumption behind this is that in order to tunnel the  
DOM fragment
-   // through the token stream intact (which, please note, has 
already
-   // been through all token transforms), we just need to figure 
out how
-   // the edges of the DOM fragment interact with the rest of the 
DOM that
-   // will get built up. In order to do this, we can just look at 
the edges
-   // of the fragment as represented by the first and last nodes.
var tokens = makeWrapperForNode(firstNode, wrapperType);
-   if (lastNode) {
-   tokens = tokens.concat(makeWrapperForNode(lastNode, 
wrapperType));
-   }
 
// Remove the typeof attribute from the first token. It will be
// replaced with mw:DOMFragment.
diff --git a/lib/wt2html/pp/handlers/unpackDOMFragments.js 
b/lib/wt2html/pp/handlers/unpackDOMFragments.js
index 7c7932b..b10cee2 100644
--- a/lib/wt2html/pp/handlers/unpackDOMFragments.js
+++ b/lib/wt2html/pp/handlers/unpackDOMFragments.js
@@ -159,14 +159,6 @@
});
DU.visitDOM(dummyNode, DU.loadDataAttribs);
 
-   // get rid of the wrapper sibling (simplifies logic below)
-   var sibling = node.nextSibling;
-   if (about !== null && sibling && DU.isElt(sibling) &&
-   sibling.getAttribute('about') === about) {
-   // remove optional second element added by wrapper tokens
-   DU.deleteNode(sibling);
-   }
-
var contentNode = dummyNode.firstChild;
 
if (/(?:^|\s)mw:Transclusion(?=$|\s)/.test(typeOf)) {
diff --git a/tests/parserTests.txt b/tests/parserTests.txt
index 1048200..e269a9b 100644
--- a/tests/parserTests.txt
+++ b/tests/parserTests.txt
@@ -28529,6 +28529,14 @@
 !! end
 
 !! test
+Catch regression when unpacking with trailing content
+!! wikitext
+{{echo|Foo  

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: T61840: Remove top level dependence on jsDiff

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/391707 )

Change subject: T61840: Remove top level dependence on jsDiff
..

T61840: Remove top level dependence on jsDiff

 * Finishes off the work to switch to simplediff, which is not without
   its own issues, see T134198, but depending on only one diff
   implementation seems like a good.

 * refineDiff is dropped since it's been commented out forever.  Feel
   free to restore it from the git history if needs be.

 * Mocha still uses it so nothing to update in the shrinkwrap.

Change-Id: I013bc79f50ebf46791c1d8d496049f5f7fe8b2e5
---
M bin/roundtrip-test.js
M lib/utils/Diff.js
M package.json
M tests/parserTests.utils.js
4 files changed, 140 insertions(+), 150 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/07/391707/1

diff --git a/bin/roundtrip-test.js b/bin/roundtrip-test.js
index 09d5ad6..f3280c6 100755
--- a/bin/roundtrip-test.js
+++ b/bin/roundtrip-test.js
@@ -438,8 +438,8 @@
// Check if this is really a semantic diff
var oldHTML = getMatchingHTML(oldBody, offset[0], nlDiffs);
var newHTML = getMatchingHTML(newBody, offset[1], nlDiffs);
-   var diff = Diff.htmlDiff(oldHTML, newHTML, false, true, true);
-   if (diff.length > 0) {
+   var diff = Diff.patchDiff(oldHTML, newHTML);
+   if (diff !== null) {
// Normalize wts to check if we really have a semantic 
diff
var wt1 = 
normalizeWikitext(oldWt.substring(offset[0].start, offset[0].end));
var wt2 = 
normalizeWikitext(newWt.substring(offset[1].start, offset[1].end));
diff --git a/lib/utils/Diff.js b/lib/utils/Diff.js
index 6243cae..ede63a3 100644
--- a/lib/utils/Diff.js
+++ b/lib/utils/Diff.js
@@ -1,129 +1,10 @@
 'use strict';
 
 var simpleDiff = require('simplediff');
-var jsDiff = require('diff');
 
 var Util = require('./Util.js').Util;
 
 var Diff = {};
-
-/**
- * Perform word-based diff on a line-based diff. The word-based algorithm is
- * practically unusable for inputs > 5k bytes, so we only perform it on the
- * output of the more efficient line-based diff.
- *
- * @method
- * @param {Array} diff The diff to refine
- * @return {Array} The refined diff
- */
-// var refineDiff = function ( diff ) {
-// // Attempt to accumulate consecutive add-delete pairs
-// // with short text separating them (short = 2 chars right now)
-// //
-// // This is equivalent to the  ...  minimization
-// // to expand range of  and  tags, except there is no optimal
-// // solution except as determined by heuristics ("short text" = <= 2 
chars).
-// function mergeConsecutiveSegments(wordDiffs) {
-// var n = wordDiffs.length,
-// currIns = null, currDel = null,
-// newDiffs = [];
-// for (var i = 0; i < n; i++) {
-// var d = wordDiffs[i],
-// dVal = d.value;
-// if (d.added) {
-// // Attempt to accumulate
-// if (currIns === null) {
-// currIns = d;
-// } else {
-// currIns.value = currIns.value + dVal;
-// }
-// } else if (d.removed) {
-// // Attempt to accumulate
-// if (currDel === null) {
-// currDel = d;
-// } else {
-// currDel.value = currDel.value + dVal;
-// }
-// } else if (((dVal.length < 4) || !dVal.match(/\s/)) && 
currIns && currDel) {
-// // Attempt to accumulate
-// currIns.value = currIns.value + dVal;
-// currDel.value = currDel.value + dVal;
-// } else {
-// // Accumulation ends. Purge!
-// if (currIns !== null) {
-// newDiffs.push(currIns);
-// currIns = null;
-// }
-// if (currDel !== null) {
-// newDiffs.push(currDel);
-// currDel = null;
-// }
-// newDiffs.push(d);
-// }
-// }
-
-// // Purge buffered diffs
-// if (currIns !== null) {
-// newDiffs.push(currIns);
-// }
-// if (currDel !== null) {
-// 

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Sync parserTests with core

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/391570 )

Change subject: Sync parserTests with core
..

Sync parserTests with core

Change-Id: I32928a0a78f2715f43711f9c12a2b7a0ff55232f
---
M tests/parserTests.json
M tests/parserTests.txt
2 files changed, 18 insertions(+), 4 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/70/391570/1

diff --git a/tests/parserTests.json b/tests/parserTests.json
index 302e56c..52f750d 100644
--- a/tests/parserTests.json
+++ b/tests/parserTests.json
@@ -2,8 +2,8 @@
"parserTests.txt": {
"repo": "/wikimedia/mediawiki/",
"path": "tests/parser/parserTests.txt",
-   "expectedSHA1": "e25c2ac3dfd7e954c4dfc6e42b0ec5afacd8205d",
-   "latestCommit": "1b13888ed6bd09731f10045650714a3392bb55df"
+   "expectedSHA1": "9ebf253ac9685ced9ef827c574ca23da8e1ddc47",
+   "latestCommit": "673d788b0f03b62ca3a466bf1ac6832fed6f8038"
},
"citeParserTests.txt": {
"repo": "/wikimedia/mediawiki-extensions-Cite/",
diff --git a/tests/parserTests.txt b/tests/parserTests.txt
index e203702..a505cde 100644
--- a/tests/parserTests.txt
+++ b/tests/parserTests.txt
@@ -15176,9 +15176,9 @@
 !! options
 parsoid=wt2html,wt2wt,html2html
 !! wikitext
-[[File:Foobar.svg|thumb|caption|lang=invalid.language.code]]
+[[File:Foobar.svg|thumb|caption|lang=invalid:language:code]]
 !! html/php
-http://example.com/images/thumb/f/ff/Foobar.svg/180px-Foobar.svg.png; 
width="180" height="135" class="thumbimage" 
srcset="http://example.com/images/thumb/f/ff/Foobar.svg/270px-Foobar.svg.png 
1.5x, http://example.com/images/thumb/f/ff/Foobar.svg/360px-Foobar.svg.png 2x" 
/>  lang=invalid.language.code
+http://example.com/images/thumb/f/ff/Foobar.svg/180px-Foobar.svg.png; 
width="180" height="135" class="thumbimage" 
srcset="http://example.com/images/thumb/f/ff/Foobar.svg/270px-Foobar.svg.png 
1.5x, http://example.com/images/thumb/f/ff/Foobar.svg/360px-Foobar.svg.png 2x" 
/>  lang=invalid:language:code
 
 !! html/parsoid
 lang=invalid.language.code
@@ -18542,6 +18542,20 @@
 !! end
 
 !! test
+Language converter glossary rules inside attributes (T119158)
+!! options
+language=sr variant=sr-el
+!! wikitext
+-{H|foAjrjvi=>sr-el:" onload="alert(1)" data-foo="}-
+
+[[File:Foobar.jpg|alt=-{}-foAjrjvi-{}-]]
+!! html
+
+http://example.com/images/3/3a/Foobar.jpg; width="1941" height="220" />
+
+!! end
+
+!! test
 Self closed html pairs (T7487)
 !! wikitext
 Centered text

-- 
To view, visit https://gerrit.wikimedia.org/r/391570
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I32928a0a78f2715f43711f9c12a2b7a0ff55232f
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki/core[master]: Sync up with Parsoid parserTests.txt

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/391560 )

Change subject: Sync up with Parsoid parserTests.txt
..

Sync up with Parsoid parserTests.txt

This now aligns with Parsoid commit 3048db625dca69d8a89cde4cbabee0105f2975f5

Change-Id: I2bf07d6582367f25d9f69712dc4350982b627851
---
M tests/parser/parserTests.txt
1 file changed, 50 insertions(+), 14 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core 
refs/changes/60/391560/1

diff --git a/tests/parser/parserTests.txt b/tests/parser/parserTests.txt
index 82054c4..a505cde 100644
--- a/tests/parser/parserTests.txt
+++ b/tests/parser/parserTests.txt
@@ -6350,7 +6350,7 @@
 
 !! html/parsoid
 
- Foo!! 
+ Foo!! 
 
 !! end
 
@@ -6626,7 +6626,7 @@
 !! html/parsoid
 
 
-[ftp://%7Cx; 
data-parsoid='{"stx":"url","a":{"href":"ftp://%7Cx"},"sa":{"href":"ftp://|x"}}'>ftp://%7Cx]" 
onmouseover="alert(document.cookie)">test
+[ftp://%7Cx; 
data-parsoid='{"stx":"url","a":{"href":"ftp://%7Cx"},"sa":{"href":"ftp://|x"}}'>ftp://%7Cx]" 
onmouseover="alert(document.cookie)">test
 !! end
 
 !! test
@@ -6683,7 +6683,7 @@
 
 !! html/parsoid
 
- style="color: red !important;" data-contrived="put this here 

 foo
+ style="color: red !important;" data-contrived="put this here 

 foo
 
 !! end
 
@@ -11492,9 +11492,9 @@
 
 !! html/parsoid
 
- testing two 
| three  
four
-
 three  four
- testing="two"
 | three  
four
+ testing two 
| three  four
+
 three  four
+ testing="two"
 | three  
four
 
 !! end
 
@@ -14831,7 +14831,43 @@
 http://example.com/images/thumb/3/3a/Foobar.jpg/180px-Foobar.jpg; 
width="180" height="20" class="thumbimage" 
srcset="http://example.com/images/thumb/3/3a/Foobar.jpg/270px-Foobar.jpg 1.5x, 
http://example.com/images/thumb/3/3a/Foobar.jpg/360px-Foobar.jpg 2x" />  
This is the image 
caption
 
 !! html/parsoid
-This 
is the image caption
+This 
is the image caption
+!! end
+
+!! test
+Image with table with attributes in caption
+!! options
+parsoid=wt2html,html2html
+!! wikitext
+[[File:Foobar.jpg|thumb|
+{| class="123" |
+|- class="456" |
+| ha
+|}
+]]
+!! html/parsoid
+
+
+
+ ha
+
+
+!! end
+
+!! test
+Image with table with rows from templates in caption
+!! wikitext
+[[File:Foobar.jpg|thumb|
+{|
+{{echo|{{!}} hi}}
+|}
+]]
+!! html/parsoid
+
+
+ hi
+
+
 !! end
 
 !! test
@@ -22285,7 +22321,7 @@
 Nested: Hello Hong Kong!
 
 !! html/parsoid
-Nested: !
+Nested: !
 !! end
 
 !! test
@@ -22298,7 +22334,7 @@
 A
 
 !! html/parsoid
-
+
 !! end
 
 !! test
@@ -22311,7 +22347,7 @@
 A
 
 !! html/parsoid
-
+
 !! end
 
 # Parsoid and PHP disagree on how to parse this example: Parsoid
@@ -28319,17 +28355,17 @@
 
 a
 b
-c
+c
 x
-y
+y
 
 
 
 a
 b
-c
+c
 x
-y
+y
 
 !! wikitext
 {|

-- 
To view, visit https://gerrit.wikimedia.org/r/391560
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I2bf07d6582367f25d9f69712dc4350982b627851
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/core
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Move out internal routes

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/391556 )

Change subject: Move out internal routes
..

Move out internal routes

Change-Id: Ib2bcaa1f56663e2cafe91c8c5d27ccf338f28cf8
---
M lib/api/ParsoidService.js
M lib/api/apiUtils.js
A lib/api/internal.js
M lib/api/routes.js
4 files changed, 229 insertions(+), 212 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/56/391556/1

diff --git a/lib/api/ParsoidService.js b/lib/api/ParsoidService.js
index ffb218b..0515c97 100644
--- a/lib/api/ParsoidService.js
+++ b/lib/api/ParsoidService.js
@@ -75,9 +75,6 @@
var workers = workerFarm(farmOptions, require.resolve('../parse.js'));
var promiseWorkers = Promise.promisify(workers);
 
-   // Load routes
-   var routes = require('./routes')(parsoidConfig, processLogger, 
parsoidOptions, promiseWorkers);
-
var app = express();
 
// Default express to production.
@@ -192,9 +189,10 @@
 
// Routes
 
+   var routes = require('./routes')(parsoidConfig, processLogger, 
parsoidOptions, promiseWorkers);
+
var a = routes.acceptable;
var p = routes.parserEnvMw;
-   var i = routes.internal;
var v3 = routes.v3Middle;
 
app.get('/', routes.home);
@@ -202,21 +200,23 @@
app.get('/version', routes.version);
app.get('/_version', routes.version);  // for backwards compat.
 
-   // private routes
-   if (parsoidConfig.devAPI) {
-   app.get('/_html/:prefix?/:title?', i, p, routes.html2wtForm);
-   app.get('/_wikitext/:prefix?/:title?', i, p, 
routes.wt2htmlForm);
-   app.get('/_rt/:prefix?/:title?', i, p, routes.roundtripTesting);
-   app.get('/_rtve/:prefix?/:title?', i, p, 
routes.roundtripTestingNL);
-   app.get('/_rtselser/:prefix?/:title?', i, p, 
routes.roundtripSelser);
-   app.get('/_rtform/:prefix?/:title?', i, p, routes.getRtForm);
-   app.post('/_rtform/:prefix?/:title?', i, p, routes.postRtForm);
-   }
-
// v3 API routes
app.get('/:domain/v3/page/:format/:title/:revision?', v3, p, a, 
routes.v3Get);
app.post('/:domain/v3/transform/:from/to/:format/:title?/:revision?', 
v3, p, a, routes.v3Post);
 
+   // private routes
+   if (parsoidConfig.devAPI) {
+   var internal = require('./internal')(parsoidConfig, 
processLogger);
+   var i = internal.middle;
+   app.get('/_html/:prefix?/:title?', i, p, internal.html2wtForm);
+   app.get('/_wikitext/:prefix?/:title?', i, p, 
internal.wt2htmlForm);
+   app.get('/_rt/:prefix?/:title?', i, p, 
internal.roundtripTesting);
+   app.get('/_rtve/:prefix?/:title?', i, p, 
internal.roundtripTestingNL);
+   app.get('/_rtselser/:prefix?/:title?', i, p, 
internal.roundtripSelser);
+   app.get('/_rtform/:prefix?/:title?', i, p, internal.getRtForm);
+   app.post('/_rtform/:prefix?/:title?', i, p, 
internal.postRtForm);
+   }
+
var server;
return new Promise(function(resolve, reject) {
app.on('error', function(err) {
diff --git a/lib/api/apiUtils.js b/lib/api/apiUtils.js
index a18557b..7f4f530 100644
--- a/lib/api/apiUtils.js
+++ b/lib/api/apiUtils.js
@@ -2,18 +2,15 @@
 
 require('../../core-upgrade.js');
 
-var domino = require('domino');
 var util = require('util');
 var semver = require('semver');
 var qs = require('querystring');
 var cType = require('content-type');
 
-var Diff = require('../utils/Diff.js').Diff;
 var DU = require('../utils/DOMUtils.js').DOMUtils;
 var Util = require('../utils/Util.js').Util;
 var PegTokenizer = require('../wt2html/tokenizer.js').PegTokenizer;
 var PHPParseRequest = require('../mw/ApiRequest.js').PHPParseRequest;
-
 
 /**
  * @class apiUtils
@@ -172,47 +169,6 @@
env.log('info', util.format(
'completed %s in %s ms', str, Date.now() - res.locals.start
));
-};
-
-apiUtils.rtResponse = function(env, req, res, data) {
-   apiUtils.renderResponse(res, 'roundtrip', data);
-   apiUtils.logTime(env, res, 'parsing');
-};
-
-apiUtils.roundTripDiff = function(env, req, res, useSelser, doc) {
-   // Re-parse the HTML to uncover foster-parenting issues
-   doc = domino.createDocument(doc.outerHTML);
-
-   var handler = env.getContentHandler();
-   return handler.fromHTML(env, doc.body, useSelser).then(function(out) {
-   // Strip selser trigger comment
-   out = out.replace(/\n*$/, '');
-
-   // Emit base href so all relative urls resolve properly
-   var headNodes = "";
-   for (var hNode = doc.head.firstChild; hNode; hNode = 
hNode.nextSibling) {
-   if (hNode.nodeName.toLowerCase() === 'base') {
-  

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Move expansion reuse to the env so cpu workers can partake

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/391555 )

Change subject: Move expansion reuse to the env so cpu workers can partake
..

Move expansion reuse to the env so cpu workers can partake

Change-Id: Ib643c208ed30fe103a0995ba75bef3817197f3fc
---
M lib/api/apiUtils.js
M lib/api/routes.js
M lib/config/MWParserEnvironment.js
M lib/parse.js
4 files changed, 48 insertions(+), 35 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/55/391555/1

diff --git a/lib/api/apiUtils.js b/lib/api/apiUtils.js
index 53a0009..d6a1d35 100644
--- a/lib/api/apiUtils.js
+++ b/lib/api/apiUtils.js
@@ -483,44 +483,13 @@
 };
 
 /**
- * See if we can reuse transclusion or extension expansions.
- *
- * @method
- * @param {MWParserEnvironment} env
- */
-apiUtils.reuseExpansions = function(env, revision, updates) {
-   updates = updates || {};
-   var doc = DU.parseHTML(revision.html.body);
-   var pb = apiUtils.extractPageBundle(revision);
-   apiUtils.validatePageBundle(pb, env.originalVersion);
-   DU.applyPageBundle(doc, pb);
-   DU.visitDOM(doc.body, DU.loadDataAttribs);
-   var expansions = DU.extractExpansions(env, doc.body);
-   Object.keys(updates).forEach(function(mode) {
-   switch (mode) {
-   case 'transclusions':
-   case 'media':
-   // Truthy values indicate that these need 
updating,
-   // so don't reuse them.
-   if (updates[mode]) {
-   expansions[mode] = {};
-   }
-   break;
-   default:
-   throw new Error('Received an unexpected update 
mode.');
-   }
-   });
-   env.setCaches(expansions);
-};
-
-/**
  * Downgrade content from 2.x to 1.x
  *
  * @method
  * @param {MWParserEnvironment} env
  * @param {Object} revision
  * @param {Response} res
- * @param [String] contentmodel
+ * @param {String} [contentmodel]
  */
 apiUtils.downgrade2to1 = function(env, revision, res, contentmodel) {
var doc = DU.parseHTML(revision.html.body);
@@ -545,7 +514,7 @@
  * @param {MWParserEnvironment} env
  * @param {Object} revision
  * @param {Response} res
- * @param [String] contentmodel
+ * @param {String} [contentmodel]
  */
 apiUtils.updateRedLinks = function(env, revision, res, contentmodel) {
var doc = DU.parseHTML(revision.html.body);
@@ -569,7 +538,7 @@
  * @param {Object} res
  * @param {String} html
  * @param {Object} pb
- * @param [String] contentmodel
+ * @param {String} [contentmodel]
  */
 apiUtils.wt2htmlRes = function(env, res, html, pb, contentmodel) {
if (pb) {
diff --git a/lib/api/routes.js b/lib/api/routes.js
index 5283765..8036e45 100644
--- a/lib/api/routes.js
+++ b/lib/api/routes.js
@@ -728,7 +728,15 @@
// FIXME(arlolra): Maybe assert(revision === 
opts.original)
return apiUtils.updateRedLinks(env, revision, 
res, contentmodel);
} else {
-   apiUtils.reuseExpansions(env, revision, 
opts.updates);
+   var doc = DU.parseHTML(revision.html.body);
+   var pb = apiUtils.extractPageBundle(revision);
+   apiUtils.validatePageBundle(pb, 
env.originalVersion);
+   DU.applyPageBundle(doc, pb);
+   var reuseExpansions = {
+   updates: opts.updates,
+   html: DU.toXML(doc),
+   };
+   env.reuseExpansions(reuseExpansions);
return wt2html(req, res);
}
} else {
diff --git a/lib/config/MWParserEnvironment.js 
b/lib/config/MWParserEnvironment.js
index 0c1172c..28aa180 100644
--- a/lib/config/MWParserEnvironment.js
+++ b/lib/config/MWParserEnvironment.js
@@ -9,6 +9,7 @@
 var ConfigRequest = require('../mw/ApiRequest.js').ConfigRequest;
 var Batcher = require('../mw/Batcher.js').Batcher;
 var Util = require('../utils/Util.js').Util;
+var DU = require('../utils/DOMUtils.js').DOMUtils;
 var ParserPipelineFactory = 
require('../wt2html/parser.js').ParserPipelineFactory;
 var LintLogger = require('../logger/LintLogger.js').LintLogger;
 var ParsoidLogger = require('../logger/ParsoidLogger.js').ParsoidLogger;
@@ -327,6 +328,36 @@
 /**
  * @method
  *
+ * See if we can reuse transclusion or extension expansions.
+ *
+ * @param {Object} obj
+ * @param {String} obj.html
+ * @param {Object} obj.updates Update mode
+ */
+MWParserEnvironment.prototype.reuseExpansions = function(obj) {
+   

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: T178253: Permit table rows from templates in figure captions

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/391360 )

Change subject: T178253: Permit table rows from templates in figure captions
..

T178253: Permit table rows from templates in figure captions

 * Despite what the large comment introduced in 581d315 says, no table
   line content from templates was allowed, not just at start and end
   positions.

 * Fixes 
http://localhost:8000/ru.wikipedia.org/v3/page/html/%D0%A5%D1%80%D0%BE%D0%BD%D0%BE%D0%BB%D0%BE%D0%B3%D0%B8%D1%8F_%D0%BF%D0%B5%D1%80%D0%B2%D1%8B%D1%85_%D0%BA%D0%BE%D1%81%D0%BC%D0%B8%D1%87%D0%B5%D1%81%D0%BA%D0%B8%D1%85_%D0%B7%D0%B0%D0%BF%D1%83%D1%81%D0%BA%D0%BE%D0%B2_%D0%BF%D0%BE_%D1%81%D1%82%D1%80%D0%B0%D0%BD%D0%B0%D0%BC/86675549

Change-Id: If6de8824828f729ce7226c398d11e8f3a74760b5
---
M lib/wt2html/pegTokenizer.pegjs
M tests/parserTests.txt
2 files changed, 17 insertions(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/60/391360/1

diff --git a/lib/wt2html/pegTokenizer.pegjs b/lib/wt2html/pegTokenizer.pegjs
index 2e6575f..3dfa339 100644
--- a/lib/wt2html/pegTokenizer.pegjs
+++ b/lib/wt2html/pegTokenizer.pegjs
@@ -1735,7 +1735,7 @@
 table_start_tag optionalNewlines
 // Accept multiple end tags since a nested table may have been
 // opened in the table content line.
-((sol table_content_line optionalNewlines)*
+((sol (table_content_line / tplarg_or_template) optionalNewlines)*
 sol table_end_tag)+
 ){
 stops.pop('linkdesc');
diff --git a/tests/parserTests.txt b/tests/parserTests.txt
index 545a628..f17bbc1 100644
--- a/tests/parserTests.txt
+++ b/tests/parserTests.txt
@@ -14855,6 +14855,22 @@
 !! end
 
 !! test
+Image with table with rows from templates in caption
+!! wikitext
+[[File:Foobar.jpg|thumb|
+{|
+{{echo|{{!}} hi}}
+|}
+]]
+!! html/parsoid
+
+
+ hi
+
+
+!! end
+
+!! test
 Image with nested tables in caption
 !! wikitext
 [[File:Foobar.jpg|thumb|Foo

-- 
To view, visit https://gerrit.wikimedia.org/r/391360
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: If6de8824828f729ce7226c398d11e8f3a74760b5
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...deploy[master]: Bump src/ to e71937d0 for deploy

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/391246 )

Change subject: Bump src/ to e71937d0 for deploy
..

Bump src/ to e71937d0 for deploy

Change-Id: If64a72bf79e97a8b023ab90d2e5754bdad3dd180
---
M src
1 file changed, 1 insertion(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid/deploy 
refs/changes/46/391246/1

diff --git a/src b/src
index ab4dc20..e71937d 16
--- a/src
+++ b/src
@@ -1 +1 @@
-Subproject commit ab4dc2094dced1267fb13944cc70bfd5a508a65a
+Subproject commit e71937d03430ea993585b948f1215966b823563e

-- 
To view, visit https://gerrit.wikimedia.org/r/391246
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: If64a72bf79e97a8b023ab90d2e5754bdad3dd180
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid/deploy
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...deploy[master]: Remove unnecessary local information from package.json

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/391087 )

Change subject: Remove unnecessary local information from package.json
..

Remove unnecessary local information from package.json

 * The result of the removeNPMAbsolutePaths utility.

Change-Id: I77ab22494778c0f411527b7f6e626d901d516cbf
---
M node_modules/abbrev/package.json
M node_modules/accepts/node_modules/mime-db/package.json
M node_modules/accepts/node_modules/mime-types/package.json
M node_modules/accepts/package.json
M node_modules/acorn-jsx/node_modules/acorn/package.json
M node_modules/acorn-jsx/package.json
M node_modules/acorn/package.json
M node_modules/ajv-keywords/package.json
M node_modules/ajv/package.json
M node_modules/alea/package.json
M node_modules/amdefine/package.json
M node_modules/ansi-escapes/package.json
M node_modules/ansi-regex/package.json
M node_modules/ansi-styles/package.json
M node_modules/argparse/package.json
M node_modules/array-flatten/package.json
M node_modules/array-union/package.json
M node_modules/array-uniq/package.json
M node_modules/arrify/package.json
M node_modules/asap/package.json
M node_modules/asn1/package.json
M node_modules/assert-plus/package.json
M node_modules/assertion-error/package.json
M node_modules/async/package.json
M node_modules/asynckit/package.json
M node_modules/aws-sign2/package.json
M node_modules/aws4/package.json
M node_modules/babel-code-frame/package.json
M node_modules/babybird/package.json
M node_modules/balanced-match/package.json
M node_modules/bl/node_modules/isarray/package.json
M node_modules/bl/node_modules/readable-stream/package.json
M node_modules/bl/node_modules/string_decoder/package.json
M node_modules/bl/package.json
M node_modules/bluebird/package.json
M node_modules/body-parser/node_modules/content-type/package.json
M node_modules/body-parser/node_modules/debug/package.json
M node_modules/body-parser/node_modules/ms/package.json
M node_modules/body-parser/node_modules/qs/package.json
M node_modules/body-parser/package.json
M node_modules/boom/package.json
M node_modules/brace-expansion/package.json
M node_modules/buffer-shims/package.json
M node_modules/builtin-modules/package.json
M node_modules/bunyan-syslog-udp/package.json
M node_modules/bunyan/package.json
M node_modules/busboy/package.json
M node_modules/bytes/package.json
M node_modules/caller-path/package.json
M node_modules/callsites/package.json
M node_modules/camelcase/package.json
M node_modules/caseless/package.json
M node_modules/chai/package.json
M node_modules/chalk/package.json
M node_modules/circular-json/package.json
M node_modules/clarinet/package.json
M node_modules/cli-cursor/package.json
M node_modules/cli-width/package.json
M node_modules/cli/node_modules/glob/package.json
M node_modules/cli/package.json
M node_modules/cliui/package.json
M node_modules/co/package.json
M node_modules/code-point-at/package.json
M node_modules/colors/package.json
M node_modules/combined-stream/package.json
M node_modules/commander/package.json
M node_modules/comment-parser/node_modules/isarray/package.json
M node_modules/comment-parser/node_modules/readable-stream/package.json
M node_modules/comment-parser/node_modules/string_decoder/package.json
M node_modules/comment-parser/package.json
M node_modules/component-emitter/package.json
M node_modules/compressible/node_modules/mime-db/package.json
M node_modules/compressible/package.json
M node_modules/compression/node_modules/debug/package.json
M node_modules/compression/node_modules/ms/package.json
M node_modules/compression/package.json
M node_modules/concat-map/package.json
M node_modules/concat-stream/node_modules/isarray/package.json
M node_modules/concat-stream/node_modules/readable-stream/package.json
M node_modules/concat-stream/node_modules/string_decoder/package.json
M node_modules/concat-stream/package.json
M node_modules/connect-busboy/package.json
M node_modules/console-browserify/package.json
M node_modules/content-disposition/package.json
M node_modules/content-type/package.json
M node_modules/cookie-signature/package.json
M node_modules/cookie/package.json
M node_modules/cookiejar/package.json
M node_modules/core-js/package.json
M node_modules/core-util-is/package.json
M node_modules/coveralls/node_modules/caseless/package.json
M node_modules/coveralls/node_modules/esprima/package.json
M node_modules/coveralls/node_modules/har-validator/package.json
M node_modules/coveralls/node_modules/js-yaml/package.json
M node_modules/coveralls/node_modules/minimist/package.json
M node_modules/coveralls/node_modules/qs/package.json
M node_modules/coveralls/node_modules/request/package.json
M node_modules/coveralls/node_modules/tunnel-agent/package.json
M node_modules/coveralls/node_modules/uuid/package.json
M node_modules/coveralls/package.json
M node_modules/cryptiles/package.json
M node_modules/d/package.json
M 

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Sync parserTests with core

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/391059 )

Change subject: Sync parserTests with core
..

Sync parserTests with core

 * Also, fix ExtResourceURLPatterns to allow non-proto-relative links to
   serialize as magic links so that the updated tests pass.

Change-Id: I1fcdb9bc58db32bb3f62fc37c27cc0a6aa7508da
---
M lib/config/WikiConfig.js
M lib/wt2html/pegTokenizer.pegjs
M tests/parserTests-blacklist.js
M tests/parserTests.json
M tests/parserTests.txt
5 files changed, 47 insertions(+), 43 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/59/391059/1

diff --git a/lib/config/WikiConfig.js b/lib/config/WikiConfig.js
index 60dadf3..318d2d8 100644
--- a/lib/config/WikiConfig.js
+++ b/lib/config/WikiConfig.js
@@ -850,8 +850,14 @@
prefix: "(?:(?:[.][.]?/)*)",
re: 'Special(?:%3[Aa]|:)Book[Ss]ources(?:%2[Ff]|/)%isbn',
},
-   'RFC': { re: '//tools.ietf.org/html/rfc%s' },
-   'PMID': { re: '//www.ncbi.nlm.nih.gov/pubmed/%s?dopt=Abstract' },
+   'RFC': {
+   prefix: '[^\/]*',
+   re: '//tools.ietf.org/html/rfc%s',
+   },
+   'PMID': {
+   prefix: '[^\/]*',
+   re: '//www.ncbi.nlm.nih.gov/pubmed/%s?dopt=Abstract',
+   },
 };
 
 var unispace = /[ \u00A0\u1680\u2000-\u200A\u202F\u205F\u3000]+/g;
diff --git a/lib/wt2html/pegTokenizer.pegjs b/lib/wt2html/pegTokenizer.pegjs
index c4029cb..c8fc9d7 100644
--- a/lib/wt2html/pegTokenizer.pegjs
+++ b/lib/wt2html/pegTokenizer.pegjs
@@ -628,11 +628,9 @@
   = ref:('RFC' / 'PMID') sp:space_or_nbsp+ identifier:$[0-9]+ end_of_word
 {
 var base_urls = {
-  'RFC': '//tools.ietf.org/html/rfc%s',
+  'RFC': 'https://tools.ietf.org/html/rfc%s',
   'PMID': '//www.ncbi.nlm.nih.gov/pubmed/%s?dopt=Abstract',
 };
-var url = tu.sprintf(base_urls[ref], identifier);
-
 return [
 new SelfclosingTagTk('extlink', [
new KV('href', tu.sprintf(base_urls[ref], identifier)),
diff --git a/tests/parserTests-blacklist.js b/tests/parserTests-blacklist.js
index b541e44..14b908e 100644
--- a/tests/parserTests-blacklist.js
+++ b/tests/parserTests-blacklist.js
@@ -443,7 +443,7 @@
 add("html2html", "Special:Search page linking.", "Special:Search\n");
 add("html2html", "Say the magic word", " 
Parser test\n 
Parser_test\n Parser 
test\n Parser_test\n Parser test\n Parser_test\n Parser test\n Parser_test\n Parser test\n Parser_test\n Talk:Parser test\n Talk:Parser_test\n Parser test\n Parser_test\n \n \n 0\n Talk\n Talk\n \n \n 
[/index.php?title=Template:Dynamicaction=editredlink=1 
Template:Dynamic]\n");
 add("html2html", "Gallery override link with absolute external link with 
LanguageConverter", "\t\t*  
\t\t\thttp://example.com/images/thumb/3/3a/Foobar.jpg/120px-Foobar.jpg\; 
alt=\"120px-Foobar.jpg\" rel=\"mw:externalImage\" 
data-parsoid='{\"dsr\":[106,170,null,null]}'/> \t\t\t 
caption  \t\t\t \t\t\n");
-add("html2html", "Double RFC with a wiki link", "RFC 
[/index.php?title=RFC_1234action=editredlink=1 RFC 
1234]\n");
+add("html2html", "Double RFC with a wiki link", "RFC 
[/index.php?title=RFC_1234action=editredlink=1 https://tools.ietf.org/html/rfc1234\; rel=\"mw:ExtLink\" 
data-parsoid='{\"stx\":\"magiclink\",\"dsr\":[53,61,0,0]}'>RFC 
1234]\n");
 add("html2html", "Centre-aligned image", "http://example.com/images/3/3a/Foobar.jpg\; alt=\"Foobar.jpg\" 
rel=\"mw:externalImage\" 
data-parsoid='{\"dsr\":[43,84,null,null]}'/>\n");
 add("html2html", "None-aligned image", "http://example.com/images/3/3a/Foobar.jpg\; alt=\"Foobar.jpg\" 
rel=\"mw:externalImage\" data-parsoid='{\"dsr\":[23,64,null,null]}'/>\n");
 add("html2html", "Width + Height sized image (using px) (height is ignored)", 
"http://example.com/images/thumb/3/3a/Foobar.jpg/640px-Foobar.jpg\; 
alt=\"640px-Foobar.jpg\" rel=\"mw:externalImage\" 
data-parsoid='{\"dsr\":[0,64,null,null]}'/>\n");
diff --git a/tests/parserTests.json b/tests/parserTests.json
index 78334ff..302e56c 100644
--- a/tests/parserTests.json
+++ b/tests/parserTests.json
@@ -2,8 +2,8 @@
"parserTests.txt": {
"repo": "/wikimedia/mediawiki/",
"path": "tests/parser/parserTests.txt",
-   "expectedSHA1": "81d2ac31b54ef284bc3ac333763e8b7ddb1218bd",
-   "latestCommit": "9ac4ed4b4a28e4f3b42ac3f3b76875e488f25cd9"
+   "expectedSHA1": "e25c2ac3dfd7e954c4dfc6e42b0ec5afacd8205d",
+   "latestCommit": "1b13888ed6bd09731f10045650714a3392bb55df"
},
"citeParserTests.txt": {
"repo": "/wikimedia/mediawiki-extensions-Cite/",
diff --git a/tests/parserTests.txt b/tests/parserTests.txt
index cef935c..ff574d1 100644
--- a/tests/parserTests.txt
+++ b/tests/parserTests.txt
@@ -5706,7 +5706,7 @@
 
 !! html/php
 

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Bump to mediawiki-title@0.6.5

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/391042 )

Change subject: Bump to mediawiki-title@0.6.5
..

Bump to mediawiki-title@0.6.5

 * Bonus, make use of getPrefixedText in one instance.

Change-Id: Ieb21006c281a5be5cea6096f152f281a46e1b768
---
M lib/wt2html/tt/LinkHandler.js
M npm-shrinkwrap.json
M package.json
3 files changed, 5 insertions(+), 6 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/42/391042/1

diff --git a/lib/wt2html/tt/LinkHandler.js b/lib/wt2html/tt/LinkHandler.js
index 691026e..b6d7aa8 100644
--- a/lib/wt2html/tt/LinkHandler.js
+++ b/lib/wt2html/tt/LinkHandler.js
@@ -521,8 +521,7 @@
 
// Add title unless it's just a fragment
if (target.href[0] !== '#') {
-   // Normalize title according to how PHP parser does it currently
-   newTk.setAttribute('title', 
target.title.getPrefixedDBKey().replace(/_/g, ' '));
+   newTk.setAttribute('title', target.title.getPrefixedText());
}
 
cb({ tokens: [newTk].concat(content, [new EndTagTk('a')]) });
diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json
index fc4f4e7..963a753 100644
--- a/npm-shrinkwrap.json
+++ b/npm-shrinkwrap.json
@@ -1686,9 +1686,9 @@
   "resolved": 
"https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz;
 },
 "mediawiki-title": {
-  "version": "0.6.4",
-  "from": "mediawiki-title@0.6.4",
-  "resolved": 
"https://registry.npmjs.org/mediawiki-title/-/mediawiki-title-0.6.4.tgz;
+  "version": "0.6.5",
+  "from": "mediawiki-title@0.6.5",
+  "resolved": 
"https://registry.npmjs.org/mediawiki-title/-/mediawiki-title-0.6.5.tgz;
 },
 "merge": {
   "version": "1.2.0",
diff --git a/package.json b/package.json
index 6ea27ce..244d751 100644
--- a/package.json
+++ b/package.json
@@ -18,7 +18,7 @@
 "express-handlebars": "^3.0.0",
 "finalhandler": "^1.0.6",
 "js-yaml": "^3.8.3",
-"mediawiki-title": "^0.6.4",
+"mediawiki-title": "^0.6.5",
 "negotiator": 
"git+https://github.com/arlolra/negotiator#full-parse-access;,
 "pegjs": "git+https://github.com/tstarling/pegjs#fork;,
 "prfun": "^2.1.4",

-- 
To view, visit https://gerrit.wikimedia.org/r/391042
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ieb21006c281a5be5cea6096f152f281a46e1b768
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Update the rest of the baseconfigs to match arwiki

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/390360 )

Change subject: Update the rest of the baseconfigs to match arwiki
..

Update the rest of the baseconfigs to match arwiki

 * And fix the default `responsiveReferences` value for parserTests
   which didn't make too much sense because it's a per wiki config.
   Keep it disabled by default so we don't need to update all tests.

Change-Id: Ia331429c9eb648481696b6414ad759a8d4742bfb
---
M bin/parserTests.js
M lib/config/baseconfig/arwiki.json
M lib/config/baseconfig/be-taraskwiki.json
M lib/config/baseconfig/cawiki.json
M lib/config/baseconfig/cswiki.json
M lib/config/baseconfig/dewiki.json
M lib/config/baseconfig/enwiki.json
M lib/config/baseconfig/eswiki.json
M lib/config/baseconfig/fawiki.json
M lib/config/baseconfig/fiwiki.json
M lib/config/baseconfig/frwiki.json
M lib/config/baseconfig/iswiki.json
M lib/config/baseconfig/kaawiki.json
M lib/config/baseconfig/lnwiki.json
M lib/config/baseconfig/nlwiki.json
M lib/config/baseconfig/srwiki.json
M lib/config/baseconfig/trwiki.json
M lib/config/baseconfig/zhwiki.json
18 files changed, 3,489 insertions(+), 1,506 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/60/390360/1


-- 
To view, visit https://gerrit.wikimedia.org/r/390360
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ia331429c9eb648481696b6414ad759a8d4742bfb
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki/core[master]: Sync up with Parsoid parserTests.txt

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/390349 )

Change subject: Sync up with Parsoid parserTests.txt
..

Sync up with Parsoid parserTests.txt

This now aligns with Parsoid commit 1d6c39d8f6f5972e72974f8d64e7a0a5c2288bf2

Change-Id: I38d9d47c9cd74257b9bedc892baad90146885ef4
---
M tests/parser/parserTests.txt
1 file changed, 125 insertions(+), 20 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core 
refs/changes/49/390349/1

diff --git a/tests/parser/parserTests.txt b/tests/parser/parserTests.txt
index 9c92da0..cef935c 100644
--- a/tests/parser/parserTests.txt
+++ b/tests/parser/parserTests.txt
@@ -1913,6 +1913,33 @@
 !! end
 
 !! test
+No p-wrappable content
+!! wikitext
+x
+x
+x
+!! html+tidy
+x
+x
+x
+!! html/parsoid
+x
+x
+x
+!! end
+
+# T177612: Parsoid-only test
+!! test
+Transclusion meta tags shouldn't trip Parsoid's useless p-wrapper stripping 
code
+!! wikitext
+{{echo|x}}
+x
+!! html/parsoid
+x
+x
+!! end
+
+!! test
 Block tag on one line ()
 !! wikitext
 a foo
@@ -4806,8 +4833,11 @@
 
 !! end
 
+## html2wt and html2html will fail because we will prefer the :en: interwiki 
prefix over wikipedia:
 !! test
 External links: with no contents
+!! options
+parsoid=wt2html,wt2wt
 !! wikitext
 [http://en.wikipedia.org/wiki/Foo]
 
@@ -5935,11 +5965,11 @@
 !! wikitext
 [[Foo|Bar]]
 [[Foo|Bar]]
-[[wikipedia:Foo|Bar]]
-[[wikipedia:Foo|Bar]]
+[[:en:Foo|Bar]]
+[[:en:Foo|Bar]]
 
-[[wikipedia:European_Robin|European Robin]]
-[[wikipedia:European_Robin|European Robin]]
+[[:en:European_Robin|European Robin]]
+[[:en:European_Robin|European Robin]]
 !! end
 
 !! test
@@ -8518,6 +8548,31 @@
 !! end
 
 !! test
+Parsoid link bracket escaping
+!! options
+parsoid=html2wt,html2html
+!! html/parsoid
+Test
+[Test]
+[[Test]]
+[[[Test]]]
+Test
+[Test]
+!! wikitext
+[[Test]]
+
+[[[Test]]]
+
+Test
+
+[Test]
+
+[[Test]]
+
+[[[Test]]]
+!! end
+
+!! test
 Parsoid-centric test: Whitespace in ext- and wiki-links should be preserved
 !! wikitext
 [[Foo|  bar]]
@@ -8584,8 +8639,11 @@
 http://www.usemod.com/cgi-bin/mb.pl?; 
title="meatball:">MeatBall:
 !! end
 
+## html2wt and html2html will fail because we will prefer the :en: interwiki 
prefix over wikipedia:
 !! test
 Interwiki link encoding conversion (T3636)
+!! options
+parsoid=wt2html,wt2wt
 !! wikitext
 *[[Wikipedia:ro:Oltenia]]
 *[[Wikipedia:ro:Oltenia]]
@@ -8597,6 +8655,11 @@
 
 http://en.wikipedia.org/wiki/ro:Olteni%C5%A3a; class="extiw" 
title="wikipedia:ro:Olteniţa">Wikipedia:ro:Olteniţa
 http://en.wikipedia.org/wiki/ro:Olteni%C5%A3a; class="extiw" 
title="wikipedia:ro:Olteniţa">Wikipedia:ro:Olteniţa
+
+!! html/parsoid
+
+http://en.wikipedia.org/wiki/ro:Olteniţa; 
title="wikipedia:ro:Olteniţa">Wikipedia:ro:Olteniţa
+http://en.wikipedia.org/wiki/ro:Olteniţa; 
title="wikipedia:ro:Olteniţa">Wikipedia:ro:Olteniţa
 
 !! end
 
@@ -9411,7 +9474,7 @@
 !! html/parsoid
 
 
-
+
 
 
 
@@ -11322,6 +11385,15 @@
 !! html/parsoid
 foo
  item 1
+!! end
+
+## Regression test; the output here isn't really that interesting.
+!! test
+Templates with templated name and top level template args
+!! wikitext
+{{1{{2{{{3}}}|4=5
+!! html/parsoid
+{{1{{2{{{3}}}|4=5
 !! end
 
 # Parsoid markup is deliberate "broken". This is an edge case.
@@ -14762,6 +14834,28 @@
 This 
is the image caption
 !! end
 
+!! test
+Image with nested tables in caption
+!! wikitext
+[[File:Foobar.jpg|thumb|Foo
+{|
+|
+{|
+|z
+|}
+|}
+]]
+!! html/parsoid
+Foo
+
+
+
+z
+
+
+
+!! end
+
 ###
 # Conflicting image format options.
 # First option specified should 'win'.
@@ -15615,9 +15709,9 @@
 
 
 !! html/parsoid
-
+
 
-↑  foo
+↑  foo
 !! end
 
 !! test
@@ -15627,9 +15721,9 @@
 
 
 !! html/parsoid
-
+
 
-↑  foo
+↑  foo
 !! end
 
 ###
@@ -18272,18 +18366,16 @@
 ### Sanitizer
 ###
 
-# HTML+Tidy effectively strips out the empty tags completely
-# But since Parsoid doesn't it wraps the  tags in p-tags
-# which Tidy would have done for the PHP parser had there been content inside 
it.
+# HTML+Tidy strips out empty tags completely. Parsoid doesn't.
+# FIXME: Wikitext for this first test doesn't match its title.
 !! test
 Sanitizer: Closing of open tags
 !! wikitext
 
-!! html
-
+!! html/php+tidy
 
 !! html/parsoid
-
+
 !! end
 
 !! test
@@ -19993,7 +20085,7 @@
 '
 !! html/php
 !! html/parsoid
-
+
 !! end
 
 # same html as previous, but wikitext adjusted to match parsoid html2wt
@@ -22298,7 +22390,7 @@
 |}
 !! end
 
-# Tests LanguageVariantText._fromSelser
+# Tests LanguageVariantText._fromSelSer
 !! test
 LanguageConverter selser (4)
 !! options
@@ -22670,6 +22762,21 @@
 abc
 a:b=c 0;zh-tw:bar
 
+!! end
+
+!! test
+T179579: Nowiki and lc interaction
+!! options
+parsoid=wt2html
+language=sr
+!! wikitext
+-{123}-
+
+-{123|456}-
+!! html/parsoid
+
+
+
 !! end
 
 !! test
@@ -24448,9 +24555,7 @@
 !! wikitext
 

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: T178253: Handle pipe ending table attributes in figure captions

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/390348 )

Change subject: T178253: Handle pipe ending table attributes in figure captions
..

T178253: Handle pipe ending table attributes in figure captions

 * Fixes 
http://localhost:8000/nl.wikipedia.org/v3/page/html/Klimaatclassificatie_van_K%C3%B6ppen/50339930

 * This isn't entirely correct, but let's see if we can justify it.  In the
   php parser, `doTableStuff` happens before `replaceInternalLinks` so none of
   the table syntax pipes should ever be breaking for a "linkdesc".

   However, the only place this is really an issue currently are the two
   "table_attributes" instances in "table_row_tag" and "table_start_tag" where
   we don't normally break on pipes, since they're optional because the php
   parser considers the rest of the line as attributes. That permits nonsense
   like,

   {| testing | class="four"
   | ha
   |}

   In the common case, it would be sufficient to just optionally capture pipes
   there at those places with `(spaces* pipe)?` and be done with it.  But this
   more permissive solution seems slightly more robust.

 * An interesting (to me) case is what happens when the suppressed "linkdesc"
   in this patch prevents legitimate breaking, as in,

   [[File:Foobar.jpg|thumb|
   {|
   | hi
 ho | jo
   |}
   ]]

   This manages to parse identically to the php parser because of support for
   templates returning options as pipe-separated strings in `renderFile`.

   But that's pretty broken wikitext.

Change-Id: I88f54399094d21a1a9db769cd46a1258691459a9
---
M lib/wt2html/pegTokenizer.pegjs
M lib/wt2html/tt/LinkHandler.js
M tests/parserTests.txt
3 files changed, 31 insertions(+), 2 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/48/390348/1

diff --git a/lib/wt2html/pegTokenizer.pegjs b/lib/wt2html/pegTokenizer.pegjs
index c4029cb..ac78b39 100644
--- a/lib/wt2html/pegTokenizer.pegjs
+++ b/lib/wt2html/pegTokenizer.pegjs
@@ -1727,7 +1727,11 @@
 full_table_in_link_caption
   = (! inline_breaks / & '{{!}}' )
 r:(
-& { return stops.push('table', true); }
+// Note that "linkdesc" is suppressed here to allow pipes in the
+// `table_start_tag` and `table_row_tag` attributes.  It's perhaps
+// overly permissive but in those cases, the wikitext is likely to
+// be pretty broken to begin with.
+& { stops.push('linkdesc', false); return stops.push('table', true); }
 tbl:(
 table_start_tag optionalNewlines
 // Accept multiple end tags since a nested table may have been
@@ -1735,10 +1739,11 @@
 ((sol table_content_line optionalNewlines)*
 sol table_end_tag)+
 ){
+stops.pop('linkdesc');
 stops.pop('table');
 return tbl;
 }
-  / & { return stops.pop('table'); }
+  / & { stops.pop('linkdesc'); return stops.pop('table'); }
 ) { return r; }
 
 table_lines
diff --git a/lib/wt2html/tt/LinkHandler.js b/lib/wt2html/tt/LinkHandler.js
index 691026e..bef4d56 100644
--- a/lib/wt2html/tt/LinkHandler.js
+++ b/lib/wt2html/tt/LinkHandler.js
@@ -1674,6 +1674,10 @@
// image options as a pipe-separated string. We 
aren't
// really providing editing support for this 
yet, or
// ever, maybe.
+   //
+   // Tables in captions also make use of this as 
a fallback
+   // since they suppress breaking on "linkdesc" 
pipes.  But
+   // that's very broken wikitext that doesn't 
need support.
var pieces = oText.split("|").map(function(s) {
return new KV("mw:maybeContent", s);
});
diff --git a/tests/parserTests.txt b/tests/parserTests.txt
index 178691c..75f161f 100644
--- a/tests/parserTests.txt
+++ b/tests/parserTests.txt
@@ -14835,6 +14835,26 @@
 !! end
 
 !! test
+Image with table with attributes in caption
+!! options
+parsoid=wt2html,html2html
+!! wikitext
+[[File:Foobar.jpg|thumb|
+{| class="123" |
+|- class="456" |
+| ha
+|}
+]]
+!! html/parsoid
+
+
+
+ ha
+
+
+!! end
+
+!! test
 Image with nested tables in caption
 !! wikitext
 [[File:Foobar.jpg|thumb|Foo

-- 
To view, visit https://gerrit.wikimedia.org/r/390348
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I88f54399094d21a1a9db769cd46a1258691459a9
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Improve error reporting in rt

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/390271 )

Change subject: Improve error reporting in rt
..

Improve error reporting in rt

Make this more useful,
https://parsoid-rt-tests.wikimedia.org/resultFlagNew/6cb7710469d9a90174dc914fd5b22ed2f4a02edf/2887b5adf392df1d511ceabf6da6d1c881a50e9a/ruwiki/%D0%9F%D1%81%D0%BA%D0%BE%D0%B2

Change-Id: I484c827fbe852d93b4b90d29ef4b0f37d9dbfa74
---
M lib/utils/Util.js
1 file changed, 1 insertion(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/71/390271/1

diff --git a/lib/utils/Util.js b/lib/utils/Util.js
index 65e2042..412cc08 100644
--- a/lib/utils/Util.js
+++ b/lib/utils/Util.js
@@ -1498,7 +1498,7 @@
.spread(function(res, body) {
if (res.statusCode !== 200) {
throw new Error('Got status code: ' + res.statusCode +
-   '; body: ' + body);
+   '; body: ' + JSON.stringify(body));
}
return Array.from(arguments);
});

-- 
To view, visit https://gerrit.wikimedia.org/r/390271
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I484c827fbe852d93b4b90d29ef4b0f37d9dbfa74
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...deploy[master]: Bump src/ to 2887b5ad + update dependencies

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/390171 )

Change subject: Bump src/ to 2887b5ad + update dependencies
..

Bump src/ to 2887b5ad + update dependencies

 * domino to 2.0.0
 * service-runner to 2.4.0

Change-Id: I7c14819342daacfd2eaa34edcf42e60650b1e67c
---
D node_modules/bl/node_modules/safe-buffer/.travis.yml
D node_modules/bl/node_modules/safe-buffer/LICENSE
D node_modules/bl/node_modules/safe-buffer/README.md
D node_modules/bl/node_modules/safe-buffer/index.js
D node_modules/bl/node_modules/safe-buffer/package.json
D node_modules/bl/node_modules/safe-buffer/test.js
M node_modules/bl/package.json
M node_modules/bluebird/README.md
M node_modules/bluebird/js/browser/bluebird.core.js
M node_modules/bluebird/js/browser/bluebird.core.min.js
M node_modules/bluebird/js/browser/bluebird.js
M node_modules/bluebird/js/browser/bluebird.min.js
M node_modules/bluebird/js/release/debuggability.js
M node_modules/bluebird/js/release/promise.js
M node_modules/bluebird/js/release/util.js
M node_modules/bluebird/package.json
M node_modules/content-type/package.json
M node_modules/domino/.travis.yml
M node_modules/domino/CHANGELOG.md
M node_modules/domino/README.md
M node_modules/domino/lib/CharacterData.js
M node_modules/domino/lib/ChildNode.js
M node_modules/domino/lib/Comment.js
A node_modules/domino/lib/ContainerNode.js
M node_modules/domino/lib/Document.js
M node_modules/domino/lib/DocumentFragment.js
M node_modules/domino/lib/DocumentType.js
M node_modules/domino/lib/Element.js
M node_modules/domino/lib/FilteredElementList.js
M node_modules/domino/lib/Leaf.js
A node_modules/domino/lib/LinkedList.js
M node_modules/domino/lib/Node.js
M node_modules/domino/lib/NonDocumentTypeChildNode.js
M node_modules/domino/lib/ProcessingInstruction.js
M node_modules/domino/lib/Text.js
M node_modules/domino/lib/utils.js
M node_modules/domino/package.json
M node_modules/domino/test/mocha.opts
A node_modules/hot-shots/#CHANGES.md#
M node_modules/hot-shots/CHANGES.md
M node_modules/hot-shots/README.md
M node_modules/hot-shots/lib/statsd.js
A node_modules/hot-shots/package-lock.json
M node_modules/hot-shots/package.json
M node_modules/js-yaml/package.json
D node_modules/kad-fs/node_modules/safe-buffer/.travis.yml
D node_modules/kad-fs/node_modules/safe-buffer/LICENSE
D node_modules/kad-fs/node_modules/safe-buffer/README.md
D node_modules/kad-fs/node_modules/safe-buffer/index.js
D node_modules/kad-fs/node_modules/safe-buffer/package.json
D node_modules/kad-fs/node_modules/safe-buffer/test.js
M node_modules/kad-fs/package.json
D node_modules/kad-memstore/node_modules/safe-buffer/.travis.yml
D node_modules/kad-memstore/node_modules/safe-buffer/LICENSE
D node_modules/kad-memstore/node_modules/safe-buffer/README.md
D node_modules/kad-memstore/node_modules/safe-buffer/index.js
D node_modules/kad-memstore/node_modules/safe-buffer/package.json
D node_modules/kad-memstore/node_modules/safe-buffer/test.js
M node_modules/kad-memstore/package.json
M node_modules/kad/package.json
D node_modules/limitation/node_modules/safe-buffer/.travis.yml
D node_modules/limitation/node_modules/safe-buffer/LICENSE
D node_modules/limitation/node_modules/safe-buffer/README.md
D node_modules/limitation/node_modules/safe-buffer/index.js
D node_modules/limitation/node_modules/safe-buffer/package.json
D node_modules/limitation/node_modules/safe-buffer/test.js
M node_modules/limitation/package.json
D node_modules/msgpack5/.npmignore
M node_modules/msgpack5/lib/encoder.js
D node_modules/msgpack5/node_modules/safe-buffer/.travis.yml
D node_modules/msgpack5/node_modules/safe-buffer/LICENSE
D node_modules/msgpack5/node_modules/safe-buffer/README.md
D node_modules/msgpack5/node_modules/safe-buffer/index.js
D node_modules/msgpack5/node_modules/safe-buffer/package.json
D node_modules/msgpack5/node_modules/safe-buffer/test.js
M node_modules/msgpack5/package.json
A node_modules/msgpack5/test/1-byte-length-uint8arrays.js
A node_modules/msgpack5/test/2-bytes-length-uint8arrays.js
A node_modules/msgpack5/test/4-bytes-length-uint8arrays.js
M node_modules/negotiator/package.json
M node_modules/pegjs/package.json
M node_modules/safe-buffer/package.json
M node_modules/service-runner/lib/worker.js
M node_modules/service-runner/package.json
M node_modules/service-runner/service-runner.js
M node_modules/sshpk/package.json
M src
87 files changed, 1,148 insertions(+), 4,639 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid/deploy 
refs/changes/71/390171/1


-- 
To view, visit https://gerrit.wikimedia.org/r/390171
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I7c14819342daacfd2eaa34edcf42e60650b1e67c
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid/deploy
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Bump to domino@2.0.0

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/390170 )

Change subject: Bump to domino@2.0.0
..

Bump to domino@2.0.0

Bug: T176728
Change-Id: I286d7f90da2b5d417935cc7661709feb3436797e
---
M npm-shrinkwrap.json
M package.json
2 files changed, 4 insertions(+), 4 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/70/390170/1

diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json
index 0d2a3fc..dfb50e0 100644
--- a/npm-shrinkwrap.json
+++ b/npm-shrinkwrap.json
@@ -730,9 +730,9 @@
   "dev": true
 },
 "domino": {
-  "version": "1.0.30",
-  "from": "domino@1.0.30",
-  "resolved": "https://registry.npmjs.org/domino/-/domino-1.0.30.tgz;
+  "version": "2.0.0",
+  "from": "domino@2.0.0",
+  "resolved": "https://registry.npmjs.org/domino/-/domino-2.0.0.tgz;
 },
 "domutils": {
   "version": "1.5.1",
diff --git a/package.json b/package.json
index 95f9bcd..a50c4a1 100644
--- a/package.json
+++ b/package.json
@@ -12,7 +12,7 @@
 "content-type": "git+https://github.com/wikimedia/content-type#master;,
 "core-js": "^2.5.1",
 "diff": "^1.0.7",
-"domino": "^1.0.30",
+"domino": "^2.0.0",
 "entities": "^1.1.1",
 "express": "^4.15.5",
 "express-handlebars": "^3.0.0",

-- 
To view, visit https://gerrit.wikimedia.org/r/390170
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I286d7f90da2b5d417935cc7661709feb3436797e
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: T180010: Broken wikitext for link inside square brackets

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/390045 )

Change subject: T180010: Broken wikitext for link inside square brackets
..

T180010: Broken wikitext for link inside square brackets

Change-Id: I638897ab96b43253dea4bc63e6500285fc6a26ca
---
M lib/html2wt/ConstrainedText.js
M tests/parserTests.txt
2 files changed, 34 insertions(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/45/390045/1

diff --git a/lib/html2wt/ConstrainedText.js b/lib/html2wt/ConstrainedText.js
index cd2a78c..f591dea 100644
--- a/lib/html2wt/ConstrainedText.js
+++ b/lib/html2wt/ConstrainedText.js
@@ -17,6 +17,7 @@
 var util = require('util');
 var DU = require('../utils/DOMUtils.js').DOMUtils;
 var Util = require('../utils/Util.js').Util;
+var JSUtils = require('../utils/jsutils.js').JSUtils;
 
 
 /*
@@ -328,10 +329,17 @@
 var WikiLinkText = function WikiLinkText(text, node, wikiConfig, type) {
// category links/external links/images don't use link trails or 
prefixes
var noTrails = !/^mw:(Wiki|Ext)Link$/.test(type);
+   var noTrailsPrefix = wikiConfig.linkPrefixRegex;
+   if (type === 'mw:WikiLink') {
+   var bracketRegexp = /(^|[^\[])(\[\[)*\[$/;
+   noTrailsPrefix = noTrailsPrefix
+   ? JSUtils.rejoin('(', noTrailsPrefix, ')|(', 
bracketRegexp, ')')
+   : bracketRegexp;
+   }
WikiLinkText.super_.call(this, {
text: text,
node: node,
-   badPrefix: noTrails ? undefined : wikiConfig.linkPrefixRegex,
+   badPrefix: noTrails ? undefined : noTrailsPrefix,
badSuffix: noTrails ? undefined : wikiConfig.linkTrailRegex,
});
// We match link trails greedily when they exist.
diff --git a/tests/parserTests.txt b/tests/parserTests.txt
index 0b5d060..178691c 100644
--- a/tests/parserTests.txt
+++ b/tests/parserTests.txt
@@ -8548,6 +8548,31 @@
 !! end
 
 !! test
+Parsoid link bracket escaping
+!! options
+parsoid=html2wt,html2html
+!! html/parsoid
+Test
+[Test]
+[[Test]]
+[[[Test]]]
+Test
+[Test]
+!! wikitext
+[[Test]]
+
+[[[Test]]]
+
+Test
+
+[Test]
+
+[[Test]]
+
+[[[Test]]]
+!! end
+
+!! test
 Parsoid-centric test: Whitespace in ext- and wiki-links should be preserved
 !! wikitext
 [[Foo|  bar]]

-- 
To view, visit https://gerrit.wikimedia.org/r/390045
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I638897ab96b43253dea4bc63e6500285fc6a26ca
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Update sitematrix

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/389558 )

Change subject: Update sitematrix
..

Update sitematrix

Change-Id: I2af04c6aa0ea98352397d574b0ace773d817c18c
---
M lib/config/sitematrix.json
1 file changed, 9 insertions(+), 2 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/58/389558/1

diff --git a/lib/config/sitematrix.json b/lib/config/sitematrix.json
index f17f76a..c87fde4 100644
--- a/lib/config/sitematrix.json
+++ b/lib/config/sitematrix.json
@@ -5680,7 +5680,7 @@
},
"233": {
"code": "se",
-   "name": "sámegiella",
+   "name": "davvisámegiella",
"site": [
{
"url": "https://se.wikipedia.org;,
@@ -7398,7 +7398,7 @@
"dir": "ltr",
"localname": "Zulu"
},
-   "count": 914,
+   "count": 916,
"specials": [
{
"url": "https://advisory.wikimedia.org;,
@@ -7574,6 +7574,13 @@
"private": ""
},
{
+   "url": "https://electcom.wikimedia.org;,
+   "dbname": "electcomwiki",
+   "code": "electcom",
+   "sitename": "Wikipedia",
+   "private": ""
+   },
+   {
"url": "https://ee.wikimedia.org;,
"dbname": "etwikimedia",
"code": "etwikimedia",

-- 
To view, visit https://gerrit.wikimedia.org/r/389558
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I2af04c6aa0ea98352397d574b0ace773d817c18c
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: T178253: Parse nested tables in figure captions

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/389553 )

Change subject: T178253: Parse nested tables in figure captions
..

T178253: Parse nested tables in figure captions

 * `nested_block_in_table` permits opening table tags so that by
   the time `full_table_in_link_caption` returns from
   `table_content_line` there may be multiple end tags that need
   capturing.  By limiting it to one, subsequent end tag are
   considered figure options and tables end up as bogus captions.

Change-Id: I28c50a503efd02f7eadc2c21323a0c7dae5d1b60
---
M lib/wt2html/pegTokenizer.pegjs
M tests/parserTests.txt
2 files changed, 27 insertions(+), 3 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/53/389553/1

diff --git a/lib/wt2html/pegTokenizer.pegjs b/lib/wt2html/pegTokenizer.pegjs
index 7ab6600..0bf35ea 100644
--- a/lib/wt2html/pegTokenizer.pegjs
+++ b/lib/wt2html/pegTokenizer.pegjs
@@ -1736,9 +1736,11 @@
 & { return stops.push('table', true); }
 tbl:(
 table_start_tag optionalNewlines
-(sol table_content_line optionalNewlines)*
-sol table_end_tag)
-{
+// Accept multiple end tags since a nested table may have been
+// opened in the table content line.
+((sol table_content_line optionalNewlines)*
+sol table_end_tag)+
+){
 stops.pop('table');
 return tbl;
 }
diff --git a/tests/parserTests.txt b/tests/parserTests.txt
index 42693ff..5e30450 100644
--- a/tests/parserTests.txt
+++ b/tests/parserTests.txt
@@ -14809,6 +14809,28 @@
 This 
is the image caption
 !! end
 
+!! test
+Image with nested tables in caption
+!! wikitext
+[[File:Foobar.jpg|thumb|Foo
+{|
+|
+{|
+|z
+|}
+|}
+]]
+!! html/parsoid
+Foo
+
+
+
+z
+
+
+
+!! end
+
 ###
 # Conflicting image format options.
 # First option specified should 'win'.

-- 
To view, visit https://gerrit.wikimedia.org/r/389553
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I28c50a503efd02f7eadc2c21323a0c7dae5d1b60
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...deploy[master]: Bump src/ to 6cb77104 for deploy

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/389526 )

Change subject: Bump src/ to 6cb77104 for deploy
..

Bump src/ to 6cb77104 for deploy

Change-Id: Ic25c924f9c70e822216da0d11185bf4510e0c1fe
---
M src
1 file changed, 1 insertion(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid/deploy 
refs/changes/26/389526/1

diff --git a/src b/src
index 292633c..6cb7710 16
--- a/src
+++ b/src
@@ -1 +1 @@
-Subproject commit 292633c4ce8831b0769a8fd90492313357f15394
+Subproject commit 6cb7710469d9a90174dc914fd5b22ed2f4a02edf

-- 
To view, visit https://gerrit.wikimedia.org/r/389526
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ic25c924f9c70e822216da0d11185bf4510e0c1fe
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid/deploy
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Move some env properties to the prototype

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/388577 )

Change subject: Move some env properties to the prototype
..

Move some env properties to the prototype

Change-Id: I7ff95d881df61e03834a09dfe7322b1fb0d917b8
---
M lib/config/MWParserEnvironment.js
1 file changed, 22 insertions(+), 12 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/77/388577/1

diff --git a/lib/config/MWParserEnvironment.js 
b/lib/config/MWParserEnvironment.js
index 7673582..0c1172c 100644
--- a/lib/config/MWParserEnvironment.js
+++ b/lib/config/MWParserEnvironment.js
@@ -124,17 +124,10 @@
return new Page();
})();
 
+   Object.assign(this, options);
+
// Record time spent in various passes
this.timeProfile = {};
-
-   // A passed-in cookie, if any
-   this.cookie = options.cookie || null;
-
-   // A passed-in request id, if any
-   this.reqId = options.reqId || null;
-
-   // A passed-in user agent, if any
-   this.userAgent = options.userAgent || null;
 
// execution state
this.setCaches({});
@@ -151,9 +144,6 @@
// eliminated so `WikiConfig` can't be instantiated without a 
`resultConf`.
console.assert(parsoidConfig.mwApiMap.has(options.prefix));
this.conf.wiki = new WikiConfig(parsoidConfig, null, options.prefix);
-
-   // Sets ids on nodes and stores data-* attributes in a JSON blob
-   this.pageBundle = false;
 
this.lintLogger = new LintLogger(this);
this.configureLogging();
@@ -685,11 +675,31 @@
 };
 
 /**
+ * A passed-in cookie, if any
+ */
+MWParserEnvironment.prototype.cookie = null;
+
+/**
+ * A passed-in request id, if any
+ */
+MWParserEnvironment.prototype.reqId = null;
+
+/**
+ * A passed-in user agent, if any
+ */
+MWParserEnvironment.prototype.userAgent = null;
+
+/**
  * Apply extra normalizations before serializing DOM.
  */
 MWParserEnvironment.prototype.scrubWikitext = false;
 
 /**
+ * Sets ids on nodes and stores data-* attributes in a JSON blob
+ */
+MWParserEnvironment.prototype.pageBundle = false;
+
+/**
  * Enable editing galleries via HTML, instead of extsrc.
  */
 MWParserEnvironment.prototype.nativeGallery = false;

-- 
To view, visit https://gerrit.wikimedia.org/r/388577
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I7ff95d881df61e03834a09dfe7322b1fb0d917b8
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: T175792: Fix infinite template name expanding

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/388569 )

Change subject: T175792: Fix infinite template name expanding
..

T175792: Fix infinite template name expanding

 * Rendering is now improved for,
   
http://localhost:8000/af.wikipedia.org/v3/page/html/Sjabloon%3AOmreken%2FDuaal%2FKafAafVhEaf/1162058

 * Blacklisted tests are fixed in a follow up.

Change-Id: I28015ff3dbe85ba047477c59efbd0cbbe8a87fbc
---
M lib/wt2html/tt/AttributeExpander.js
M lib/wt2html/tt/TemplateHandler.js
M tests/parserTests-blacklist.js
M tests/parserTests.txt
4 files changed, 41 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/69/388569/1

diff --git a/lib/wt2html/tt/AttributeExpander.js 
b/lib/wt2html/tt/AttributeExpander.js
index 4604d66..a928836 100644
--- a/lib/wt2html/tt/AttributeExpander.js
+++ b/lib/wt2html/tt/AttributeExpander.js
@@ -364,6 +364,27 @@
var rule = nlTkOkay ? 
'generic_newline_attributes' : 'table_attributes';
var kvs  = /=/.test(kStr) ? 
this.tokenizer.tokenizeSync(kStr, rule) : null;
if (kvs) {
+   // At this point, 
templates should have been
+   // expanded.  Returning 
a template token here
+   // probably means that 
when we just converted to
+   // string and reparsed, 
we put back together a
+   // failed expansion.  
This can be particularly bad
+   // when we make 
iterative calls to expand template
+   // names.
+   var convertTemplates = 
function(p) {
+   return 
p.map(function(t) {
+   if 
(!Util.isTemplateToken(t)) { return t; }
+   return 
t.dataAttribs.src;
+   });
+   };
+   
kvs.forEach(function(kv) {
+   if 
(Array.isArray(kv.k)) {
+   kv.k = 
convertTemplates(kv.k);
+   }
+   if 
(Array.isArray(kv.v)) {
+   kv.v = 
convertTemplates(kv.v);
+   }
+   });
// SSS FIXME: Collect 
all keys here, not just the first key
// i.e. in a string 
like {{echo|1=id='v1' title='foo' style='..'}}
// that string is 
setting attributes for [id, title, style], not just id.
diff --git a/lib/wt2html/tt/TemplateHandler.js 
b/lib/wt2html/tt/TemplateHandler.js
index 568dc55..926aceb 100644
--- a/lib/wt2html/tt/TemplateHandler.js
+++ b/lib/wt2html/tt/TemplateHandler.js
@@ -80,6 +80,14 @@
this.ae.onToken(token, frame, function(ret) {
if (ret.tokens) {
// Force reprocessing of the token by demoting 
its rank.
+   //
+   // Note that there's some hacky code in the 
attribute expander
+   // to try and prevent it from returning 
templates in the
+   // expanded attribs.  Otherwise, we can find 
outselves in a loop
+   // here, where `hasTemplateToken` continuously 
returns true.
+   //
+   // That was happening when a template name 
depending on a top
+   // level templatearg failed to expand.
ret.tokens.rank = th.rank - 0.0001;
}
cb(ret);
diff --git a/tests/parserTests-blacklist.js b/tests/parserTests-blacklist.js
index 187710b..c793e25 100644
--- a/tests/parserTests-blacklist.js
+++ b/tests/parserTests-blacklist.js
@@ -104,6 +104,7 @@
 add("wt2html", "Template with invalid target containing 

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Shift newly parsed KVs by the old attribute offset

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/388570 )

Change subject: Shift newly parsed KVs by the old attribute offset
..

Shift newly parsed KVs by the old attribute offset

Change-Id: I6859bf705dbe09e53e69099ac4980b8d091a2516
---
M lib/wt2html/tt/AttributeExpander.js
M tests/parserTests-blacklist.js
2 files changed, 10 insertions(+), 3 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/70/388570/1

diff --git a/lib/wt2html/tt/AttributeExpander.js 
b/lib/wt2html/tt/AttributeExpander.js
index a928836..67e858a 100644
--- a/lib/wt2html/tt/AttributeExpander.js
+++ b/lib/wt2html/tt/AttributeExpander.js
@@ -384,6 +384,16 @@
if 
(Array.isArray(kv.v)) {
kv.v = 
convertTemplates(kv.v);
}
+   // Shift these 
by the old amount
+   if 
(Array.isArray(expandedA.srcOffsets)) {
+   var 
offset = expandedA.srcOffsets[0];
+   if 
(Array.isArray(kv.srcOffsets)) {
+   
kv.srcOffsets = kv.srcOffsets.map(function(n) {
+   
n += offset;
+   
return n;
+   
});
+   }
+   }
});
// SSS FIXME: Collect 
all keys here, not just the first key
// i.e. in a string 
like {{echo|1=id='v1' title='foo' style='..'}}
diff --git a/tests/parserTests-blacklist.js b/tests/parserTests-blacklist.js
index c793e25..187710b 100644
--- a/tests/parserTests-blacklist.js
+++ b/tests/parserTests-blacklist.js
@@ -104,7 +104,6 @@
 add("wt2html", "Template with invalid target containing tags", "{{ab|foo|a=b|a
 =b}}");
 add("wt2html", "Template with invalid target containing unclosed tag", "{{a|foo|a=b|a
 =b}}");
 add("wt2html", "Template with invalid target containing wikilink", "{{Main Page}}");
-add("wt2html", "Templates with templated name and top level template args", 
"{{1{{2{{{3}}}|4=5");
 add("wt2html", "Template with complex arguments", "link This is a test template with parameter (test)");
 add("wt2html", "Template infinite loop", "");
 add("wt2html", "int keyword", "Parser function 
implementation for pf_int missing in Parsoid.");
@@ -251,7 +250,6 @@
 add("wt2wt", "Template with invalid target containing wikilink", 
"{{[[Main Page]]}}\n");
 add("wt2wt", "Template with just whitespace in it, T70421", "{{echo|{{ 
");
 add("wt2wt", "Template with argument in separate line", "{{templateasargtest  
|\n templ = simple }}");
-add("wt2wt", "Templates with templated name and top level template args", 
"1{{2{{{3}}}|4=5}}");
 add("wt2wt", "T2553: link with two variables in a piped link", 
"{|\n|[[{{{1}}}|{{{2}}}]]\n|}");
 add("wt2wt", "Abort table cell attribute parsing on wikilink", "{|\n| testing 
[[one|two]] | three || four\n| testing one two | three || 
four\n| testing=\"[[one|two]]\" | three || four\n|}");
 add("wt2wt", "2. includeonly in html attr value", "v1v2\">bar\nv1v2\">bar\n");
@@ -373,7 +371,6 @@
 add("html2html", "Template with just whitespace in it, T70421", "{{echo|{{");
 add("html2html", "Template unnamed parameter", "the main page\n");
 add("html2html", "Templates with templated name", "foo\n{inner list}");
-add("html2html", "Templates with templated name and top level template args", 
"{{1{{2{{{3}}}|4=5");
 add("html2html", "Templates with invalid templated targets", "{{echo\nfoo}}");
 add("html2html", "Template with complex arguments", "link
 This is a test template with parameter (test)\n");
 add("html2html", "T2553: link with two variables in a piped link", "\n[[{{{1}}}|{{{2}}}]]\n\n");

-- 
To view, visit https://gerrit.wikimedia.org/r/388570
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I6859bf705dbe09e53e69099ac4980b8d091a2516
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: [WIP] Bump service-runner to latest

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/388270 )

Change subject: [WIP] Bump service-runner to latest
..

[WIP] Bump service-runner to latest

Change-Id: Icb44e2c144bdd5fc256a79f067ef44682f2e405e
---
M npm-shrinkwrap.json
M package.json
2 files changed, 16 insertions(+), 41 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/70/388270/1

diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json
index 0d2a3fc..6c38325 100644
--- a/npm-shrinkwrap.json
+++ b/npm-shrinkwrap.json
@@ -179,18 +179,13 @@
   "dependencies": {
 "isarray": {
   "version": "1.0.0",
-  "from": "isarray@>=1.0.0 <1.1.0",
+  "from": "isarray@~1.0.0",
   "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz;
 },
 "readable-stream": {
   "version": "2.3.3",
   "from": "readable-stream@^2.0.5",
   "resolved": 
"https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.3.tgz;
-},
-"safe-buffer": {
-  "version": "5.1.1",
-  "from": "safe-buffer@~5.1.1",
-  "resolved": 
"https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz;
 },
 "string_decoder": {
   "version": "1.0.3",
@@ -200,9 +195,9 @@
   }
 },
 "bluebird": {
-  "version": "3.5.0",
-  "from": "bluebird@>=3.4.7 <4.0.0",
-  "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.0.tgz;
+  "version": "3.5.1",
+  "from": "bluebird@>=3.5.0 <4.0.0",
+  "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz;
 },
 "body-parser": {
   "version": "1.18.2",
@@ -1266,9 +1261,9 @@
   "resolved": 
"https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.5.0.tgz;
 },
 "hot-shots": {
-  "version": "4.7.0",
+  "version": "4.8.0",
   "from": "hot-shots@>=4.4.0 <5.0.0",
-  "resolved": "https://registry.npmjs.org/hot-shots/-/hot-shots-4.7.0.tgz;
+  "resolved": "https://registry.npmjs.org/hot-shots/-/hot-shots-4.8.0.tgz;
 },
 "htmlparser2": {
   "version": "3.8.3",
@@ -1500,7 +1495,7 @@
 },
 "js-yaml": {
   "version": "3.8.3",
-  "from": "js-yaml@>=3.8.1 <4.0.0",
+  "from": "js-yaml@3.8.3",
   "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.8.3.tgz;
 },
 "jshint": {
@@ -1580,11 +1575,6 @@
   "from": "readable-stream@^2.0.4",
   "resolved": 
"https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.3.tgz;
 },
-"safe-buffer": {
-  "version": "5.1.1",
-  "from": "safe-buffer@>=5.1.1 <5.2.0",
-  "resolved": 
"https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz;
-},
 "string_decoder": {
   "version": "1.0.3",
   "from": "string_decoder@>=1.0.3 <1.1.0",
@@ -1604,18 +1594,13 @@
   "dependencies": {
 "isarray": {
   "version": "1.0.0",
-  "from": "isarray@>=1.0.0 <1.1.0",
+  "from": "isarray@~1.0.0",
   "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz;
 },
 "readable-stream": {
   "version": "2.3.3",
   "from": "readable-stream@^2.0.5",
   "resolved": 
"https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.3.tgz;
-},
-"safe-buffer": {
-  "version": "5.1.1",
-  "from": "safe-buffer@~5.1.1",
-  "resolved": 
"https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz;
 },
 "string_decoder": {
   "version": "1.0.3",
@@ -1648,18 +1633,13 @@
   "dependencies": {
 "isarray": {
   "version": "1.0.0",
-  "from": "isarray@>=1.0.0 <1.1.0",
+  "from": "isarray@~1.0.0",
   "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz;
 },
 "readable-stream": {
   "version": "2.3.3",
   "from": "readable-stream@>=2.0.5 <3.0.0",
   "resolved": 
"https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.3.tgz;
-},
-"safe-buffer": {
-  "version": "5.1.1",
-  "from": "safe-buffer@~5.1.1",
-  "resolved": 
"https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz;
 },
 "string_decoder": {
   "version": "1.0.3",
@@ -1820,24 +1800,19 @@
   "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.2.tgz;
 },
 "msgpack5": {
-  "version": "3.5.1",
+  "version": "3.6.0",
   "from": "msgpack5@>=3.3.0 <4.0.0",
-  "resolved": "https://registry.npmjs.org/msgpack5/-/msgpack5-3.5.1.tgz;,
+  "resolved": "https://registry.npmjs.org/msgpack5/-/msgpack5-3.6.0.tgz;,
   "dependencies": {
 "isarray": {
   "version": "1.0.0",
-  "from": "isarray@>=1.0.0 <1.1.0",
+ 

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: [WIP] T179579: Fix nowiki in LC markup tokenizing

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/388157 )

Change subject: [WIP] T179579: Fix nowiki in LC markup tokenizing
..

[WIP] T179579: Fix nowiki in LC markup tokenizing

 * Two issues here: "lang_variant_flag" is eating nowikis, and
   "nowiki_text" doesn't handle the possibility of being returned
   an EndTagTk.

 * Fixes 
http://localhost:8000/sr.wikipedia.org/v3/page/html/%D0%90%D1%83%D1%80%D0%B8%D0%BA%D0%B0%D0%BB%D1%86%D0%B8%D1%82/10716583

Change-Id: Iff6edae49b0cb6e99cddf2f356c0a452d514123e
---
M lib/wt2html/pegTokenizer.pegjs
1 file changed, 8 insertions(+), 3 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/57/388157/1

diff --git a/lib/wt2html/pegTokenizer.pegjs b/lib/wt2html/pegTokenizer.pegjs
index 8dad166..7b53e0d 100644
--- a/lib/wt2html/pegTokenizer.pegjs
+++ b/lib/wt2html/pegTokenizer.pegjs
@@ -1159,7 +1159,7 @@
 lang_variant_flag
   = f:[-+A-Z]   { return { flag: f }; }
   / v:lang_variant_name { return { variant: v }; }
-  / b:(!space_or_newline [^{}|;])+ { return { bogus: b.join('') }; /* bad flag 
*/}
+  / b:(!space_or_newline !nowiki [^{}|;])+ { return { bogus: b.join('') }; /* 
bad flag */}
 
 lang_variant_name // language variant name, like zh, zh-cn, etc.
   = h:[a-z] t:[-a-z]+ { return h + t.join(''); }
@@ -1379,8 +1379,13 @@
 // conversion strings.
 nowiki_text
   = extToken:nowiki {
-var txt = Util.getArgInfo(extToken).dict.body.extsrc;
-return Util.decodeEntities(txt);
+if (extToken.constructor === EndTagTk) {
+// An unmatched closing tag
+return extToken;
+} else {
+var txt = Util.getArgInfo(extToken).dict.body.extsrc;
+return Util.decodeEntities(txt);
+}
   }
 
 /* Generic XML-like tags

-- 
To view, visit https://gerrit.wikimedia.org/r/388157
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Iff6edae49b0cb6e99cddf2f356c0a452d514123e
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Be explicit about returning contentmodel

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/387802 )

Change subject: Be explicit about returning contentmodel
..

Be explicit about returning contentmodel

 * Don't rely on the environment being present.

Change-Id: I2de9e3563bc730f3ba36ee0cf5f082c36d5a4a3c
---
M lib/api/apiUtils.js
M lib/api/routes.js
2 files changed, 14 insertions(+), 17 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/02/387802/1

diff --git a/lib/api/apiUtils.js b/lib/api/apiUtils.js
index b1129fa..c645192 100644
--- a/lib/api/apiUtils.js
+++ b/lib/api/apiUtils.js
@@ -527,8 +527,9 @@
  * @param {MWParserEnvironment} env
  * @param {Object} revision
  * @param {Response} res
+ * @param [String] contentmodel
  */
-apiUtils.downgrade2to1 = function(env, revision, res) {
+apiUtils.downgrade2to1 = function(env, revision, res, contentmodel) {
var doc = DU.parseHTML(revision.html.body);
var pb = apiUtils.extractPageBundle(revision);
apiUtils.validatePageBundle(pb, env.originalVersion);
@@ -541,7 +542,7 @@
var html = DU.toXML(res.locals.bodyOnly ? doc.body : doc, {
innerXML: res.locals.bodyOnly,
});
-   apiUtils.wt2htmlRes(env, res, html, pb);
+   apiUtils.wt2htmlRes(env, res, html, pb, contentmodel);
 };
 
 /**
@@ -551,8 +552,9 @@
  * @param {MWParserEnvironment} env
  * @param {Object} revision
  * @param {Response} res
+ * @param [String] contentmodel
  */
-apiUtils.updateRedLinks = function(env, revision, res) {
+apiUtils.updateRedLinks = function(env, revision, res, contentmodel) {
var doc = DU.parseHTML(revision.html.body);
var pb = apiUtils.extractPageBundle(revision);
apiUtils.validatePageBundle(pb, env.originalVersion);
@@ -562,7 +564,7 @@
var html = DU.toXML(res.locals.bodyOnly ? doc.body : doc, {
innerXML: res.locals.bodyOnly,
});
-   apiUtils.wt2htmlRes(env, res, html, pb);
+   apiUtils.wt2htmlRes(env, res, html, pb, contentmodel);
});
 };
 
@@ -574,11 +576,12 @@
  * @param {Object} res
  * @param {String} html
  * @param {Object} pb
+ * @param [String] contentmodel
  */
-apiUtils.wt2htmlRes = function(env, res, html, pb) {
+apiUtils.wt2htmlRes = function(env, res, html, pb, contentmodel) {
if (pb) {
var response = {
-   contentmodel: env.page.meta.revision.contentmodel,
+   contentmodel: contentmodel,
html: {
headers: { 'content-type': 
apiUtils.htmlContentType(env) },
body: html,
diff --git a/lib/api/routes.js b/lib/api/routes.js
index 5d143f8..5283765 100644
--- a/lib/api/routes.js
+++ b/lib/api/routes.js
@@ -541,7 +541,8 @@
if (opts.format === 'lint') {
apiUtils.jsonResponse(res, 
env.lintLogger.buffer);
} else {
-   apiUtils.wt2htmlRes(env, res, out.html, 
out.pb);
+   var contentmodel = 
env.page.meta.revision.contentmodel;
+   apiUtils.wt2htmlRes(env, res, out.html, 
out.pb, contentmodel);
env.log('end/response');  // Flush log 
buffer for linter
}
 
@@ -707,14 +708,7 @@
return apiUtils.fatalRequest(env, 'Content-type of 
revision html is missing.', 400);
}
 
-   // Set the contentmodel here for downgrades.
-   // Reuse will overwrite it when setting the src.
-   if (!env.page.meta) {
-   env.page.meta = { revision: {} };
-   }
-   env.page.meta.revision.contentmodel =
-   (revision && revision.contentmodel) ||
-   env.page.meta.revision.contentmodel;
+   var contentmodel = (revision && revision.contentmodel);
 
// TODO(arlolra): subbu has some sage advice in T114413#2365456 
that
// we should probably be more explicit about the pb2pb 
conversion
@@ -724,7 +718,7 @@
if (revision === opts.original &&  // Maybe provide a stronger 
assertion.
semver.satisfies(env.contentVersion, '^1.0.0') 
&&
semver.satisfies(env.originalVersion, 
'^2.0.0')) {
-   return apiUtils.downgrade2to1(env, revision, res);
+   return apiUtils.downgrade2to1(env, revision, res, 
contentmodel);
// No reuse from semantically different content versions.
} else if (semver.satisfies(env.contentVersion, '^' + 
env.originalVersion)) {

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Rename property s/str/html/ in xmlserializer return value

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/387801 )

Change subject: Rename property s/str/html/ in xmlserializer return value
..

Rename property s/str/html/ in xmlserializer return value

Change-Id: I23f755b351d0e77e2ede82e19c26675af59c97a6
---
M lib/api/routes.js
M lib/utils/DOMUtils.js
M lib/wt2html/XMLSerializer.js
3 files changed, 18 insertions(+), 17 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/01/387801/1

diff --git a/lib/api/routes.js b/lib/api/routes.js
index 82ae566..5d143f8 100644
--- a/lib/api/routes.js
+++ b/lib/api/routes.js
@@ -525,26 +525,27 @@
})
// .timeout(REQ_TIMEOUT)
.then(function(doc) {
-   var html, pb;
+   var out;
if (pageBundle) {
-   var out = 
DU.extractDpAndSerialize(res.locals.bodyOnly ? doc.body : doc, {
+   out = 
DU.extractDpAndSerialize(res.locals.bodyOnly ? doc.body : doc, {
innerXML: res.locals.bodyOnly,
});
-   html = out.str;
-   pb = out.pb;
} else {
-   html = DU.toXML(res.locals.bodyOnly ? 
doc.body : doc, {
-   innerXML: res.locals.bodyOnly,
-   });
+   out = {
+   html: 
DU.toXML(res.locals.bodyOnly ? doc.body : doc, {
+   innerXML: 
res.locals.bodyOnly,
+   }),
+   };
}
 
if (opts.format === 'lint') {
apiUtils.jsonResponse(res, 
env.lintLogger.buffer);
} else {
-   apiUtils.wt2htmlRes(env, res, html, pb);
+   apiUtils.wt2htmlRes(env, res, out.html, 
out.pb);
env.log('end/response');  // Flush log 
buffer for linter
}
 
+   var html = out.html;
if (metrics) {
if 
(startTimers.has('wt2html.wt.parse')) {

metrics.endTiming('wt2html.wt.parse',
diff --git a/lib/utils/DOMUtils.js b/lib/utils/DOMUtils.js
index a2db0eb..042fd2b 100644
--- a/lib/utils/DOMUtils.js
+++ b/lib/utils/DOMUtils.js
@@ -2235,7 +2235,7 @@
  * @return {string}
  */
 DOMUtils.toXML = function(node, options) {
-   return XMLSerializer.serialize(node, options).str;
+   return XMLSerializer.serialize(node, options).html;
 };
 
 /**
diff --git a/lib/wt2html/XMLSerializer.js b/lib/wt2html/XMLSerializer.js
index 2c2a993..5f0ac2a 100644
--- a/lib/wt2html/XMLSerializer.js
+++ b/lib/wt2html/XMLSerializer.js
@@ -172,9 +172,9 @@
}
 
if (DU.isBody(node)) {
-   out.str += bit;
+   out.html += bit;
if (flag === 'start') {
-   out.start = out.str.length;
+   out.start = out.html.length;
} else if (flag === 'end') {
out.start = null;
out.uid = null;
@@ -183,7 +183,7 @@
// In case you're wondering, out.start may never be set if body
// isn't a child of the node passed to serializeToString, or if 
it
// is the node itself but options.innerXML is true.
-   out.str += bit;
+   out.html += bit;
if (out.uid !== null) {
out.offsets[out.uid].html[1] += bit.length;
}
@@ -205,10 +205,10 @@
}
console.assert(out.uid !== null);
if (!out.offsets.hasOwnProperty(out.uid)) {
-   var dt = out.str.length - out.start;
+   var dt = out.html.length - out.start;
out.offsets[out.uid] = { html: [dt, dt] };
}
-   out.str += bit;
+   out.html += bit;
out.offsets[out.uid].html[1] += bit.length;
}
 };
@@ -234,9 +234,9 @@
if (node.nodeName === '#document') {
node = node.documentElement;
}
-   var out = { str: '', start: null, offsets: {}, uid: null, last: null };
+   var out = { html: '', start: null, offsets: {}, uid: null, 

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Use scriptpath instead of wgScriptPath

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/387684 )

Change subject: Use scriptpath instead of wgScriptPath
..

Use scriptpath instead of wgScriptPath

 * Also, get rid of this as a cli argument since its usecase seems
   long gone.

Change-Id: I2c8771d25c27581b8ef160f77fb206e254b8f1ef
---
M bin/parse.js
M bin/parserTests.js
M lib/wt2html/tt/ParserFunctions.js
3 files changed, 2 insertions(+), 9 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/84/387684/1

diff --git a/bin/parse.js b/bin/parse.js
index 4988209..effdc81 100755
--- a/bin/parse.js
+++ b/bin/parse.js
@@ -259,11 +259,6 @@
env.bumpTimeUse("Init", initTime);
}
 
-   // fetch templates from enwiki by default.
-   if (argv.wgScriptPath) {
-   env.conf.wiki.wgScriptPath = argv.wgScriptPath;
-   }
-
// Enable wikitext scrubbing
env.scrubWikitext = argv.scrubWikitext;
 
diff --git a/bin/parserTests.js b/bin/parserTests.js
index 01e2ac5..e855292 100755
--- a/bin/parserTests.js
+++ b/bin/parserTests.js
@@ -1443,7 +1443,7 @@
wikiConf.fakeTimestamp = 123;
wikiConf.timezoneOffset = 0; // force utc for parsertests
wikiConf.server = 'http://example.org';
-   wikiConf.wgScriptPath = '/';
+   wikiConf.scriptpath = '/';
wikiConf.script = '/index.php';
wikiConf.articlePath = '/wiki/$1';
wikiConf.interwikiMap.clear();
diff --git a/lib/wt2html/tt/ParserFunctions.js 
b/lib/wt2html/tt/ParserFunctions.js
index ca86752..4304d2a 100644
--- a/lib/wt2html/tt/ParserFunctions.js
+++ b/lib/wt2html/tt/ParserFunctions.js
@@ -704,8 +704,6 @@
}
cb({
tokens: [
-   // FIXME! Figure out correct prefix to 
use
-   // this.env.conf.wiki.wgScriptPath +
env.conf.wiki.script + '?title=' +
env.normalizedTitleKey(target) + '&' +
expandedArgs.join('&'),
@@ -845,7 +843,7 @@
cb({ tokens: [this.env.page.name || ''] });
 };
 ParserFunctions.prototype.pf_scriptpath = function(token, frame, cb, args) {
-   cb({ tokens: [this.env.conf.wiki.wgScriptPath] });
+   cb({ tokens: [this.env.conf.wiki.scriptpath] });
 };
 ParserFunctions.prototype.pf_server = function(token, frame, cb, args) {
var dataAttribs = Util.clone(token.dataAttribs);

-- 
To view, visit https://gerrit.wikimedia.org/r/387684
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I2c8771d25c27581b8ef160f77fb206e254b8f1ef
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: [WIP] Bring back request timeouts

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/387458 )

Change subject: [WIP] Bring back request timeouts
..

[WIP] Bring back request timeouts

Change-Id: I39761bff7679a3cfbbde27951a6f094043a67c6b
---
M lib/api/ParsoidService.js
M lib/api/apiUtils.js
M lib/api/routes.js
3 files changed, 13 insertions(+), 43 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/58/387458/1

diff --git a/lib/api/ParsoidService.js b/lib/api/ParsoidService.js
index 5944e9d..48f37e6 100644
--- a/lib/api/ParsoidService.js
+++ b/lib/api/ParsoidService.js
@@ -63,6 +63,7 @@
var farmOptions = {
maxConcurrentWorkers: numWorkers,
maxConcurrentCallsPerWorker: 1,
+   maxCallTime: parsoidConfig.timeouts.request,
// Crashes will retry, timeouts won't, as far as testing showed,
// but it's documented differently.  Anyways, we don't want 
retries.
maxRetries: 0,
@@ -191,7 +192,6 @@
var a = routes.acceptable;
var p = routes.parserEnvMw;
var i = routes.internal;
-   var u = routes.updateActiveRequests;
var v3 = routes.v3Middle;
 
app.get('/', routes.home);
@@ -211,8 +211,8 @@
}
 
// v3 API routes
-   app.get('/:domain/v3/page/:format/:title/:revision?', v3, u, p, a, 
routes.v3Get);
-   app.post('/:domain/v3/transform/:from/to/:format/:title?/:revision?', 
v3, u, p, a, routes.v3Post);
+   app.get('/:domain/v3/page/:format/:title/:revision?', v3, p, a, 
routes.v3Get);
+   app.post('/:domain/v3/transform/:from/to/:format/:title?/:revision?', 
v3, p, a, routes.v3Post);
 
var server;
return new Promise(function(resolve, reject) {
diff --git a/lib/api/apiUtils.js b/lib/api/apiUtils.js
index 207b074..fbc7e62 100644
--- a/lib/api/apiUtils.js
+++ b/lib/api/apiUtils.js
@@ -12,7 +12,6 @@
 var DU = require('../utils/DOMUtils.js').DOMUtils;
 var Util = require('../utils/Util.js').Util;
 var PegTokenizer = require('../wt2html/tokenizer.js').PegTokenizer;
-var Promise = require('../utils/promise.js');
 var PHPParseRequest = require('../mw/ApiRequest.js').PHPParseRequest;
 
 
@@ -162,7 +161,7 @@
  * @param {Error} err
  */
 apiUtils.timeoutResp = function(env, err) {
-   if (err instanceof Promise.TimeoutError) {
+   if (err.type === 'TimeoutError') {
err = new Error('Request timed out.');
err.suppressLoggingStack = true;
}
diff --git a/lib/api/routes.js b/lib/api/routes.js
index 1355d1c..33d2dba 100644
--- a/lib/api/routes.js
+++ b/lib/api/routes.js
@@ -4,7 +4,6 @@
 
 var childProcess = require('child_process');
 var corepath = require('path');
-var uuid = require('uuid/v4');
 var Negotiator = require('negotiator');
 var semver = require('semver');
 
@@ -22,7 +21,6 @@
 module.exports = function(parsoidConfig, processLogger, parsoidOptions, 
promiseWorkers) {
var routes = {};
var metrics = parsoidConfig.metrics;
-   var REQ_TIMEOUT = parsoidConfig.timeouts.request;
 
// This helper is only to be used in middleware, before an environment
// is setup.  The logger doesn't emit the expected location info.
@@ -138,36 +136,6 @@
}
 
res.locals.opts = opts;
-   next();
-   };
-
-   var activeRequests = new Map();
-   routes.updateActiveRequests = function(req, res, next) {
-   var buf = new Buffer(16);
-   uuid(null, buf);
-   var id = buf.toString('hex');
-   var location = res.locals.iwp + '/' + res.locals.pageName +
-   (res.locals.oldid ? '?oldid=' + res.locals.oldid : '');
-   activeRequests.set(id, {
-   location: location,
-   timeout: setTimeout(function() {
-   // This is pretty harsh but was, in effect, 
what we were doing
-   // before with the cpu timeouts.  Shoud be 
removed with
-   // T123446 and T110961.
-   processLogger.log('fatal', 'Timed out 
processing: ' + location);
-   // `processLogger` is async; give it some time 
to deliver the msg.
-   setTimeout(function() { process.exit(1); }, 
100);
-   }, REQ_TIMEOUT),
-   });
-   var current = [];
-   activeRequests.forEach(function(val) {
-   current.push(val.location);
-   });
-   process.emit('service_status', current);
-   res.once('finish', function() {
-   clearTimeout(activeRequests.get(id).timeout);
-   activeRequests.delete(id);
-   });
next();
};
 
@@ 

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Fix page title normalization when oldid is supplied

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/387443 )

Change subject: Fix page title normalization when oldid is supplied
..

Fix page title normalization when oldid is supplied

 * This was introduced in 6d704298

Change-Id: I5747976bdb7724c38566d0d399aed280bd218769
---
M lib/mw/ApiRequest.js
1 file changed, 18 insertions(+), 20 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/43/387443/1

diff --git a/lib/mw/ApiRequest.js b/lib/mw/ApiRequest.js
index b63d884..129f545 100644
--- a/lib/mw/ApiRequest.js
+++ b/lib/mw/ApiRequest.js
@@ -543,9 +543,6 @@
  * @param {Object} data The response from the server - parsed JSON object
  */
 TemplateRequest.prototype._handleJSON = function(error, data) {
-   var regex, title, location, iwstr, interwiki;
-   var metadata = { title: this.title };
-
logAPIWarnings(this, data);
 
if (!error && !data.query) {
@@ -557,22 +554,18 @@
return;
}
 
-   if (data.query.normalized && data.query.normalized.length) {
-   // update title (ie, "foo_Bar" -> "Foo Bar")
-   metadata.title = data.query.normalized[0].to;
-   }
-
+   var metadata;
if (!data.query.pages) {
if (data.query.interwiki) {
// Essentially redirect, but don't actually redirect.
-   interwiki = data.query.interwiki[0];
-   title = interwiki.title;
-   regex = new RegExp('^' + interwiki.iw + ':');
+   var interwiki = data.query.interwiki[0];
+   var title = interwiki.title;
+   var regex = new RegExp('^' + interwiki.iw + ':');
title = title.replace(regex, '');
-   iwstr = 
this.env.conf.wiki.interwikiMap.get(interwiki.iw).url ||
+   var iwstr = 
this.env.conf.wiki.interwikiMap.get(interwiki.iw).url ||

this.env.conf.parsoid.mwApiMap.get(interwiki.iw).uri ||
'/' + interwiki.iw + '/' + '$1';
-   location = iwstr.replace('$1', title);
+   var location = iwstr.replace('$1', title);
error = new DoesNotExistError('The page at ' + 
this.title +
' can be found at a different location: ' + 
location);
} else {
@@ -589,13 +582,18 @@
if (!page || !page.revisions || !page.revisions.length) 
{
return false;
}
-   metadata.id = page.pageid;
-   metadata.ns = page.ns;
-   metadata.latest = page.lastrevid;
-   metadata.revision = page.revisions[0];
-   metadata.pagelanguage = page.pagelanguage;
-   metadata.pagelanguagedir = page.pagelanguagedir;
-
+   metadata = {
+   id: page.pageid,
+   // If we requested by `oldid`, the title 
normalization won't be
+   // returned in `data.query.normalized`, so use 
the page property
+   // uniformly.
+   title: page.title,
+   ns: page.ns,
+   latest: page.lastrevid,
+   revision: page.revisions[0],
+   pagelanguage: page.pagelanguage,
+   pagelanguagedir: page.pagelanguagedir,
+   };
if (metadata.revision.texthidden || 
!metadata.revision.hasOwnProperty("*")) {
error = new DoesNotExistError("Source is hidden 
for " + self.title);
}

-- 
To view, visit https://gerrit.wikimedia.org/r/387443
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I5747976bdb7724c38566d0d399aed280bd218769
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...deploy[master]: Bump src/ to 292633c4 for deploy

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/387267 )

Change subject: Bump src/ to 292633c4 for deploy
..

Bump src/ to 292633c4 for deploy

Change-Id: Ifabec91d938dd8e45c62dcfca107667b4af4dbdb
---
M src
1 file changed, 1 insertion(+), 1 deletion(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid/deploy 
refs/changes/67/387267/1

diff --git a/src b/src
index 8e99708..292633c 16
--- a/src
+++ b/src
@@ -1 +1 @@
-Subproject commit 8e99708a3c00f4ce287ea05d61c37dfef1722640
+Subproject commit 292633c4ce8831b0769a8fd90492313357f15394

-- 
To view, visit https://gerrit.wikimedia.org/r/387267
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ifabec91d938dd8e45c62dcfca107667b4af4dbdb
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid/deploy
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Add a test for ref in ref

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/386767 )

Change subject: Add a test for ref in ref
..

Add a test for ref in ref

 * Follow up to d05ddd3

 * This would increases coverage and would have caught that regression.

Change-Id: I16a2ea9cf914dd40cbc2fccf3cd1c14673b64080
---
M tests/citeParserTests.txt
1 file changed, 16 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/67/386767/1

diff --git a/tests/citeParserTests.txt b/tests/citeParserTests.txt
index a5ad0ee..ce0539b 100644
--- a/tests/citeParserTests.txt
+++ b/tests/citeParserTests.txt
@@ -10,6 +10,12 @@
 {{{1}}}
 !! endarticle
 
+!! article
+Template:refinref
+!! text
+ho
+!! endarticle
+
 !! test
 Simple , no 
 !! wikitext
@@ -1185,6 +1191,16 @@
 foo
 !! end
 
+!! test
+Ref in ref
+!! wikitext
+test hi {{refinref}}
+
+!! html/parsoid
+test [1]
+↑  hi [2]↑  ho
+!! end
+
 ## Parsoid responsive references tests
 
 !! test

-- 
To view, visit https://gerrit.wikimedia.org/r/386767
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I16a2ea9cf914dd40cbc2fccf3cd1c14673b64080
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

[MediaWiki-commits] [Gerrit] mediawiki...parsoid[master]: Fix processing ref in ref

[Arlolra (Code Review)]

Arlolra has uploaded a new change for review. ( 
https://gerrit.wikimedia.org/r/386764 )

Change subject: Fix processing ref in ref
..

Fix processing ref in ref

 * Follow up to d47d5abb

 * Fixes the regression found in rt:

   node bin/roundtrip-test.js --domain ru.wikipedia.org "Феодосия"

 * Simple test case (do we really not have one!?):

   test haha{{#tag:ref|ok}}

Change-Id: Ie0a53a8e885a6d94769034cce4ef432773635842
---
M lib/ext/Cite/index.js
1 file changed, 5 insertions(+), 2 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid 
refs/changes/64/386764/1

diff --git a/lib/ext/Cite/index.js b/lib/ext/Cite/index.js
index 0192263..5ae6236 100644
--- a/lib/ext/Cite/index.js
+++ b/lib/ext/Cite/index.js
@@ -417,7 +417,7 @@
var refName = dp.name;
var about = node.getAttribute("about");
var ref = refsData.add(env, group, refName, about, nestedInReferences);
-   var nodeType = (node.getAttribute("typeof") || 
'').replace(/mw:Extension\/ref\/Marker/, '');
+   var nodeType = (node.getAttribute("typeof") || '').replace(/ 
mw:Extension\/ref\/Marker/, '');
 
// Add ref-index linkback
var doc = node.ownerDocument;
@@ -427,7 +427,10 @@
var body;
 
if (dp.hasRefInRef) {
-   _processRefs(cite, refsData, env.fragmentMap.get(content));
+   var c = env.fragmentMap.get(content);
+   DU.visitDOM(c, DU.loadDataAttribs);
+   _processRefs(cite, refsData, c);
+   DU.visitDOM(c, DU.storeDataAttribs);
}
 
if (content) {

-- 
To view, visit https://gerrit.wikimedia.org/r/386764
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ie0a53a8e885a6d94769034cce4ef432773635842
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Arlolra 

___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits