Cscott has uploaded a new change for review.
https://gerrit.wikimedia.org/r/243235
Change subject: WIP: Allow configuring the number of tokens processed before
yielding
......................................................................
WIP: Allow configuring the number of tokens processed before yielding
Change-Id: Id114183a54ea47acc5b8d67492e94b79dfa7e06e
---
M lib/mediawiki.tokenizer.peg.js
1 file changed, 8 insertions(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid
refs/changes/35/243235/1
diff --git a/lib/mediawiki.tokenizer.peg.js b/lib/mediawiki.tokenizer.peg.js
index d0900a7..4672bc3 100644
--- a/lib/mediawiki.tokenizer.peg.js
+++ b/lib/mediawiki.tokenizer.peg.js
@@ -22,6 +22,8 @@
// allow dumping tokenizer rules (only) to disk, for linting.
var PARSOID_DUMP_TOKENIZER_RULES = process.env.PARSOID_DUMP_TOKENIZER_RULES ||
false;
+var TOKENS_PER_IO = (+process.env.PARSOID_TOKENS) || 0;
+
/**
* Includes passed to the tokenizer, so that it does not need to require those
* on each call. They are available as pegArgs.pegIncludes, and are unpacked
@@ -292,6 +294,7 @@
var iterator;
var pegTokenizer = this;
+ var count = 0;
var tokenizeChunk = function() {
var next;
@@ -304,8 +307,12 @@
if (next.done) {
pegTokenizer.onEnd();
- } else {
+ } else if (count < TOKENS_PER_IO) {
+ count++;
process.nextTick(tokenizeChunk);
+ } else {
+ count = 0;
+ setImmediate(tokenizeChunk);
}
};
--
To view, visit https://gerrit.wikimedia.org/r/243235
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Id114183a54ea47acc5b8d67492e94b79dfa7e06e
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Cscott <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
