Cscott has uploaded a new change for review.
https://gerrit.wikimedia.org/r/97014
Change subject: Refactor revision fetch code to occur in (limited) parallel.
......................................................................
Refactor revision fetch code to occur in (limited) parallel.
Change-Id: Ideaeea33506a7e4d0e42662d814910a719ba06a6
---
M lib/revisions.js
1 file changed, 13 insertions(+), 10 deletions(-)
git pull
ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Collection/OfflineContentGenerator/bundler
refs/changes/14/97014/1
diff --git a/lib/revisions.js b/lib/revisions.js
index cab04f9..1c5fb93 100644
--- a/lib/revisions.js
+++ b/lib/revisions.js
@@ -2,12 +2,16 @@
// this is a particularly grody file, so we also store this in a more
// sane manner as revisions.db. hopefully we can deprecate the ugliness.
var fs = require('fs');
+var guard = require('when/guard');
var path = require('path');
var when = require('when');
var Api = require('./api');
var Db = require('./db');
var P = require('./p');
+
+// limit the # of concurrent image requests
+var REVISION_REQUEST_LIMIT = 5;
var fetchOne = function(wikis, wiki, title, revid) {
var api = new Api(wikis);
@@ -74,22 +78,21 @@
var revStream = pediapress_compat ?
fs.createWriteStream(path.join(outdir, 'revisions-1.txt')) :
null;
- var p = when.resolve();
- titles.forEach(function(t) {
- p = p.then(function() {
- log('Fetching revision info for', t.title);
- return fetchOne(wikis, t.wiki, t.canontitle || t.title,
t.revid);
- }).then(function(data) {
- return data===null ? null : writeOne(data, revStream,
revDb);
- });
+
+ var fetchAndWriteOne = guard(guard.n(REVISION_REQUEST_LIMIT),
function(t) {
+ log('Fetching revision info for', t.title);
+ return fetchOne(wikis, t.wiki, t.canontitle || t.title,
t.revid).
+ then(function(data) {
+ return data===null ? null : writeOne(data,
revStream, revDb);
+ });
});
- p = p.then(function() {
+
+ return when.all(titles.map(fetchAndWriteOne)).then(function() {
return revDb.close();
}).then(function() {
return (revStream===null) ? null :
P.call(revStream.end, revStream);
});
- return p;
};
module.exports = {
--
To view, visit https://gerrit.wikimedia.org/r/97014
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ideaeea33506a7e4d0e42662d814910a719ba06a6
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/Collection/OfflineContentGenerator/bundler
Gerrit-Branch: master
Gerrit-Owner: Cscott <canan...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
