Cscott has uploaded a new change for review.
https://gerrit.wikimedia.org/r/96547
Change subject: Properly support foreign file repos (particular commons).
......................................................................
Properly support foreign file repos (particular commons).
The wiki configuration information now includes the filerepoinfo,
which includes commonswiki if InstantCommons is in use.
Change-Id: Ib6f4c7575c11d421b07ae624975c248b7c457fe8
---
M bin/mw-bundler
M lib/index.js
2 files changed, 109 insertions(+), 35 deletions(-)
git pull
ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Collection/OfflineContentGenerator/bundler
refs/changes/47/96547/1
diff --git a/bin/mw-bundler b/bin/mw-bundler
index 3c45805..81a54e4 100755
--- a/bin/mw-bundler
+++ b/bin/mw-bundler
@@ -10,6 +10,7 @@
var url = require('url');
var when = require('when');
+var Api = require('../lib/api');
var Siteinfo = require('../lib/siteinfo');
program
@@ -72,19 +73,42 @@
title: "",
type: "collection",
version: 1,
- wikis: [
- {
- baseurl: null, // filled in below
- imagesize: 1200,
- keep_tmpfiles: false,
- script_extension: ".php",
- format: "nuwiki",
- type: "wikiconf",
- // our extra fields
- parsoid: program.api,
- prefix: program.prefix
- }
- ]
+ wikis: [{
+ baseurl: null, // API endpoint for the wiki, filled in below
+ imagesize: 1200,
+ keep_tmpfiles: false,
+ script_extension: ".php",
+ format: "nuwiki",
+ type: "wikiconf",
+ // our extra fields
+ parsoid: program.api,
+ prefix: program.prefix,
+ filerepos: undefined // link to commons, etc; filled in below
+ // the filerepos field looks something like:
+ /*
+ filerepos: [{
+ name: 'local',
+ displayname: 'Wikimedia Commons',
+ rootUrl: "//upload.wikimedia.org/wikipedia/commons",
+ local: true,
+ apiUrl: 'http://commons.wikimedia.org/w/api.php',
+ wiki: 0 // pointer to an entry in metabook.wikis
+ }]
+ */
+ }]
+};
+
+var COMMONSWIKI = {
+ baseurl: 'http://commons.wikimedia.org/w/api.php',
+ imagesize: 1200,
+ keep_tmpfiles: false,
+ script_extension: ".php",
+ format: "nuwiki",
+ type: "wikiconf",
+ // our extra fields
+ parsoid: program.api,
+ prefix: 'commonswiki',
+ filerepos: undefined
};
// Load metabook.json
@@ -131,27 +155,75 @@
});
}
+var needsCommonsWiki = false;
+// promise to fetch missing 'filerepos' field in wiki config
+var fetchFileRepos = function(wiki) {
+ var api = new Api(metabook.wikis);
+ var w = metabook.wikis[wiki];
+ var p = when.resolve();
+ if (!w.filerepos) {
+ w.filerepos = [];
+ p = p.then(function() {
+ return api.request(wiki, {
+ action: 'query',
+ meta: 'filerepoinfo'
+ });
+ }).then(function(resp) {
+ resp.query.repos.forEach(function(repo) {
+ w.filerepos.push(repo);
+ // link to a wiki # in metabooks.json
+ if (repo.local) {
+ repo.wiki = wiki;
+ return;
+ }
+ // xxx note that apiUrl isn't (yet) part of the
+ // filerepoinfo response.
+ if (!repo.apiUrl) {
+ repo.apiUrl = COMMONSWIKI.baseurl;
+ }
+ for (var i=0; i<metabook.wikis.length; i++) {
+ if (metabook.wikis[i].baseurl ===
repo.apiUrl) {
+ repo.wiki = i;
+ return;
+ }
+ }
+ // fudge a pointer to commons
+ needsCommonsWiki = true;
+ repo.wiki = metabook.wikis.length;
+ });
+ });
+ }
+ return p;
+};
// add our extension fields, if missing
p = p.then(function() {
- metabook.wikis.forEach(function(w) {
+ var pp = when.resolve();
+ metabook.wikis.forEach(function(w, idx) {
if (!w.parsoid) {
w.parsoid = program.api;
}
if (!w.prefix) {
- // look for baseurl in interwikimap
- var url = w.baseurl.replace(/\/w$/, '/wiki/$1');
- interwikimap.forEach(function(ww) {
- if (/^(w|wikipedia)$/.test(ww.prefix)) {
- // filter out some aliases (which
aren't supported by
- // parsoid)
- return;
- }
- if (url === ww.url) {
- w.prefix = ww.prefix + 'wiki';
- }
+ // look up siteid in siteinfo
+ pp = pp.then(function() {
+ return Siteinfo.fetch(metabook.wikis, idx);
+ }).then(function(resp) {
+ w.prefix = resp.general.wikiid;
+ });
+ }
+ if (!w.filerepos) {
+ pp = pp.then(function() {
+ return fetchFileRepos(idx);
});
}
});
+ return pp;
+});
+// fudge a pointer to commonswiki if needed
+p = p.then(function() {
+ if (needsCommonsWiki) {
+ metabook.wikis.push(COMMONSWIKI);
+ return fetchFileRepos(metabook.wikis.length - 1);
+ }
});
// Create/load nfo.json
diff --git a/lib/index.js b/lib/index.js
index 97459c3..dd76269 100644
--- a/lib/index.js
+++ b/lib/index.js
@@ -165,16 +165,18 @@
});
// ... all image pages
imageMap.forEach(function(img) {
- if (img.imagerepository !== 'local') {
- // xxx fetch from commons
- /* jshint noempty: false */ // we know this is
empty!
- } else {
- titles.push({
- wiki: img.wiki,
- title: img.short
- // images are always the 'latest'
revision
- });
- }
+ // look up appropriate wiki (may fetch from commons)
+ var w = metabook.wikis[img.wiki], iwiki = img.wiki;
+ w.filerepos.forEach(function(repo) {
+ if (img.imagerepository === repo.name) {
+ iwiki = repo.wiki;
+ }
+ });
+ titles.push({
+ wiki: iwiki,
+ title: img.short
+ // images are always the 'latest' revision
+ });
});
return Revisions.fetchAndWrite(
metabook.wikis, titles, options.output, log,
PEDIAPRESS_COMPAT
--
To view, visit https://gerrit.wikimedia.org/r/96547
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib6f4c7575c11d421b07ae624975c248b7c457fe8
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/Collection/OfflineContentGenerator/bundler
Gerrit-Branch: master
Gerrit-Owner: Cscott <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
