Cscott has uploaded a new change for review. https://gerrit.wikimedia.org/r/96713
Change subject: Add travis test runner; add test case; update dependencies. ...................................................................... Add travis test runner; add test case; update dependencies. Change-Id: If7314a4db3f1c212ad1ccc7f60fc3df111c4b5ea --- A .travis.yml M README.md A samples/hurricanes.json A samples/taoism.json A samples/us.json A test/mocha.opts A test/samples.js 7 files changed, 387 insertions(+), 1 deletion(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Collection/OfflineContentGenerator/bundler refs/changes/13/96713/1 diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..17956ee --- /dev/null +++ b/.travis.yml @@ -0,0 +1,7 @@ +language: node_js +node_js: + - "0.10" + - "0.8" +before_install: + - sudo apt-get update -qq + - sudo apt-get install -qq zip diff --git a/README.md b/README.md index 56a4979..c6ab8da 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # mw-bundler +[![Build Status][1]][2] [![dependency status][3]][4] [![dev dependency status][5]][6] + A mediawiki article spider tool. This tool grabs all the dependencies for a given set of articles and @@ -8,11 +10,16 @@ ## Installation -Tested with node 0.10. Probably any recent node will work. +Node version 0.8 and 0.10 are tested to work. Install the node package depdendencies with: ``` npm install +``` + +Install other system dependencies. +``` +apt-get install zip ``` ## Running @@ -38,3 +45,10 @@ Copyright (c) 2013 C. Scott Ananian Licensed under GPLv2. + +[1]: https://travis-ci.org/cscott/mw-bundler.png +[2]: https://travis-ci.org/cscott/mw-bundler +[3]: https://david-dm.org/cscott/mw-bundler.png +[4]: https://david-dm.org/cscott/mw-bundler +[5]: https://david-dm.org/cscott/mw-bundler/dev-status.png +[6]: https://david-dm.org/cscott/mw-bundler#info=devDependencies diff --git a/samples/hurricanes.json b/samples/hurricanes.json new file mode 100644 index 0000000..761faf9 --- /dev/null +++ b/samples/hurricanes.json @@ -0,0 +1,178 @@ +{ + "items": [ + { + "items": [ + { + "content_type": "text/x-wiki", + "currentVersion": 1, + "latest": 579329188, + "revision": "579329188", + "timestamp": "1383073948", + "title": "1933 Atlantic hurricane season", + "type": "article", + "url": "http://en.wikipedia.org/wiki/1933_Atlantic_hurricane_season";, + "wiki": 0 + } + ], + "title": "Introduction", + "type": "chapter" + }, + { + "items": [ + { + "content_type": "text/x-wiki", + "currentVersion": 1, + "latest": 550867472, + "revision": "550867472", + "timestamp": "1366230514", + "title": "1933 Trinidad hurricane", + "type": "article", + "url": "http://en.wikipedia.org/wiki/1933_Trinidad_hurricane";, + "wiki": 0 + }, + { + "content_type": "text/x-wiki", + "currentVersion": 1, + "latest": 572402424, + "revision": "572402424", + "timestamp": "1378848251", + "title": "1933 Texas tropical storm", + "type": "article", + "url": "http://en.wikipedia.org/wiki/1933_Texas_tropical_storm";, + "wiki": 0 + }, + { + "content_type": "text/x-wiki", + "currentVersion": 1, + "latest": 579390421, + "revision": "579390421", + "timestamp": "1383099157", + "title": "1933 Chesapeake–Potomac hurricane", + "type": "article", + "url": "http://en.wikipedia.org/wiki/1933_Chesapeake%E2%80%93Potomac_hurricane";, + "wiki": 0 + }, + { + "content_type": "text/x-wiki", + "currentVersion": 1, + "latest": 572402369, + "revision": "572402369", + "timestamp": "1378848216", + "title": "1933 Cuba–Brownsville hurricane", + "type": "article", + "url": "http://en.wikipedia.org/wiki/1933_Cuba%E2%80%93Brownsville_hurricane";, + "wiki": 0 + }, + { + "content_type": "text/x-wiki", + "currentVersion": 1, + "latest": 575674769, + "revision": "575674769", + "timestamp": "1380856363", + "title": "1933 Treasure Coast hurricane", + "type": "article", + "url": "http://en.wikipedia.org/wiki/1933_Treasure_Coast_hurricane";, + "wiki": 0 + }, + { + "content_type": "text/x-wiki", + "currentVersion": 1, + "latest": 576090149, + "revision": "576090149", + "timestamp": "1381119599", + "title": "1933 Outer Banks hurricane", + "type": "article", + "url": "http://en.wikipedia.org/wiki/1933_Outer_Banks_hurricane";, + "wiki": 0 + }, + { + "content_type": "text/x-wiki", + "currentVersion": 1, + "latest": 567480201, + "revision": "567480201", + "timestamp": "1375840931", + "title": "1933 Tampico hurricane", + "type": "article", + "url": "http://en.wikipedia.org/wiki/1933_Tampico_hurricane";, + "wiki": 0 + }, + { + "content_type": "text/x-wiki", + "currentVersion": 1, + "latest": 578977949, + "revision": "578977949", + "timestamp": "1382894555", + "title": "1933 Cuba–Bahamas hurricane", + "type": "article", + "url": "http://en.wikipedia.org/wiki/1933_Cuba%E2%80%93Bahamas_hurricane";, + "wiki": 0 + } + ], + "title": "Storms", + "type": "chapter" + } + ], + "licenses": [ + { + "mw_rights_icon": "//creativecommons.org/images/public/somerights20.png", + "mw_rights_text": "Creative Commons Attribution-Share Alike 3.0", + "mw_rights_url": "//creativecommons.org/licenses/by-sa/3.0/", + "name": "License", + "type": "license" + } + ], + "subtitle": "", + "summary": "", + "title": "1933 Atlantic hurricane season", + "type": "collection", + "version": 1, + "wikis": [ + { + "baseurl": "http://en.wikipedia.org/w";, + "imagesize": 1200, + "keep_tmpfiles": false, + "output": "/home/anomie/test.zip", + "script_extension": ".php", + "status_file": "/home/anomie/statusfile", + "type": "wikiconf", + "parsoid": "http://parsoid-lb.eqiad.wikimedia.org/";, + "prefix": "enwiki", + "filerepos": [ + { + "name": "shared", + "displayname": "Wikimedia Commons", + "rootUrl": "//upload.wikimedia.org/wikipedia/commons", + "local": false, + "apiUrl": "http://commons.wikimedia.org/w/api.php";, + "wiki": 1 + }, + { + "name": "local", + "displayname": null, + "rootUrl": "//upload.wikimedia.org/wikipedia/en", + "local": true, + "wiki": 0 + } + ] + }, + { + "baseurl": "http://commons.wikimedia.org/w/api.php";, + "imagesize": 1200, + "keep_tmpfiles": false, + "script_extension": ".php", + "format": "nuwiki", + "type": "wikiconf", + "parsoid": "http://parsoid-lb.eqiad.wikimedia.org/";, + "prefix": "commonswiki", + "filerepos": [ + { + "name": "local", + "displayname": null, + "rootUrl": "//upload.wikimedia.org/wikipedia/commons", + "local": true, + "wiki": 1 + } + ] + } + ] +} \ No newline at end of file diff --git a/samples/taoism.json b/samples/taoism.json new file mode 100644 index 0000000..a96dda9 --- /dev/null +++ b/samples/taoism.json @@ -0,0 +1,75 @@ +{ + "items": [ + { + "content_type": "text/x-wiki", + "title": "Taoism", + "type": "article", + "url": "http://simple.wikipedia.org/wiki/Taoism";, + "wiki": 0, + "revision": "4585542" + } + ], + "licenses": [ + { + "mw_rights_icon": "", + "mw_rights_page": "", + "mw_rights_text": "", + "mw_rights_url": "", + "name": "License", + "type": "license" + } + ], + "subtitle": "", + "summary": "", + "title": "", + "type": "collection", + "version": 1, + "wikis": [ + { + "baseurl": "http://simple.wikipedia.org/w";, + "imagesize": 1200, + "keep_tmpfiles": false, + "script_extension": ".php", + "format": "nuwiki", + "type": "wikiconf", + "parsoid": "http://parsoid-lb.eqiad.wikimedia.org/";, + "prefix": "simplewiki", + "filerepos": [ + { + "name": "shared", + "displayname": "Wikimedia Commons", + "rootUrl": "//upload.wikimedia.org/wikipedia/commons", + "local": false, + "apiUrl": "http://commons.wikimedia.org/w/api.php";, + "wiki": 1 + }, + { + "name": "local", + "displayname": null, + "rootUrl": "//upload.wikimedia.org/wikipedia/simple", + "local": true, + "wiki": 0 + } + ] + }, + { + "baseurl": "http://commons.wikimedia.org/w/api.php";, + "imagesize": 1200, + "keep_tmpfiles": false, + "script_extension": ".php", + "format": "nuwiki", + "type": "wikiconf", + "parsoid": "http://parsoid-lb.eqiad.wikimedia.org/";, + "prefix": "commonswiki", + "filerepos": [ + { + "name": "local", + "displayname": null, + "rootUrl": "//upload.wikimedia.org/wikipedia/commons", + "local": true, + "wiki": 1 + } + ] + } + ] +} \ No newline at end of file diff --git a/samples/us.json b/samples/us.json new file mode 100644 index 0000000..a600921 --- /dev/null +++ b/samples/us.json @@ -0,0 +1,75 @@ +{ + "items": [ + { + "content_type": "text/x-wiki", + "title": "United States", + "type": "article", + "url": "http://en.wikipedia.org/wiki/United States", + "wiki": 0, + "revision": "581396893" + } + ], + "licenses": [ + { + "mw_rights_icon": "", + "mw_rights_page": "", + "mw_rights_text": "", + "mw_rights_url": "", + "name": "License", + "type": "license" + } + ], + "subtitle": "", + "summary": "", + "title": "", + "type": "collection", + "version": 1, + "wikis": [ + { + "baseurl": "http://en.wikipedia.org/w";, + "imagesize": 1200, + "keep_tmpfiles": false, + "script_extension": ".php", + "format": "nuwiki", + "type": "wikiconf", + "parsoid": "http://parsoid-lb.eqiad.wikimedia.org/";, + "prefix": "enwiki", + "filerepos": [ + { + "name": "shared", + "displayname": "Wikimedia Commons", + "rootUrl": "//upload.wikimedia.org/wikipedia/commons", + "local": false, + "apiUrl": "http://commons.wikimedia.org/w/api.php";, + "wiki": 1 + }, + { + "name": "local", + "displayname": null, + "rootUrl": "//upload.wikimedia.org/wikipedia/en", + "local": true, + "wiki": 0 + } + ] + }, + { + "baseurl": "http://commons.wikimedia.org/w/api.php";, + "imagesize": 1200, + "keep_tmpfiles": false, + "script_extension": ".php", + "format": "nuwiki", + "type": "wikiconf", + "parsoid": "http://parsoid-lb.eqiad.wikimedia.org/";, + "prefix": "commonswiki", + "filerepos": [ + { + "name": "local", + "displayname": null, + "rootUrl": "//upload.wikimedia.org/wikipedia/commons", + "local": true, + "wiki": 1 + } + ] + } + ] +} \ No newline at end of file diff --git a/test/mocha.opts b/test/mocha.opts new file mode 100644 index 0000000..af53e24 --- /dev/null +++ b/test/mocha.opts @@ -0,0 +1,2 @@ +--check-leaks +--reporter spec diff --git a/test/samples.js b/test/samples.js new file mode 100644 index 0000000..e49e350 --- /dev/null +++ b/test/samples.js @@ -0,0 +1,35 @@ +var assert = require("assert"); +var fs = require('fs'); +var nodefn = require("when/node/function"); +var path = require('path'); + +var bundler = require('../'); + +// ensure that we don't crash on any of our sample inputs +describe("Basic crash test", function() { + ['taoism.json', 'hurricanes.json' /*, 'us.json'*/].forEach(function(name) { + describe(name, function() { + it('should bundle', function(done) { + this.timeout(0); + var filename = path.join(__dirname, '..', 'samples', name); + return nodefn.call(fs.readFile.bind(fs), filename, 'utf8') + .then(function(metabook) { + metabook = JSON.parse(metabook); + return bundler.bundle(metabook, { + output: filename + '.zip', + verbose: false + }); + }).then(function(statusCode) { + assert.equal(statusCode, 0); + }).ensure(function() { + try { + fs.unlinkSync(filename + '.zip'); + } catch (e) { } + }).done( + function() { done(); }, + function(err) { done(err); } + ); + }); + }); + }); +}); -- To view, visit https://gerrit.wikimedia.org/r/96713 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: If7314a4db3f1c212ad1ccc7f60fc3df111c4b5ea Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/extensions/Collection/OfflineContentGenerator/bundler Gerrit-Branch: master Gerrit-Owner: Cscott <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
