MaxSem has uploaded a new change for review.
https://gerrit.wikimedia.org/r/253060
Change subject: WIP: Stats thingie
......................................................................
WIP: Stats thingie
Change-Id: I497364fc18c2183fc0b793d0d1e7d158ef094cd4
---
M package.json
A site-stats.js
A tests/site-stats-test.js
3 files changed, 206 insertions(+), 3 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/wikimedia/portals
refs/changes/60/253060/1
diff --git a/package.json b/package.json
index f74db3e..6573784 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,10 @@
{
- "dependencies": {},
- "devDependencies": {
+ "name": "portals",
+ "version": "1.0.0",
+ "description": "",
+ "main": "gulpfile.js",
+ "dependencies": {
+ "bluebird": "^3.0.5",
"gulp": "^3.9.0",
"gulp-imagemin": "^2.3.0",
"gulp-inline": "^0.1.0",
@@ -10,6 +14,19 @@
"gulp-minify-css": "^1.2.1",
"gulp-uglify": "^1.4.2",
"imagemin-pngquant": "^4.2.0",
+ "moment": "^2.10.6",
+ "preq": "^0.4.6",
+ "underscore": "^1.8.3",
"yargs": "^3.29.0"
- }
+ },
+ "devDependencies": {},
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "repository": {
+ "type": "git",
+ "url": "ssh://[email protected]:29418/wikimedia/portals"
+ },
+ "author": "",
+ "license": "ISC"
}
diff --git a/site-stats.js b/site-stats.js
new file mode 100644
index 0000000..205b748
--- /dev/null
+++ b/site-stats.js
@@ -0,0 +1,171 @@
+var preq = require( 'preq' );
+var BBPromise = require( 'bluebird' );
+var moment = require( 'moment' );
+var _ = require( 'underscore' );
+
+
+var projects = {
+ 'wikipedia': /wiki$/,
+ 'wiktionary': /wiktionary$/,
+ 'wikinews': /wikinews$/,
+ 'wikiquote': /wikiquote$/,
+ 'wikisource': /wikisource$/,
+ 'wikibooks': /wikibooks$/,
+ 'wikiversity': /wikiversity$/,
+ 'wikivoyage': /wikivoyage$/
+};
+
+var codeMapping = {
+ b: 'wikibooks',
+ d: 'wiktionary',
+ f: 'foundationwiki',
+ m: 'wiki',
+ mw: 'wiki',
+ n: 'wikinews',
+ q: 'wikiquote',
+ s: 'wikisource',
+ v: 'wikiversity',
+ voy: 'wikivoyage',
+ w: 'wiki',
+ wd: 'wikidatawiki',
+ zero: 'wiki'
+};
+
+var soloProjects = {
+ f: 'foundationwiki',
+ wd: 'wikidatawiki'
+};
+
+function httpGet( url ) {
+ //console.log( url );
+ var preqOptions = { headers: { 'User-Agent': 'Hi Öłïvér!' } };
+
+ return preq.get( url, preqOptions )
+ .then( function( request ) {
+ return BBPromise.resolve( request.body );
+ } )
+ .catch( function( err ) {
+ // I can haz error message that makes sense?
+ var msg = err.toString() + ' requesting ' + url;
+ console.error( msg );
+ BBPromise.reject( msg );
+ } );
+}
+
+function getPageCounts() {
+ if ( process.env.MEDIAWIKI_DEPLOYMENT_DIR && false ) {
+ // Production
+ // TODO:
+ } else {
+ // Developer's machine
+ return httpGet(
'https://tools.wmflabs.org/pagecounts/pagecounts.json' )
+ .then( function( pagecounts ) {
+ var stats = {};
+
+ _.each( pagecounts, function( wiki, code ) {
+ stats[code] = wiki.contentPages;
+ } );
+ return BBPromise.resolve( stats );
+ } );
+ }
+}
+
+function getSiteMatrix() {
+ return httpGet(
'https://meta.wikimedia.org/w/api.php?action=sitematrix&format=json&smtype=language&smlangprop=code%7Csite&smsiteprop=url%7Cdbname%7Ccode'
);
+}
+
+function parseProjectString( str ) {
+ var parts = str.split( '.' );
+ var name = parts.shift(); //
+ if ( parts.length > 1 ) {
+ parts.shift(); // Qualifier like mobile or zero
+ }
+ if ( parts.length ) {
+ if ( !codeMapping[ parts[0] ] ) {
+ throw 'Cannot parse wiki ' + str;
+ }
+ name += codeMapping[ parts[0] ];
+ } else {
+ name += 'wiki';
+ }
+ if ( parts.length > 1 ) {
+ throw 'Cannot parse wiki ' + str;
+ }
+ return name;
+}
+
+function getProjectViews() {
+ var yesterday = moment.utc().subtract( 1, 'days' ).startOf( 'day' );
+ var baseUrl = 'https://dumps.wikimedia.org/other/pageviews/' +
yesterday.format( 'YYYY/YYYY-MM' ) + '/projectviews-' + yesterday.format(
'YYYYMMDD' );
+ var promises = [];
+
+ for ( var i = 0; i <= 23; i++ ) {
+ var url = baseUrl + '-' + ( i < 10 ? '0' : '' ) + i.toString()
+ '0000';
+ promises.push( httpGet( url ) );
+ }
+
+ return BBPromise
+ .all( promises )
+ .then( function( hourlies ) {
+ var views = {};
+
+ _.each( hourlies, function( hourly ) {
+ if ( !hourly ) {
+ return;
+ }
+ var lines = hourly.toString().split( '\n' );
+ _.each( lines, function( line ) {
+ var parts = line.split( /\s+-?\s*/ );
+ var wiki = parseProjectString( parts[0]
);
+ views[wiki] = views[wiki] || 0;
+ views[wiki] += parseInt( parts[1] );
+ } );
+ } );
+ if ( !views ) {
+ // We permit some hourly files to fail to be
downloaded, but all of them missing
+ // is a sign of a problem
+ return BBPromise.reject( 'No hourly project
views file was successfully loaded' );
+ }
+ return BBPromise.resolve( views );
+ } )
+ .error( function() {} ); // Do nothing, last file can be being
generated right now
+}
+
+
+function getSiteStats() {
+ var stats = {};
+
+ BBPromise.all( [ getPageCounts(), getSiteMatrix(), getProjectViews() ] )
+ .then( function( data ) {
+ var counts = data[0];
+ var siteMatrix = data[1].sitematrix;
+ var views = data[2];
+
+ _.each( siteMatrix, function( lang, propName ) {
+ if ( !/^\d+$/.test( propName ) ) {
+ return; // Not a language... Fuck, this
API's output is ugly
+ }
+ _.each( lang.site, function( site ) {
+ stats[site.code] = stats[site.code] ||
{};
+ stats[site.code][lang.code] = {
+ url: site.url,
+ articles: counts[site.dbname]
|| 0,
+ views: views[site.dbname] || 0,
+ closed: site.closed !==
undefined
+ };
+ } );
+ } );
+
+ //console.log(stats);
+ } );
+
+}
+
+module.exports = {
+ get: getSiteStats(),
+ parseProjectString: parseProjectString
+};
+
+console.log( parseProjectString( 'm.f' ) );
+
+getSiteStats();
diff --git a/tests/site-stats-test.js b/tests/site-stats-test.js
new file mode 100644
index 0000000..0cb589c
--- /dev/null
+++ b/tests/site-stats-test.js
@@ -0,0 +1,15 @@
+var siteStats = require( '../site-stats' );
+
+module.exports.testParseProjectString = function( test ) {
+ var parseProjectString = siteStats.parseProjectString;
+
+ test.equals( 'enwiki', parseProjectString( 'en' ) );
+ test.equals( 'enwiki', parseProjectString( 'en.zero' ) );
+ test.equals( 'enwiki', parseProjectString( 'en.m' ) );
+ test.equals( 'enwikibooks', parseProjectString( 'en.m.b' ) );
+
+ test.equals( 'foundationwiki', parseProjectString( 'm.f' ) );
+ test.equals( 'foundationwiki', parseProjectString( 'f' ) );
+
+ test.done();
+};
\ No newline at end of file
--
To view, visit https://gerrit.wikimedia.org/r/253060
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I497364fc18c2183fc0b793d0d1e7d158ef094cd4
Gerrit-PatchSet: 1
Gerrit-Project: wikimedia/portals
Gerrit-Branch: master
Gerrit-Owner: MaxSem <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
