Nuria has uploaded a new change for review.
https://gerrit.wikimedia.org/r/270867
Change subject: [WIP] Dashiki gets pageview data from pageview API
......................................................................
[WIP] Dashiki gets pageview data from pageview API
TODO: Mobile breakdowns, testing
Bug:T124063
Change-Id: I1d403595cd3b25a099721227f6a1cf0b5fddd34d
---
M package.json
M src/app/apis/pageview-api.js
M src/app/config.js
M src/app/data-converters/factory.js
A src/app/data-converters/pageview-api-response.js
M src/app/data-converters/wikimetrics-timeseries.js
M src/app/require.config.js
A src/app/sitematrix.js
M src/components/wikimetrics-visualizer/wikimetrics-visualizer.js
A src/lib/pageviews.js
10 files changed, 671 insertions(+), 75 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/dashiki
refs/changes/67/270867/1
diff --git a/package.json b/package.json
index 326b2e4..649ef8f 100644
--- a/package.json
+++ b/package.json
@@ -2,6 +2,7 @@
"name": "dashiki",
"version": "0.0.0",
"devDependencies": {
+ "pageviews": "~1.0.2",
"chalk": "~0.4.0",
"cheerio": "^0.19.0",
"deeply": "~0.1.0",
diff --git a/src/app/apis/pageview-api.js b/src/app/apis/pageview-api.js
index e2767bf..2f3fed5 100644
--- a/src/app/apis/pageview-api.js
+++ b/src/app/apis/pageview-api.js
@@ -12,7 +12,10 @@
dataConverterFactory = require('dataConverterFactory'),
uri = require('uri/URI'),
logger = require('logger'),
- TimeseriesData = require('converters.timeseries');
+ moment = require('moment'),
+ pageviews = require('pageviews'),
+ config = require('config'),
+ sitematrix = require('sitematrix');
require('uri/URITemplate');
@@ -35,20 +38,20 @@
PageviewApi.prototype.getData = function (metric, project, showBreakdown) {
var deferred = new $.Deferred();
- //using christian's endpoint
- // http://quelltextlich.at/wmf/projectcounts/daily/enwiki.csv
- var address =
uri.expand('https://{root}/static/public/datafiles/{metric}/{project}.csv', {
- root: this.root,
- metric: metric.name,
- project: project
- }).toString();
+ var endDate = moment().format('YYYYMMDDHH');
- $.ajax({
- //let jquery decide datatype, otherwise things do not work when
retrieving cvs
- url: address
- }).done(function (data) {
- var converter = this.getDataConverter(),
- opt = {
+ sitematrix.getProjectUrl(config, project).done(function (projectUrl) {
+
+ pageviews.getAggregatedPageviews({
+ project: projectUrl,
+ granularity: 'daily',
+ start: '2015010100',
+ end: endDate
+ }).then(function (data) {
+ // console.log(JSON.stringify(result, null, 2));
+
+ var converter = this.dataConverter;
+ var opt = {
label: project,
allColumns: showBreakdown,
varyColors: false,
@@ -57,16 +60,18 @@
startDate: '2014-01-01'
};
- deferred.resolve(converter(opt, data));
- }.bind(this))
- .fail(function (error) {
- // resolve as done with empty results and log the error
- // to avoid crashing the ui when a metric has problems
- deferred.resolve(new TimeseriesData());
- logger.error(error);
- });
+ deferred.resolve(converter(opt, data));
+
+ }.bind(this)).catch(function (error) {
+ console.log(error);
+ });
+
+
+ }.bind(this));
return deferred.promise();
+
+
};
PageviewApi.prototype.getDataConverter = function () {
@@ -74,4 +79,4 @@
};
return new PageviewApi(siteConfig);
-});
+});
\ No newline at end of file
diff --git a/src/app/config.js b/src/app/config.js
index c3ba165..0b23223 100644
--- a/src/app/config.js
+++ b/src/app/config.js
@@ -29,10 +29,9 @@
},
- //placeholder for now, note this is coming from a temporary domain
pageviewApi: {
- endpoint: 'metrics.wmflabs.org', // needs to support https
- format: 'csv'
+ endpoint: '', // not used
+ format: 'pageview-api-response'
},
@@ -41,9 +40,13 @@
format: 'tsv'
},
+ // this doc does not have
+ sitematrix: {
+ endpoint:
'https://meta.wikimedia.org/w/api.php?action=sitematrix&formatversion=2&format=json&maxage=3600&smaxage=3600'
+ }
//urlProjectLanguageChoices:
'/stubs/fake-wikimetrics/projectLanguageChoices.json',
//urlCategorizedMetrics:
'/stubs/fake-wikimetrics/categorizedMetrics.json',
//urlDefaultDashboard: '/stubs/fake-wikimetrics/defaultDashboard.json',
};
-});
+});
\ No newline at end of file
diff --git a/src/app/data-converters/factory.js
b/src/app/data-converters/factory.js
index 31aa751..18d8b60 100644
--- a/src/app/data-converters/factory.js
+++ b/src/app/data-converters/factory.js
@@ -6,7 +6,8 @@
'use strict';
var separatedValues = require('converters.separated-values'),
- wikimetricsTimeseries = require('converters.wikimetrics-timeseries');
+ wikimetricsTimeseries = require('converters.wikimetrics-timeseries'),
+ pageviewApiResponse = require('converters.pageview-api-response');
function ConverterFactory() {
return;
@@ -33,6 +34,8 @@
case 'json':
return wikimetricsTimeseries();
+ case 'pageview-api-response':
+ return pageviewApiResponse();
}
};
diff --git a/src/app/data-converters/pageview-api-response.js
b/src/app/data-converters/pageview-api-response.js
new file mode 100644
index 0000000..ac06564
--- /dev/null
+++ b/src/app/data-converters/pageview-api-response.js
@@ -0,0 +1,64 @@
+/**
+ * This module returns a method that knows how to translate json data from
+ * pageview API to the canonical format understood by dashiki
+
+ * Responses look like:
+{"items":[
+{"project":"en.wikipedia","access":"all-access","agent":"all-agents","granularity":"daily","timestamp":"2015120200","views":291268249},
+{"project":"en.wikipedia","access":"all-access","agent":"all-agents","granularity":"daily","timestamp":"2015120300","views":284829416},
+{"project":"en.wikipedia","access":"all-access","agent":"all-agents","granularity":"daily","timestamp":"2015120400","views":280259970}
+]
+*/
+define(function (require) {
+ 'use strict';
+
+ var _ = require('lodash'),
+ moment = require('moment'),
+ sitematrix = require('sitematrix'),
+ TimeseriesData = require('converters.timeseries');
+
+ /**
+ * Parameters
+ * options : a dictionary of options. Required options:
+ * defaultSubmetrics - dictionary of metric names to default
submetrics to use
+ *
+ * rawData : json data, as fetched from the pageview API
public endpoint
+ * Returns
+ * A TimeseriesData instance
+ */
+
+ return function () {
+
+ return function (options, rawData) {
+
+ if (!_.has(rawData, 'items')) {
+ return new TimeseriesData([]);
+ }
+
+ var parameters = rawData.parameters,
+ metricName = 'pageviews',
+ project;
+
+
+
+ // transform array of items into hash so TimeSeries data can
digest it
+ var dict = {};
+ _.forEach(rawData.items, function (value, index) {
+ var ts = moment(value.timestamp,
"YYYYMMDDHH").format('YYYY-MM-DD');
+
+ dict[ts] = [
+ [value.views ? parseFloat(value.views) : null]
+ ]
+ });
+
+ return new TimeseriesData(
+ // labels and colors can use the cohort name
+ ["Total"], //TODO
+ // wrap the values in an array to match the header
+ dict, [project],
+ // but keep patterns globally constant
+ [0]
+ );
+ };
+ };
+});
\ No newline at end of file
diff --git a/src/app/data-converters/wikimetrics-timeseries.js
b/src/app/data-converters/wikimetrics-timeseries.js
index 113b300..a6abff3 100644
--- a/src/app/data-converters/wikimetrics-timeseries.js
+++ b/src/app/data-converters/wikimetrics-timeseries.js
@@ -1,6 +1,19 @@
/**
* This module returns a method that knows how to translate json data from
- * wikimetrics to the canonical timeseries format understood by dashiki
+ * wikimetrics to the canonical timeseries format understood by dashiki
+ * wikimetrics format is as follows:
+ * {{
+ "result": {
+ "Sum": {
+ "edits": {
+ "2015-06-12 00:00:00": 24625.0,
+ "2015-09-18 00:00:00": 20972.0,
+ "2015-09-01 00:00:00": 30450.0,
+ "2015-12-29 00:00:00": 26637.0,
+ "2015-08-14 00:00:00": 21502.0,
+ "2015-08-22 00:00:00": 21239.0,
+
+ }
*/
define(function (require) {
'use strict';
@@ -40,7 +53,7 @@
// labels and colors can use the cohort name
[parameters.Cohort],
// wrap the values in an array to match the header
- _.forEach(rawData.result.Sum[submetric], function (value, key,
dict) {
+ _.forEach(rawData.items, function (value, key, dict) {
dict[key] = [[value ? parseFloat(value) : null]];
}),
[parameters.Cohort],
diff --git a/src/app/require.config.js b/src/app/require.config.js
index 5e94fe2..df74ca1 100644
--- a/src/app/require.config.js
+++ b/src/app/require.config.js
@@ -5,63 +5,66 @@
var require = {
baseUrl: '/src',
paths: {
- 'jquery' : 'bower_modules/jquery/dist/jquery',
- 'lodash' : 'bower_modules/lodash/main',
+ 'jquery': 'bower_modules/jquery/dist/jquery',
+ 'lodash': 'bower_modules/lodash/main',
// NOTE: the minified ko build is broken in 3.2.0
// (Issue reported https://github.com/knockout/knockout/issues/1528)
- 'knockout' : 'bower_modules/knockout/dist/knockout.debug',
- 'text' : 'bower_modules/requirejs-text/text',
- 'd3' : 'bower_modules/d3/d3',
- 'vega' : 'bower_modules/vega/vega',
- 'topojson' : 'bower_modules/topojson/topojson',
- 'moment' : 'bower_modules/moment/moment',
- 'semantic-dropdown' :
'bower_modules/semantic/build/uncompressed/modules/dropdown',
- 'semantic-popup' :
'bower_modules/semantic/build/uncompressed/modules/popup',
- 'mediawiki-storage' :
'bower_modules/mediawiki-storage/dist/mediawiki-storage',
- 'marked' : 'bower_modules/marked/lib/marked',
- 'twix' : 'bower_modules/twix/bin/twix',
- 'dygraphs' : 'bower_modules/dygraphs/dygraph-combined',
- 'nvd3' : 'bower_modules/nvd3/build/nv.d3',
- 'rickshaw' : 'bower_modules/rickshaw/rickshaw',
+ 'knockout': 'bower_modules/knockout/dist/knockout.debug',
+ 'text': 'bower_modules/requirejs-text/text',
+ 'd3': 'bower_modules/d3/d3',
+ 'vega': 'bower_modules/vega/vega',
+ 'topojson': 'bower_modules/topojson/topojson',
+ 'moment': 'bower_modules/moment/moment',
+ 'semantic-dropdown':
'bower_modules/semantic/build/uncompressed/modules/dropdown',
+ 'semantic-popup':
'bower_modules/semantic/build/uncompressed/modules/popup',
+ 'mediawiki-storage':
'bower_modules/mediawiki-storage/dist/mediawiki-storage',
+ 'marked': 'bower_modules/marked/lib/marked',
+ 'twix': 'bower_modules/twix/bin/twix',
+ 'dygraphs': 'bower_modules/dygraphs/dygraph-combined',
+ 'nvd3': 'bower_modules/nvd3/build/nv.d3',
+ 'rickshaw': 'bower_modules/rickshaw/rickshaw',
// NOTE: if you want functions like uri.expand, you must include both
// URI and URITemplate like define(['uri/URI', 'uri/URITemplate'] ...
// because URITemplate modifies URI when it's parsed
- 'uri' : 'bower_modules/URIjs/src',
- 'config' : 'app/config',
- 'logger' : 'lib/logger',
+ 'uri': 'bower_modules/URIjs/src',
+ 'config': 'app/config',
+ 'logger': 'lib/logger',
- 'api-finder' : 'app/apis/api-finder',
- 'dataConverterFactory' : 'app/data-converters/factory',
- 'typeahead' :
'bower_modules/typeahead.js/dist/typeahead.bundle',
- 'ajaxWrapper' : 'lib/ajax-wrapper',
- 'utils' : 'lib/utils',
- 'window' : 'lib/window',
- 'stateManager' : 'lib/state-manager',
+ 'api-finder': 'app/apis/api-finder',
+ 'dataConverterFactory': 'app/data-converters/factory',
+ 'typeahead': 'bower_modules/typeahead.js/dist/typeahead.bundle',
+ 'ajaxWrapper': 'lib/ajax-wrapper',
+ 'utils': 'lib/utils',
+ 'window': 'lib/window',
+ 'stateManager': 'lib/state-manager',
+ 'pageviews': 'lib/pageviews',
+ 'sitematrix': 'app/sitematrix',
// *** viewmodels
- 'viewmodels.copy-params' :
'app/ko-extensions/common-viewmodels/copy-params',
- 'viewmodels.single-select' :
'app/ko-extensions/common-viewmodels/single-select',
+ 'viewmodels.copy-params':
'app/ko-extensions/common-viewmodels/copy-params',
+ 'viewmodels.single-select':
'app/ko-extensions/common-viewmodels/single-select',
// *** custom observables
- 'observables.async' : 'app/ko-extensions/async-observables',
+ 'observables.async': 'app/ko-extensions/async-observables',
// *** apis
- 'apis.wikimetrics' : 'app/apis/wikimetrics',
- 'apis.annotations' : 'app/apis/annotations-api',
- 'apis.pageview' : 'app/apis/pageview-api',
- 'apis.datasets' : 'app/apis/datasets-api',
- 'apis.config' : 'app/apis/config-api',
+ 'apis.wikimetrics': 'app/apis/wikimetrics',
+ 'apis.annotations': 'app/apis/annotations-api',
+ 'apis.pageview': 'app/apis/pageview-api',
+ 'apis.datasets': 'app/apis/datasets-api',
+ 'apis.config': 'app/apis/config-api',
// *** converters
- 'converters.separated-values' :
'app/data-converters/separated-values',
+ 'converters.separated-values': 'app/data-converters/separated-values',
'converters.simple-separated-values':
'app/data-converters/simple-separated-values',
- 'converters.wikimetrics-timeseries' :
'app/data-converters/wikimetrics-timeseries',
- 'converters.funnel-data' :
'app/data-converters/funnel-data',
- 'converters.timeseries' :
'app/data-converters/timeseries-data',
- 'converters.annotations' :
'app/data-converters/annotations-data',
+ 'converters.wikimetrics-timeseries':
'app/data-converters/wikimetrics-timeseries',
+ 'converters.funnel-data': 'app/data-converters/funnel-data',
+ 'converters.timeseries': 'app/data-converters/timeseries-data',
+ 'converters.annotations': 'app/data-converters/annotations-data',
+ 'converters.pageview-api-response':
'app/data-converters/pageview-api-response',
// *** lib
- 'lib.polyfills' : 'lib/polyfills',
+ 'lib.polyfills': 'lib/polyfills',
},
shim: {
'ajaxWrapper': {
@@ -74,13 +77,15 @@
//typeahead
deps: ['jquery']
},
- d3: { exports: 'd3' },
+ d3: {
+ exports: 'd3'
+ },
nvd3: {
- exports: 'nv',
- deps: ['d3']
+ exports: 'nv',
+ deps: ['d3']
},
'semantic-popup': {
- deps: ['jquery']
+ deps: ['jquery']
},
}
};
diff --git a/src/app/sitematrix.js b/src/app/sitematrix.js
new file mode 100644
index 0000000..56a6627
--- /dev/null
+++ b/src/app/sitematrix.js
@@ -0,0 +1,87 @@
+/**
+ * Mdule that gets the sitematrix and parses it.
+ * Site matrix location is on config.js
+ * Once initialized this class is just a singleton that holds an application
scoped cache
+ *
https://meta.wikimedia.org/w/api.php?action=sitematrix&formatversion=2&format=json&&maxage=3600&smaxage=3600
+ * Format:
+ * {"sitematrix":{"count":894,
+ * "0":{"code":"aa","name":"Qafár af","
+
site":[{"url":"https://aa.wikipedia.org","dbname":"aawiki","code":"wiki","sitename":"Wikipedia","closed":""},{"url":"https://aa.wiktionary.org","dbname":"aawiktionary","code":"wiktionary","sitename":"Wiktionary","closed":""},{"url":"https://aa.wikibooks.org","dbname":"aawikibooks","code":"wikibooks","sitename":"Wikibooks","closed":""}],"localname":"Afar"},
+ * "1":{"code":"ab","name":"Аҧсшәа",
+
"site":[{"url":"https://ab.wikipedia.org","dbname":"abwiki","code":"wiki","sitename":"Авикипедиа"},
+ */
+define(function (require) {
+ 'use strict';
+
+ var config = require('config'),
+ _ = require('lodash');
+
+
+ function Sitematrix() {}
+
+ Sitematrix.cache = null;
+ /**
+ * Jsonp request for sitematrix, wikimedia api doesn't allow CORS
+ * from non-whitelisted domains
+ */
+ Sitematrix.prototype.getProjectUrl = function (config, dbname) {
+ var endpoint = config.sitematrix.endpoint;
+ var deferred = new $.Deferred();
+
+ // jsonp request for sitematrix, cors is not allowed
+ $.ajax({
+ url: endpoint,
+ // Tell jQuery we're expecting JSONP
+ dataType: "jsonp",
+ //otherwise jquery takes the liberty of not caching your jsonp
requests
+ cache: true
+
+ }).then(function (data) {
+
+ // transform sitematrix in structure that allows easy o(1)
lookups
+ var _sitematrix = {};
+ _.forEach(data.sitematrix, function (value, key) {
+
+ _.forEach(value.site, function (_value, index) {
+
+ /*
+ Each record is like:
+ closed: ""
+ code: "wiki"
+ dbname: "aawiki"
+ sitename: "Wikipedia"
+ url: "https://aa.wikipedia.org";
+ url needs to be transform to: aa.wikipedia
+ */
+ var urlEndpoint = _value.url.replace("https://";,
"").replace(".org", "");
+
+ //building lookup both ways
+ _sitematrix[_value.dbname] = urlEndpoint;
+ _sitematrix[urlEndpoint] = _value.dbname;
+
+ });
+
+ });
+ // can we populate an application wide cache now? or is this
bad practice?
+ if (!Sitematrix.cache) {
+ Sitematrix.cache = _sitematrix;
+ }
+
+
+ deferred.resolve(_sitematrix[dbname]);
+ })
+ .fail(function (error) {
+
+ logger.error(error);
+ deferred.reject(error);
+ });
+
+
+
+ return deferred.promise();
+ };
+
+
+
+ return new Sitematrix();
+});
\ No newline at end of file
diff --git a/src/components/wikimetrics-visualizer/wikimetrics-visualizer.js
b/src/components/wikimetrics-visualizer/wikimetrics-visualizer.js
index a633cbe..b058eb0 100644
--- a/src/components/wikimetrics-visualizer/wikimetrics-visualizer.js
+++ b/src/components/wikimetrics-visualizer/wikimetrics-visualizer.js
@@ -43,10 +43,14 @@
promises = projects.map(function (project) {
return api.getData(metric, project.database,
showBreakdown);
});
+
+ //invoqued when all promises are done
$.when.apply(this, promises).then(function () {
+
this.mergedData(TimeseriesData.mergeAll(_.toArray(arguments)));
this.applyColors(projects);
}.bind(this));
+
} else {
this.mergedData(new TimeseriesData([]));
}
@@ -88,4 +92,4 @@
viewModel: WikimetricsVisualizer,
template: templateMarkup
};
-});
+});
\ No newline at end of file
diff --git a/src/lib/pageviews.js b/src/lib/pageviews.js
new file mode 100644
index 0000000..a051c27
--- /dev/null
+++ b/src/lib/pageviews.js
@@ -0,0 +1,411 @@
+/**
+ * @license
+ * Copyright 2015 Thomas Steiner (@tomayac). All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+define([], function () {
+ 'use strict';
+
+ var request;
+
+ var USER_AGENT = 'pageviews.js';
+
+ //TODO pull request bower module
+
+ // Dynamically adapt to the runtime environment
+ var environment = typeof window === 'undefined' ? 'node' : 'browser';
+ if (environment === 'node') {
+ // Node.js
+ request = require('request');
+ // browser chokes on this due to stric
+ //TODO pull request
+ //var package = require('./package.json');
+ // The user agent to use
+ USER_AGENT = 'pageviews.js';
+ } else {
+ // Browser
+ request = function (options, callback) {
+ var xhr = new XMLHttpRequest();
+ xhr.addEventListener('load', function () {
+ return callback(null, {
+ statusCode: this.status
+ }, this.responseText);
+ });
+ xhr.addEventListener('error', function (e) {
+ return callback(e);
+ });
+ xhr.open('GET', options.url);
+ xhr.send();
+ };
+ }
+
+ var pageviews = (function () {
+ // The Pageviews base URL
+ var BASE_URL = 'https://wikimedia.org/api/rest_v1';
+
+ var _access = {
+ default: 'all-access',
+ allowed: ['all-access', 'desktop', 'mobile-web', 'mobile-app']
+ };
+
+ var _agent = {
+ default: 'all-agents',
+ allowed: ['all-agents', 'user', 'spider', 'bot']
+ };
+
+ var _granularityAggregated = {
+ default: 'hourly',
+ allowed: ['daily', 'hourly', 'monthly']
+ };
+
+ var _granularityPerArticle = {
+ default: 'daily',
+ allowed: ['daily']
+ };
+
+ /**
+ * Checks the input parameters for validity.
+ */
+ var _checkParams = function (params, caller) {
+ if (!params) {
+ return new Error('Required parameters missing.');
+ }
+ // Required: project or projects
+ if ((!params.project) && (!params.projects)) {
+ if (caller === 'getAggregatedPageviews' || caller ===
'getTopPageviews') {
+ return new Error('Required parameter "project" or "projects"
missing.');
+ } else {
+ return new Error('Required parameter "project" missing.');
+ }
+ }
+ if ((params.project) && (params.project.indexOf('.') === -1)) {
+ return new Error('Required parameter "project" invalid.');
+ }
+ if ((caller === 'getAggregatedPageviews') ||
+ (caller === 'getTopPageviews')) {
+ if (params.projects) {
+ if ((!Array.isArray(params.projects)) || (!params.projects.length) ||
+ (params.projects.filter(function (project) {
+ return project.indexOf('.') === -1;
+ }).length)
+ ) {
+ return new Error('Required parameter "projects" invalid.');
+ }
+ }
+ }
+ // Required: article or articles
+ if (caller === 'getPerArticlePageviews') {
+ if ((!params.article) && (!params.articles)) {
+ return new Error('Required parameter "article" or "articles"
missing.');
+ }
+ if (params.articles) {
+ if ((!Array.isArray(params.articles)) || (!params.articles.length)) {
+ return new Error('Required parameter "articles" invalid.');
+ }
+ }
+ }
+ if (caller === 'getPerArticlePageviews') {
+ // Required: start
+ if ((!params.start) ||
+ (!/^(?:19|20)\d\d[- /.]?(?:0[1-9]|1[012])[-
/.]?(?:0[1-9]|[12][0-9]|3[01])$/.test(params.start))) {
+ return new Error('Required parameter "start" missing or invalid.');
+ }
+ // Required: end
+ if ((!params.end) ||
+ (!/^(19|20)\d\d[- /.]?(0[1-9]|1[012])[-
/.]?(0[1-9]|[12][0-9]|3[01])$/.test(params.end))) {
+ return new Error('Required parameter "end" missing or invalid.');
+ }
+ } else if (caller === 'getAggregatedPageviews') {
+ // Required: start
+ if ((!params.start) ||
+ (!/^(?:19|20)\d\d[- /.]?(?:0[1-9]|1[012])[-
/.]?(?:0[1-9]|[12][0-9]|3[01])[- /.]?(?:[012][0-9])$/.test(params.start))) {
+ return new Error('Required parameter "start" missing or invalid.');
+ }
+ // Required: end
+ if ((!params.end) ||
+ (!/^(19|20)\d\d[- /.]?(0[1-9]|1[012])[-
/.]?(0[1-9]|[12][0-9]|3[01])[- /.]?(?:[012][0-9])$/.test(params.end))) {
+ return new Error('Required parameter "end" missing or invalid.');
+ }
+ }
+ if (caller === 'getTopPageviews') {
+ // Required: year
+ if ((!params.year) || (!/^(?:19|20)\d\d$/.test(params.year))) {
+ return new Error('Required parameter "year" missing or invalid.');
+ }
+ // Required: month
+ if ((!params.month) || (!/^(?:0?[1-9]|1[012])$/.test(params.month))) {
+ return new Error('Required parameter "month" missing or invalid.');
+ }
+ // Required: day
+ if ((!params.day) ||
(!/^(?:0?[1-9]|[12][0-9]|3[01])$/.test(params.day))) {
+ return new Error('Required parameter "day" missing or invalid.');
+ }
+ if ((params.limit) && !/^\d+$/.test(params.limit) &&
+ (0 < params.limit) && (params.limit <= 1000)) {
+ return new Error('Invalid optional parameter "limit".');
+ }
+ }
+ // Optional: access
+ if ((params.access) && (_access.allowed.indexOf(params.access) === -1)) {
+ return new Error('Invalid optional parameter "access".');
+ }
+ // Optional: agent
+ if ((params.agent) && (_agent.allowed.indexOf(params.agent) === -1)) {
+ return new Error('Invalid optional parameter "agent".');
+ }
+ // Optional: granularity
+ if (params.granularity) {
+ if (caller === 'getAggregatedPageviews') {
+ if (_granularityAggregated.allowed.indexOf(params.granularity) ===
-1) {
+ return new Error('Invalid optional parameter "granularity".');
+ }
+ } else if (caller === 'getPerArticlePageviews') {
+ if (_granularityPerArticle.allowed.indexOf(params.granularity) ===
-1) {
+ return new Error('Invalid optional parameter "granularity".');
+ }
+ }
+ }
+ return params;
+ };
+
+ /**
+ * Checks the results for validity, in case of success returns the parsed
+ * data, else returns the error details.
+ */
+ var _checkResult = function (error, response, body) {
+ var data;
+ if (error || response.statusCode !== 200) {
+ if (error) {
+ return error;
+ }
+ if (response.statusCode === 404) {
+ try {
+ data = JSON.parse(body);
+ return new Error(data.detail);
+ } catch (e) {
+ return new Error(e);
+ }
+ }
+ return new Error('Status code ' + response.statusCode);
+ }
+ try {
+ data = JSON.parse(body);
+ } catch (e) {
+ return new Error(e);
+ }
+ return data;
+ };
+
+ var _getPerArticlePageviews = function (params) {
+ return new Promise(function (resolve, reject) {
+ params = _checkParams(params, 'getPerArticlePageviews');
+ if (params.stack) {
+ return reject(params);
+ }
+ // Call yourself recursively in case of multiple articles
+ if (params.articles) {
+ var promises = [];
+ params.articles.map(function (article, i) {
+ var newParams = params;
+ delete newParams.articles;
+ newParams.article = article;
+ promises[i] = _getPerArticlePageviews(newParams);
+ });
+ return resolve(Promise.all(promises));
+ }
+ // Required params
+ var project = params.project;
+ var article = encodeURIComponent(params.article.replace(/\s/g, '_'));
+ var start = params.start;
+ var end = params.end;
+ // Optional params
+ var access = params.access ? params.access : _access.default;
+ var agent = params.agent ? params.agent : _agent.default;
+ var granularity = params.granularity ?
+ params.granularity : _granularityPerArticle.default;
+
+ var options = {
+ url: BASE_URL + '/metrics/pageviews/per-article' +
+ '/' + project +
+ '/' + access +
+ '/' + agent +
+ '/' + article +
+ '/' + granularity +
+ '/' + start +
+ '/' + end,
+ headers: {
+ 'User-Agent': USER_AGENT
+ }
+ };
+ request(options, function (error, response, body) {
+ var result = _checkResult(error, response, body);
+ if (result.stack) {
+ return reject(result);
+ }
+ return resolve(result);
+ });
+ });
+ };
+
+ var _getAggregatedPageviews = function (params) {
+ return new Promise(function (resolve, reject) {
+ params = _checkParams(params, 'getAggregatedPageviews');
+ if (params.stack) {
+ return reject(params);
+ }
+ // Call yourself recursively in case of multiple projects
+ if (params.projects) {
+ var promises = [];
+ params.projects.map(function (project, i) {
+ var newParams = params;
+ delete newParams.projects;
+ newParams.project = project;
+ promises[i] = _getAggregatedPageviews(newParams);
+ });
+ return resolve(Promise.all(promises));
+ }
+ // Required params
+ var project = params.project;
+ var start = params.start;
+ var end = params.end;
+ // Optional params
+ var access = params.access ? params.access : _access.default;
+ var agent = params.agent ? params.agent : _agent.default;
+ var granularity = params.granularity ?
+ params.granularity : _granularityAggregated.default;
+ var options = {
+ url: BASE_URL + '/metrics/pageviews/aggregate' +
+ '/' + project +
+ '/' + access +
+ '/' + agent +
+ '/' + granularity +
+ '/' + start +
+ '/' + end,
+ headers: {
+ 'User-Agent': USER_AGENT
+ }
+ };
+ request(options, function (error, response, body) {
+ var result = _checkResult(error, response, body);
+ if (result.stack) {
+ return reject(result);
+ }
+ return resolve(result);
+ });
+ });
+ };
+
+ var _getTopPageviews = function (params) {
+ return new Promise(function (resolve, reject) {
+ params = _checkParams(params, 'getTopPageviews');
+ if (params.stack) {
+ return reject(params);
+ }
+ // Call yourself recursively in case of multiple projects
+ if (params.projects) {
+ var promises = [];
+ params.projects.map(function (project, i) {
+ var newParams = params;
+ delete newParams.projects;
+ newParams.project = project;
+ promises[i] = _getTopPageviews(newParams);
+ });
+ return resolve(Promise.all(promises));
+ }
+ // Required params
+ var project = params.project;
+ var year = params.year;
+ var month = typeof params.month === 'number' && params.month < 10 ?
+ '0' + params.month : params.month;
+ var day = typeof params.day === 'number' && params.day < 10 ?
+ '0' + params.day : params.day;
+ var limit = params.limit || false;
+ // Optional params
+ var access = params.access ? params.access : _access.default;
+ var options = {
+ url: BASE_URL + '/metrics/pageviews/top' +
+ '/' + project +
+ '/' + access +
+ '/' + year +
+ '/' + month +
+ '/' + day,
+ headers: {
+ 'User-Agent': USER_AGENT
+ }
+ };
+ request(options, function (error, response, body) {
+ var result = _checkResult(error, response, body);
+ if (result.stack) {
+ return reject(result);
+ }
+ if (limit) {
+ result.items[0].articles = result.items[0].articles.slice(0,
limit);
+ }
+ return resolve(result);
+ });
+ });
+ };
+
+ var _getPageviewsDimensions = function () {
+ return new Promise(function (resolve, reject) {
+ var options = {
+ url: BASE_URL + '/metrics/pageviews/',
+ headers: {
+ 'User-Agent': USER_AGENT
+ }
+ };
+ request(options, function (error, response, body) {
+ var result = _checkResult(error, response, body);
+ if (result.stack) {
+ return reject(result);
+ }
+ return resolve(result);
+ });
+ });
+ };
+
+ return {
+ /**
+ * This is the root of all pageview data endpoints. The list of paths
that
+ * this returns includes ways to query by article, project, top articles,
+ * etc. If browsing the interactive documentation, see the specifics for
+ * each endpoint below.
+ */
+ getPageviewsDimensions: _getPageviewsDimensions,
+
+ /**
+ * Given a Mediawiki article and a date range, returns a daily
timeseries of
+ * its pageview counts. You can also filter by access method and/or agent
+ * type.
+ */
+ getPerArticlePageviews: _getPerArticlePageviews,
+
+ /**
+ * Given a date range, returns a timeseries of pageview counts. You can
+ * filter by project, access method and/or agent type. You can choose
+ * between daily and hourly granularity as well.
+ */
+ getAggregatedPageviews: _getAggregatedPageviews,
+
+ /**
+ * Lists the 1000 most viewed articles for a given project and timespan
+ * (year, month or day). You can filter by access method.
+ */
+ getTopPageviews: _getTopPageviews
+ };
+ });
+
+ return pageviews();
+});
\ No newline at end of file
--
To view, visit https://gerrit.wikimedia.org/r/270867
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I1d403595cd3b25a099721227f6a1cf0b5fddd34d
Gerrit-PatchSet: 1
Gerrit-Project: analytics/dashiki
Gerrit-Branch: master
Gerrit-Owner: Nuria <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
