Mforns has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/393591 )
Change subject: Add pageviews by country endpoint
......................................................................
Add pageviews by country endpoint
Adds a new tops endpoint to aqs that returns top countries by number
of pageviews for a given project, access type and time range.
Bug: T181520
Change-Id: I88c28812b8f2a854ed9ec5abacbc5c42c85db194
---
M lib/aqsUtil.js
M sys/pageviews.js
M sys/pageviews.yaml
M test/aqs_test_module.yaml
M test/features/pageviews/pageviews.js
M v1/pageviews.yaml
6 files changed, 216 insertions(+), 0 deletions(-)
Approvals:
Mforns: Verified; Looks good to me, approved
diff --git a/lib/aqsUtil.js b/lib/aqsUtil.js
index 4eac56b..3ef429d 100644
--- a/lib/aqsUtil.js
+++ b/lib/aqsUtil.js
@@ -185,6 +185,22 @@
throwIfNeeded(errors);
};
+aqsUtil.validateYearMonth = function(rp) {
+ const errors = [];
+
+ aqsUtil.normalizeProject(rp);
+
+ // fake a timestamp in the YYYYMMDDHH format so we can reuse the validator
+ const day = '01';
+ const validDate =
aqsUtil.validateTimestamp(`${rp.year}${rp.month}${day}00`);
+
+ if (!validDate) {
+ errors.push('Given year/month is invalid date');
+ }
+
+ throwIfNeeded(errors);
+};
+
aqsUtil.convertTimestampToDate = function(timestamp) {
const year = parseInt(timestamp.substring(0, 4), 10);
const month = parseInt(timestamp.substring(4, 6), 10) - 1;
diff --git a/sys/pageviews.js b/sys/pageviews.js
index a9ef7cd..6db613a 100644
--- a/sys/pageviews.js
+++ b/sys/pageviews.js
@@ -31,6 +31,7 @@
articleFlat: 'pageviews.per.article.flat',
project_v2: 'pageviews.per.project.v2',
tops: 'top.pageviews',
+ bycountry: 'top.bycountry'
};
const tableSchemas = {
@@ -95,6 +96,23 @@
{ attribute: 'year', type: 'hash' },
{ attribute: 'month', type: 'hash' },
{ attribute: 'day', type: 'hash' },
+ ]
+ },
+ bycountry: {
+ table: tables.bycountry,
+ version: 1,
+ attributes: {
+ project: 'string',
+ access: 'string',
+ year: 'string',
+ month: 'string',
+ countriesJSON: 'json'
+ },
+ index: [
+ { attribute: 'project', type: 'hash' },
+ { attribute: 'access', type: 'hash' },
+ { attribute: 'year', type: 'hash' },
+ { attribute: 'month', type: 'hash' }
]
}
};
@@ -309,6 +327,37 @@
});
};
+PJVS.prototype.pageviewsByCountry = function(hyper, req) {
+ const rp = req.params;
+
+ aqsUtil.validateYearMonth(rp);
+
+ const dataRequest = hyper.get({
+ uri: tableURI(rp.domain, tables.bycountry),
+ body: {
+ table: tables.bycountry,
+ attributes: {
+ project: rp.project,
+ access: rp.access,
+ year: rp.year,
+ month: rp.month
+ }
+ }
+
+ }).catch(aqsUtil.notFoundCatcher);
+
+ return dataRequest.then(aqsUtil.normalizeResponse).then((res) => {
+ if (res.body.items) {
+ res.body.items.forEach((item) => {
+ item.countries = item.countriesJSON;
+ delete item.countriesJSON;
+ });
+ }
+
+ return res;
+ });
+};
+
module.exports = function(options) {
const pjvs = new PJVS(options);
@@ -319,6 +368,7 @@
pageviewsForArticle: pjvs.pageviewsForArticleFlat.bind(pjvs),
pageviewsForProjects: pjvs.pageviewsForProjects.bind(pjvs),
pageviewsForTops: pjvs.pageviewsForTops.bind(pjvs),
+ pageviewsByCountry: pjvs.pageviewsByCountry.bind(pjvs),
},
resources: [
{
@@ -332,6 +382,10 @@
// top pageviews table
uri: `/{domain}/sys/table/${tables.tops}`,
body: tableSchemas.tops,
+ }, {
+ // pageviews by country table
+ uri: `/{domain}/sys/table/${tables.bycountry}`,
+ body: tableSchemas.bycountry,
}
]
};
diff --git a/sys/pageviews.yaml b/sys/pageviews.yaml
index cfb824a..811acfc 100644
--- a/sys/pageviews.yaml
+++ b/sys/pageviews.yaml
@@ -11,3 +11,7 @@
get:
summary: query top pageviews
operationId: pageviewsForTops
+ /top-by-country/{project}/{access}/{year}/{month}:
+ get:
+ summary: query top countries by pageviews
+ operationId: pageviewsByCountry
diff --git a/test/aqs_test_module.yaml b/test/aqs_test_module.yaml
index 6d3f0bf..82cde2d 100644
--- a/test/aqs_test_module.yaml
+++ b/test/aqs_test_module.yaml
@@ -96,6 +96,23 @@
articlesJSON: '{{request.body.articles}}'
x-monitor: false
+
/pageviews/insert-top-by-country/{project}/{access}/{year}/{month}:
+ post:
+ x-request-handler:
+ - put_to_storage:
+ request:
+ method: 'put'
+ uri: '/{domain}/sys/table/top.bycountry/'
+ body:
+ table: 'top.bycountry'
+ attributes:
+ project: '{{request.params.project}}'
+ access: '{{request.params.access}}'
+ year: '{{request.params.year}}'
+ month: '{{request.params.month}}'
+ countriesJSON: '{{request.body.countries}}'
+ x-monitor: false
+
/legacy/pagecounts/insert-aggregate/{project}/{access-site}/{granularity}/{timestamp}/{count}:
post:
x-request-handler:
diff --git a/test/features/pageviews/pageviews.js
b/test/features/pageviews/pageviews.js
index f8aca78..474dd9a 100644
--- a/test/features/pageviews/pageviews.js
+++ b/test/features/pageviews/pageviews.js
@@ -31,6 +31,10 @@
top: {
all: '/pageviews/top/en.wikipedia/mobile-web/2015/01/all-days',
insert:
'/pageviews/insert-top/en.wikipedia/mobile-web/2015/01/all-days'
+ },
+ bycountry: {
+ all: '/pageviews/top-by-country/en.wikipedia/all-access/2015/01',
+ insert:
'/pageviews/insert-top-by-country/en.wikipedia/all-access/2015/01'
}
}
var projectEndpointStrip =
'/pageviews/aggregate/www.en.wikipedia.org/all-access/all-agents/hourly/1969010100/1971010100';
@@ -381,4 +385,35 @@
assert.deepEqual(res.body.items[0].articles[1].article, 'two\\');
});
});
+
+ // By country test
+
+ it('should return the correct countries after insertion', function () {
+ return preq.post({
+ uri: server.config.aqsURL + endpoints.bycountry.insert,
+ body: {
+ countries: [{
+ rank: 1,
+ country: 'Republic of Mriiii\'duuh',
+ views: 2000
+ },{
+ rank: 2,
+ country: 'Kingdom of OOOOOOOOOOH',
+ views: 1000
+ }
+ ]
+ },
+ headers: { 'content-type': 'application/json' }
+
+ }).then(function() {
+ return preq.get({
+ uri: server.config.aqsURL + endpoints.bycountry.all
+ });
+ }).then(function(res) {
+ assert.deepEqual(res.body.items.length, 1);
+ assert.deepEqual(res.body.items[0].countries[0].country, 'Republic
of Mriiii\'duuh');
+ assert.deepEqual(res.body.items[0].countries[1].views, 1000);
+ assert.deepEqual(res.body.items[0].countries[1].country, 'Kingdom
of OOOOOOOOOOH');
+ });
+ })
});
diff --git a/v1/pageviews.yaml b/v1/pageviews.yaml
index 3f6ed18..44be4af 100644
--- a/v1/pageviews.yaml
+++ b/v1/pageviews.yaml
@@ -282,6 +282,78 @@
views: 0
rank: 1
+ /top-by-country/{project}/{access}/{year}/{month}:
+ get:
+ tags:
+ - Pageviews data
+ summary: Get pageviews by country and access method.
+ description: |
+ Lists the pageviews to this project, split by country of origin for a
given month.
+ Because of privacy reasons, pageviews are given in a bucketed format,
and countries
+ with less than 100 views do not get reported.
+ Stability:
[experimental](https://www.mediawiki.org/wiki/API_versioning#Experimental)
+ produces:
+ - application/json
+ parameters:
+ - name: project
+ in: path
+ description: The domain of any Wikimedia project (.org optional),
like gl.wikimedia.
+ type: string
+ required: true
+ - name: access
+ in: path
+ description: The method of access to this project. Use all-access
for all methods.
+ type: string
+ enum: ['all-access', 'desktop', 'mobile-app', 'mobile-web']
+ required: true
+ - name: year
+ in: path
+ description: The year for which to retrieve top countries, in YYYY
format.
+ type: string
+ required: true
+ - name: month
+ in: path
+ description: The month for which to retrieve top countries, in MM
format.
+ type: string
+ required: true
+ responses:
+ '200':
+ description: The list of top countries by pageviews in the project
+ schema:
+ $ref: '#/definitions/by-country'
+ default:
+ description: Error
+ schema:
+ $ref: '#/definitions/problem'
+ x-request-handler:
+ - get_from_backend:
+ request:
+ uri:
/{domain}/sys/pageviews/top-by-country/{project}/{access}/{year}/{month}
+ x-monitor: true
+ x-amples:
+ - title: Get top countries by page views
+ request:
+ params:
+ domain: wikimedia.org
+ project: en.wikipedia
+ access: all-access
+ year: '1970'
+ month: '01'
+ response:
+ status: 200
+ headers:
+ content-type: application/json
+ body:
+ items:
+ - project: en.wikipedia
+ access: all-access
+ year: '1970'
+ month: '01'
+ countries:
+ - country: '-'
+ views: 0
+ rank: 1
+
definitions:
# A https://tools.ietf.org/html/draft-nottingham-http-problem
problem:
@@ -370,3 +442,21 @@
articles:
# format for this is a json array: [{rank: 1, article:
<<title>>, views: 123}, ...]
type: string
+
+ by-country:
+ properties:
+ items:
+ type: array
+ items:
+ properties:
+ project:
+ type: string
+ access:
+ type : string
+ year:
+ type: string
+ month:
+ type: string
+ countries:
+ # format for this is a json array: [{rank: 1, country: <<name>>,
views: 123}, ...]
+ type: string
--
To view, visit https://gerrit.wikimedia.org/r/393591
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I88c28812b8f2a854ed9ec5abacbc5c42c85db194
Gerrit-PatchSet: 9
Gerrit-Project: analytics/aqs
Gerrit-Branch: master
Gerrit-Owner: Fdans <[email protected]>
Gerrit-Reviewer: Fdans <[email protected]>
Gerrit-Reviewer: Joal <[email protected]>
Gerrit-Reviewer: Mforns <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
