Milimetric has submitted this change and it was merged.
Change subject: Add unique_devices endpoint to AQS
......................................................................
Add unique_devices endpoint to AQS
Change pageviews endpoint to facilitate other endpoint integration.
Add unique devices front and sys specifications and js code.
Correct small misconception in lib extracted function.
Bug: T129518
Change-Id: Ife9547679bfdbda8ccc1de47aede30290180ebc9
---
M config.test.yaml
M lib/aqsUtil.js
M projects/aqs_default.yaml
A sys/unique-devices.js
A sys/unique-devices.yaml
M test/aqs_test_module.yaml
A test/features/unique-devices/unique-devices.js
M v1/pageviews.yaml
A v1/unique-devices.yaml
9 files changed, 344 insertions(+), 18 deletions(-)
Approvals:
Mobrovac: Looks good to me, but someone else must approve
Milimetric: Verified; Looks good to me, approved
diff --git a/config.test.yaml b/config.test.yaml
index 5e50bf2..f89f281 100644
--- a/config.test.yaml
+++ b/config.test.yaml
@@ -1,4 +1,4 @@
-# A synthetic domain to test pageviews module data
+# A synthetic domain to test aqs modules data
aqs.wikimedia.org: &analytics.wikimedia.org
x-modules:
/:
@@ -28,7 +28,7 @@
services:
- name: aqs
module: hyperswitch
- conf:
+ conf:
port: 7231
spec: *spec_root
salt: secret
diff --git a/lib/aqsUtil.js b/lib/aqsUtil.js
index 59ed927..b66bb96 100644
--- a/lib/aqsUtil.js
+++ b/lib/aqsUtil.js
@@ -49,12 +49,11 @@
/**
* Cleans the project parameter so it can be passed in as either
en.wikipedia.org or en.wikipedia
*/
-var stripOrgFromProject = function(rp) {
+aqsUtil.stripOrgFromProject = function(rp) {
rp.project = rp.project.replace(/\.org$/, '');
};
-aqsUtil.stripOrgFromProject = stripOrgFromProject;
-var validateTimestamp = function(timestamp, opts) {
+aqsUtil.validateTimestamp = function(timestamp, opts) {
opts = opts || {};
if (timestamp && timestamp.length > 10) {
@@ -79,8 +78,6 @@
return false;
}
};
-aqsUtil.validateTimestamp = validateTimestamp;
-
aqsUtil.validateStartAndEnd = function(rp, opts) {
opts = opts || {};
@@ -90,12 +87,12 @@
'invalid, must be a valid date in YYYYMMDD format' :
'invalid, must be a valid timestamp in YYYYMMDDHH format';
- stripOrgFromProject(rp);
+ aqsUtil.stripOrgFromProject(rp);
- if (!validateTimestamp(rp.start, opts)) {
+ if (!aqsUtil.validateTimestamp(rp.start, opts)) {
errors.push('start timestamp is ' + invalidMessage);
}
- if (!validateTimestamp(rp.end, opts)) {
+ if (!aqsUtil.validateTimestamp(rp.end, opts)) {
errors.push('end timestamp is ' + invalidMessage);
}
@@ -109,7 +106,7 @@
aqsUtil.validateYearMonthDay = function(rp) {
var errors = [];
- stripOrgFromProject(rp);
+ aqsUtil.stripOrgFromProject(rp);
if (rp.month === 'all-months' && rp.day !== 'all-days') {
errors.push('day must be "all-days" when passing "all-months"');
@@ -119,7 +116,7 @@
throwIfNeeded(errors);
// fake a timestamp in the YYYYMMDDHH format so we can reuse the validator
- var validDate = validateTimestamp(
+ var validDate = aqsUtil.validateTimestamp(
rp.year +
((rp.month === 'all-months') ? '01' : rp.month) +
((rp.day === 'all-days') ? '01' : rp.day) +
diff --git a/projects/aqs_default.yaml b/projects/aqs_default.yaml
index 95c9b44..540b267 100644
--- a/projects/aqs_default.yaml
+++ b/projects/aqs_default.yaml
@@ -13,8 +13,10 @@
x-host-basePath: /api/rest_v1
x-modules:
- /:
+ /pageviews:
- path: v1/pageviews.yaml
+ /unique-devices:
+ - path: v1/unique-devices.yaml
/{api:sys}:
swagger: 2.0
@@ -28,3 +30,5 @@
conf: '{{options.table}}'
/pageviews:
- path: sys/pageviews.js
+ /unique-devices:
+ - path: sys/unique-devices.js
diff --git a/sys/unique-devices.js b/sys/unique-devices.js
new file mode 100644
index 0000000..cec0e67
--- /dev/null
+++ b/sys/unique-devices.js
@@ -0,0 +1,92 @@
+'use strict';
+
+/**
+ * Unique devices API module
+ *
+ * This API serves pre-aggregated unique devices statistics from Cassandra
+ */
+
+var HyperSwitch = require('hyperswitch');
+var path = require('path');
+var HTTPError = HyperSwitch.HTTPError;
+var URI = HyperSwitch.URI;
+
+var aqsUtil = require('../lib/aqsUtil');
+
+var spec = HyperSwitch.utils.loadSpec(path.join(__dirname,
'unique-devices.yaml'));
+
+// Unique devices Service
+function UDVS(options) {
+ this.options = options;
+}
+
+
+var tables = {
+ project: 'unique.devices',
+};
+var tableURI = function(domain, tableName) {
+ return new URI([domain, 'sys', 'table', tableName, '']);
+};
+var tableSchemas = {
+ project: {
+ table: tables.project,
+ version: 1,
+ attributes: {
+ project: 'string',
+ 'access-site': 'string',
+ granularity: 'string',
+ // the hourly timestamp will be stored as YYYYMMDDHH
+ timestamp: 'string',
+ devices: 'long'
+ },
+ index: [
+ { attribute: 'project', type: 'hash' },
+ { attribute: 'access-site', type: 'hash' },
+ { attribute: 'granularity', type: 'hash' },
+ { attribute: 'timestamp', type: 'range', order: 'asc' },
+ ]
+ }
+};
+
+
+UDVS.prototype.uniqueDevices = function(hyper, req) {
+ var rp = req.params;
+
+ // dates are passed in as YYYYMMDD but we need the HH to be validated using
+ // our generic data validator, so we pass "fakeHour"
+ aqsUtil.validateStartAndEnd(rp, { fakeHour: true });
+
+ var dataRequest = hyper.get({
+ uri: tableURI(rp.domain, tables.project),
+ body: {
+ table: tables.project,
+ attributes: {
+ project: rp.project,
+ 'access-site': rp['access-site'],
+ granularity: rp.granularity,
+ timestamp: { between: [rp.start, rp.end] },
+ }
+ }
+
+ }).catch(aqsUtil.notFoundCatcher);
+
+ return dataRequest.then(aqsUtil.normalizeResponse);
+};
+
+module.exports = function(options) {
+ var udvs = new UDVS(options);
+
+ return {
+ spec: spec,
+ operations: {
+ uniqueDevices: udvs.uniqueDevices.bind(udvs)
+ },
+ resources: [
+ {
+ // unique devices per project table
+ uri: '/{domain}/sys/table/' + tables.project,
+ body: tableSchemas.project,
+ }
+ ]
+ };
+};
diff --git a/sys/unique-devices.yaml b/sys/unique-devices.yaml
new file mode 100644
index 0000000..ad56b44
--- /dev/null
+++ b/sys/unique-devices.yaml
@@ -0,0 +1,5 @@
+paths:
+ /{project}/{access-site}/{granularity}/{start}/{end}:
+ get:
+ summary: query unique devices per project
+ operationId: uniqueDevices
diff --git a/test/aqs_test_module.yaml b/test/aqs_test_module.yaml
index 557f010..fa93af9 100644
--- a/test/aqs_test_module.yaml
+++ b/test/aqs_test_module.yaml
@@ -6,7 +6,7 @@
swagger: '2.0'
info:
version: 1.0.0-beta
- title: Wikimedia testing APIs
+ title: AQS testing APIs
x-is-api-root: true
paths:
/pageviews/insert-per-article-flat/{project}/{article}/{granularity}/{timestamp}/{views}:
@@ -94,3 +94,20 @@
day: '{{request.params.day}}'
articlesJSON: '{{request.body.articles}}'
x-monitor: false
+
+
/unique-devices/insert/{project}/{access-site}/{granularity}/{timestamp}/{devices}:
+ post:
+ x-request-handler:
+ - put_to_storage:
+ request:
+ method: 'put'
+ uri: '/{domain}/sys/table/unique.devices/'
+ body:
+ table: 'unique.devices'
+ attributes:
+ project: '{{request.params.project}}'
+ 'access-site': '{{request.params.access-site}}'
+ granularity: '{{request.params.granularity}}'
+ timestamp: '{{request.params.timestamp}}'
+ devices: '{{request.params.devices}}'
+ x-monitor: false
diff --git a/test/features/unique-devices/unique-devices.js
b/test/features/unique-devices/unique-devices.js
new file mode 100644
index 0000000..53845d7
--- /dev/null
+++ b/test/features/unique-devices/unique-devices.js
@@ -0,0 +1,82 @@
+'use strict';
+
+// mocha defines to avoid JSHint breakage
+/* global describe, it, before, beforeEach, afterEach */
+
+var assert = require('../../utils/assert.js');
+var preq = require('preq');
+var server = require('../../utils/server.js');
+
+
+describe('unique-devices endpoints', function () {
+ this.timeout(20000);
+
+ //Start server before running tests
+ before(function () { return server.start(); });
+
+ // NOTE: this tests using the projects/aqs_default.yaml config, so
+ // it doesn't know about the /metrics root like the prod config does
+ var endpoint =
'/unique-devices/en.wikipedia/all-sites/daily/19690101/19710101';
+
+ // Fake data insertion endpoints
+ var insertEndpoint =
'/unique-devices/insert/en.wikipedia/all-sites/daily/19700101';
+
+ function fix(b, s, u) { return b.replace(s, s + u); }
+
+ // Test Endpoint
+
+ it('should return 400 when parameters are wrong', function () {
+ return preq.get({
+ uri: server.config.aqsURL + endpoint.replace('19710101',
'20150701000000')
+ }).catch(function(res) {
+ assert.deepEqual(res.status, 400);
+ });
+ });
+
+ it('should return 400 when start is before end', function () {
+ return preq.get({
+ uri: server.config.aqsURL + endpoint.replace('19690101',
'20160701')
+ }).catch(function(res) {
+ assert.deepEqual(res.status, 400);
+ });
+ });
+
+ it('should return 400 when timestamp is invalid', function () {
+ return preq.get({
+ uri: server.config.aqsURL + endpoint.replace('19710101',
'20150229')
+ }).catch(function(res) {
+ assert.deepEqual(res.status, 400);
+ });
+ });
+
+ // WARNING: the data created in this test is used exactly as created
+ // by the monitoring tests.
+ it('should return the expected aggregate data after insertion', function
() {
+ return preq.post({
+ uri: server.config.aqsURL + insertEndpoint + '/0'
+ }).then(function() {
+ return preq.get({
+ uri: server.config.aqsURL + endpoint
+ });
+ }).then(function(res) {
+ assert.deepEqual(res.body.items.length, 1);
+ assert.deepEqual(res.body.items[0]['devices'], 0);
+ });
+ });
+
+ it('should parse the v column string into an int', function () {
+ return preq.post({
+ uri: server.config.aqsURL +
+ fix(insertEndpoint, 'en.wikipedia', '3') +
+ '/9007199254740991'
+ }).then(function() {
+ return preq.get({
+ uri: server.config.aqsURL +
+ fix(endpoint, 'en.wikipedia', '3')
+ });
+ }).then(function(res) {
+ assert.deepEqual(res.body.items.length, 1);
+ assert.deepEqual(res.body.items[0]['devices'], 9007199254740991);
+ });
+ });
+});
diff --git a/v1/pageviews.yaml b/v1/pageviews.yaml
index 7fd3730..fd97016 100644
--- a/v1/pageviews.yaml
+++ b/v1/pageviews.yaml
@@ -11,7 +11,7 @@
name: Apache licence, v2
url: https://www.apache.org/licenses/LICENSE-2.0
paths:
- /pageviews/:
+ /:
get:
tags:
- Pageviews data
@@ -32,7 +32,7 @@
$ref: '#/definitions/problem'
x-monitor: false
-
/pageviews/per-article/{project}/{access}/{agent}/{article}/{granularity}/{start}/{end}:
+
/per-article/{project}/{access}/{agent}/{article}/{granularity}/{start}/{end}:
get:
tags:
- Pageviews data
@@ -95,7 +95,7 @@
uri:
/{domain}/sys/pageviews/per-article/{project}/{access}/{agent}/{article}/{granularity}/{start}/{end}
x-monitor: false
- /pageviews/aggregate/{project}/{access}/{agent}/{granularity}/{start}/{end}:
+ /aggregate/{project}/{access}/{agent}/{granularity}/{start}/{end}:
get:
tags:
- Pageviews data
@@ -176,7 +176,7 @@
timestamp: 1970010100
views: 0
- /pageviews/top/{project}/{access}/{year}/{month}/{day}:
+ /top/{project}/{access}/{year}/{month}/{day}:
get:
tags:
- Pageviews data
diff --git a/v1/unique-devices.yaml b/v1/unique-devices.yaml
new file mode 100644
index 0000000..2c75950
--- /dev/null
+++ b/v1/unique-devices.yaml
@@ -0,0 +1,129 @@
+swagger: '2.0'
+info:
+ version: '1.0.0-beta'
+ title: Analytics Unique Devices API
+ description: Analytics Unique Devicess API
+ contact:
+ name: Analytics
+ email: [email protected]
+ url: https://www.mediawiki.org/wiki/Analytics
+ license:
+ name: Apache licence, v2
+ url: https://www.apache.org/licenses/LICENSE-2.0
+paths:
+ /{project}/{access-site}/{granularity}/{start}/{end}:
+ get:
+ tags:
+ - Unique devices data
+ description: |
+ Given a project and a date range, returns a timeseries of unique
devices counts. You need to specify a project, and can filter by accessed site
(mobile or desktop). You can choose between daily and hourly granularity as well
+ Stability:
[experimental](https://www.mediawiki.org/wiki/API_versioning#Experimental)
+ produces:
+ - application/json
+ parameters:
+ - name: project
+ in: path
+ description: The name of any Wikimedia project formatted like
{language code}.{project name}, for example en.wikipedia. You may pass
en.wikipedia.org and the .org will be stripped off. For projects like commons
without language codes, use commons.wikimedia
+ type: string
+ required: true
+ - name: access-site
+ in: path
+ description: If you want to filter by accessed site, use one of
desktop-site or mobile-site. If you are interested in unique devices regardless
of accessed site, use or all-sites
+ type: string
+ enum: ['all-sites', 'desktop-site', 'mobile-site']
+ required: true
+ - name: granularity
+ in: path
+ description: The time unit for the response data. As of today, the
supported granularities for this endpoint are daily and monthly
+ type: string
+ enum: ['daily', 'monthly']
+ required: true
+ - name: start
+ in: path
+ description: The timestamp of the first day/month to include, in
YYYYMMDD format
+ type: string
+ required: true
+ - name: end
+ in: path
+ description: The timestamp of the last day/month to include, in
YYYYMMDD format
+ type: string
+ required: true
+ responses:
+ '200':
+ description: The list of values
+ schema:
+ $ref: '#/definitions/project'
+ default:
+ description: Error
+ schema:
+ $ref: '#/definitions/problem'
+ x-request-handler:
+ - get_from_backend:
+ request:
+ uri:
/{domain}/sys/unique-devices/{project}/{access-site}/{granularity}/{start}/{end}
+ x-monitor: true
+ x-amples:
+ - title: Get unique devices
+ request:
+ params:
+ domain: wikimedia.org
+ project: en.wikipedia
+ access-site: all-sites
+ granularity: daily
+ start: 19700101
+ end: 19700101
+ response:
+ status: 200
+ headers:
+ content-type: application/json
+ body:
+ items:
+ - project: en.wikipedia
+ access-site: all-sites
+ granularity: daily
+ timestamp: 19700101
+ views: 0
+
+definitions:
+ # A https://tools.ietf.org/html/draft-nottingham-http-problem
+ problem:
+ required:
+ - type
+ properties:
+ type:
+ type: string
+ title:
+ type: string
+ detail:
+ type: string
+ instance:
+ type: string
+
+ listing:
+ description: The result format for listings
+ required:
+ - items
+ properties:
+ items:
+ type: array
+ items:
+ type: string
+
+ project:
+ properties:
+ items:
+ type: array
+ items:
+ properties:
+ project:
+ type : string
+ access-site:
+ type : string
+ granularity:
+ type: string
+ timestamp:
+ # the daily timestamp will be stored as YYYYMMDD
+ type: string
+ devices:
+ type: integer
+ format: int64
--
To view, visit https://gerrit.wikimedia.org/r/277784
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ife9547679bfdbda8ccc1de47aede30290180ebc9
Gerrit-PatchSet: 2
Gerrit-Project: analytics/aqs
Gerrit-Branch: master
Gerrit-Owner: Joal <[email protected]>
Gerrit-Reviewer: Elukey <[email protected]>
Gerrit-Reviewer: Joal <[email protected]>
Gerrit-Reviewer: Milimetric <[email protected]>
Gerrit-Reviewer: Mobrovac <[email protected]>
Gerrit-Reviewer: Nuria <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
