Joal has uploaded a new change for review.
https://gerrit.wikimedia.org/r/277784
Change subject: Add unique_devices endpoint to AQS
......................................................................
Add unique_devices endpoint to AQS
Change pageviews endpoint to facilitate other endpoint integration.
Add unique devices front and sys specifications and js code.
Bug: T129518
Change-Id: Ife9547679bfdbda8ccc1de47aede30290180ebc9
---
M config.test.yaml
M projects/aqs_default.yaml
A sys/unique-devices.js
A sys/unique-devices.yaml
M test/aqs_test_module.yaml
A test/features/unique-devices/unique-devices.js
M v1/pageviews.yaml
A v1/unique-devices.yaml
8 files changed, 353 insertions(+), 8 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/aqs
refs/changes/84/277784/1
diff --git a/config.test.yaml b/config.test.yaml
index 5e50bf2..f89f281 100644
--- a/config.test.yaml
+++ b/config.test.yaml
@@ -1,4 +1,4 @@
-# A synthetic domain to test pageviews module data
+# A synthetic domain to test aqs modules data
aqs.wikimedia.org: &analytics.wikimedia.org
x-modules:
/:
@@ -28,7 +28,7 @@
services:
- name: aqs
module: hyperswitch
- conf:
+ conf:
port: 7231
spec: *spec_root
salt: secret
diff --git a/projects/aqs_default.yaml b/projects/aqs_default.yaml
index 95c9b44..540b267 100644
--- a/projects/aqs_default.yaml
+++ b/projects/aqs_default.yaml
@@ -13,8 +13,10 @@
x-host-basePath: /api/rest_v1
x-modules:
- /:
+ /pageviews:
- path: v1/pageviews.yaml
+ /unique-devices:
+ - path: v1/unique-devices.yaml
/{api:sys}:
swagger: 2.0
@@ -28,3 +30,5 @@
conf: '{{options.table}}'
/pageviews:
- path: sys/pageviews.js
+ /unique-devices:
+ - path: sys/unique-devices.js
diff --git a/sys/unique-devices.js b/sys/unique-devices.js
new file mode 100644
index 0000000..b70fa5a
--- /dev/null
+++ b/sys/unique-devices.js
@@ -0,0 +1,108 @@
+'use strict';
+
+/**
+ * Unique devices API module
+ *
+ * This API serves pre-aggregated unique devices statistics from Cassandra
+ */
+
+var HyperSwitch = require('hyperswitch');
+var path = require('path');
+var HTTPError = HyperSwitch.HTTPError;
+var URI = HyperSwitch.URI;
+
+var aqsUtil = require('../lib/aqsUtil');
+
+var spec = HyperSwitch.utils.loadSpec(path.join(__dirname,
'unique-devices.yaml'));
+
+// Unique devices Service
+function UDVS(options) {
+ this.options = options;
+}
+
+
+var tables = {
+ project: 'unique-devices.per.project',
+};
+var tableURI = function(domain, tableName) {
+ return new URI([domain, 'sys', 'table', tableName, '']);
+};
+var tableSchemas = {
+ project: {
+ table: tables.project,
+ version: 2,
+ attributes: {
+ project: 'string',
+ site: 'string',
+ granularity: 'string',
+ // the hourly timestamp will be stored as YYYYMMDDHH
+ timestamp: 'string',
+ v: 'long'
+ },
+ index: [
+ { attribute: 'project', type: 'hash' },
+ { attribute: 'site', type: 'hash' },
+ { attribute: 'granularity', type: 'hash' },
+ { attribute: 'timestamp', type: 'range', order: 'asc' },
+ ]
+ }
+};
+
+
+UDVS.prototype.uniqueDevicesForProjects = function(hyper, req) {
+ var rp = req.params;
+
+ // dates are passed in as YYYYMMDD but we need the HH to match the loaded
data
+ // which was originally planned at hourly resolution, so we pass "fakeHour"
+ aqsUtil.validateStartAndEnd(rp, { fakeHour: true });
+
+ var dataRequest = hyper.get({
+ uri: tableURI(rp.domain, tables.project),
+ body: {
+ table: tables.project,
+ attributes: {
+ project: rp.project,
+ site: rp.site,
+ granularity: rp.granularity,
+ timestamp: { between: [rp.start, rp.end] },
+ }
+ }
+
+ }).catch(aqsUtil.notFoundCatcher);
+
+ return dataRequest.then(aqsUtil.normalizeResponse).then(function(res) {
+ if (res.body.items) {
+ res.body.items.forEach(function(item) {
+ // transform site to site-version
+ item['site-version'] = item.site;
+ delete item.site;
+ // transform v to unique-devices int
+ try {
+ item['unique-devices'] = parseInt(item.v, 10);
+ } catch (e) {
+ item['unique-devices'] = null;
+ }
+ delete item.v;
+ });
+ }
+ return res;
+ });
+};
+
+module.exports = function(options) {
+ var udvs = new UDVS(options);
+
+ return {
+ spec: spec,
+ operations: {
+ uniqueDevicesForProjects: udvs.uniqueDevicesForProjects.bind(udvs)
+ },
+ resources: [
+ {
+ // unique devices per project table
+ uri: '/{domain}/sys/table/' + tables.project,
+ body: tableSchemas.project,
+ }
+ ]
+ };
+};
diff --git a/sys/unique-devices.yaml b/sys/unique-devices.yaml
new file mode 100644
index 0000000..f296078
--- /dev/null
+++ b/sys/unique-devices.yaml
@@ -0,0 +1,5 @@
+paths:
+ /per-project/{project}/{site}/{granularity}/{start}/{end}:
+ get:
+ summary: query unique devices per project
+ operationId: uniqueDevicesForProjects
diff --git a/test/aqs_test_module.yaml b/test/aqs_test_module.yaml
index 557f010..46782a7 100644
--- a/test/aqs_test_module.yaml
+++ b/test/aqs_test_module.yaml
@@ -6,7 +6,7 @@
swagger: '2.0'
info:
version: 1.0.0-beta
- title: Wikimedia testing APIs
+ title: AQS testing APIs
x-is-api-root: true
paths:
/pageviews/insert-per-article-flat/{project}/{article}/{granularity}/{timestamp}/{views}:
@@ -94,3 +94,20 @@
day: '{{request.params.day}}'
articlesJSON: '{{request.body.articles}}'
x-monitor: false
+
+
/unique-devices/insert-per-project/{project}/{site}/{granularity}/{timestamp}/{v}:
+ post:
+ x-request-handler:
+ - put_to_storage:
+ request:
+ method: 'put'
+ uri: '/{domain}/sys/table/unique-devices.per.project/'
+ body:
+ table: 'unique-devices.per.project'
+ attributes:
+ project: '{{request.params.project}}'
+ site: '{{request.params.site}}'
+ granularity: '{{request.params.granularity}}'
+ timestamp: '{{request.params.timestamp}}'
+ v: '{{request.params.v}}'
+ x-monitor: false
diff --git a/test/features/unique-devices/unique-devices.js
b/test/features/unique-devices/unique-devices.js
new file mode 100644
index 0000000..3c1e5a9
--- /dev/null
+++ b/test/features/unique-devices/unique-devices.js
@@ -0,0 +1,82 @@
+'use strict';
+
+// mocha defines to avoid JSHint breakage
+/* global describe, it, before, beforeEach, afterEach */
+
+var assert = require('../../utils/assert.js');
+var preq = require('preq');
+var server = require('../../utils/server.js');
+
+
+describe('unique-devices endpoints', function () {
+ this.timeout(20000);
+
+ //Start server before running tests
+ before(function () { return server.start(); });
+
+ // NOTE: this tests using the projects/aqs_default.yaml config, so
+ // it doesn't know about the /metrics root like the prod config does
+ var endpoint =
'/unique-devices/en.wikipedia/all-sites/daily/19690101/19710101';
+
+ // Fake data insertion endpoints
+ var insertEndpoint =
'/unique-devices/insert-per-project/en.wikipedia/all-sites/daily/1970010100';
+
+ function fix(b, s, u) { return b.replace(s, s + u); }
+
+ // Test Endpoint
+
+ it('should return 400 when parameters are wrong', function () {
+ return preq.get({
+ uri: server.config.aqsURL + endpoint.replace('19710101',
'20150701000000')
+ }).catch(function(res) {
+ assert.deepEqual(res.status, 400);
+ });
+ });
+
+ it('should return 400 when start is before end', function () {
+ return preq.get({
+ uri: server.config.aqsURL + endpoint.replace('19690101',
'2016070100')
+ }).catch(function(res) {
+ assert.deepEqual(res.status, 400);
+ });
+ });
+
+ it('should return 400 when timestamp is invalid', function () {
+ return preq.get({
+ uri: server.config.aqsURL + endpoint.replace('19710101',
'2015022900')
+ }).catch(function(res) {
+ assert.deepEqual(res.status, 400);
+ });
+ });
+
+ // WARNING: the data created in this test is used exactly as created
+ // by the monitoring tests.
+ it('should return the expected aggregate data after insertion', function
() {
+ return preq.post({
+ uri: server.config.aqsURL + insertEndpoint + '/0'
+ }).then(function() {
+ return preq.get({
+ uri: server.config.aqsURL + endpoint
+ });
+ }).then(function(res) {
+ assert.deepEqual(res.body.items.length, 1);
+ assert.deepEqual(res.body.items[0]['unique-devices'], 0);
+ });
+ });
+
+ it('should parse the v column string into an int', function () {
+ return preq.post({
+ uri: server.config.aqsURL +
+ fix(insertEndpoint, 'en.wikipedia', '3') +
+ '/9007199254740991'
+ }).then(function() {
+ return preq.get({
+ uri: server.config.aqsURL +
+ fix(endpoint, 'en.wikipedia', '3')
+ });
+ }).then(function(res) {
+ assert.deepEqual(res.body.items.length, 1);
+ assert.deepEqual(res.body.items[0]['unique-devices'],
9007199254740991);
+ });
+ });
+});
diff --git a/v1/pageviews.yaml b/v1/pageviews.yaml
index 7fd3730..fd97016 100644
--- a/v1/pageviews.yaml
+++ b/v1/pageviews.yaml
@@ -11,7 +11,7 @@
name: Apache licence, v2
url: https://www.apache.org/licenses/LICENSE-2.0
paths:
- /pageviews/:
+ /:
get:
tags:
- Pageviews data
@@ -32,7 +32,7 @@
$ref: '#/definitions/problem'
x-monitor: false
-
/pageviews/per-article/{project}/{access}/{agent}/{article}/{granularity}/{start}/{end}:
+
/per-article/{project}/{access}/{agent}/{article}/{granularity}/{start}/{end}:
get:
tags:
- Pageviews data
@@ -95,7 +95,7 @@
uri:
/{domain}/sys/pageviews/per-article/{project}/{access}/{agent}/{article}/{granularity}/{start}/{end}
x-monitor: false
- /pageviews/aggregate/{project}/{access}/{agent}/{granularity}/{start}/{end}:
+ /aggregate/{project}/{access}/{agent}/{granularity}/{start}/{end}:
get:
tags:
- Pageviews data
@@ -176,7 +176,7 @@
timestamp: 1970010100
views: 0
- /pageviews/top/{project}/{access}/{year}/{month}/{day}:
+ /top/{project}/{access}/{year}/{month}/{day}:
get:
tags:
- Pageviews data
diff --git a/v1/unique-devices.yaml b/v1/unique-devices.yaml
new file mode 100644
index 0000000..ac885b7
--- /dev/null
+++ b/v1/unique-devices.yaml
@@ -0,0 +1,129 @@
+swagger: '2.0'
+info:
+ version: '1.0.0-beta'
+ title: Analytics Unique Devices API
+ description: Analytics Unique Devicess API
+ contact:
+ name: Analytics
+ email: [email protected]
+ url: https://www.mediawiki.org/wiki/Analytics
+ license:
+ name: Apache licence, v2
+ url: https://www.apache.org/licenses/LICENSE-2.0
+paths:
+ /{project}/{site-version}/{granularity}/{start}/{end}:
+ get:
+ tags:
+ - Unique devices data
+ description: |
+ Given a project and a date range, returns a timeseries of unique
devices counts. You need to specify a project, and can filter by site-version.
You can choose between daily and hourly granularity as well
+ Stability:
[experimental](https://www.mediawiki.org/wiki/API_versioning#Experimental)
+ produces:
+ - application/json
+ parameters:
+ - name: project
+ in: path
+ description: The name of any Wikimedia project formatted like
{language code}.{project name}, for example en.wikipedia. You may pass
en.wikipedia.org and the .org will be stripped off. For projects like commons
without language codes, use commons.wikimedia
+ type: string
+ required: true
+ - name: site-version
+ in: path
+ description: If you want to filter by site version, use one of
desktop-site, mobile-site. If you are interested in unique devices regardless
of site version, use or all-sites
+ type: string
+ enum: ['all-sites', 'desktop-site', 'mobile-site']
+ required: true
+ - name: granularity
+ in: path
+ description: The time unit for the response data. As of today, the
supported granularities for this endpoint are daily and monthly
+ type: string
+ enum: ['daily', 'monthly']
+ required: true
+ - name: start
+ in: path
+ description: The timestamp of the first day/month to include, in
YYYYMMDD format
+ type: string
+ required: true
+ - name: end
+ in: path
+ description: The timestamp of the last day/month to include, in
YYYYMMDD format
+ type: string
+ required: true
+ responses:
+ '200':
+ description: The list of values
+ schema:
+ $ref: '#/definitions/project'
+ default:
+ description: Error
+ schema:
+ $ref: '#/definitions/problem'
+ x-request-handler:
+ - get_from_backend:
+ request:
+ uri:
/{domain}/sys/unique-devices/per-project/{project}/{site-version}/{granularity}/{start}/{end}
+ x-monitor: true
+ x-amples:
+ - title: Get unique devices
+ request:
+ params:
+ domain: wikimedia.org
+ project: en.wikipedia
+ site-version: all-sites
+ granularity: daily
+ start: 19700101
+ end: 19700101
+ response:
+ status: 200
+ headers:
+ content-type: application/json
+ body:
+ items:
+ - project: en.wikipedia
+ site-version: all-sites
+ granularity: daily
+ timestamp: 19700101
+ views: 0
+
+definitions:
+ # A https://tools.ietf.org/html/draft-nottingham-http-problem
+ problem:
+ required:
+ - type
+ properties:
+ type:
+ type: string
+ title:
+ type: string
+ detail:
+ type: string
+ instance:
+ type: string
+
+ listing:
+ description: The result format for listings
+ required:
+ - items
+ properties:
+ items:
+ type: array
+ items:
+ type: string
+
+ project:
+ properties:
+ items:
+ type: array
+ items:
+ properties:
+ project:
+ type : string
+ site-version:
+ type : string
+ granularity:
+ type: string
+ timestamp:
+ # the hourly timestamp will be stored as YYYYMMDDHH
+ type: string
+ unique-devices:
+ type: integer
+ format: int64
--
To view, visit https://gerrit.wikimedia.org/r/277784
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ife9547679bfdbda8ccc1de47aede30290180ebc9
Gerrit-PatchSet: 1
Gerrit-Project: analytics/aqs
Gerrit-Branch: master
Gerrit-Owner: Joal <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
