Joal has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/384590 )
Change subject: Upgrade restbase-modules to latest
......................................................................
Upgrade restbase-modules to latest
It makes month we've not done that, we are many versions
behind for core modules hyperswtich, service-runner, and
restbase-mod-table-[cassandra|sqlite].
This patch also adds linting and corrects code accordingly.
Bug: T178312
Change-Id: Ie7847c1de76cb7e6cc868a1d3b94af68785f86bc
---
A .eslintrc.yml
M .jscs.json
M .jshintrc
M config.example.wikimedia.yaml
M config.test.yaml
M lib/aqsUtil.js
M lib/druidUtil.js
M package.json
M projects/aqs_default.yaml
M sys/legacy/pagecounts.js
M sys/mediawiki-history-metrics.js
M sys/pageviews.js
A sys/table.js
M sys/unique-devices.js
M test/aqs_test_module.yaml
M test/features/mediawiki-history-metrics/mediawiki-history-metrics.js
M test/features/pageviews/pageviews.js
M test/index.js
M test/utils/run_tests.sh
M test/utils/server.js
20 files changed, 546 insertions(+), 539 deletions(-)
Approvals:
Ppchelko: Looks good to me, but someone else must approve
Joal: Verified; Looks good to me, approved
diff --git a/.eslintrc.yml b/.eslintrc.yml
new file mode 100644
index 0000000..9e2c225
--- /dev/null
+++ b/.eslintrc.yml
@@ -0,0 +1 @@
+extends: 'eslint-config-node-services'
\ No newline at end of file
diff --git a/.jscs.json b/.jscs.json
index bf87d75..8a4d757 100644
--- a/.jscs.json
+++ b/.jscs.json
@@ -10,7 +10,9 @@
"catch"
],
"validateIndentation": 4,
- "requireCamelCaseOrUpperCaseIdentifiers": "ignoreProperties",
+ "requireCamelCaseOrUpperCaseIdentifiers": {
+ "ignoreProperties": true
+ },
"requireCapitalizedComments": null,
"maximumLineLength": 100,
"validateQuoteMarks": null,
@@ -23,6 +25,7 @@
"node_modules/**",
"test/**",
"coverage/**",
- "test.db.**"
+ "test.db.**",
+ "maintenance/**"
]
-}
\ No newline at end of file
+}
diff --git a/.jshintrc b/.jshintrc
index 5288038..8acdd9d 100644
--- a/.jshintrc
+++ b/.jshintrc
@@ -1,38 +1,37 @@
{
- "predef": [
- "ve",
+ "predef": [
+ "ve",
- "setImmediate",
+ "setImmediate",
- "QUnit",
+ "QUnit",
- "Map",
- "Set"
- ],
+ "Map",
+ "Set"
+ ],
- "bitwise": true,
- "laxbreak": true,
- "curly": true,
- "eqeqeq": true,
- "immed": true,
- "latedef": "nofunc",
- "maxlen":false,
- "newcap": true,
- "noarg": true,
- "noempty": true,
- "nonew": true,
- "regexp": false,
- "undef": true,
- "strict": true,
- "trailing": true,
+ "bitwise": true,
+ "laxbreak": true,
+ "curly": true,
+ "eqeqeq": true,
+ "immed": true,
+ "latedef": "nofunc",
+ "newcap": true,
+ "noarg": true,
+ "noempty": true,
+ "nonew": true,
+ "regexp": false,
+ "undef": true,
+ "strict": true,
+ "trailing": true,
- "smarttabs": true,
- "multistr": true,
+ "smarttabs": true,
+ "multistr": true,
- "node": true,
+ "node": true,
- "nomen": false,
- "loopfunc": true,
- "esnext": true
- //"onevar": true
+ "nomen": false,
+ "loopfunc": true,
+ "esnext": true
+ //"onevar": true
}
diff --git a/config.example.wikimedia.yaml b/config.example.wikimedia.yaml
index b61aad9..39f7d33 100644
--- a/config.example.wikimedia.yaml
+++ b/config.example.wikimedia.yaml
@@ -1,23 +1,23 @@
# Analytics Query Service config
aqs_project: &aqs_project
x-modules:
- /:
- - path: projects/aqs_default.yaml
- options: &default_options
- table:
- hosts: [localhost]
- keyspace: system
- username: cassandra
- password: cassandra
- defaultConsistency: one # or 'localQuorum' for production
- storage_groups:
- - name: default.group.local
- domains: /./
- druid:
- scheme: http
- host: druid1004.eqiad.wmnet
- port: 8082
- query_path: /druid/v2/
+ - path: projects/aqs_default.yaml
+ options: &default_options
+ table:
+ backend: cassandra
+ hosts: [localhost]
+ keyspace: system
+ username: cassandra
+ password: cassandra
+ defaultConsistency: one # or 'localQuorum' for production
+ storage_groups:
+ - name: default.group.local
+ domains: /./
+ druid:
+ scheme: http
+ host: druid1004.eqiad.wmnet
+ port: 8082
+ query_path: /druid/v2/
# Swagger spec root.
spec: &spec_root
diff --git a/config.test.yaml b/config.test.yaml
index 965ddc0..dc49ed6 100644
--- a/config.test.yaml
+++ b/config.test.yaml
@@ -1,23 +1,23 @@
# A synthetic domain to test aqs modules data
aqs.wikimedia.org: &analytics.wikimedia.org
x-modules:
- /:
- - path: test/aqs_test_module.yaml
- - path: projects/aqs_default.yaml
- options: &default_options
- table:
- hosts: [localhost]
- keyspace: system
- username: cassandra
- password: cassandra
- defaultConsistency: one # or 'localQuorum' for production
- storage_groups:
- - name: test.group.local
- domains: /./
- dbname: test.db.sqlite3 # ignored in cassandra, but useful in
SQLite testing
- druid:
- # Use a fake internal endpoint in restbase to test
- query_path: /analytics.wikimedia.org/sys/fake-druid/druid/v2
+ - path: test/aqs_test_module.yaml
+ - path: projects/aqs_default.yaml
+ options: &default_options
+ table:
+ backend: cassandra
+ hosts: [ localhost ]
+ keyspace: system
+ username: cassandra
+ password: cassandra
+ defaultConsistency: one # or 'localQuorum' for production
+ storage_groups:
+ - name: test.group.local
+ domains: /./
+ dbname: test.db.sqlite3 # ignored in cassandra, but useful in SQLite
testing
+ druid:
+ # Use a fake internal endpoint in restbase to test
+ query_path: /analytics.wikimedia.org/sys/fake-druid/druid/v2
spec_root: &spec_root
title: "The Analytics Query Service root"
diff --git a/lib/aqsUtil.js b/lib/aqsUtil.js
index b07e369..4eac56b 100644
--- a/lib/aqsUtil.js
+++ b/lib/aqsUtil.js
@@ -1,10 +1,9 @@
'use strict';
-var HyperSwitch = require('hyperswitch');
-var HTTPError = HyperSwitch.HTTPError;
-var druidUtil = require('./druidUtil');
+const HyperSwitch = require('hyperswitch');
+const HTTPError = HyperSwitch.HTTPError;
-var aqsUtil = {};
+const aqsUtil = {};
aqsUtil.notFoundCatcher = function(e) {
@@ -39,7 +38,7 @@
* Parameter validators
* Only needed internally, not exposed
*/
-var throwIfNeeded = function(errors) {
+function throwIfNeeded(errors) {
if (errors && errors.length) {
throw new HTTPError({
status: 400,
@@ -49,7 +48,7 @@
}
});
}
-};
+}
/**
* Normalizes the project parameter to "en.wikipedia"
@@ -78,19 +77,18 @@
}
try {
- var year = timestamp.substring(0, 4);
- var month = timestamp.substring(4, 6);
- var day = timestamp.substring(6, 8);
- var hour = opts.fakeHour ? '00' : timestamp.substring(8, 10);
+ const year = timestamp.substring(0, 4);
+ const month = timestamp.substring(4, 6);
+ const day = timestamp.substring(6, 8);
+ const hour = opts.fakeHour ? '00' : timestamp.substring(8, 10);
- var dt = new Date([year, month, day].join('-') + ' '
- + hour + ':00:00 UTC');
+ const dt = new Date(`${year}-${month}-${day} ${hour}:00:00 UTC`);
return dt.toString() !== 'Invalid Date'
&& dt.getUTCFullYear() === parseInt(year, 10)
&& dt.getUTCMonth() === (parseInt(month, 10) - 1)
&& dt.getUTCDate() === parseInt(day, 10)
- && dt.getUTCHours() === parseInt(hour);
+ && dt.getUTCHours() === parseInt(hour, 10);
} catch (e) {
return false;
@@ -115,18 +113,17 @@
aqsUtil.validateStartAndEnd = function(rp, opts) {
opts = opts || {};
- var errors = [];
- var invalidMessage = opts.fakeHour ?
- 'invalid, must be a valid date in YYYYMMDD format' :
- 'invalid, must be a valid timestamp in YYYYMMDDHH format';
+ const errors = [];
+ const messageFormat = opts.fakeHour ? 'YYYYMMDD' : 'YYYYMMDDHH';
+ const invalidMessage = `invalid, must be a valid date in ${messageFormat}
format`;
aqsUtil.normalizeProject(rp);
if (!aqsUtil.validateTimestamp(rp.start, opts)) {
- errors.push('start timestamp is ' + invalidMessage);
+ errors.push(`start timestamp is ${invalidMessage}`);
}
if (!aqsUtil.validateTimestamp(rp.end, opts)) {
- errors.push('end timestamp is ' + invalidMessage);
+ errors.push(`end timestamp is ${invalidMessage}`);
}
if (rp.start > rp.end) {
@@ -173,16 +170,13 @@
};
aqsUtil.validateYearMonthDay = function(rp) {
- var errors = [];
+ const errors = [];
aqsUtil.normalizeProject(rp);
// fake a timestamp in the YYYYMMDDHH format so we can reuse the validator
- var validDate = aqsUtil.validateTimestamp(
- rp.year + rp.month +
- ((rp.day === 'all-days') ? '01' : rp.day) +
- '00'
- );
+ const day = ((rp.day === 'all-days') ? '01' : rp.day);
+ const validDate =
aqsUtil.validateTimestamp(`${rp.year}${rp.month}${day}00`);
if (!validDate) {
errors.push('Given year/month/day is invalid date');
@@ -192,20 +186,21 @@
};
aqsUtil.convertTimestampToDate = function(timestamp) {
- var year = parseInt(timestamp.substring(0, 4), 10);
- var month = parseInt(timestamp.substring(4, 6), 10) - 1;
- var day = parseInt(timestamp.substring(6, 8), 10);
+ const year = parseInt(timestamp.substring(0, 4), 10);
+ const month = parseInt(timestamp.substring(4, 6), 10) - 1;
+ const day = parseInt(timestamp.substring(6, 8), 10);
return new Date(Date.UTC(year, month, day));
};
aqsUtil.convertDateToTimestamp = function(date) {
- return date.getUTCFullYear().toString() +
- ('0' + (date.getUTCMonth() + 1)).slice(-2) +
- ('0' + date.getUTCDate()).slice(-2);
+ const year = date.getUTCFullYear().toString();
+ const month = (`0${(date.getUTCMonth() + 1)}`).slice(-2);
+ const day = (`0${date.getUTCDate()}`).slice(-2);
+ return `${year}${month}${day}`;
};
aqsUtil.getFirstFullMonthFirstDay = function(startDate) {
- var dt = aqsUtil.convertTimestampToDate(startDate);
+ const dt = aqsUtil.convertTimestampToDate(startDate);
if (dt.getUTCDate() === 1) {
return startDate;
@@ -218,8 +213,8 @@
};
aqsUtil.getLastFullMonthLastDay = function(endDate) {
- var dt = aqsUtil.convertTimestampToDate(endDate);
- var lastDayOfCurrentMonth = new Date(Date.UTC(dt.getUTCFullYear(),
dt.getUTCMonth() + 1, 0));
+ const dt = aqsUtil.convertTimestampToDate(endDate);
+ const lastDayOfCurrentMonth = new Date(Date.UTC(dt.getUTCFullYear(),
dt.getUTCMonth() + 1, 0));
if (dt.getUTCDate() === lastDayOfCurrentMonth.getUTCDate()) {
return endDate;
@@ -231,7 +226,7 @@
};
aqsUtil.getDayAfterLastFullMonth = function(endDate) {
- var dt = aqsUtil.convertTimestampToDate(endDate);
+ const dt = aqsUtil.convertTimestampToDate(endDate);
dt.setUTCDate(1);
return aqsUtil.convertDateToTimestamp(dt);
};
diff --git a/lib/druidUtil.js b/lib/druidUtil.js
index cb5579d..fd5a492 100644
--- a/lib/druidUtil.js
+++ b/lib/druidUtil.js
@@ -1,39 +1,31 @@
'use strict';
-var druidUtil = {};
+const druidUtil = {};
/*
- * Intervals building functions
+ * Intervals building function
*/
-var makeInterval = function(start, end) {
- return start + '/' + end ;
-};
-
druidUtil.makeInterval = function(start, end) {
- return [ makeInterval(start, end) ];
+ return [ `${start}/${end}` ];
};
/*
* Filters building functions
*/
druidUtil.makeSelectorFilter = function(dimension, value) {
- return { type: 'selector', dimension: dimension, value: value };
+ return { type: 'selector', dimension, value };
};
-
druidUtil.makeRegexFilter = function(dimension, pattern) {
- return { type: 'regex', dimension: dimension, pattern: pattern };
+ return { type: 'regex', dimension, pattern };
};
-
druidUtil.makeAndFilter = function(fields) {
- return { type: 'and', fields: fields };
+ return { type: 'and', fields };
};
-
druidUtil.makeOrFilter = function(fields) {
- return { type: 'or', fields: fields };
+ return { type: 'or', fields };
};
-
druidUtil.makeNotFilter = function(field) {
- return { type: 'not', field: field };
+ return { type: 'not', field };
};
@@ -41,12 +33,11 @@
* Aggregations building functions
*/
druidUtil.makeCount = function(name) {
- return { type: 'count', name: name };
+ return { type: 'count', name };
};
-
-var makeSimpleTypeAggregation = function(type, name, fieldName) {
- return { type: type, name: name, fieldName: fieldName };
-};
+function makeSimpleTypeAggregation(type, name, fieldName) {
+ return { type, name, fieldName };
+}
// Sum
druidUtil.makeLongSum = function(name, fieldName) {
return makeSimpleTypeAggregation('longSum', name, fieldName);
@@ -80,13 +71,11 @@
druidUtil.makeDoubleLast = function(name, fieldName) {
return makeSimpleTypeAggregation('doubleLast', name, fieldName);
};
-
druidUtil.makeCardinalityAggregation = function(name, fields) {
- return { type: 'cardinality', name: name, fields: fields };
+ return { type: 'cardinality', name, fields };
};
-
druidUtil.makeFilteredAggregation = function(filter, aggregator) {
- return { type: 'filtered', filter: filter, aggregator: aggregator };
+ return { type: 'filtered', filter, aggregator };
};
@@ -94,29 +83,23 @@
* PostAggregations building functions
*/
druidUtil.makeFieldAccessor = function(fieldName) {
- return { type: 'fieldAccess', fieldName: fieldName };
+ return { type: 'fieldAccess', fieldName };
};
-
-var makeArithmeticPostAggregation = function(fn, name, fields) {
- return { type: 'arithmetic', name: name, fn: fn, fields: fields };
-};
-
+function makeArithmeticPostAggregation(fn, name, fields) {
+ return { type: 'arithmetic', name, fn, fields };
+}
druidUtil.makePlusPostAggregation = function(name, fields) {
return makeArithmeticPostAggregation('+', name, fields);
};
-
druidUtil.makeMinusPostAggregation = function(name, fields) {
return makeArithmeticPostAggregation('-', name, fields);
};
-
druidUtil.makeTimesPostAggregation = function(name, fields) {
return makeArithmeticPostAggregation('*', name, fields);
};
-
druidUtil.makeDividePostAggregation = function(name, fields) {
return makeArithmeticPostAggregation('/', name, fields);
};
-
druidUtil.makeQuotientPostAggregation = function(name, fields) {
return makeArithmeticPostAggregation('quotient', name, fields);
};
@@ -128,42 +111,38 @@
const JSON_HEADER = { 'content-type': 'application/json' };
-var makeQuery = function(uri, body) {
- return { uri: uri, headers: JSON_HEADER, body: body };
+function makeQuery(uri, body) {
+ return { uri, headers: JSON_HEADER, body };
+}
+
+druidUtil.makeTimeseriesQuery = function(uri, dataSource, granularity,
+ filter, aggregations,
postAggregations, intervals) {
+ return makeQuery(uri, {
+ queryType: 'timeseries',
+ dataSource,
+ granularity,
+ filter,
+ aggregations,
+ postAggregations,
+ intervals
+ });
};
-druidUtil.makeTimeseriesQuery = function(uri, datasource, granularity,
- filters, aggregations,
postAggregations, intervals) {
- return makeQuery(uri,
- {
- queryType: 'timeseries',
- dataSource: datasource,
- granularity: granularity,
- filter: filters,
- aggregations: aggregations,
- postAggregations: postAggregations,
- intervals: intervals
- }
- );
-};
-
-druidUtil.makeTopN = function(uri, datasource, granularity,
+druidUtil.makeTopN = function(uri, dataSource, granularity,
dimension, threshold, metric,
- filters, aggregations, postAggregations,
intervals) {
- return makeQuery(uri,
- {
- queryType: 'topN',
- dataSource: datasource,
- granularity: granularity,
- dimension: dimension,
- threshold: threshold,
- metric: metric,
- filter: filters,
- aggregations: aggregations,
- postAggregations: postAggregations,
- intervals: intervals
- }
- );
+ filter, aggregations, postAggregations,
intervals) {
+ return makeQuery(uri, {
+ queryType: 'topN',
+ dataSource,
+ granularity,
+ dimension,
+ threshold,
+ metric,
+ filter,
+ aggregations,
+ postAggregations,
+ intervals
+ });
};
module.exports = druidUtil;
diff --git a/package.json b/package.json
index 20f3f50..06bf72c 100644
--- a/package.json
+++ b/package.json
@@ -24,26 +24,35 @@
"url": "https://phabricator.wikimedia.org/tag/analytics/";
},
"dependencies": {
- "bluebird": "^3.1.1",
- "restbase-mod-table-cassandra": "^0.8.15",
- "service-runner": "^1.1.0",
- "hyperswitch": "^0.1.1"
+ "bluebird": "^3.5.0",
+ "restbase-mod-table-cassandra": "^0.11.2",
+ "service-runner": "^2.3.0",
+ "hyperswitch": "^0.9.1"
},
"devDependencies": {
- "js-yaml": "^3.5.2",
- "preq": "^0.4.8",
- "bunyan": "^1.5.1",
- "coveralls": "^2.11.6",
- "istanbul": "^0.4.2",
- "mocha": "^2.3.4",
- "mocha-jscs": "^4.0.0",
- "mocha-jshint": "^2.2.6",
- "mocha-lcov-reporter": "^1.0.0",
- "restbase-mod-table-sqlite": "^0.1.14",
- "temp": "^0.8.3"
+ "ajv": "^5.1.5",
+ "bunyan": "^1.8.10",
+ "coveralls": "^2.13.1",
+ "eslint-config-node-services": "^2.2.2",
+ "eslint-config-wikimedia": "^0.4.0",
+ "eslint-plugin-jsdoc": "^3.1.0",
+ "eslint-plugin-json": "^1.2.0",
+ "istanbul": "^0.4.5",
+ "js-yaml": "^3.8.4",
+ "mocha": "^3.4.2",
+ "mocha-eslint": "^3.0.1",
+ "mocha-jscs": "^5.0.1",
+ "mocha-jshint": "^2.3.1",
+ "mocha-lcov-reporter": "^1.3.0",
+ "mocha.parallel": "^0.15.2",
+ "preq": "^0.5.2",
+ "restbase-mod-table-sqlite": "^0.1.20"
+ },
+ "engines": {
+ "node": ">=4"
},
"deploy": {
- "node": "4.4.6",
+ "node": "6.11.1",
"target": "debian",
"dependencies": {
"_all": []
diff --git a/projects/aqs_default.yaml b/projects/aqs_default.yaml
index d7fb8bf..78d2454 100644
--- a/projects/aqs_default.yaml
+++ b/projects/aqs_default.yaml
@@ -1,52 +1,64 @@
-swagger: '2.0'
paths:
/{api:v1}: &default_project_paths_v1
- swagger: '2.0'
- # swagger options, overriding the shared ones from the merged specs (?)
- info:
- version: 1.0.0-beta
- title: Analytics REST API
- description: >
- Analytics Query Service setup
- x-is-api-root: true
-
- x-host-basePath: /api/rest_v1
-
x-modules:
- /pageviews:
- - path: v1/pageviews.yaml
- /legacy/pagecounts:
- - path: v1/legacy/pagecounts.yaml
- /unique-devices:
- - path: v1/unique-devices.yaml
- /edited-pages:
- - path: v1/edited-pages.yaml
- /editors:
- - path: v1/editors.yaml
- /registered-users:
- - path: v1/registered-users.yaml
- /edits:
- - path: v1/edits.yaml
- /bytes-difference:
- - path: v1/bytes-difference.yaml
+ - spec:
+ # Careful - 2 indentations here !
+ info:
+ version: 1.0.0-beta
+ title: Analytics REST API
+ description: Analytics Query Service setup
+
+ x-is-api-root: true
+ x-host-basePath: /api/rest_v1
+
+ paths:
+ /pageviews:
+ x-modules:
+ - path: v1/pageviews.yaml
+ /legacy/pagecounts:
+ x-modules:
+ - path: v1/legacy/pagecounts.yaml
+ /unique-devices:
+ x-modules:
+ - path: v1/unique-devices.yaml
+ /edited-pages:
+ x-modules:
+ - path: v1/edited-pages.yaml
+ /editors:
+ x-modules:
+ - path: v1/editors.yaml
+ /registered-users:
+ x-modules:
+ - path: v1/registered-users.yaml
+ /edits:
+ x-modules:
+ - path: v1/edits.yaml
+ /bytes-difference:
+ x-modules:
+ - path: v1/bytes-difference.yaml
/{api:sys}:
- swagger: 2.0
- info:
- x-is-api-root: true
x-modules:
- /table:
- - type: npm
- name: restbase-mod-table-cassandra
- options:
- conf: '{{options.table}}'
- /pageviews:
- - path: sys/pageviews.js
- /legacy/pagecounts:
- - path: sys/legacy/pagecounts.js
- /unique-devices:
- - path: sys/unique-devices.js
- /mediawiki-history-metrics:
- - path: sys/mediawiki-history-metrics.js
- options:
- druid: '{{options.druid}}'
+ - spec:
+ # Careful - 2 indentations here !
+ paths:
+ /table:
+ x-modules:
+ - path: sys/table.js
+ options:
+ conf: '{{options.table}}'
+ /pageviews:
+ x-modules:
+ - path: sys/pageviews.js
+ /legacy/pagecounts:
+ x-modules:
+ - path: sys/legacy/pagecounts.js
+ /unique-devices:
+ x-modules:
+ - path: sys/unique-devices.js
+ /mediawiki-history-metrics:
+ x-modules:
+ - path: sys/mediawiki-history-metrics.js
+ options:
+ druid: '{{options.druid}}'
+ options: '{{options}}'
diff --git a/sys/legacy/pagecounts.js b/sys/legacy/pagecounts.js
index d7aa458..b8a6983 100644
--- a/sys/legacy/pagecounts.js
+++ b/sys/legacy/pagecounts.js
@@ -8,26 +8,27 @@
* https://wikitech.wikimedia.org/wiki/Analytics/Data/Pagecounts-raw
*/
-var HyperSwitch = require('hyperswitch');
-var path = require('path');
-var URI = HyperSwitch.URI;
+const HyperSwitch = require('hyperswitch');
+const path = require('path');
+const URI = HyperSwitch.URI;
-var aqsUtil = require('../../lib/aqsUtil');
+const aqsUtil = require('../../lib/aqsUtil');
-var spec = HyperSwitch.utils.loadSpec(path.join(__dirname, 'pagecounts.yaml'));
+const spec = HyperSwitch.utils.loadSpec(path.join(__dirname,
'pagecounts.yaml'));
// Pagecounts service
function LPCS(options) {
this.options = options;
}
-var tables = {
+function tableURI(domain, tableName) {
+ return new URI([domain, 'sys', 'table', tableName, '']);
+}
+
+const tables = {
project: 'lgc.pagecounts.per.project'
};
-var tableURI = function(domain, tableName) {
- return new URI([domain, 'sys', 'table', tableName, '']);
-};
-var tableSchemas = {
+const tableSchemas = {
project: {
table: tables.project,
version: 1,
@@ -48,14 +49,14 @@
};
LPCS.prototype.pagecountsPerProject = function(hyper, req) {
- var rp = req.params;
+ const rp = req.params;
aqsUtil.validateStartAndEnd(rp, {
zeroHour: rp.granularity !== 'hourly',
fullMonths: rp.granularity === 'monthly'
});
- var dataRequest = hyper.get({
+ const dataRequest = hyper.get({
uri: tableURI(rp.domain, tables.project),
body: {
table: tables.project,
@@ -69,9 +70,9 @@
}).catch(aqsUtil.notFoundCatcher);
// Parse long from string to int
- return dataRequest.then(aqsUtil.normalizeResponse).then(function(res) {
+ return dataRequest.then(aqsUtil.normalizeResponse).then((res) => {
if (res.body.items) {
- res.body.items.forEach(function(item) {
+ res.body.items.forEach((item) => {
if (item.count !== null) {
try {
item.count = parseInt(item.count, 10);
@@ -87,16 +88,16 @@
};
module.exports = function(options) {
- var lpcs = new LPCS(options);
+ const lpcs = new LPCS(options);
return {
- spec: spec,
+ spec,
operations: {
pagecountsPerProject: lpcs.pagecountsPerProject.bind(lpcs)
},
resources: [
{
- uri: '/{domain}/sys/table/' + tables.project,
+ uri: `/{domain}/sys/table/${tables.project}`,
body: tableSchemas.project
}
]
diff --git a/sys/mediawiki-history-metrics.js b/sys/mediawiki-history-metrics.js
index 54e9fec..4d6068e 100644
--- a/sys/mediawiki-history-metrics.js
+++ b/sys/mediawiki-history-metrics.js
@@ -10,20 +10,18 @@
*
*/
-var HyperSwitch = require('hyperswitch');
-var HTTPError = HyperSwitch.HTTPError;
-var path = require('path');
-var URI = HyperSwitch.URI;
+const HyperSwitch = require('hyperswitch');
+const HTTPError = HyperSwitch.HTTPError;
+const path = require('path');
-var aqsUtil = require('../lib/aqsUtil');
-var druidUtil = require('../lib/druidUtil');
+const aqsUtil = require('../lib/aqsUtil');
+const druidUtil = require('../lib/druidUtil');
-var spec = HyperSwitch.utils.loadSpec(path.join(__dirname,
'mediawiki-history-metrics.yaml'));
-var schemas = HyperSwitch.utils.loadSpec(path.join(__dirname,
'mediawiki-history-schemas.yaml'));
-var D = schemas.druid;
-var A2D = schemas.aqs2druid;
+const spec = HyperSwitch.utils.loadSpec(path.join(__dirname,
'mediawiki-history-metrics.yaml'));
+const schemas = HyperSwitch.utils.loadSpec(path.join(__dirname,
'mediawiki-history-schemas.yaml'));
+const D = schemas.druid;
+const A2D = schemas.aqs2druid;
-const ALL = 'all';
// How many results to return in topN queries
const TOP_THRESHOLD = 100;
@@ -38,15 +36,14 @@
this.druid = options.druid;
}
-var requestURI = function(druid) {
+function requestURI(druid) {
if (druid) {
- var uri = '';
- uri += (druid.scheme) ? druid.scheme + '://' : '';
- uri += druid.host || '';
- uri += (druid.port) ? ':' + druid.port : '';
- uri += druid.query_path || '';
+ const scheme = (druid.scheme) ? `${druid.scheme}://` : '';
+ const host = druid.host || '';
+ const port = (druid.port) ? `:${druid.port}` : '';
+ const path = druid.query_path || '';
- return uri;
+ return `${scheme}${host}${port}${path}`;
} else { // Fail with 500 if druid conf is not set
throw new HTTPError({
status: 500,
@@ -56,7 +53,7 @@
}
});
}
-};
+}
const druidQueriesBlocks = {
@@ -81,7 +78,7 @@
};
-var validateRequestParams = function(requestParams, opts) {
+function validateRequestParams(requestParams, opts) {
opts = opts || {};
aqsUtil.normalizeProject(requestParams, opts);
@@ -96,7 +93,7 @@
// Druid uses ISO date format, so convert to YYYY-MM-DD
isoDateFormat: true
}));
-};
+}
/*
@@ -105,22 +102,23 @@
* With those events, metrics are additive and therefore 'all'
* parameter values always mean 'no filtering'
*/
-var eventsFiltersFromRequestParams = function(requestParams) {
- return AQS_PARAMS.filter(paramName => {
+function eventsFiltersFromRequestParams(requestParams) {
+ return AQS_PARAMS.filter((paramName) => {
return requestParams[paramName] && // Parameter is defined in request
A2D.filter[paramName] && // Parameter has a related filter
function
- !A2D.all.hasOwnProperty(requestParams[paramName]); //
Parameter value is not ALL
- }).map(paramName => {
+ // Parameter value is not ALL
+ !Object.prototype.hasOwnProperty.call(A2D.all,
requestParams[paramName]);
+ }).map((paramName) => {
// Get filter function by name from schemas
- var makeFilter = druidUtil[A2D.filter[paramName]];
- var filterDim = A2D.dimension[paramName];
- var filterVal = requestParams[paramName];
- if (A2D.hasOwnProperty(paramName)) { // Convert or keep same
+ const makeFilter = druidUtil[A2D.filter[paramName]];
+ const filterDim = A2D.dimension[paramName];
+ let filterVal = requestParams[paramName];
+ if (Object.prototype.hasOwnProperty.call(A2D, paramName)) { // Convert
or keep same
filterVal = A2D[paramName][requestParams[paramName]];
}
return makeFilter(filterDim, filterVal);
});
-};
+}
/*
@@ -132,81 +130,66 @@
* Since the digests are currently not fully optimized, some special
* cases also apply (see below)
*/
-var digestsFiltersFromRequestParams = function(requestParams) {
- return AQS_PARAMS.filter(paramName => {
+function digestsFiltersFromRequestParams(requestParams) {
+ return AQS_PARAMS.filter((paramName) => {
// Special case:
// request[project] = all-projects --> No filter
// request[activity-level] = all-activity-levels --> No filter
// request[editor-type] = all-editor-types --> Filter
// request[page-type] = all-page-types --> Filter
- if ({ project: true, 'activity-level': true
}.hasOwnProperty(paramName)) {
+ if (paramName === 'project' || paramName === 'activity-level') {
return requestParams[paramName] && // Parameter is defined in
request
A2D.filter[paramName] && // Parameter has a related filter
function
- !A2D.all.hasOwnProperty(requestParams[paramName]); //
Parameter value is not ALL
+ // Parameter value is not ALL
+ !Object.prototype.hasOwnProperty.call(A2D.all,
requestParams[paramName]);
} else {
return requestParams[paramName] && // Parameter is defined in
request
A2D.dimension[paramName] && // Parameter has a related druid
dimension
A2D.filter[paramName]; // Parameter has a related filter
function
}
- }).map(paramName => {
+ }).map((paramName) => {
// Get filter function by name from schemas
- var makeFilter = druidUtil[A2D.filter[paramName]];
- var filterDim = A2D.dimension[paramName];
- var filterVal = requestParams[paramName];
- if (A2D.hasOwnProperty(paramName)) { // Convert or keep same
+ const makeFilter = druidUtil[A2D.filter[paramName]];
+ const filterDim = A2D.dimension[paramName];
+ let filterVal = requestParams[paramName];
+ if (Object.prototype.hasOwnProperty.call(A2D, paramName)) { // Convert
or keep same
filterVal = A2D[paramName][requestParams[paramName]];
}
return makeFilter(filterDim, filterVal);
});
-};
+}
-var digestGranularityFilter = function(granularity) {
+function digestGranularityFilter(granularity) {
return druidUtil.makeSelectorFilter(
D.dimension.eventType,
A2D['granularity-digest'][granularity]
);
-};
+}
-
-/*
-
-Currently not used - We have decided to go for metrics without
-deletion drift at first, in order to be able to explain it and
-show it correctly in the future. Keeping this function for later
-
-var deletedCurrentFilters = function(granularity) {
- var deletedCurrents = A2D['granularity-deleted_currents'][granularity];
- return deletedCurrents.map(tag => {
- return druidUtil.makeNotFilter(
- druidUtil.makeSelectorFilter(D.dimension.otherTags, tag));
- });
-};
-*/
-
-var eventsCountingAggregation = function(outputMetricName) {
+function eventsCountingAggregation(outputMetricName) {
return druidUtil.makeLongSum(outputMetricName, D.metric.events);
-};
+}
-var convertDruidResultToAqsResult = function(druidResult, requestParams,
keyFilters, isTop) {
+function convertDruidResultToAqsResult(druidResult, requestParams, keyFilters,
isTop) {
if (druidResult.status === 200) {
// Overwrite body: Druid result is an array of results,
// we send a single item with an array of results
- var coreItem = {};
+ const coreItem = {};
- AQS_PARAMS.forEach(paramName => {
+ AQS_PARAMS.forEach((paramName) => {
if (requestParams[paramName]) {
coreItem[paramName] = requestParams[paramName];
}
});
- coreItem.results = druidResult.body.map(druidRes => {
- var aqsRes = { timestamp: druidRes.timestamp };
+ coreItem.results = druidResult.body.map((druidRes) => {
+ const aqsRes = { timestamp: druidRes.timestamp };
if (isTop) {
// Just copy the result array to top field
aqsRes.top = druidRes.result;
} else {
// Iterate over result keys and keep/convert only needed ones
- Object.keys(druidRes.result).forEach(k => {
+ Object.keys(druidRes.result).forEach((k) => {
if ((keyFilters === undefined) || (keyFilters[k] !==
undefined)) {
// Results from Druid can be floats in case of
// count-distinct approximations. We convert them to
int
@@ -220,13 +203,15 @@
// Return a single item
druidResult.body = { items: [ coreItem ] };
} else {
- druidResult.body = 'Druid server error.\nIt would be great if you
could send us ' +
- 'an email ([email protected]) with a copy of this message.\n
' +
- 'Thanks a lot !\n' + druidResult.body;
+ druidResult.body = ```Druid server error.
+It would be great if you could send us an email ([email protected])
+with a copy of this message.
+Thanks a lot!
+${druidResult.body}```;
}
return druidResult;
-};
+}
/*
@@ -235,10 +220,10 @@
MHMS.prototype.newPagesTimeseries = function(hyper, req) {
// Validate request parameters in place
- var rp = req.params;
+ const rp = req.params;
validateRequestParams(rp);
- var druidRequest = druidUtil.makeTimeseriesQuery(
+ const druidRequest = druidUtil.makeTimeseriesQuery(
requestURI(this.druid),
D.datasource,
A2D.granularity[rp.granularity],
@@ -270,13 +255,13 @@
druidUtil.makeInterval(rp.start, rp.end)
);
- var keyFilters = {};
+ const keyFilters = {};
keyFilters[D.outputMetric.newPages] = true;
return hyper
.post(druidRequest)
.catch(aqsUtil.notFoundCatcher)
- .then(aqsUtil.normalizeResponse).then(res => {
+ .then(aqsUtil.normalizeResponse).then((res) => {
// Need to filter out some druid results fields
// we only want the post-aggregation one
return convertDruidResultToAqsResult(res, rp, keyFilters);
@@ -289,10 +274,10 @@
MHMS.prototype.newlyRegisteredUsersTimeseries = function(hyper, req) {
// Validate request parameters in place
- var rp = req.params;
+ const rp = req.params;
validateRequestParams(rp);
- var druidRequest = druidUtil.makeTimeseriesQuery(
+ const druidRequest = druidUtil.makeTimeseriesQuery(
requestURI(this.druid),
D.datasource,
A2D.granularity[rp.granularity],
@@ -312,7 +297,7 @@
return hyper
.post(druidRequest)
.catch(aqsUtil.notFoundCatcher)
- .then(aqsUtil.normalizeResponse).then(res => {
+ .then(aqsUtil.normalizeResponse).then((res) => {
return convertDruidResultToAqsResult(res, rp);
});
};
@@ -326,14 +311,14 @@
MHMS.prototype.digestsTimeseries = function(hyper, req) {
// Validate request parameters in place
- var rp = req.params;
+ const rp = req.params;
validateRequestParams(rp, {
noAllProjects: true // Don't accept all-projects aggregation
});
// editors or edited-pages specific parts
- var eventEntityFilter;
- var outputMetric;
+ let eventEntityFilter;
+ let outputMetric;
if (rp['digest-type'] === 'editors') {
eventEntityFilter = druidQueriesBlocks.filter.users;
outputMetric = D.outputMetric.editors;
@@ -344,7 +329,7 @@
throw new Error('Internal error - Invalid digest-type parameter for
digestsTimeseries');
}
- var druidRequest = druidUtil.makeTimeseriesQuery(
+ const druidRequest = druidUtil.makeTimeseriesQuery(
requestURI(this.druid),
D.datasource,
A2D.granularity[rp.granularity],
@@ -359,7 +344,7 @@
return hyper
.post(druidRequest)
.catch(aqsUtil.notFoundCatcher)
- .then(aqsUtil.normalizeResponse).then(res => {
+ .then(aqsUtil.normalizeResponse).then((res) => {
return convertDruidResultToAqsResult(res, rp);
});
};
@@ -373,7 +358,7 @@
MHMS.prototype.revisionsTimeseries = function(hyper, req) {
// Validate request parameters in place
- var rp = req.params;
+ const rp = req.params;
validateRequestParams(rp,
// Accept all-projects aggregation if not grouping by page-id or
editor-id
(rp['page-id'] || rp['editor-id']) ? { noAllProjects: true } : {}
@@ -381,7 +366,7 @@
// edits, net-bytes-diff or abs-bytes-diff specific parts
- var aggregation;
+ let aggregation;
if (rp.metric === 'edits') {
aggregation = eventsCountingAggregation(D.outputMetric.edits);
} else if (rp.metric === 'net-bytes-diff') {
@@ -394,7 +379,7 @@
throw new Error('Internal error - Invalid metric parameter for
revisionsTimeseries');
}
- var druidRequest = druidUtil.makeTimeseriesQuery(
+ const druidRequest = druidUtil.makeTimeseriesQuery(
requestURI(this.druid),
D.datasource,
A2D.granularity[rp.granularity],
@@ -409,7 +394,7 @@
return hyper
.post(druidRequest)
.catch(aqsUtil.notFoundCatcher)
- .then(aqsUtil.normalizeResponse).then(res => {
+ .then(aqsUtil.normalizeResponse).then((res) => {
return convertDruidResultToAqsResult(res, rp);
});
};
@@ -423,13 +408,13 @@
MHMS.prototype.revisionsTop = function(hyper, req) {
// Validate request parameters in place
- var rp = req.params;
+ const rp = req.params;
validateRequestParams(rp, {
noAllProjects: true // Don't accept all-projects aggregation
});
// editors or edited-pages specific parts
- var topDimension;
+ let topDimension;
if (rp['top-type'] === 'editors') {
topDimension = D.dimension.userId;
} else if (rp['top-type'] === 'edited-pages') {
@@ -439,8 +424,8 @@
}
// edits, net-bytes-diff or abs-bytes-diff specific parts
- var aggregation;
- var outputMetric;
+ let aggregation;
+ let outputMetric;
if (rp.metric === 'edits') {
outputMetric = D.outputMetric.edits;
aggregation = eventsCountingAggregation(outputMetric);
@@ -454,7 +439,7 @@
throw new Error('Internal error - Invalid metric parameter for
revisionsTop');
}
- var druidRequest = druidUtil.makeTopN(
+ const druidRequest = druidUtil.makeTopN(
requestURI(this.druid),
D.datasource,
A2D.granularity[rp.granularity],
@@ -472,17 +457,17 @@
return hyper
.post(druidRequest)
.catch(aqsUtil.notFoundCatcher)
- .then(aqsUtil.normalizeResponse).then(res => {
+ .then(aqsUtil.normalizeResponse).then((res) => {
return convertDruidResultToAqsResult(res, rp, undefined, true);
});
};
module.exports = function(options) {
- var mhms = new MHMS(options);
+ const mhms = new MHMS(options);
return {
- spec: spec,
+ spec,
operations: {
newPagesTimeseries: mhms.newPagesTimeseries.bind(mhms),
newlyRegisteredUsersTimeseries:
mhms.newlyRegisteredUsersTimeseries.bind(mhms),
diff --git a/sys/pageviews.js b/sys/pageviews.js
index 0f8b151..a9ef7cd 100644
--- a/sys/pageviews.js
+++ b/sys/pageviews.js
@@ -6,14 +6,14 @@
* This API serves pre-aggregated pageview statistics from Cassandra
*/
-var HyperSwitch = require('hyperswitch');
-var path = require('path');
-var HTTPError = HyperSwitch.HTTPError;
-var URI = HyperSwitch.URI;
+const HyperSwitch = require('hyperswitch');
+const path = require('path');
+const HTTPError = HyperSwitch.HTTPError;
+const URI = HyperSwitch.URI;
-var aqsUtil = require('../lib/aqsUtil');
+const aqsUtil = require('../lib/aqsUtil');
-var spec = HyperSwitch.utils.loadSpec(path.join(__dirname, 'pageviews.yaml'));
+const spec = HyperSwitch.utils.loadSpec(path.join(__dirname,
'pageviews.yaml'));
const MONTHLY = 'monthly';
const DAILY = 'daily';
@@ -23,16 +23,17 @@
this.options = options;
}
+function tableURI(domain, tableName) {
+ return new URI([domain, 'sys', 'table', tableName, '']);
+}
-var tables = {
+const tables = {
articleFlat: 'pageviews.per.article.flat',
project_v2: 'pageviews.per.project.v2',
tops: 'top.pageviews',
};
-var tableURI = function(domain, tableName) {
- return new URI([domain, 'sys', 'table', tableName, '']);
-};
-var tableSchemas = {
+
+const tableSchemas = {
articleFlat: {
table: tables.articleFlat,
version: 3,
@@ -99,7 +100,7 @@
};
-var viewCountColumnsForArticleFlat = {
+const viewCountColumnsForArticleFlat = {
views_all_access_all_agents: 'aa', // views for all-access, all-agents
views_all_access_bot: 'ab', // views for all-access, bot
views_all_access_spider: 'as', // views for all-access, spider
@@ -125,13 +126,13 @@
// view count column in the dictionary above, using its short name.
// The short name saves space because cassandra stores the column name with
// each record.
-Object.keys(viewCountColumnsForArticleFlat).forEach(function(k) {
+Object.keys(viewCountColumnsForArticleFlat).forEach((k) => {
tableSchemas.articleFlat.attributes[viewCountColumnsForArticleFlat[k]] =
'int';
});
PJVS.prototype.pageviewsForArticleFlat = function(hyper, req) {
- var rp = req.params;
+ const rp = req.params;
// dates are passed in as YYYYMMDD but we need the HH to match the loaded
data
// which was originally planned at hourly resolution, so we pass "fakeHour"
@@ -142,7 +143,7 @@
fullMonths: rp.granularity === MONTHLY
});
- var dataRequest = hyper.get({
+ const dataRequest = hyper.get({
uri: tableURI(rp.domain, tables.articleFlat),
body: {
table: tables.articleFlat,
@@ -157,23 +158,23 @@
}).catch(aqsUtil.notFoundCatcher);
function viewKey(access, agent) {
- var ret = ['views', access, agent].join('_');
+ const ret = ['views', access, agent].join('_');
return viewCountColumnsForArticleFlat[ret.replace(/-/g, '_')];
}
function removeDenormalizedColumns(item) {
- Object.keys(viewCountColumnsForArticleFlat).forEach(function(k) {
+ Object.keys(viewCountColumnsForArticleFlat).forEach((k) => {
delete item[viewCountColumnsForArticleFlat[k]];
});
}
- return dataRequest.then(aqsUtil.normalizeResponse).then(function(res) {
+ return dataRequest.then(aqsUtil.normalizeResponse).then((res) => {
if (res.body.items) {
- var monthViews = {};
+ const monthViews = {};
const aggregateMonthly = rp.granularity === MONTHLY;
- res.body.items.forEach(function(item) {
- var yearAndMonth = item.timestamp.substring(0, 6);
+ res.body.items.forEach((item) => {
+ const yearAndMonth = item.timestamp.substring(0, 6);
item.access = rp.access;
item.agent = rp.agent;
@@ -185,12 +186,12 @@
removeDenormalizedColumns(item);
if (aggregateMonthly) {
- if (!monthViews.hasOwnProperty(yearAndMonth)) {
- var newMonth = {
+ if (!Object.prototype.hasOwnProperty.call(monthViews,
yearAndMonth)) {
+ const newMonth = {
project: item.project,
article: item.article,
granularity: MONTHLY,
- timestamp: yearAndMonth + '0100',
+ timestamp: `${yearAndMonth}0100`,
access: rp.access,
agent: rp.agent,
views: 0
@@ -204,9 +205,9 @@
});
if (aggregateMonthly) {
- var sortedMonths = Object.keys(monthViews);
+ const sortedMonths = Object.keys(monthViews);
sortedMonths.sort();
- res.body.items = sortedMonths.map(function(month) {
+ res.body.items = sortedMonths.map((month) => {
return monthViews[month];
});
}
@@ -217,7 +218,7 @@
};
PJVS.prototype.pageviewsForProjects = function(hyper, req) {
- var rp = req.params;
+ const rp = req.params;
aqsUtil.validateStartAndEnd(rp, {
fakeHour: (rp.granularity === MONTHLY || rp.granularity === DAILY),
@@ -225,7 +226,7 @@
fullMonths: rp.granularity === MONTHLY
});
- var dataRequest = hyper.get({
+ const dataRequest = hyper.get({
uri: tableURI(rp.domain, tables.project_v2),
body: {
table: tables.project_v2,
@@ -240,9 +241,9 @@
}).catch(aqsUtil.notFoundCatcher);
- return dataRequest.then(aqsUtil.normalizeResponse).then(function(res) {
+ return dataRequest.then(aqsUtil.normalizeResponse).then((res) => {
if (res.body.items) {
- res.body.items.forEach(function(item) {
+ res.body.items.forEach((item) => {
// prefer the v column if it's loaded
if (item.v !== null) {
try {
@@ -260,11 +261,11 @@
};
PJVS.prototype.pageviewsForTops = function(hyper, req) {
- var rp = req.params;
+ const rp = req.params;
aqsUtil.validateYearMonthDay(rp);
- var dataRequest = hyper.get({
+ const dataRequest = hyper.get({
uri: tableURI(rp.domain, tables.tops),
body: {
table: tables.tops,
@@ -279,9 +280,9 @@
}).catch(aqsUtil.notFoundCatcher);
- return dataRequest.then(aqsUtil.normalizeResponse).then(function(res) {
+ return dataRequest.then(aqsUtil.normalizeResponse).then((res) => {
if (res.body.items) {
- res.body.items.forEach(function(item) {
+ res.body.items.forEach((item) => {
// prefer the articlesJSON column if it's loaded
if (item.articlesJSON !== null) {
item.articles = item.articlesJSON;
@@ -310,10 +311,10 @@
module.exports = function(options) {
- var pjvs = new PJVS(options);
+ const pjvs = new PJVS(options);
return {
- spec: spec,
+ spec,
operations: {
pageviewsForArticle: pjvs.pageviewsForArticleFlat.bind(pjvs),
pageviewsForProjects: pjvs.pageviewsForProjects.bind(pjvs),
@@ -321,15 +322,15 @@
},
resources: [
{
- uri: '/{domain}/sys/table/' + tables.articleFlat,
+ uri: `/{domain}/sys/table/${tables.articleFlat}`,
body: tableSchemas.articleFlat,
}, {
// table where per-project data will be stored with fixed
timestamps (T156312)
- uri: '/{domain}/sys/table/' + tables.project_v2,
+ uri: `/{domain}/sys/table/${tables.project_v2}`,
body: tableSchemas.project_v2,
}, {
// top pageviews table
- uri: '/{domain}/sys/table/' + tables.tops,
+ uri: `/{domain}/sys/table/${tables.tops}`,
body: tableSchemas.tops,
}
]
diff --git a/sys/table.js b/sys/table.js
new file mode 100644
index 0000000..3bbd576
--- /dev/null
+++ b/sys/table.js
@@ -0,0 +1,17 @@
+"use strict";
+
+/*
+ * A simple wrapper module over storage modules which allows to switch between
storage
+ * implementation using a config option.
+ */
+
+module.exports = (options) => {
+ options.conf.backend = options.conf.backend || 'cassandra';
+
+ if (options.conf.backend !== 'cassandra'
+ && options.conf.backend !== 'sqlite') {
+ throw new Error(`Unsupported backend version specified:
${options.conf.backend}`);
+ }
+
+ return require(`restbase-mod-table-${options.conf.backend}`)(options);
+};
diff --git a/sys/unique-devices.js b/sys/unique-devices.js
index 2b33727..6536c78 100644
--- a/sys/unique-devices.js
+++ b/sys/unique-devices.js
@@ -6,27 +6,28 @@
* This API serves pre-aggregated unique devices statistics from Cassandra
*/
-var HyperSwitch = require('hyperswitch');
-var path = require('path');
-var URI = HyperSwitch.URI;
+const HyperSwitch = require('hyperswitch');
+const path = require('path');
+const URI = HyperSwitch.URI;
-var aqsUtil = require('../lib/aqsUtil');
+const aqsUtil = require('../lib/aqsUtil');
-var spec = HyperSwitch.utils.loadSpec(path.join(__dirname,
'unique-devices.yaml'));
+const spec = HyperSwitch.utils.loadSpec(path.join(__dirname,
'unique-devices.yaml'));
// Unique devices Service
function UDVS(options) {
this.options = options;
}
+function tableURI(domain, tableName) {
+ return new URI([domain, 'sys', 'table', tableName, '']);
+}
-var tables = {
+const tables = {
project: 'unique.devices',
};
-var tableURI = function(domain, tableName) {
- return new URI([domain, 'sys', 'table', tableName, '']);
-};
-var tableSchemas = {
+
+const tableSchemas = {
project: {
table: tables.project,
version: 1,
@@ -49,7 +50,7 @@
UDVS.prototype.uniqueDevices = function(hyper, req) {
- var rp = req.params;
+ const rp = req.params;
aqsUtil.validateStartAndEnd(rp, {
// YYYYMMDD dates are allowed, but need an hour to pass validation
@@ -59,7 +60,7 @@
stripHour: true,
});
- var dataRequest = hyper.get({
+ const dataRequest = hyper.get({
uri: tableURI(rp.domain, tables.project),
body: {
table: tables.project,
@@ -74,9 +75,9 @@
}).catch(aqsUtil.notFoundCatcher);
// Parse long from string to int
- return dataRequest.then(aqsUtil.normalizeResponse).then(function(res) {
+ return dataRequest.then(aqsUtil.normalizeResponse).then((res) => {
if (res.body.items) {
- res.body.items.forEach(function(item) {
+ res.body.items.forEach((item) => {
if (item.devices !== null) {
try {
item.devices = parseInt(item.devices, 10);
@@ -92,17 +93,17 @@
};
module.exports = function(options) {
- var udvs = new UDVS(options);
+ const udvs = new UDVS(options);
return {
- spec: spec,
+ spec,
operations: {
uniqueDevices: udvs.uniqueDevices.bind(udvs)
},
resources: [
{
// unique devices per project table
- uri: '/{domain}/sys/table/' + tables.project,
+ uri: `/{domain}/sys/table/${tables.project}`,
body: tableSchemas.project,
}
]
diff --git a/test/aqs_test_module.yaml b/test/aqs_test_module.yaml
index 5ed46a2..6d3f0bf 100644
--- a/test/aqs_test_module.yaml
+++ b/test/aqs_test_module.yaml
@@ -1,139 +1,141 @@
# Simple test spec
-swagger: 2.0
paths:
/{api:v1}:
- swagger: '2.0'
- info:
- version: 1.0.0-beta
- title: AQS testing APIs
- x-is-api-root: true
- paths:
-
/pageviews/insert-per-article-flat/{project}/{article}/{granularity}/{timestamp}/{views}:
- post:
- x-request-handler:
- - put_to_storage:
- request:
- method: 'put'
- uri: '/{domain}/sys/table/pageviews.per.article.flat/'
- body:
- table: 'pageviews.per.article.flat'
- attributes:
- project: '{{request.params.project}}'
- article: '{{request.params.article}}'
- granularity: '{{request.params.granularity}}'
- timestamp: '{{request.params.timestamp}}'
- aa: '{{request.params.views}}1'
- ab: '{{request.params.views}}2'
- as: '{{request.params.views}}3'
- au: '{{request.params.views}}4'
- da: '{{request.params.views}}5'
- db: '{{request.params.views}}6'
- ds: '{{request.params.views}}7'
- du: null
- maa: '{{request.params.views}}9'
- mab: '{{request.params.views}}10'
- mas: '{{request.params.views}}11'
- mau: '{{request.params.views}}12'
- mwa: '{{request.params.views}}13'
- mwb: '{{request.params.views}}14'
- mws: '{{request.params.views}}15'
- mwu: '{{request.params.views}}16'
- x-monitor: false
+ x-modules:
+ - spec:
+ # Careful - 2 indentations here !
+ info:
+ version: 1.0.0-beta
+ title: AQS testing APIs
+ x-is-api-root: true
+ paths:
+
/pageviews/insert-per-article-flat/{project}/{article}/{granularity}/{timestamp}/{views}:
+ post:
+ x-request-handler:
+ - put_to_storage:
+ request:
+ method: 'put'
+ uri:
'/{domain}/sys/table/pageviews.per.article.flat/'
+ body:
+ table: 'pageviews.per.article.flat'
+ attributes:
+ project: '{{request.params.project}}'
+ article: '{{request.params.article}}'
+ granularity: '{{request.params.granularity}}'
+ timestamp: '{{request.params.timestamp}}'
+ aa: '{{request.params.views}}1'
+ ab: '{{request.params.views}}2'
+ as: '{{request.params.views}}3'
+ au: '{{request.params.views}}4'
+ da: '{{request.params.views}}5'
+ db: '{{request.params.views}}6'
+ ds: '{{request.params.views}}7'
+ du: null
+ maa: '{{request.params.views}}9'
+ mab: '{{request.params.views}}10'
+ mas: '{{request.params.views}}11'
+ mau: '{{request.params.views}}12'
+ mwa: '{{request.params.views}}13'
+ mwb: '{{request.params.views}}14'
+ mws: '{{request.params.views}}15'
+ mwu: '{{request.params.views}}16'
+ x-monitor: false
-
/pageviews/insert-aggregate/{project}/{access}/{agent}/{granularity}/{timestamp}/{views}:
- post:
- x-request-handler:
- - put_to_storage:
- request:
- method: 'put'
- uri: '/{domain}/sys/table/pageviews.per.project.v2/'
- body:
- table: 'pageviews.per.project.v2'
- attributes:
- project: '{{request.params.project}}'
- access: '{{request.params.access}}'
- agent: '{{request.params.agent}}'
- granularity: '{{request.params.granularity}}'
- timestamp: '{{request.params.timestamp}}'
- views: '{{request.params.views}}'
- x-monitor: false
+
/pageviews/insert-aggregate/{project}/{access}/{agent}/{granularity}/{timestamp}/{views}:
+ post:
+ x-request-handler:
+ - put_to_storage:
+ request:
+ method: 'put'
+ uri: '/{domain}/sys/table/pageviews.per.project.v2/'
+ body:
+ table: 'pageviews.per.project.v2'
+ attributes:
+ project: '{{request.params.project}}'
+ access: '{{request.params.access}}'
+ agent: '{{request.params.agent}}'
+ granularity: '{{request.params.granularity}}'
+ timestamp: '{{request.params.timestamp}}'
+ views: '{{request.params.views}}'
+ x-monitor: false
-
/pageviews/insert-aggregate-long/{project}/{access}/{agent}/{granularity}/{timestamp}/{v}:
- post:
- x-request-handler:
- - put_to_storage:
- request:
- method: 'put'
- uri: '/{domain}/sys/table/pageviews.per.project.v2/'
- body:
- table: 'pageviews.per.project.v2'
- attributes:
- project: '{{request.params.project}}'
- access: '{{request.params.access}}'
- agent: '{{request.params.agent}}'
- granularity: '{{request.params.granularity}}'
- timestamp: '{{request.params.timestamp}}'
- v: '{{request.params.v}}'
- x-monitor: false
+
/pageviews/insert-aggregate-long/{project}/{access}/{agent}/{granularity}/{timestamp}/{v}:
+ post:
+ x-request-handler:
+ - put_to_storage:
+ request:
+ method: 'put'
+ uri: '/{domain}/sys/table/pageviews.per.project.v2/'
+ body:
+ table: 'pageviews.per.project.v2'
+ attributes:
+ project: '{{request.params.project}}'
+ access: '{{request.params.access}}'
+ agent: '{{request.params.agent}}'
+ granularity: '{{request.params.granularity}}'
+ timestamp: '{{request.params.timestamp}}'
+ v: '{{request.params.v}}'
+ x-monitor: false
- /pageviews/insert-top/{project}/{access}/{year}/{month}/{day}:
- post:
- x-request-handler:
- - put_to_storage:
- request:
- method: 'put'
- uri: '/{domain}/sys/table/top.pageviews/'
- body:
- table: 'top.pageviews'
- attributes:
- project: '{{request.params.project}}'
- access: '{{request.params.access}}'
- year: '{{request.params.year}}'
- month: '{{request.params.month}}'
- day: '{{request.params.day}}'
- articlesJSON: '{{request.body.articles}}'
- x-monitor: false
+ /pageviews/insert-top/{project}/{access}/{year}/{month}/{day}:
+ post:
+ x-request-handler:
+ - put_to_storage:
+ request:
+ method: 'put'
+ uri: '/{domain}/sys/table/top.pageviews/'
+ body:
+ table: 'top.pageviews'
+ attributes:
+ project: '{{request.params.project}}'
+ access: '{{request.params.access}}'
+ year: '{{request.params.year}}'
+ month: '{{request.params.month}}'
+ day: '{{request.params.day}}'
+ articlesJSON: '{{request.body.articles}}'
+ x-monitor: false
-
/legacy/pagecounts/insert-aggregate/{project}/{access-site}/{granularity}/{timestamp}/{count}:
- post:
- x-request-handler:
- - put_to_storage:
- request:
- method: 'put'
- uri: '/{domain}/sys/table/lgc.pagecounts.per.project/'
- body:
- table: 'lgc.pagecounts.per.project'
- attributes:
- project: '{{request.params.project}}'
- 'access-site': '{{request.params.access-site}}'
- granularity: '{{request.params.granularity}}'
- timestamp: '{{request.params.timestamp}}'
- count: '{{request.params.count}}'
- x-monitor: false
+
/legacy/pagecounts/insert-aggregate/{project}/{access-site}/{granularity}/{timestamp}/{count}:
+ post:
+ x-request-handler:
+ - put_to_storage:
+ request:
+ method: 'put'
+ uri:
'/{domain}/sys/table/lgc.pagecounts.per.project/'
+ body:
+ table: 'lgc.pagecounts.per.project'
+ attributes:
+ project: '{{request.params.project}}'
+ 'access-site': '{{request.params.access-site}}'
+ granularity: '{{request.params.granularity}}'
+ timestamp: '{{request.params.timestamp}}'
+ count: '{{request.params.count}}'
+ x-monitor: false
-
/unique-devices/insert/{project}/{access-site}/{granularity}/{timestamp}/{devices}:
- post:
- x-request-handler:
- - put_to_storage:
- request:
- method: 'put'
- uri: '/{domain}/sys/table/unique.devices/'
- body:
- table: 'unique.devices'
- attributes:
- project: '{{request.params.project}}'
- 'access-site': '{{request.params.access-site}}'
- granularity: '{{request.params.granularity}}'
- timestamp: '{{request.params.timestamp}}'
- devices: '{{request.params.devices}}'
- x-monitor: false
+
/unique-devices/insert/{project}/{access-site}/{granularity}/{timestamp}/{devices}:
+ post:
+ x-request-handler:
+ - put_to_storage:
+ request:
+ method: 'put'
+ uri: '/{domain}/sys/table/unique.devices/'
+ body:
+ table: 'unique.devices'
+ attributes:
+ project: '{{request.params.project}}'
+ 'access-site': '{{request.params.access-site}}'
+ granularity: '{{request.params.granularity}}'
+ timestamp: '{{request.params.timestamp}}'
+ devices: '{{request.params.devices}}'
+ x-monitor: false
# Fake internal endpoint simulating a druid cluster for testing
/{api:sys}:
- swagger: 2.0
- info:
- x-is-api-root: true
x-modules:
- /fake-druid:
- - path: test/features/mediawiki-history-metrics/fake-druid.js
+ - spec:
+ # Careful - 2 indentations here !
+ paths:
+ /fake-druid:
+ x-modules:
+ - path: test/features/mediawiki-history-metrics/fake-druid.js
diff --git
a/test/features/mediawiki-history-metrics/mediawiki-history-metrics.js
b/test/features/mediawiki-history-metrics/mediawiki-history-metrics.js
index e166b15..a91c1d7 100644
--- a/test/features/mediawiki-history-metrics/mediawiki-history-metrics.js
+++ b/test/features/mediawiki-history-metrics/mediawiki-history-metrics.js
@@ -23,10 +23,8 @@
this.timeout(20000);
// Start server before running tests
- before('before-suite', setupDone => {
- return server.start().then(() => {
- setupDone();
- });
+ before('before-suite', () => {
+ return server.start();
});
var makeTest = function(fixture) {
diff --git a/test/features/pageviews/pageviews.js
b/test/features/pageviews/pageviews.js
index 95f78b7..f8aca78 100644
--- a/test/features/pageviews/pageviews.js
+++ b/test/features/pageviews/pageviews.js
@@ -40,7 +40,7 @@
// Start server before running tests
// insert here data that tests assume exists on db to start working
before('before-suite', function(setupDone) {
- return server.start().then(function() {
+ server.start().then(function() {
const dataToInsert = {
2015070200: 100,
2015072100: 200,
diff --git a/test/index.js b/test/index.js
index b21a583..c126552 100644
--- a/test/index.js
+++ b/test/index.js
@@ -5,3 +5,8 @@
require('mocha-jshint')();
// Run jscs as part of normal testing
require('mocha-jscs')();
+require('mocha-eslint')([
+ 'lib',
+ 'sys',
+ 'v1'
+]);
\ No newline at end of file
diff --git a/test/utils/run_tests.sh b/test/utils/run_tests.sh
index d6807c1..a471729 100644
--- a/test/utils/run_tests.sh
+++ b/test/utils/run_tests.sh
@@ -11,6 +11,13 @@
rm -f test.db.sqlite3
else
echo "Running with Cassandra backend"
+ if [ `nc -z localhost 9042 < /dev/null; echo $?` != 0 ]; then
+ echo "Waiting for Cassandra to start..."
+ while [ `nc -z localhost 9042; echo $?` != 0 ]; do
+ sleep 1
+ done
+ echo "Cassandra is ready."
+ fi
export RB_TEST_BACKEND=cassandra
sh ./test/utils/cleandb.sh
fi
@@ -28,7 +35,7 @@
if [ "$?" -eq 0 ]; then
runTest "cassandra" $1
else
- echo "Cassandra not available. Using SQLite backed for tests"
+ echo "Cassandra not available. Using SQLite backend for tests"
runTest "sqlite" $1
fi
elif [ "$2" = "sqlite" ]; then
diff --git a/test/utils/server.js b/test/utils/server.js
index 3ae3240..076902e 100644
--- a/test/utils/server.js
+++ b/test/utils/server.js
@@ -8,7 +8,6 @@
var fs = require('fs');
var assert = require('./assert');
var yaml = require('js-yaml');
-var temp = require('temp').track();
var hostPort = 'http://localhost:7231';
var aqsURL = hostPort + '/analytics.wikimedia.org/v1';
@@ -21,14 +20,7 @@
throw new Error('Invalid RB_TEST_BACKEND env variable value.
Allowed values: "cassandra", "sqlite"');
}
if (backendImpl === 'sqlite') {
- // First, replace the module in all projects and move them to the
temp directory
- var tempDir = temp.mkdirSync('tempProjects');
- fs.readdirSync(__dirname +
'/../../projects').forEach(function(fileName) {
- var fileStr = fs.readFileSync(__dirname + '/../../projects/' +
fileName).toString()
- .replace(/restbase\-mod\-table\-cassandra/g,
'restbase-mod-table-sqlite');
- fs.writeFileSync(tempDir + '/' + fileName, fileStr);
- });
- confString = confString.replace(/projects\//g, tempDir + '/');
+ confString = confString.replace(/backend: cassandra/, "backend:
sqlite");
}
}
return yaml.safeLoad(confString);
--
To view, visit https://gerrit.wikimedia.org/r/384590
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ie7847c1de76cb7e6cc868a1d3b94af68785f86bc
Gerrit-PatchSet: 3
Gerrit-Project: analytics/aqs
Gerrit-Branch: master
Gerrit-Owner: Joal <[email protected]>
Gerrit-Reviewer: Joal <[email protected]>
Gerrit-Reviewer: Milimetric <[email protected]>
Gerrit-Reviewer: Mobrovac <[email protected]>
Gerrit-Reviewer: Ppchelko <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
