Joal has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/384590 )
Change subject: [WIP] Upgrade restbase-modules to latest
......................................................................
[WIP] Upgrade restbase-modules to latest
It makes month we've not done that, we are many versions
behind.
This ipatch also adds linting and correct code accordingly.
Bug: T178312
Change-Id: Ie7847c1de76cb7e6cc868a1d3b94af68785f86bc
---
A .eslintrc.yml
M .jscs.json
M .jshintrc
M config.example.wikimedia.yaml
M config.test.yaml
M lib/aqsUtil.js
M package.json
M projects/aqs_default.yaml
A sys/table.js
M test/aqs_test_module.yaml
M test/features/mediawiki-history-metrics/mediawiki-history-metrics.js
M test/features/pageviews/pageviews.js
M test/index.js
M test/utils/run_tests.sh
M test/utils/server.js
15 files changed, 345 insertions(+), 304 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/aqs
refs/changes/90/384590/1
diff --git a/.eslintrc.yml b/.eslintrc.yml
new file mode 100644
index 0000000..9e2c225
--- /dev/null
+++ b/.eslintrc.yml
@@ -0,0 +1 @@
+extends: 'eslint-config-node-services'
\ No newline at end of file
diff --git a/.jscs.json b/.jscs.json
index bf87d75..8a4d757 100644
--- a/.jscs.json
+++ b/.jscs.json
@@ -10,7 +10,9 @@
"catch"
],
"validateIndentation": 4,
- "requireCamelCaseOrUpperCaseIdentifiers": "ignoreProperties",
+ "requireCamelCaseOrUpperCaseIdentifiers": {
+ "ignoreProperties": true
+ },
"requireCapitalizedComments": null,
"maximumLineLength": 100,
"validateQuoteMarks": null,
@@ -23,6 +25,7 @@
"node_modules/**",
"test/**",
"coverage/**",
- "test.db.**"
+ "test.db.**",
+ "maintenance/**"
]
-}
\ No newline at end of file
+}
diff --git a/.jshintrc b/.jshintrc
index 5288038..8acdd9d 100644
--- a/.jshintrc
+++ b/.jshintrc
@@ -1,38 +1,37 @@
{
- "predef": [
- "ve",
+ "predef": [
+ "ve",
- "setImmediate",
+ "setImmediate",
- "QUnit",
+ "QUnit",
- "Map",
- "Set"
- ],
+ "Map",
+ "Set"
+ ],
- "bitwise": true,
- "laxbreak": true,
- "curly": true,
- "eqeqeq": true,
- "immed": true,
- "latedef": "nofunc",
- "maxlen":false,
- "newcap": true,
- "noarg": true,
- "noempty": true,
- "nonew": true,
- "regexp": false,
- "undef": true,
- "strict": true,
- "trailing": true,
+ "bitwise": true,
+ "laxbreak": true,
+ "curly": true,
+ "eqeqeq": true,
+ "immed": true,
+ "latedef": "nofunc",
+ "newcap": true,
+ "noarg": true,
+ "noempty": true,
+ "nonew": true,
+ "regexp": false,
+ "undef": true,
+ "strict": true,
+ "trailing": true,
- "smarttabs": true,
- "multistr": true,
+ "smarttabs": true,
+ "multistr": true,
- "node": true,
+ "node": true,
- "nomen": false,
- "loopfunc": true,
- "esnext": true
- //"onevar": true
+ "nomen": false,
+ "loopfunc": true,
+ "esnext": true
+ //"onevar": true
}
diff --git a/config.example.wikimedia.yaml b/config.example.wikimedia.yaml
index b61aad9..39f7d33 100644
--- a/config.example.wikimedia.yaml
+++ b/config.example.wikimedia.yaml
@@ -1,23 +1,23 @@
# Analytics Query Service config
aqs_project: &aqs_project
x-modules:
- /:
- - path: projects/aqs_default.yaml
- options: &default_options
- table:
- hosts: [localhost]
- keyspace: system
- username: cassandra
- password: cassandra
- defaultConsistency: one # or 'localQuorum' for production
- storage_groups:
- - name: default.group.local
- domains: /./
- druid:
- scheme: http
- host: druid1004.eqiad.wmnet
- port: 8082
- query_path: /druid/v2/
+ - path: projects/aqs_default.yaml
+ options: &default_options
+ table:
+ backend: cassandra
+ hosts: [localhost]
+ keyspace: system
+ username: cassandra
+ password: cassandra
+ defaultConsistency: one # or 'localQuorum' for production
+ storage_groups:
+ - name: default.group.local
+ domains: /./
+ druid:
+ scheme: http
+ host: druid1004.eqiad.wmnet
+ port: 8082
+ query_path: /druid/v2/
# Swagger spec root.
spec: &spec_root
diff --git a/config.test.yaml b/config.test.yaml
index 965ddc0..dc49ed6 100644
--- a/config.test.yaml
+++ b/config.test.yaml
@@ -1,23 +1,23 @@
# A synthetic domain to test aqs modules data
aqs.wikimedia.org: &analytics.wikimedia.org
x-modules:
- /:
- - path: test/aqs_test_module.yaml
- - path: projects/aqs_default.yaml
- options: &default_options
- table:
- hosts: [localhost]
- keyspace: system
- username: cassandra
- password: cassandra
- defaultConsistency: one # or 'localQuorum' for production
- storage_groups:
- - name: test.group.local
- domains: /./
- dbname: test.db.sqlite3 # ignored in cassandra, but useful in
SQLite testing
- druid:
- # Use a fake internal endpoint in restbase to test
- query_path: /analytics.wikimedia.org/sys/fake-druid/druid/v2
+ - path: test/aqs_test_module.yaml
+ - path: projects/aqs_default.yaml
+ options: &default_options
+ table:
+ backend: cassandra
+ hosts: [ localhost ]
+ keyspace: system
+ username: cassandra
+ password: cassandra
+ defaultConsistency: one # or 'localQuorum' for production
+ storage_groups:
+ - name: test.group.local
+ domains: /./
+ dbname: test.db.sqlite3 # ignored in cassandra, but useful in SQLite
testing
+ druid:
+ # Use a fake internal endpoint in restbase to test
+ query_path: /analytics.wikimedia.org/sys/fake-druid/druid/v2
spec_root: &spec_root
title: "The Analytics Query Service root"
diff --git a/lib/aqsUtil.js b/lib/aqsUtil.js
index b07e369..e548475 100644
--- a/lib/aqsUtil.js
+++ b/lib/aqsUtil.js
@@ -1,10 +1,9 @@
'use strict';
-var HyperSwitch = require('hyperswitch');
-var HTTPError = HyperSwitch.HTTPError;
-var druidUtil = require('./druidUtil');
+const HyperSwitch = require('hyperswitch');
+const HTTPError = HyperSwitch.HTTPError;
-var aqsUtil = {};
+const aqsUtil = {};
aqsUtil.notFoundCatcher = function(e) {
@@ -39,7 +38,7 @@
* Parameter validators
* Only needed internally, not exposed
*/
-var throwIfNeeded = function(errors) {
+function throwIfNeeded(errors) {
if (errors && errors.length) {
throw new HTTPError({
status: 400,
@@ -49,7 +48,7 @@
}
});
}
-};
+}
/**
* Normalizes the project parameter to "en.wikipedia"
@@ -78,19 +77,19 @@
}
try {
- var year = timestamp.substring(0, 4);
- var month = timestamp.substring(4, 6);
- var day = timestamp.substring(6, 8);
- var hour = opts.fakeHour ? '00' : timestamp.substring(8, 10);
+ const year = timestamp.substring(0, 4);
+ const month = timestamp.substring(4, 6);
+ const day = timestamp.substring(6, 8);
+ const hour = opts.fakeHour ? '00' : timestamp.substring(8, 10);
- var dt = new Date([year, month, day].join('-') + ' '
- + hour + ':00:00 UTC');
+ const dtDay = [year, month, day].join('-');
+ const dt = new Date(`${dtDay} ${hour}:00:00 UTC`);
return dt.toString() !== 'Invalid Date'
&& dt.getUTCFullYear() === parseInt(year, 10)
&& dt.getUTCMonth() === (parseInt(month, 10) - 1)
&& dt.getUTCDate() === parseInt(day, 10)
- && dt.getUTCHours() === parseInt(hour);
+ && dt.getUTCHours() === parseInt(hour, 10);
} catch (e) {
return false;
@@ -115,18 +114,17 @@
aqsUtil.validateStartAndEnd = function(rp, opts) {
opts = opts || {};
- var errors = [];
- var invalidMessage = opts.fakeHour ?
- 'invalid, must be a valid date in YYYYMMDD format' :
- 'invalid, must be a valid timestamp in YYYYMMDDHH format';
+ const errors = [];
+ const messageFormat = opts.fakeHour ? 'YYYYMMDD' : 'YYYYMMDDHH';
+ const invalidMessage = `invalid, must be a valid date in ${messageFormat}
format`;
aqsUtil.normalizeProject(rp);
if (!aqsUtil.validateTimestamp(rp.start, opts)) {
- errors.push('start timestamp is ' + invalidMessage);
+ errors.push(`start timestamp is ${invalidMessage}`);
}
if (!aqsUtil.validateTimestamp(rp.end, opts)) {
- errors.push('end timestamp is ' + invalidMessage);
+ errors.push(`end timestamp is ${invalidMessage}`);
}
if (rp.start > rp.end) {
@@ -173,16 +171,13 @@
};
aqsUtil.validateYearMonthDay = function(rp) {
- var errors = [];
+ const errors = [];
aqsUtil.normalizeProject(rp);
// fake a timestamp in the YYYYMMDDHH format so we can reuse the validator
- var validDate = aqsUtil.validateTimestamp(
- rp.year + rp.month +
- ((rp.day === 'all-days') ? '01' : rp.day) +
- '00'
- );
+ const day = ((rp.day === 'all-days') ? '01' : rp.day);
+ const validDate =
aqsUtil.validateTimestamp(`${rp.year}${rp.month}${day}00`);
if (!validDate) {
errors.push('Given year/month/day is invalid date');
@@ -192,20 +187,21 @@
};
aqsUtil.convertTimestampToDate = function(timestamp) {
- var year = parseInt(timestamp.substring(0, 4), 10);
- var month = parseInt(timestamp.substring(4, 6), 10) - 1;
- var day = parseInt(timestamp.substring(6, 8), 10);
+ const year = parseInt(timestamp.substring(0, 4), 10);
+ const month = parseInt(timestamp.substring(4, 6), 10) - 1;
+ const day = parseInt(timestamp.substring(6, 8), 10);
return new Date(Date.UTC(year, month, day));
};
aqsUtil.convertDateToTimestamp = function(date) {
- return date.getUTCFullYear().toString() +
- ('0' + (date.getUTCMonth() + 1)).slice(-2) +
- ('0' + date.getUTCDate()).slice(-2);
+ const year = date.getUTCFullYear().toString();
+ const month = (`0${(date.getUTCMonth() + 1)}`).slice(-2);
+ const day = (`0${date.getUTCDate()}`).slice(-2);
+ return `${year}${month}${day}`;
};
aqsUtil.getFirstFullMonthFirstDay = function(startDate) {
- var dt = aqsUtil.convertTimestampToDate(startDate);
+ const dt = aqsUtil.convertTimestampToDate(startDate);
if (dt.getUTCDate() === 1) {
return startDate;
@@ -218,8 +214,8 @@
};
aqsUtil.getLastFullMonthLastDay = function(endDate) {
- var dt = aqsUtil.convertTimestampToDate(endDate);
- var lastDayOfCurrentMonth = new Date(Date.UTC(dt.getUTCFullYear(),
dt.getUTCMonth() + 1, 0));
+ const dt = aqsUtil.convertTimestampToDate(endDate);
+ const lastDayOfCurrentMonth = new Date(Date.UTC(dt.getUTCFullYear(),
dt.getUTCMonth() + 1, 0));
if (dt.getUTCDate() === lastDayOfCurrentMonth.getUTCDate()) {
return endDate;
@@ -231,7 +227,7 @@
};
aqsUtil.getDayAfterLastFullMonth = function(endDate) {
- var dt = aqsUtil.convertTimestampToDate(endDate);
+ const dt = aqsUtil.convertTimestampToDate(endDate);
dt.setUTCDate(1);
return aqsUtil.convertDateToTimestamp(dt);
};
diff --git a/package.json b/package.json
index 20f3f50..06bf72c 100644
--- a/package.json
+++ b/package.json
@@ -24,26 +24,35 @@
"url": "https://phabricator.wikimedia.org/tag/analytics/";
},
"dependencies": {
- "bluebird": "^3.1.1",
- "restbase-mod-table-cassandra": "^0.8.15",
- "service-runner": "^1.1.0",
- "hyperswitch": "^0.1.1"
+ "bluebird": "^3.5.0",
+ "restbase-mod-table-cassandra": "^0.11.2",
+ "service-runner": "^2.3.0",
+ "hyperswitch": "^0.9.1"
},
"devDependencies": {
- "js-yaml": "^3.5.2",
- "preq": "^0.4.8",
- "bunyan": "^1.5.1",
- "coveralls": "^2.11.6",
- "istanbul": "^0.4.2",
- "mocha": "^2.3.4",
- "mocha-jscs": "^4.0.0",
- "mocha-jshint": "^2.2.6",
- "mocha-lcov-reporter": "^1.0.0",
- "restbase-mod-table-sqlite": "^0.1.14",
- "temp": "^0.8.3"
+ "ajv": "^5.1.5",
+ "bunyan": "^1.8.10",
+ "coveralls": "^2.13.1",
+ "eslint-config-node-services": "^2.2.2",
+ "eslint-config-wikimedia": "^0.4.0",
+ "eslint-plugin-jsdoc": "^3.1.0",
+ "eslint-plugin-json": "^1.2.0",
+ "istanbul": "^0.4.5",
+ "js-yaml": "^3.8.4",
+ "mocha": "^3.4.2",
+ "mocha-eslint": "^3.0.1",
+ "mocha-jscs": "^5.0.1",
+ "mocha-jshint": "^2.3.1",
+ "mocha-lcov-reporter": "^1.3.0",
+ "mocha.parallel": "^0.15.2",
+ "preq": "^0.5.2",
+ "restbase-mod-table-sqlite": "^0.1.20"
+ },
+ "engines": {
+ "node": ">=4"
},
"deploy": {
- "node": "4.4.6",
+ "node": "6.11.1",
"target": "debian",
"dependencies": {
"_all": []
diff --git a/projects/aqs_default.yaml b/projects/aqs_default.yaml
index d7fb8bf..78d2454 100644
--- a/projects/aqs_default.yaml
+++ b/projects/aqs_default.yaml
@@ -1,52 +1,64 @@
-swagger: '2.0'
paths:
/{api:v1}: &default_project_paths_v1
- swagger: '2.0'
- # swagger options, overriding the shared ones from the merged specs (?)
- info:
- version: 1.0.0-beta
- title: Analytics REST API
- description: >
- Analytics Query Service setup
- x-is-api-root: true
-
- x-host-basePath: /api/rest_v1
-
x-modules:
- /pageviews:
- - path: v1/pageviews.yaml
- /legacy/pagecounts:
- - path: v1/legacy/pagecounts.yaml
- /unique-devices:
- - path: v1/unique-devices.yaml
- /edited-pages:
- - path: v1/edited-pages.yaml
- /editors:
- - path: v1/editors.yaml
- /registered-users:
- - path: v1/registered-users.yaml
- /edits:
- - path: v1/edits.yaml
- /bytes-difference:
- - path: v1/bytes-difference.yaml
+ - spec:
+ # Careful - 2 indentations here !
+ info:
+ version: 1.0.0-beta
+ title: Analytics REST API
+ description: Analytics Query Service setup
+
+ x-is-api-root: true
+ x-host-basePath: /api/rest_v1
+
+ paths:
+ /pageviews:
+ x-modules:
+ - path: v1/pageviews.yaml
+ /legacy/pagecounts:
+ x-modules:
+ - path: v1/legacy/pagecounts.yaml
+ /unique-devices:
+ x-modules:
+ - path: v1/unique-devices.yaml
+ /edited-pages:
+ x-modules:
+ - path: v1/edited-pages.yaml
+ /editors:
+ x-modules:
+ - path: v1/editors.yaml
+ /registered-users:
+ x-modules:
+ - path: v1/registered-users.yaml
+ /edits:
+ x-modules:
+ - path: v1/edits.yaml
+ /bytes-difference:
+ x-modules:
+ - path: v1/bytes-difference.yaml
/{api:sys}:
- swagger: 2.0
- info:
- x-is-api-root: true
x-modules:
- /table:
- - type: npm
- name: restbase-mod-table-cassandra
- options:
- conf: '{{options.table}}'
- /pageviews:
- - path: sys/pageviews.js
- /legacy/pagecounts:
- - path: sys/legacy/pagecounts.js
- /unique-devices:
- - path: sys/unique-devices.js
- /mediawiki-history-metrics:
- - path: sys/mediawiki-history-metrics.js
- options:
- druid: '{{options.druid}}'
+ - spec:
+ # Careful - 2 indentations here !
+ paths:
+ /table:
+ x-modules:
+ - path: sys/table.js
+ options:
+ conf: '{{options.table}}'
+ /pageviews:
+ x-modules:
+ - path: sys/pageviews.js
+ /legacy/pagecounts:
+ x-modules:
+ - path: sys/legacy/pagecounts.js
+ /unique-devices:
+ x-modules:
+ - path: sys/unique-devices.js
+ /mediawiki-history-metrics:
+ x-modules:
+ - path: sys/mediawiki-history-metrics.js
+ options:
+ druid: '{{options.druid}}'
+ options: '{{options}}'
diff --git a/sys/table.js b/sys/table.js
new file mode 100644
index 0000000..3bbd576
--- /dev/null
+++ b/sys/table.js
@@ -0,0 +1,17 @@
+"use strict";
+
+/*
+ * A simple wrapper module over storage modules which allows to switch between
storage
+ * implementation using a config option.
+ */
+
+module.exports = (options) => {
+ options.conf.backend = options.conf.backend || 'cassandra';
+
+ if (options.conf.backend !== 'cassandra'
+ && options.conf.backend !== 'sqlite') {
+ throw new Error(`Unsupported backend version specified:
${options.conf.backend}`);
+ }
+
+ return require(`restbase-mod-table-${options.conf.backend}`)(options);
+};
diff --git a/test/aqs_test_module.yaml b/test/aqs_test_module.yaml
index 5ed46a2..6d3f0bf 100644
--- a/test/aqs_test_module.yaml
+++ b/test/aqs_test_module.yaml
@@ -1,139 +1,141 @@
# Simple test spec
-swagger: 2.0
paths:
/{api:v1}:
- swagger: '2.0'
- info:
- version: 1.0.0-beta
- title: AQS testing APIs
- x-is-api-root: true
- paths:
-
/pageviews/insert-per-article-flat/{project}/{article}/{granularity}/{timestamp}/{views}:
- post:
- x-request-handler:
- - put_to_storage:
- request:
- method: 'put'
- uri: '/{domain}/sys/table/pageviews.per.article.flat/'
- body:
- table: 'pageviews.per.article.flat'
- attributes:
- project: '{{request.params.project}}'
- article: '{{request.params.article}}'
- granularity: '{{request.params.granularity}}'
- timestamp: '{{request.params.timestamp}}'
- aa: '{{request.params.views}}1'
- ab: '{{request.params.views}}2'
- as: '{{request.params.views}}3'
- au: '{{request.params.views}}4'
- da: '{{request.params.views}}5'
- db: '{{request.params.views}}6'
- ds: '{{request.params.views}}7'
- du: null
- maa: '{{request.params.views}}9'
- mab: '{{request.params.views}}10'
- mas: '{{request.params.views}}11'
- mau: '{{request.params.views}}12'
- mwa: '{{request.params.views}}13'
- mwb: '{{request.params.views}}14'
- mws: '{{request.params.views}}15'
- mwu: '{{request.params.views}}16'
- x-monitor: false
+ x-modules:
+ - spec:
+ # Careful - 2 indentations here !
+ info:
+ version: 1.0.0-beta
+ title: AQS testing APIs
+ x-is-api-root: true
+ paths:
+
/pageviews/insert-per-article-flat/{project}/{article}/{granularity}/{timestamp}/{views}:
+ post:
+ x-request-handler:
+ - put_to_storage:
+ request:
+ method: 'put'
+ uri:
'/{domain}/sys/table/pageviews.per.article.flat/'
+ body:
+ table: 'pageviews.per.article.flat'
+ attributes:
+ project: '{{request.params.project}}'
+ article: '{{request.params.article}}'
+ granularity: '{{request.params.granularity}}'
+ timestamp: '{{request.params.timestamp}}'
+ aa: '{{request.params.views}}1'
+ ab: '{{request.params.views}}2'
+ as: '{{request.params.views}}3'
+ au: '{{request.params.views}}4'
+ da: '{{request.params.views}}5'
+ db: '{{request.params.views}}6'
+ ds: '{{request.params.views}}7'
+ du: null
+ maa: '{{request.params.views}}9'
+ mab: '{{request.params.views}}10'
+ mas: '{{request.params.views}}11'
+ mau: '{{request.params.views}}12'
+ mwa: '{{request.params.views}}13'
+ mwb: '{{request.params.views}}14'
+ mws: '{{request.params.views}}15'
+ mwu: '{{request.params.views}}16'
+ x-monitor: false
-
/pageviews/insert-aggregate/{project}/{access}/{agent}/{granularity}/{timestamp}/{views}:
- post:
- x-request-handler:
- - put_to_storage:
- request:
- method: 'put'
- uri: '/{domain}/sys/table/pageviews.per.project.v2/'
- body:
- table: 'pageviews.per.project.v2'
- attributes:
- project: '{{request.params.project}}'
- access: '{{request.params.access}}'
- agent: '{{request.params.agent}}'
- granularity: '{{request.params.granularity}}'
- timestamp: '{{request.params.timestamp}}'
- views: '{{request.params.views}}'
- x-monitor: false
+
/pageviews/insert-aggregate/{project}/{access}/{agent}/{granularity}/{timestamp}/{views}:
+ post:
+ x-request-handler:
+ - put_to_storage:
+ request:
+ method: 'put'
+ uri: '/{domain}/sys/table/pageviews.per.project.v2/'
+ body:
+ table: 'pageviews.per.project.v2'
+ attributes:
+ project: '{{request.params.project}}'
+ access: '{{request.params.access}}'
+ agent: '{{request.params.agent}}'
+ granularity: '{{request.params.granularity}}'
+ timestamp: '{{request.params.timestamp}}'
+ views: '{{request.params.views}}'
+ x-monitor: false
-
/pageviews/insert-aggregate-long/{project}/{access}/{agent}/{granularity}/{timestamp}/{v}:
- post:
- x-request-handler:
- - put_to_storage:
- request:
- method: 'put'
- uri: '/{domain}/sys/table/pageviews.per.project.v2/'
- body:
- table: 'pageviews.per.project.v2'
- attributes:
- project: '{{request.params.project}}'
- access: '{{request.params.access}}'
- agent: '{{request.params.agent}}'
- granularity: '{{request.params.granularity}}'
- timestamp: '{{request.params.timestamp}}'
- v: '{{request.params.v}}'
- x-monitor: false
+
/pageviews/insert-aggregate-long/{project}/{access}/{agent}/{granularity}/{timestamp}/{v}:
+ post:
+ x-request-handler:
+ - put_to_storage:
+ request:
+ method: 'put'
+ uri: '/{domain}/sys/table/pageviews.per.project.v2/'
+ body:
+ table: 'pageviews.per.project.v2'
+ attributes:
+ project: '{{request.params.project}}'
+ access: '{{request.params.access}}'
+ agent: '{{request.params.agent}}'
+ granularity: '{{request.params.granularity}}'
+ timestamp: '{{request.params.timestamp}}'
+ v: '{{request.params.v}}'
+ x-monitor: false
- /pageviews/insert-top/{project}/{access}/{year}/{month}/{day}:
- post:
- x-request-handler:
- - put_to_storage:
- request:
- method: 'put'
- uri: '/{domain}/sys/table/top.pageviews/'
- body:
- table: 'top.pageviews'
- attributes:
- project: '{{request.params.project}}'
- access: '{{request.params.access}}'
- year: '{{request.params.year}}'
- month: '{{request.params.month}}'
- day: '{{request.params.day}}'
- articlesJSON: '{{request.body.articles}}'
- x-monitor: false
+ /pageviews/insert-top/{project}/{access}/{year}/{month}/{day}:
+ post:
+ x-request-handler:
+ - put_to_storage:
+ request:
+ method: 'put'
+ uri: '/{domain}/sys/table/top.pageviews/'
+ body:
+ table: 'top.pageviews'
+ attributes:
+ project: '{{request.params.project}}'
+ access: '{{request.params.access}}'
+ year: '{{request.params.year}}'
+ month: '{{request.params.month}}'
+ day: '{{request.params.day}}'
+ articlesJSON: '{{request.body.articles}}'
+ x-monitor: false
-
/legacy/pagecounts/insert-aggregate/{project}/{access-site}/{granularity}/{timestamp}/{count}:
- post:
- x-request-handler:
- - put_to_storage:
- request:
- method: 'put'
- uri: '/{domain}/sys/table/lgc.pagecounts.per.project/'
- body:
- table: 'lgc.pagecounts.per.project'
- attributes:
- project: '{{request.params.project}}'
- 'access-site': '{{request.params.access-site}}'
- granularity: '{{request.params.granularity}}'
- timestamp: '{{request.params.timestamp}}'
- count: '{{request.params.count}}'
- x-monitor: false
+
/legacy/pagecounts/insert-aggregate/{project}/{access-site}/{granularity}/{timestamp}/{count}:
+ post:
+ x-request-handler:
+ - put_to_storage:
+ request:
+ method: 'put'
+ uri:
'/{domain}/sys/table/lgc.pagecounts.per.project/'
+ body:
+ table: 'lgc.pagecounts.per.project'
+ attributes:
+ project: '{{request.params.project}}'
+ 'access-site': '{{request.params.access-site}}'
+ granularity: '{{request.params.granularity}}'
+ timestamp: '{{request.params.timestamp}}'
+ count: '{{request.params.count}}'
+ x-monitor: false
-
/unique-devices/insert/{project}/{access-site}/{granularity}/{timestamp}/{devices}:
- post:
- x-request-handler:
- - put_to_storage:
- request:
- method: 'put'
- uri: '/{domain}/sys/table/unique.devices/'
- body:
- table: 'unique.devices'
- attributes:
- project: '{{request.params.project}}'
- 'access-site': '{{request.params.access-site}}'
- granularity: '{{request.params.granularity}}'
- timestamp: '{{request.params.timestamp}}'
- devices: '{{request.params.devices}}'
- x-monitor: false
+
/unique-devices/insert/{project}/{access-site}/{granularity}/{timestamp}/{devices}:
+ post:
+ x-request-handler:
+ - put_to_storage:
+ request:
+ method: 'put'
+ uri: '/{domain}/sys/table/unique.devices/'
+ body:
+ table: 'unique.devices'
+ attributes:
+ project: '{{request.params.project}}'
+ 'access-site': '{{request.params.access-site}}'
+ granularity: '{{request.params.granularity}}'
+ timestamp: '{{request.params.timestamp}}'
+ devices: '{{request.params.devices}}'
+ x-monitor: false
# Fake internal endpoint simulating a druid cluster for testing
/{api:sys}:
- swagger: 2.0
- info:
- x-is-api-root: true
x-modules:
- /fake-druid:
- - path: test/features/mediawiki-history-metrics/fake-druid.js
+ - spec:
+ # Careful - 2 indentations here !
+ paths:
+ /fake-druid:
+ x-modules:
+ - path: test/features/mediawiki-history-metrics/fake-druid.js
diff --git
a/test/features/mediawiki-history-metrics/mediawiki-history-metrics.js
b/test/features/mediawiki-history-metrics/mediawiki-history-metrics.js
index e166b15..a91c1d7 100644
--- a/test/features/mediawiki-history-metrics/mediawiki-history-metrics.js
+++ b/test/features/mediawiki-history-metrics/mediawiki-history-metrics.js
@@ -23,10 +23,8 @@
this.timeout(20000);
// Start server before running tests
- before('before-suite', setupDone => {
- return server.start().then(() => {
- setupDone();
- });
+ before('before-suite', () => {
+ return server.start();
});
var makeTest = function(fixture) {
diff --git a/test/features/pageviews/pageviews.js
b/test/features/pageviews/pageviews.js
index 95f78b7..f8aca78 100644
--- a/test/features/pageviews/pageviews.js
+++ b/test/features/pageviews/pageviews.js
@@ -40,7 +40,7 @@
// Start server before running tests
// insert here data that tests assume exists on db to start working
before('before-suite', function(setupDone) {
- return server.start().then(function() {
+ server.start().then(function() {
const dataToInsert = {
2015070200: 100,
2015072100: 200,
diff --git a/test/index.js b/test/index.js
index b21a583..c126552 100644
--- a/test/index.js
+++ b/test/index.js
@@ -5,3 +5,8 @@
require('mocha-jshint')();
// Run jscs as part of normal testing
require('mocha-jscs')();
+require('mocha-eslint')([
+ 'lib',
+ 'sys',
+ 'v1'
+]);
\ No newline at end of file
diff --git a/test/utils/run_tests.sh b/test/utils/run_tests.sh
index d6807c1..a471729 100644
--- a/test/utils/run_tests.sh
+++ b/test/utils/run_tests.sh
@@ -11,6 +11,13 @@
rm -f test.db.sqlite3
else
echo "Running with Cassandra backend"
+ if [ `nc -z localhost 9042 < /dev/null; echo $?` != 0 ]; then
+ echo "Waiting for Cassandra to start..."
+ while [ `nc -z localhost 9042; echo $?` != 0 ]; do
+ sleep 1
+ done
+ echo "Cassandra is ready."
+ fi
export RB_TEST_BACKEND=cassandra
sh ./test/utils/cleandb.sh
fi
@@ -28,7 +35,7 @@
if [ "$?" -eq 0 ]; then
runTest "cassandra" $1
else
- echo "Cassandra not available. Using SQLite backed for tests"
+ echo "Cassandra not available. Using SQLite backend for tests"
runTest "sqlite" $1
fi
elif [ "$2" = "sqlite" ]; then
diff --git a/test/utils/server.js b/test/utils/server.js
index 3ae3240..076902e 100644
--- a/test/utils/server.js
+++ b/test/utils/server.js
@@ -8,7 +8,6 @@
var fs = require('fs');
var assert = require('./assert');
var yaml = require('js-yaml');
-var temp = require('temp').track();
var hostPort = 'http://localhost:7231';
var aqsURL = hostPort + '/analytics.wikimedia.org/v1';
@@ -21,14 +20,7 @@
throw new Error('Invalid RB_TEST_BACKEND env variable value.
Allowed values: "cassandra", "sqlite"');
}
if (backendImpl === 'sqlite') {
- // First, replace the module in all projects and move them to the
temp directory
- var tempDir = temp.mkdirSync('tempProjects');
- fs.readdirSync(__dirname +
'/../../projects').forEach(function(fileName) {
- var fileStr = fs.readFileSync(__dirname + '/../../projects/' +
fileName).toString()
- .replace(/restbase\-mod\-table\-cassandra/g,
'restbase-mod-table-sqlite');
- fs.writeFileSync(tempDir + '/' + fileName, fileStr);
- });
- confString = confString.replace(/projects\//g, tempDir + '/');
+ confString = confString.replace(/backend: cassandra/, "backend:
sqlite");
}
}
return yaml.safeLoad(confString);
--
To view, visit https://gerrit.wikimedia.org/r/384590
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ie7847c1de76cb7e6cc868a1d3b94af68785f86bc
Gerrit-PatchSet: 1
Gerrit-Project: analytics/aqs
Gerrit-Branch: master
Gerrit-Owner: Joal <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
