Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/199250
Change subject: [BREAKING] Make Citoid use service-template-node and
service-runner
......................................................................
[BREAKING] Make Citoid use service-template-node and service-runner
This patch converts Citoid into a service-template-node-based service,
thus easing its development and deployment. Amongst other things, this
new basis brings:
- unified command-line arguments
- unified configuration management; localsettings.js is gone, the
configuration is to be managed by altering config.dev.yaml
- support for process clustering; this should improve performance and
availability, as until now only one process per host has been executed
- support for process management (OOM prevents and the like)
- support for logging directly to logstash (no more grepping and tailing)
- support for metrics reporting, both to a StatsD server, as well as to
the logger when run locally during development
- support for auto-magic error handling (with or without promises)
- code coverage reporting
- support for API versioning (yet to be used in Citoid)
This patch also structures and adds some more tests. Grunt has been
removed in favour of mocha. Note that 'npm test' still reports only
jshint results as a compatibility requirement with Jenkins. To run all
of the tests locally, use the 'mocha' command instead.
Note that, due to the disruptive nature of the patch, the following
commands should be run:
rm localsettings.js
rm -rf node_modules
npm install
Bug: T75993
Change-Id: I7370c9a91e67c263a291aab4b8ae9014a23efa5f
---
M .gitignore
M .jshintignore
M .jshintrc
D Gruntfile.js
A app.js
A config.dev.yaml
A config.yaml
M lib/CitoidService.js
M lib/Scraper.js
M lib/ZoteroService.js
M lib/pubMedRequest.js
M lib/translators/general.js
M lib/translators/openGraph.js
M lib/unshorten.js
A lib/util.js
D localsettings.js.sample
M package.json
A routes/root.js
M server.js
A test/features/app/index.js
A test/features/errors/index.js
A test/features/scraping/index.js
A test/features/scraping/lang.js
M test/index.js
A test/mocha.opts
A test/utils/assert.js
A test/utils/logStream.js
A test/utils/server.js
28 files changed, 1,154 insertions(+), 543 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/citoid
refs/changes/50/199250/1
diff --git a/.gitignore b/.gitignore
index c88c5d5..ea81c00 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,5 @@
*~
.DS_Store
*.log
-
-localsettings.js
-
node_modules
+coverage
diff --git a/.jshintignore b/.jshintignore
index 79e6ea4..1c69eee 100644
--- a/.jshintignore
+++ b/.jshintignore
@@ -1,2 +1,3 @@
+coverage
node_modules
-test
\ No newline at end of file
+test
diff --git a/.jshintrc b/.jshintrc
index bae215e..6e8ef79 100644
--- a/.jshintrc
+++ b/.jshintrc
@@ -1,4 +1,10 @@
{
+
+ "predef": [
+ "Map",
+ "Set"
+ ],
+
// Enforcing
"bitwise": true,
"eqeqeq": true,
@@ -6,7 +12,6 @@
"noarg": true,
"nonew": true,
"undef": true,
- "unused": true,
"curly": true,
"newcap": true,
@@ -33,5 +38,7 @@
"module": true,
"global": true
},
+ "node": true,
"esnext": true
+
}
diff --git a/Gruntfile.js b/Gruntfile.js
deleted file mode 100644
index 7178f57..0000000
--- a/Gruntfile.js
+++ /dev/null
@@ -1,39 +0,0 @@
-module.exports = function( grunt ) {
-
- // These plugins provide necessary tasks.
- grunt.loadNpmTasks('grunt-contrib-jshint');
- grunt.loadNpmTasks('grunt-simple-mocha');
-
- // Project configuration.
- grunt.initConfig({
- // Task configuration.
- jshint: {
- options: {
- jshintrc: true
- },
- all: [
- '*.js',
- 'localsettings.js.sample',
- 'lib/*.js',
- 'lib/translators/*.js',
- 'test/*.js'
- ]
- },
- simplemocha: {
- options: {
- globals: ['describe', 'its'],
- timeout: 20000,
- ignoreLeaks: false,
- ui: 'bdd',
- reporter: 'tap'
- },
- all: { src: ['test/*.js'] }
- }
- });
-
- // Default task.
- grunt.registerTask('test', ['jshint:all']);
- grunt.registerTask('all', ['jshint:all', 'simplemocha']);
- grunt.registerTask('default', 'test');
-
-};
diff --git a/app.js b/app.js
new file mode 100644
index 0000000..d636a6b
--- /dev/null
+++ b/app.js
@@ -0,0 +1,148 @@
+'use strict';
+
+
+var http = require('http');
+var BBPromise = require('bluebird');
+var express = require('express');
+var compression = require('compression');
+var bodyParser = require('body-parser');
+var fs = BBPromise.promisifyAll(require('fs'));
+var sUtil = require('./lib/util');
+var CitoidService = require('./lib/CitoidService');
+var packageInfo = require('./package.json');
+
+
+/**
+ * Creates an express app and initialises it
+ * @param {Object} options the options to initialise the app with
+ * @return {bluebird} the promise resolving to the app object
+ */
+function initApp(options) {
+
+ // the main application object
+ var app = express();
+
+ // get the options and make them available in the app
+ app.logger = options.logger; // the logging device
+ app.metrics = options.metrics; // the metrics
+ app.conf = options.config; // this app's config options
+ app.info = packageInfo; // this app's package info
+
+ // ensure some sane defaults
+ if(!app.conf.port) { app.conf.port = 1970; }
+ if(!app.conf.interface) { app.conf.interface = '0.0.0.0'; }
+ if(!app.conf.compression_level) { app.conf.compression_level = 3; }
+
+ // set outgoing proxy
+ if(app.conf.proxy) {
+ process.env.HTTP_PROXY = app.conf.proxy;
+ if(!app.conf.zoteroUseProxy) {
+ // don't use proxy for accessing Zotero unless
specified in settings
+ process.env.NO_PROXY = app.conf.zoteroInterface;
+ }
+ }
+
+ // set the CORS headers
+ app.all('*', function(req, res, next) {
+ res.header("Access-Control-Allow-Origin", app.conf.cors);
+ res.header("Access-Control-Allow-Headers", "X-Requested-With,
Content-Type");
+ next();
+ });
+ // disable the X-Powered-By header
+ app.set('x-powered-by', false);
+ // disable the ETag header - users should provide them!
+ app.set('etag', false);
+ // enable compression
+ app.use(compression({level: app.conf.compression_level}));
+ // use the JSON body parser
+ app.use(bodyParser.json());
+ // use the application/x-www-form-urlencoded parser
+ app.use(bodyParser.urlencoded({extended: true}));
+ // serve static files from static/
+ app.use(express.static(__dirname + '/static'));
+
+ // init the Citoid service object
+ app.citoid = new CitoidService(app.conf, app.logger, app.metrics);
+
+ return BBPromise.resolve(app);
+
+}
+
+
+/**
+ * Loads all routes declared in routes/ into the app
+ * @param {Application} app the application object to load routes into
+ * @returns {bluebird} a promise resolving to the app object
+ */
+function loadRoutes (app) {
+
+ // get the list of files in routes/
+ return fs.readdirAsync(__dirname + '/routes')
+ .map(function (fname) {
+ // ... and then load each route
+ // but only if it's a js file
+ if(!/\.js$/.test(fname)) {
+ return;
+ }
+ // import the route file
+ var route = require(__dirname + '/routes/' + fname);
+ route = route(app);
+ // check that the route exports the object we need
+ if(route.constructor !== Object || !route.path || !route.router
|| !(route.api_version || route.skip_domain)) {
+ throw new TypeError('routes/' + fname + ' does not
export the correct object!');
+ }
+ // wrap the route handlers with Promise.try() blocks
+ sUtil.wrapRouteHandlers(route.router, app);
+ // determine the path prefix
+ var prefix = '';
+ if(!route.skip_domain) {
+ prefix = '/:domain/v' + route.api_version;
+ }
+ // all good, use that route
+ app.use(prefix + route.path, route.router);
+ }).then(function () {
+ // catch errors
+ sUtil.setErrorHandler(app);
+ // route loading is now complete, return the app object
+ return BBPromise.resolve(app);
+ });
+
+}
+
+/**
+ * Creates and start the service's web server
+ * @param {Application} app the app object to use in the service
+ * @returns {bluebird} a promise creating the web server
+ */
+function createServer(app) {
+
+ // return a promise which creates an HTTP server,
+ // attaches the app to it, and starts accepting
+ // incoming client requests
+ return new BBPromise(function (resolve) {
+ http.createServer(app).listen(
+ app.conf.port,
+ app.conf.interface,
+ resolve
+ );
+ }).then(function () {
+ app.logger.log('info',
+ 'Worker ' + process.pid + ' listening on ' +
app.conf.interface + ':' + app.conf.port);
+ });
+
+}
+
+/**
+ * The service's entry point. It takes over the configuration
+ * options and the logger- and metrics-reporting objects from
+ * service-runner and starts an HTTP server, attaching the application
+ * object to it.
+ */
+module.exports = function(options) {
+
+ return initApp(options)
+ .then(loadRoutes)
+ .then(createServer);
+
+};
+
diff --git a/config.dev.yaml b/config.dev.yaml
new file mode 100644
index 0000000..ad74fed
--- /dev/null
+++ b/config.dev.yaml
@@ -0,0 +1,52 @@
+# Number of worker processes to spawn.
+# Set to 0 to run everything in a single process without clustering.
+# Use 'ncpu' to run as many workers as there are CPU units
+num_workers: 0
+
+# Log error messages and gracefully restart a worker if v8 reports that it
+# uses more heap (note: not RSS) than this many mb.
+worker_heap_limit_mb: 250
+
+# Logger info
+logging:
+ level: trace
+# streams:
+# # Use gelf-stream -> logstash
+# - type: gelf
+# host: logstash1003.eqiad.wmnet
+# port: 12201
+
+# Statsd metrics reporter
+metrics:
+ type: log
+ #host: localhost
+ #port: 8125
+
+services:
+ - name: citoid
+ # a relative path or the name of an npm package, if different from name
+ module: ./app.js
+ # optionally, a version constraint of the npm package
+ # version: ^0.4.0
+ # per-service config
+ conf:
+ # the port to bind to
+ port: 1970
+ # IP address to bind to, all IPs by default
+ # interface: localhost # uncomment to only listen on localhost
+ # allow cross-domain requests to the API (default '*')
+ cors: '*'
+ # to disbale use:
+ # cors: false
+ # to restrict to a particular domain, use:
+ # cors: restricted.domain.org
+ # URL of the outbound proxy to use (complete with protocol)
+ # proxy: http://127.0.0.0.1
+ # User-Agent HTTP header to use for requests
+ userAgent: null
+ # URL where to contact Zotero
+ zoteroInterface: 127.0.0.1
+ # zotero's server port
+ zoteroPort: 1969
+ # whether the proxy should be used to contact zotero
+ zoteroUseProxy: false
diff --git a/config.yaml b/config.yaml
new file mode 120000
index 0000000..c11eec8
--- /dev/null
+++ b/config.yaml
@@ -0,0 +1 @@
+config.dev.yaml
\ No newline at end of file
diff --git a/lib/CitoidService.js b/lib/CitoidService.js
index 25a6065..6fb0875 100644
--- a/lib/CitoidService.js
+++ b/lib/CitoidService.js
@@ -1,10 +1,11 @@
+'use strict';
+
/**
* Handles requests to the citoid service
*/
/* Import Modules */
-var urlParse = require('url'),
- StatsD = require('node-txstatsd');
+var urlParse = require('url');
/* Import Local Modules */
var unshorten = require('./unshorten.js'),
@@ -15,13 +16,14 @@
/**
* Constructor for CitoidService object
* @param {Object} citoidConfig configuration object
- * @param {Object} logger bunyan logger object
+ * @param {Object} logger logger object, must have a log() method
+ * @param {Object} statsd metrics object
*/
-function CitoidService(citoidConfig, logger){
- this.log = logger;
+function CitoidService(citoidConfig, logger, statsd) {
+ this.logger = logger;
this.zoteroService = new ZoteroService(citoidConfig, logger);
this.scraper = new Scraper(citoidConfig, logger);
- this.stats = new StatsD(citoidConfig.statsd || {mock: true});
+ this.stats = statsd;
}
/**
@@ -49,39 +51,38 @@
* @param {Object} opts options object containing requested url
* @param {Function} callback callback (error, statusCode, body)
*/
-CitoidService.prototype.requestFromURL = function (opts, callback){
+CitoidService.prototype.requestFromURL = function (opts, callback) {
var self = this,
- log = self.log,
+ logger = self.logger,
zoteroWebRequest =
self.zoteroService.zoteroWebRequest.bind(self.zoteroService),
requestedURL = opts.search,
format = opts.format;
- zoteroWebRequest(requestedURL, format, function(error, response, body){
- log.info("Zotero request made for: " + requestedURL);
+ zoteroWebRequest(requestedURL, format, function(error, response, body) {
+ logger.log('debug/zotero', "Zotero request made for: " +
requestedURL);
if (error) {
- log.error(error);
- self.stats.increment('citoid.zotero.req.error');
+ logger.log('warn/zotero', error);
+ self.stats.increment('zotero.req.error');
self.scrape(opts, callback);
} else if (response) {
- self.stats.increment('citoid.zotero.req.' +
response.statusCode);
+ self.stats.increment('zotero.req.' +
Math.floor(response.statusCode / 100) + 'xx');
// 501 indicates no translator available
// This is common- can indicate shortened url,
// or a website not specified in the translators
if (response.statusCode === 501){
- log.info("No Zotero translator found.");
- log.info("Looking for redirects...");
+ logger.log('trace/zotero', "No Zotero
translator found, looking for redirects");
// Try again following all redirects-
// We don't do this initially because many sites
// will redirect to a log-in screen
unshorten(requestedURL, function(detected,
expandedURL) {
if (detected) {
- log.info("Redirect detected to
"+ expandedURL);
+ logger.log('trace/zotero',
"Redirect detected to "+ expandedURL);
zoteroWebRequest(expandedURL,
format, function(error, response, body){
if (response && !error
&& response.statusCode === 200){
-
log.info("Successfully retrieved and translated body from Zotero");
+
logger.log('debug/zotero', "Successfully retrieved and translated body from
Zotero");
callback(null,
200, body);
} else {
- log.info("No
Zotero response available.");
+
logger.log('debug/zotero', "No Zotero response available.");
// Try scraping
original URL before expansion
self.scrape(opts, function(error, responseCode, body){
if
(error || responseCode !== 200){
@@ -94,12 +95,12 @@
}
});
} else {
- log.info("No redirect
detected.");
+ logger.log('debug/zotero', "No
redirect detected.");
self.scrape(opts, callback);
}
});
} else if (response.statusCode === 200){
- log.info("Successfully retrieved and translated
body from Zotero");
+ logger.log('debug/zotero', "Successfully
retrieved and translated body from Zotero");
callback (null, 200, body);
} else {
//other response codes such as 500, 300
@@ -129,13 +130,13 @@
* @param {Function} callback callback (error, statusCode, body)
*/
CitoidService.prototype.requestFromPubMedID = function(opts, callback){
- var citoidService = this;
- pubMedRequest(opts.search, function(error, obj){
- if(error){
+ var citoidService = this;
+ pubMedRequest(opts.search, this.logger, function(error, obj){
+ if(error) {
callback(error, null, null);
} else {
var doi = obj.records[0].doi;
- citoidService.log.info("Got DOI " + doi);
+ citoidService.logger.log('debug/pubmed', "Got DOI " +
doi);
opts.search = doi;
citoidService.requestFromDOI(opts, callback);
}
@@ -168,24 +169,24 @@
if (matchHTTP || matchWWW){
- this.stats.increment('citoid.input.url');
+ this.stats.increment('input.url');
callback(matchHTTP ? matchHTTP[0] : 'http://' + matchWWW[0],
this.requestFromURL.bind(this));
} else if (matchDOI) {
- this.stats.increment('citoid.input.doi');
+ this.stats.increment('input.doi');
callback(matchDOI[0], this.requestFromDOI.bind(this));
} else if (matchPMID) {
- this.stats.increment('citoid.input.pmid');
+ this.stats.increment('input.pmid');
callback(matchPMID[0], this.requestFromPubMedID.bind(this));
} else if (matchPMCID) {
- this.stats.increment('citoid.input.pmcid');
+ this.stats.increment('input.pmcid');
callback(matchPMCID[0], this.requestFromPubMedID.bind(this));
} else {
matchPMCID = search.match(rePMCID2);
if (matchPMCID) {
- this.stats.increment('citoid.input.pmcid');
+ this.stats.increment('input.pmcid');
callback('PMC' + matchPMCID[0],
this.requestFromPubMedID.bind(this));
} else {
- this.stats.increment('citoid.input.url');
+ this.stats.increment('input.url');
parsedURL = urlParse.parse(search);
if (!parsedURL.protocol){
search = 'http://'+ search;
diff --git a/lib/Scraper.js b/lib/Scraper.js
index 4e6e43c..4ddb7e4 100644
--- a/lib/Scraper.js
+++ b/lib/Scraper.js
@@ -1,4 +1,5 @@
-#!/usr/bin/env node
+'use strict';
+
/**
* https://www.mediawiki.org/wiki/citoid
*/
@@ -15,7 +16,7 @@
gen = require('./translators/general.js');
function Scraper(citoidConfig, logger){
- this.log = logger;
+ this.logger = logger;
this.userAgent = citoidConfig.userAgent || 'Citoid/0.0.0';
}
@@ -29,14 +30,14 @@
Scraper.prototype.scrape = function(opts, callback){
var chtml,
- log = this.log,
+ logger = this.logger,
scraper = this,
acceptLanguage = opts.acceptLanguage,
url = opts.search,
userAgent = this.userAgent,
citation = {url: url, title: url};
- log.info("Using native scraper on " + url);
+ logger.log('debug/scraper', "Using native scraper on " + url);
request(
{
url: url,
@@ -48,14 +49,13 @@
}, function(error, response, html){
if (error || !response || response.statusCode !== 200) {
if (error) {
- log.error(error);
+ logger.log('warn/scraper', error);
} else if (!response){
- log.error("No response from resource
server at " + url);
+ logger.log('warn/scraper', "No response
from resource server at " + url);
} else {
- log.error("Status from resource server
at " + url +
+ logger.log('warn/scraper', "Status from
resource server at " + url +
": " + response.statusCode);
}
- log.info("Unable to scrape resource at " + url);
citation.itemType = 'webpage';
callback(error, 520, [citation]);
} else {
@@ -63,12 +63,11 @@
chtml = cheerio.load(html);
citation.title = null;
scraper.parseHTML(url, chtml, citation,
function(citation){
- log.info("Sucessfully scraped
resource at " + url);
+ logger.log('debug/scraper',
"Sucessfully scraped resource at " + url);
callback(null, 200, [citation]);
});
} catch (e){
- log.error(e);
- log.info("Unable to scrape resource at
" + url);
+ logger.log('warn/scraper', e);
callback(error, 520, [citation]);
}
}
diff --git a/lib/ZoteroService.js b/lib/ZoteroService.js
index d6df076..9593859 100644
--- a/lib/ZoteroService.js
+++ b/lib/ZoteroService.js
@@ -1,4 +1,5 @@
-#!/usr/bin/env node
+'use strict';
+
/**
* https://www.mediawiki.org/wiki/citoid
*
@@ -12,8 +13,11 @@
util = require('util'),
pubMedRequest = require('./pubMedRequest.js');
+var defaultLogger;
+
function ZoteroService(citoidConfig, logger){
- this.log = logger;
+ this.logger = logger;
+ defaultLogger = logger;
var baseURL = util.format('http://%s:%s/',
citoidConfig.zoteroInterface,
citoidConfig.zoteroPort.toString());
this.webURL = baseURL + 'web';
@@ -279,7 +283,7 @@
if (!citation.PMID && citation.DOI) {
//if pmid is not found, lookup the pmid using the DOI
- pubMedRequest(citation.DOI, function (error, object){
+ pubMedRequest(citation.DOI, defaultLogger, function (error,
object){
if (!error) { //don't pass along error as it's
non-critical, and will halt the waterfall
if (object.records[0].pmid){
citation['PMID'] =
object.records[0].pmid;
diff --git a/lib/pubMedRequest.js b/lib/pubMedRequest.js
index 472318e..4ecfdbb 100644
--- a/lib/pubMedRequest.js
+++ b/lib/pubMedRequest.js
@@ -1,4 +1,5 @@
-#!/usr/bin/env node
+'use strict';
+
/**
* https://www.mediawiki.org/wiki/citoid
*
@@ -7,43 +8,42 @@
(function() {
- var request = require('request'),
- bunyan = require('bunyan'),
- log = bunyan.createLogger({name: "citoid"});
+ var request = require('request');
/**
* Requests a PubMed object using any supported identifier
* @param {String} identifier Valid PubMed identifier (PMID, PMCID,
Manuscript ID, versioned ID)
+ * @param {Object} logger logger object with log() method
* @param {Function} callback callback (error, object)
*/
- var pubMedRequest = function (identifier, callback){
- var escapedId = encodeURIComponent(identifier),
- url =
"http://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/?tool=citoid&email=citoid@mediawiki&format=json&ids=";
+ escapedId;
+ var pubMedRequest = function (identifier, logger, callback){
+ var escapedId = encodeURIComponent(identifier),
+ url =
"http://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/?tool=citoid&email=citoid@mediawiki&format=json&ids=";
+ escapedId;
request(url, function(error, response, body){
- log.info("PubMed query made for: " + url);
+ logger.log('debug/pubmed', "PubMed query made for: " +
url);
if (error) {
- log.error(error);
+ logger.log('warn/pubmed', error);
callback(error, null);
} else if (response.statusCode !== 200) {
- log.error("Unexpected HTTP status code: " +
response.statusCode);
+ logger.log('warn/pubmed', "Unexpected HTTP
status code: " + response.statusCode);
callback("Unexpected HTTP status code: " +
response.statusCode, null);
} else {
var jsonObj;
try {
jsonObj = JSON.parse(body);
} catch (error) {
- log.info("Original response: " + body);
- log.error("JSON parse error: " + error);
+ logger.log('debug/pubmed', "Original
response: " + body);
+ logger.log('warn/pubmed', "JSON parse
error: " + error);
callback("JSON parse error: " + error,
null);
}
if (jsonObj){
if (jsonObj.status !== 'ok'){
- log.error("Unexpected status
from PubMed API: " + jsonObj.status);
+ logger.log('warn/pubmed',
"Unexpected status from PubMed API: " + jsonObj.status);
callback("Unexpected status
from PubMed API: " + jsonObj.status, null);
} else if (jsonObj.records.length ===
0){
- log.error("No records from
PubMed API");
+ logger.log('warn/pubmed', "No
records from PubMed API");
callback("No records from
PubMed API", null);
} else {
callback(null, jsonObj);
diff --git a/lib/translators/general.js b/lib/translators/general.js
index c4cf42f..5b0e455 100644
--- a/lib/translators/general.js
+++ b/lib/translators/general.js
@@ -1,4 +1,5 @@
-#!/usr/bin/env node
+'use strict';
+
/**
* General field values : Zotero type field values
* @type {Object}
diff --git a/lib/translators/openGraph.js b/lib/translators/openGraph.js
index b178ffc..e34ecac 100644
--- a/lib/translators/openGraph.js
+++ b/lib/translators/openGraph.js
@@ -1,4 +1,5 @@
-#!/usr/bin/env node
+'use strict';
+
/**
* Open graph type field values : Zotero type field values
* @type {Object}
diff --git a/lib/unshorten.js b/lib/unshorten.js
index ccedf2b..e239d45 100644
--- a/lib/unshorten.js
+++ b/lib/unshorten.js
@@ -1,3 +1,5 @@
+'use strict';
+
(function() {
var request = require('request');
diff --git a/lib/util.js b/lib/util.js
new file mode 100644
index 0000000..9b735cd
--- /dev/null
+++ b/lib/util.js
@@ -0,0 +1,240 @@
+'use strict';
+
+
+var BBPromise = require('bluebird');
+var util = require('util');
+var express = require('express');
+var uuid = require('node-uuid');
+var bunyan = require('bunyan');
+
+
+/**
+ * Error instance wrapping HTTP error responses
+ */
+function HTTPError(response) {
+
+ Error.call(this);
+ Error.captureStackTrace(this, HTTPError);
+
+ if(response.constructor !== Object) {
+ // just assume this is just the error message
+ var msg = response;
+ response = {
+ status: 500,
+ type: 'internal_error',
+ title: 'InternalError',
+ detail: msg
+ };
+ }
+
+ this.name = this.constructor.name;
+ this.message = response.status + '';
+ if(response.type) {
+ this.message += ': ' + response.type;
+ }
+
+ for (var key in response) {
+ this[key] = response[key];
+ }
+
+}
+
+util.inherits(HTTPError, Error);
+
+
+/**
+ * Generates an object suitable for logging out of a request object
+ *
+ * @param {Request} the request
+ * @return {Object} an object containing the key components of the request
+ */
+function reqForLog(req) {
+
+ return {
+ url: req.originalUrl,
+ headers: req.headers,
+ method: req.method,
+ params: req.params,
+ query: req.query,
+ body: req.body,
+ remoteAddress: req.connection.remoteAddress,
+ remotePort: req.connection.remotePort
+ };
+
+}
+
+
+/**
+ * Serialises an error object in a form suitable for logging
+ *
+ * @param {Error} the error to serialise
+ * @return {Object} the serialised version of the error
+ */
+function errForLog(err) {
+
+ var ret = bunyan.stdSerializers.err(err);
+ ret.status = err.status;
+ ret.type = err.type;
+ ret.detail = err.detail;
+
+ return ret;
+
+}
+
+/**
+ * Generates a unique request ID
+ *
+ * @return {String} the generated request ID
+ */
+var reqIdBuff = new Buffer(16);
+function generateRequestId() {
+
+ uuid.v4(null, reqIdBuff);
+ return reqIdBuff.toString('hex');
+
+}
+
+
+
+/**
+ * Wraps all of the given router's handler functions with
+ * promised try blocks so as to allow catching all errors,
+ * regardless of whether a handler returns/uses promises
+ * or not.
+ *
+ * @param {Router} the router object
+ * @param {Application} the application object
+ */
+function wrapRouteHandlers(router, app) {
+
+ router.stack.forEach(function(routerLayer) {
+ routerLayer.route.stack.forEach(function(layer) {
+ var origHandler = layer.handle;
+ layer.handle = function(req, res, next) {
+ BBPromise.try(function() {
+ req.headers = req.headers || {};
+ req.headers['x-request-id'] =
req.headers['x-request-id'] || generateRequestId();
+ req.logger =
app.logger.child({request_id: req.headers['x-request-id']});
+ req.logger.log('debug/req', {req:
reqForLog(req), msg: 'incoming request'});
+ return origHandler(req, res, next);
+ })
+ .catch(next);
+ };
+ });
+ });
+
+}
+
+
+/**
+ * Generates an error handler for the given applications
+ * and installs it. Usage:
+ *
+ * @param {Application} app the application object to add the handler to
+ */
+function setErrorHandler(app) {
+
+ app.use(function(err, req, res, next) {
+ var errObj;
+ // ensure this is an HTTPError object
+ if(err.constructor === HTTPError) {
+ errObj = err;
+ } else if(err instanceof Error) {
+ // is this an HTTPError defined elsewhere? (preq)
+ if(err.constructor.name === 'HTTPError') {
+ var o = { status: err.status };
+ if(err.body && err.body.constructor === Object)
{
+
Object.keys(err.body).forEach(function(key) {
+ o[key] = err.body[key];
+ });
+ } else {
+ o.detail = err.body;
+ }
+ o.message = err.message;
+ errObj = new HTTPError(o);
+ } else {
+ // this is a standard error, convert it
+ errObj = new HTTPError({
+ status: 500,
+ type: 'internal_error',
+ title: err.name,
+ detail: err.message,
+ stack: err.stack
+ });
+ }
+ } else if(err.constructor === Object) {
+ // this is a regular object, suppose it's a response
+ errObj = new HTTPError(err);
+ } else {
+ // just assume this is just the error message
+ errObj = new HTTPError({
+ status: 500,
+ type: 'internal_error',
+ title: 'InternalError',
+ detail: err
+ });
+ }
+ // ensure some important error fields are present
+ if(!errObj.status) { errObj.status = 500; }
+ if(!errObj.type) { errObj.type = 'internal_error'; }
+ // add the offending URI and method as well
+ if(!errObj.method) { errObj.method = req.method; }
+ if(!errObj.uri) { errObj.uri = req.url; }
+ // some set 'message' or 'description' instead of 'detail'
+ errObj.detail = errObj.detail || errObj.message ||
errObj.description || '';
+ // adjust the log level based on the status code
+ var level = 'error';
+ if(Number.parseInt(errObj.status) < 400) {
+ level = 'trace';
+ } else if(Number.parseInt(errObj.status) < 500) {
+ level = 'info';
+ }
+ // log the error
+ req.logger.log(level +
+ (errObj.component ? '/' + errObj.component :
'/' + errObj.status),
+ errForLog(errObj));
+ // let through only non-sensitive info
+ var respBody = {
+ status: errObj.status,
+ type: errObj.type,
+ title: errObj.title,
+ detail: errObj.detail,
+ method: errObj.method,
+ uri: errObj.uri
+ };
+ res.status(errObj.status).json(respBody);
+ });
+
+}
+
+
+/**
+ * Creates a new router with some default options.
+ *
+ * @param {Object} opts additional options to pass to express.Router()
+ * @return {Router} a new router object
+ */
+function createRouter(opts) {
+
+ var options = {
+ mergeParams: true
+ };
+
+ if(opts && opts.constructor === Object) {
+ Object.keys(opts).forEach(function(key) {
+ options[key] = opts[key];
+ });
+ }
+
+ return express.Router(options);
+
+}
+
+
+module.exports = {
+ HTTPError: HTTPError,
+ wrapRouteHandlers: wrapRouteHandlers,
+ setErrorHandler: setErrorHandler,
+ router: createRouter
+};
+
diff --git a/localsettings.js.sample b/localsettings.js.sample
deleted file mode 100644
index 46e82e1..0000000
--- a/localsettings.js.sample
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * This is a sample configuration file.
- *
- * Copy this file to localsettings.js and edit that file to fit your needs.
- *
- * Also see the file server.js for more information.
- */
-
-var CitoidConfig = {
-
- // Allow cross-domain requests to the API (default '*')
- // Sets Access-Control-Allow-Origin header
- // enable:
- allowCORS : '*',
- // disable:
- //allowCORS : false,
- // restrict:
- //allowCORS : 'some.domain.org',
-
- // Set proxy for outgoing requests
- // proxy : '127.0.0.1',
-
- // Allow override of port/interface:
- citoidPort : 1970,
- citoidInterface : '0.0.0.0',
-
- // userAgent string for lib/Scraper.js
- userAgent : null,
-
- // Settings for Zotero server
- zoteroPort : 1969,
- zoteroInterface : '127.0.0.1',
- // Whether or not the outbound proxy should be used to access Zotero
- zoteroUseProxy : false,
-
- // Settings for StatsD client
- //statsd : {
- // host: 'statsd.server',
- // port: 8125,
- //}
-};
-
-/*Exports*/
-module.exports = CitoidConfig;
-
diff --git a/package.json b/package.json
index f05efaf..662a2ad 100644
--- a/package.json
+++ b/package.json
@@ -1,29 +1,33 @@
{
"name": "citoid",
- "version": "0.0.0",
+ "version": "0.1.0",
"description": "Converts search terms such as URL or DOI into citations.",
"scripts": {
- "test": "grunt test"
+ "start": "service-runner",
+ "test": "mocha test/index.js",
+ "coverage": "istanbul cover _mocha -- -R spec"
},
"dependencies": {
"async": "0.9.0",
- "bluebird": "2.3.11",
- "body-parser": "1.10.0",
- "bunyan": "1.2.3",
+ "bluebird": "2.9.14",
+ "body-parser": "1.12.1",
+ "bunyan": "1.3.4",
"cheerio": "0.18.0",
- "express": "4.10.4",
+ "compression": "1.4.3",
+ "express": "4.12.2",
"html-metadata": "0.1.1",
- "path": "0.4.9",
+ "js-yaml": "3.2.7",
+ "node-uuid": "1.4.3",
+ "preq": "0.3.12",
"request": "2.49.0",
- "node-txstatsd": "0.1.5",
- "xmldom": "0.1.19",
- "xpath": "0.0.7",
- "yargs": "1.3.3"
+ "service-runner": "0.1.5"
},
"devDependencies": {
- "grunt": "0.4.5",
- "grunt-contrib-jshint": "0.10.0",
- "grunt-simple-mocha": "0.4.0"
+ "assert": "1.3.0",
+ "istanbul": "0.3.8",
+ "mocha": "2.2.1",
+ "mocha-jshint": "0.0.9",
+ "mocha-lcov-reporter": "0.0.1"
},
"repository": {
"type": "git",
@@ -45,6 +49,10 @@
{
"name": "Dan Michael O. Heggø",
"email": "danmicha...@gmail.com"
+ },
+ {
+ "name": "Marko Obrovac",
+ "email": "mobro...@wikimedia.org"
}
]
}
diff --git a/routes/root.js b/routes/root.js
new file mode 100644
index 0000000..b2e9684
--- /dev/null
+++ b/routes/root.js
@@ -0,0 +1,116 @@
+'use strict';
+
+
+var sUtil = require('../lib/util');
+
+
+/**
+ * The main router object
+ */
+var router = sUtil.router();
+
+/**
+ * The main application object reported when this module is require()d
+ */
+var app;
+
+
+/**
+ * GET /robots.txt
+ * Instructs robots no indexing should occur on this domain.
+ */
+router.get('/robots.txt', function(req, res) {
+
+ res.set({
+ 'User-agent': '*',
+ 'Disallow': '/'
+ }).end();
+
+});
+
+
+/**
+ * POST /url
+ * Endpoint for retrieving citations in JSON format from a URL.
+ * Note: this endpoint is deprecated.
+ */
+router.post('/url', function(req, res) {
+
+ var opts,
+ acceptLanguage = req.headers['accept-language'],
+ format = req.body.format,
+ requestedURL = req.body.url;
+
+ // temp backwards compatibility
+ if (!format) {
+ format = 'mwDeprecated';
+ }
+
+ if (!requestedURL) {
+ res.status(400).type('text/plain');
+ res.send('"url" is a required parameter');
+ return;
+ }
+
+ opts = {
+ search: requestedURL,
+ format: format,
+ acceptLanguage: acceptLanguage
+ };
+
+ app.citoid.request(opts, function(error, responseCode, body){
+ res.status(responseCode).type('application/json');
+ res.send(body);
+ });
+
+});
+
+
+/**
+ * GET /api
+ * Endpoint for retrieving citations based on search term (URL, DOI).
+ */
+router.get('/api', function(req, res) {
+
+ var dSearch, opts,
+ acceptLanguage = req.headers['accept-language'],
+ format = req.query.format,
+ search = req.query.search;
+
+ if (!search) {
+ res.status(400).type('text/plain');
+ res.send("No 'search' value specified\n");
+ return;
+ } else if(!format) {
+ res.status(400).type('text/plain');
+ res.send("No 'format' value specified\nOptions are
'mediawiki','zotero'");
+ return;
+ }
+
+ dSearch = decodeURIComponent(search); //decode urlencoded search string
+ opts = {
+ search: dSearch,
+ format: format,
+ acceptLanguage: acceptLanguage
+ };
+
+ app.citoid.request(opts, function(error, responseCode, body) {
+ res.status(responseCode).type('application/json');
+ res.send(body);
+ });
+
+});
+
+
+module.exports = function(appObj) {
+
+ app = appObj;
+
+ return {
+ path: '/',
+ skip_domain: true,
+ router: router
+ };
+
+};
+
diff --git a/server.js b/server.js
old mode 100644
new mode 100755
index 7f8711f..da9fad8
--- a/server.js
+++ b/server.js
@@ -1,131 +1,12 @@
#!/usr/bin/env node
-/**
- * https://www.mediawiki.org/wiki/citoid
- */
-/* Import External Modules */
-var bodyParser = require('body-parser'),
- bunyan = require('bunyan'),
- express = require('express'),
- path = require('path'),
- opts = require('yargs')
- .usage('Usage: $0 [-c configfile|--config=configfile]')
- .default({
- c: __dirname + '/localsettings.js'
- })
- .alias( 'c', 'config' ),
- argv = opts.argv;
+'use strict';
-/* Import Local Modules */
-var CitoidService = require('./lib/CitoidService.js');
+// Service entry point. Try node server --help for commandline options.
-/* Import Local Settings */
-var settingsFile = path.resolve(process.cwd(), argv.c),
- citoidConfig = require(settingsFile),
- citoidPort = citoidConfig.citoidPort,
- citoidInterface = citoidConfig.citoidInterface,
- allowCORS = citoidConfig.allowCORS;
-
-// Set outgoing proxy
-if (citoidConfig.proxy) {
- process.env.HTTP_PROXY = citoidConfig.proxy;
- if (!citoidConfig.zoteroUseProxy) {
- // Don't use proxy for accessing Zotero unless specified in
settings
- process.env.NO_PROXY = citoidConfig.zoteroInterface;
- }
-}
-
-// Init citoid webserver
-var app = express();
-var log = bunyan.createLogger({name: "citoid"});
-
-// Init citoid service object
-var citoidService = new CitoidService(citoidConfig, log);
-
-// SECURITY WARNING: ALLOWS ALL REQUEST ORIGINS
-// change allowCORS in localsettings.js
-app.all('*', function(req, res, next) {
- res.header("Access-Control-Allow-Origin", allowCORS);
- res.header("Access-Control-Allow-Headers", "X-Requested-With, Content-Type");
- next();
- });
-
-app.use(bodyParser.json());
-app.use(bodyParser.urlencoded({extended: false}));
-app.use(express.static('api')); // Cache api pages
-app.use(express.static(__dirname + '/static')); // Static HTML files
-
-/* Endpoint for retrieving citations in JSON format from a URL */
-app.post('/url', function(req, res){
-
- res.type('application/json');
-
- var opts,
- acceptLanguage = req.headers['accept-language'],
- format = req.body.format,
- requestedURL = req.body.url;
-
- log.info(req);
-
- //temp backwards compatibility
- if (!format){
- format = 'mwDeprecated';
- }
-
- if (!requestedURL){
- res.statusCode = 400;
- res.setHeader("Content-Type", "text/plain");
- res.send('"url" is a required parameter');
- } else {
- opts = {
- search : requestedURL,
- format: format,
- acceptLanguage : acceptLanguage
- };
- citoidService.request(opts, function(error, responseCode, body){
- res.statusCode = responseCode;
- res.send(body);
- });
- }
-});
-
-/* Endpoint for retrieving citations based on search term (URL, DOI) */
-app.get('/api', function(req, res){
-
- res.type('application/json');
-
- var dSearch, opts,
- acceptLanguage = req.headers['accept-language'],
- format = req.query.format,
- search = req.query.search;
-
- log.info(req);
-
- if (!search){
- res.statusCode = 400;
- res.setHeader("Content-Type", "text/plain");
- res.send("No 'search' value specified\n");
- } else if(!format){
- res.statusCode = 400;
- res.setHeader("Content-Type", "text/plain");
- res.send("No 'format' value specified\nOptions are
'mediawiki','zotero'");
- } else {
- dSearch = decodeURIComponent(search); //decode urlencoded
search string
- opts = {
- search : dSearch,
- format: format,
- acceptLanguage : acceptLanguage
- };
- citoidService.request(opts, function(error, responseCode, body){
- res.statusCode = responseCode;
- res.send(body);
- });
- }
-});
-
-app.listen(citoidPort, citoidInterface);
-
-log.info('Server started on ' + citoidInterface + ':' + citoidPort);
-
-/* Exports */
-exports = module.exports = app;
+// Start the service by running service-runner, which in turn loads the config
+// (config.yaml by default, specify other path with -c). It requires the
+// module(s) specified in the config 'services' section (app.js in this
+// example).
+var ServiceRunner = require('service-runner');
+return new ServiceRunner().run();
diff --git a/test/features/app/index.js b/test/features/app/index.js
new file mode 100644
index 0000000..6c98bbf
--- /dev/null
+++ b/test/features/app/index.js
@@ -0,0 +1,40 @@
+'use strict';
+
+
+// mocha defines to avoid JSHint breakage
+/* global describe, it, before, beforeEach, after, afterEach */
+
+
+var preq = require('preq');
+var assert = require('../../utils/assert.js');
+var server = require('../../utils/server.js');
+
+
+describe('express app', function() {
+
+ this.timeout(20000);
+
+ before(function () { return server.start(); });
+
+ it('get robots.txt', function() {
+ return preq.get({
+ uri: server.config.uri + 'robots.txt'
+ }).then(function(res) {
+ assert.status(res, 200);
+ assert.deepEqual(res.headers['disallow'], '/');
+ });
+ });
+
+ it('get landing page', function() {
+ return preq.get({
+ uri: server.config.uri
+ }).then(function(res) {
+ // check that the response is present
+ assert.status(res, 200);
+ assert.contentType(res, 'text/html');
+ assert.notDeepEqual(res.body.length, 0, 'Empty
response');
+ });
+ });
+
+});
+
diff --git a/test/features/errors/index.js b/test/features/errors/index.js
new file mode 100644
index 0000000..fa5a3d7
--- /dev/null
+++ b/test/features/errors/index.js
@@ -0,0 +1,70 @@
+/**
+ * https://www.mediawiki.org/wiki/citoid
+ */
+'use strict';
+
+
+// mocha defines to avoid JSHint errors
+/* global describe, it */
+
+
+var preq = require('preq');
+var assert = require('../../utils/assert.js');
+var server = require('../../utils/server.js');
+
+
+describe('errors', function() {
+
+ this.timeout(20000);
+
+ before(function () { return server.start(); });
+
+ it('missing format', function() {
+ return preq.get({
+ uri: server.config.q_uri,
+ query: {
+ search: '123456'
+ }
+ }).then(function(res) {
+ assert.status(res, 400);
+ }, function(err) {
+ assert.status(err, 400);
+ });
+ });
+
+ it('missing search', function() {
+ return preq.get({
+ uri: server.config.q_uri,
+ query: {
+ format: 'mediawiki'
+ }
+ }).then(function(res) {
+ assert.status(res, 400);
+ }, function(err) {
+ assert.status(err, 400);
+ });
+ });
+
+ it('erroneous domain', function() {
+ return server.query('example./com', 'mediawiki', 'en')
+ .then(function(res) {
+ assert.status(res, 520);
+ }, function(err) {
+ assert.status(err, 520);
+ assert.checkCitation(err, 'http://example./com');
+ });
+ });
+
+ it('non-existent URL path', function() {
+ var url = 'http://example.com/thisurldoesntexist';
+ return server.query(url, 'mediawiki', 'en')
+ .then(function(res) {
+ assert.status(res, 520);
+ }, function(err) {
+ assert.status(err, 520);
+ assert.checkCitation(err, url);
+ });
+ });
+
+});
+
diff --git a/test/features/scraping/index.js b/test/features/scraping/index.js
new file mode 100644
index 0000000..6954720
--- /dev/null
+++ b/test/features/scraping/index.js
@@ -0,0 +1,62 @@
+/**
+ * https://www.mediawiki.org/wiki/citoid
+ */
+'use strict';
+
+
+// mocha defines to avoid JSHint errors
+/* global describe, it */
+
+
+var preq = require('preq');
+var assert = require('../../utils/assert.js');
+var server = require('../../utils/server.js');
+
+
+describe('scraping', function() {
+
+ this.timeout(40000);
+
+ before(function () { return server.start(); });
+
+ it('pmid', function() {
+ return server.query('23555203').then(function(res) {
+ assert.status(res, 200);
+ assert.checkCitation(res, 'Viral Phylodynamics');
+ });
+ });
+
+ it('example domain', function() {
+ return server.query('example.com').then(function(res) {
+ assert.status(res, 200);
+ assert.checkCitation(res, 'Example Domain');
+ });
+ });
+
+ it('doi', function() {
+ return server.query('doi:
10.1371/journal.pcbi.1002947').then(function(res) {
+ assert.status(res, 200);
+ assert.checkCitation(res);
+ assert.deepEqual(res.body[0].pages, 'e1002947', 'Wrong
pages item; expected e1002947, got ' + res.body[0].pages);
+ });
+ });
+
+ it('open graph', function() {
+ return
server.query('http://www.pbs.org/newshour/making-sense/care-peoples-kids/').then(function(res)
{
+ assert.status(res, 200);
+ assert.checkCitation(res);
+ assert.notDeepEqual(res.body[0].language, undefined,
'Missing maguage field');
+ });
+ });
+
+ it('websiteTitle + publicationTitle', function() {
+ return
server.query('http://blog.woorank.com/2013/04/dublin-core-metadata-for-seo-and-usability/').then(function(res)
{
+ assert.status(res, 200);
+ assert.checkCitation(res);
+ assert.notDeepEqual(res.body[0].websiteTitle,
undefined, 'Missing websiteTitle field');
+ assert.notDeepEqual(res.body[0].publicationTitle,
undefined, 'Missing publicationTitle field');
+ });
+ });
+
+});
+
diff --git a/test/features/scraping/lang.js b/test/features/scraping/lang.js
new file mode 100644
index 0000000..6daf768
--- /dev/null
+++ b/test/features/scraping/lang.js
@@ -0,0 +1,30 @@
+/**
+ * https://www.mediawiki.org/wiki/citoid
+ */
+'use strict';
+
+
+// mocha defines to avoid JSHint errors
+/* global describe, it */
+
+
+var preq = require('preq');
+var assert = require('../../utils/assert.js');
+var server = require('../../utils/server.js');
+
+
+describe('languages', function() {
+
+ this.timeout(20000);
+
+ before(function () { return server.start(); });
+
+ it('german twitter', function() {
+ return server.query('http://twitter.com', 'mediawiki',
'de').then(function(res) {
+ assert.status(res, 200);
+ assert.checkCitation(res, 'Willkommen bei Twitter -
Anmelden oder Registrieren');
+ });
+ });
+
+});
+
diff --git a/test/index.js b/test/index.js
index 95ad698..535299f 100644
--- a/test/index.js
+++ b/test/index.js
@@ -1,256 +1,6 @@
-#!/usr/bin/env node
-/**
- * https://www.mediawiki.org/wiki/citoid
- */
-
-// mocha defines to avoid JSHint errors
-/* global describe, it */
-
-var CitoidService = require('../lib/CitoidService.js'),
- bunyan = require('bunyan'),
- path = require('path'),
- opts = require('yargs')
- .usage('Usage: $0 [-c configfile|--config=configfile]')
- .default({
- c: __dirname + '/localsettings.js'
- })
- .alias( 'c', 'config' ),
- argv = opts.argv,
- settingsFile = path.resolve(process.cwd(), argv.c),
- log = bunyan.createLogger({name: "citoid"}),
- defaults = {
- allowCORS : '*',
- citoidPort : 1970,
- citoidInterface : 'localhost',
- userAgent : null,
- zoteroPort : 1969,
- zoteroInterface : 'localhost',
- },
- citoidConfig,
- citoidService;
-
-try {
- citoidConfig = require(settingsFile).CitoidConfig;
-} catch (e) {
- citoidConfig = defaults;
-}
-
-citoidService = new CitoidService(citoidConfig, log);
-
-describe('pmid', function() {
-
- var opts = {
- search : '23555203',
- format : 'mediawiki',
- acceptLanguage : 'en'
- },
- expectedTitle = 'Viral Phylodynamics';
-
- it('should scrape info successfully', function(done) {
- citoidService.request(opts, function(error, responseCode,
citation){
- if (error) {throw error;}
- if (responseCode !== 200){
- throw new Error('Not successful: Response code
is' + responseCode);
- }
- if (!citation) {throw new Error ('Empty body');}
- if (citation[0].title !== expectedTitle){
- throw new Error('Expected title is: ' +
expectedTitle +
- ";\nGot: " + citation[0].title);
- }
- if (!citation[0].itemType){
- throw new Error('Missing itemType');
- }
- done();
- });
- });
-});
-
-describe('200', function() {
-
- var opts = {
- search : 'example.com',
- format : 'mediawiki',
- acceptLanguage : 'en'
- },
- expectedTitle = 'Example Domain';
-
- it('should scrape info successfully', function(done) {
- citoidService.request(opts, function(error, responseCode,
citation){
- if (error) {throw error;}
- if (responseCode !== 200){
- throw new Error('Not successful: Response code
is' + responseCode);
- }
- if (!citation) {throw new Error ('Empty body');}
- if (citation[0].title !== expectedTitle){
- throw new Error('Expected title is: ' +
expectedTitle +
- ";\nGot: " + citation[0].title);
- }
- if (!citation[0].itemType){
- throw new Error('Missing itemType');
- }
- done();
- });
- });
-});
-
-describe('ENOTFOUND', function() {
-
- var url = 'example./com',
- opts = {
- search : url,
- format : 'mediawiki',
- acceptLanguage : 'en'
- },
- expectedTitle = 'http://example./com';
-
- it('should return a ENOTFOUND error, a 520 responseCode, and citation',
function(done) {
- citoidService.request(opts, function(error, responseCode,
citation){
- if (!error) {
- throw new Error('No error');
- }
- // Throw errors except the expected error, ENOTFOUND
- if (error.message !== 'getaddrinfo ENOTFOUND'){
- throw error;
- }
- if (responseCode !== 520){
- throw new Error('Should throw 520: Response
code is ' + responseCode);
- }
- if (!citation) {throw new Error ('Empty body');}
- if (citation[0].title !== expectedTitle){
- throw new Error('Expected title is: ' +
expectedTitle +
- ";\nGot: " + citation[0].title);
- }
- if (!citation[0].itemType){
- throw new Error('Missing itemType');
- }
- done();
- });
- });
-});
-
-describe('404', function() {
-
- var url = 'http://example.com/thisurldoesntexist',
- opts = {
- search : url,
- format : 'mediawiki',
- acceptLanguage : 'en'
- },
- expectedTitle = url;
-
- it('should return a 520 responseCode and citation', function(done) {
- citoidService.request(opts, function(error, responseCode,
citation){
- if (responseCode !== 520){
- throw new Error('Should throw 520: Response
code is ' + responseCode);
- }
- if (!citation) {throw new Error ('Empty body');}
- if (citation[0].title !== expectedTitle){
- throw new Error('Expected title is: ' +
expectedTitle +
- ";\nGot: " + citation[0].title);
- }
- if (!citation[0].itemType){
- throw new Error('Missing itemType');
- }
- done();
- });
- });
-});
+'use strict';
-describe('German twitter', function() {
+// Run jshint as part of normal testing
+require('mocha-jshint')();
- var opts = {
- search : 'http://twitter.com',
- format : 'mediawiki',
- acceptLanguage : 'de'
- },
- expectedTitle = 'Willkommen bei Twitter - Anmelden oder
Registrieren';
-
- it('should return the citation for twitter in German', function(done) {
- citoidService.request(opts, function(error, responseCode,
citation){
- if (error) {throw error;}
- if (responseCode !== 200){
- throw new Error('Should respond 200: Response
code is ' + responseCode);
- }
- if (!citation) {throw new Error ('Empty body');}
- if (citation[0].title !== expectedTitle){
- throw new Error('Expected title is: ' +
expectedTitle +
- ";\nGot: " + citation[0].title);
- }
- done();
- });
- });
-});
-
-describe('doi', function() {
-
- var opts = {
- search : 'doi: 10.1371/journal.pcbi.1002947',
- format : 'mediawiki',
- acceptLanguage : 'en'
- };
-
- it('should use return citation from doi', function(done) {
- citoidService.request(opts, function(error, responseCode,
citation){
- if (error) {throw error;}
- if (responseCode !== 200){
- throw new Error('Should respond 200: Response
code is ' + responseCode);
- }
- if (!citation) {throw new Error ('Empty body');}
- if (citation[0].pages !== 'e1002947'){
- throw new Error('Expected pages value should
be: e1002947; Got: '
- + citation[0].pages);
- }
- done();
- });
- });
-});
-
-
-describe('websiteTitle', function() {
-
- var opts = {
- search :
'http://blog.woorank.com/2013/04/dublin-core-metadata-for-seo-and-usability/',
- format : 'mediawiki',
- acceptLanguage : 'en'
- };
-
- it('should contain a websiteTitle and a publicationTitle',
function(done) {
- citoidService.request(opts, function(error, responseCode,
citation){
- if (error) {throw error;}
- if (responseCode !== 200){
- throw new Error('Should respond 200: Response
code is ' + responseCode);
- }
- if (!citation) {throw new Error ('Empty body');}
- if (!citation[0].publicationTitle){
- throw new Error('Should contain field
publicationTitle');
- }
- if (!citation[0].websiteTitle){
- throw new Error('Should contain field
websiteTitle');
- }
- done();
- });
- });
-});
-describe('scrape open graph', function() {
-
- var opts = {
- search :
'http://www.pbs.org/newshour/making-sense/care-peoples-kids/',
- format : 'mediawiki',
- acceptLanguage : 'en'
- };
-
- it('should correctly scrape open graph data', function(done) {
- citoidService.request(opts, function(error, responseCode,
citation){
- if (error) {throw error;}
- if (responseCode !== 200){
- throw new Error('Should respond 200: Response
code is ' + responseCode);
- }
- if (!citation) {throw new Error ('Empty body');}
- if (!citation[0].language){
- throw new Error('Should contain language code');
- }
- done();
- });
- });
-});
\ No newline at end of file
diff --git a/test/mocha.opts b/test/mocha.opts
new file mode 100644
index 0000000..4a52320
--- /dev/null
+++ b/test/mocha.opts
@@ -0,0 +1 @@
+--recursive
diff --git a/test/utils/assert.js b/test/utils/assert.js
new file mode 100644
index 0000000..a2e7db0
--- /dev/null
+++ b/test/utils/assert.js
@@ -0,0 +1,121 @@
+'use strict';
+
+
+var assert = require('assert');
+
+
+/**
+ * Asserts whether the return status was as expected
+ */
+function status(res, expected) {
+
+ deepEqual(res.status, expected,
+ 'Expected status to be ' + expected + ', but was ' + res.status);
+
+}
+
+
+/**
+ * Asserts whether content type was as expected
+ */
+function contentType(res, expected) {
+
+ var actual = res.headers['content-type'];
+ deepEqual(actual, expected,
+ 'Expected content-type to be ' + expected + ', but was ' + actual);
+
+}
+
+
+function isDeepEqual(result, expected, message) {
+
+ try {
+ if (typeof expected === 'string') {
+ assert.ok(result === expected || (new
RegExp(expected).test(result)), message);
+ } else {
+ assert.deepEqual(result, expected, message);
+ }
+ return true;
+ } catch (e) {
+ return false;
+ }
+
+}
+
+
+function deepEqual(result, expected, message) {
+
+ try {
+ if (typeof expected === 'string') {
+ assert.ok(result === expected || (new
RegExp(expected).test(result)));
+ } else {
+ assert.deepEqual(result, expected, message);
+ }
+ } catch (e) {
+ console.log('Expected:\n' + JSON.stringify(expected, null, 2));
+ console.log('Result:\n' + JSON.stringify(result, null, 2));
+ throw e;
+ }
+
+}
+
+
+function notDeepEqual(result, expected, message) {
+
+ try {
+ assert.notDeepEqual(result, expected, message);
+ } catch (e) {
+ console.log('Not expected:\n' + JSON.stringify(expected, null, 2));
+ console.log('Result:\n' + JSON.stringify(result, null, 2));
+ throw e;
+ }
+
+}
+
+
+function fails(promise, onRejected) {
+
+ var failed = false;
+
+ function trackFailure(e) {
+ failed = true;
+ return onRejected(e);
+ }
+
+ function check() {
+ if (!failed) {
+ throw new Error('expected error was not thrown');
+ }
+ }
+
+ return promise.catch(trackFailure).then(check);
+
+}
+
+
+function checkCitation(res, title) {
+
+ var cit = res.body;
+
+ if(!Array.isArray(cit) || !cit.length) {
+ throw new Error('Expected to receive an array of citations, got: ' +
JSON.stringify(cit));
+ }
+
+ cit = cit[0];
+ assert.notDeepEqual(cit.itemType, undefined, 'No itemType present');
+ if(title) {
+ assert.deepEqual(cit.title, title, 'Wrong title, expected "' + title +
'", got "' + cit.title + '"');
+ }
+
+}
+
+
+module.exports.ok = assert.ok;
+module.exports.fails = fails;
+module.exports.deepEqual = deepEqual;
+module.exports.isDeepEqual = isDeepEqual;
+module.exports.notDeepEqual = notDeepEqual;
+module.exports.contentType = contentType;
+module.exports.status = status;
+module.exports.checkCitation = checkCitation;
+
diff --git a/test/utils/logStream.js b/test/utils/logStream.js
new file mode 100644
index 0000000..f8e292d
--- /dev/null
+++ b/test/utils/logStream.js
@@ -0,0 +1,68 @@
+'use strict';
+
+var bunyan = require('bunyan');
+
+function logStream(logStdout) {
+
+ var log = [];
+ var parrot = bunyan.createLogger({
+ name: 'test-logger',
+ level: 'warn'
+ });
+
+ function write(chunk, encoding, callback) {
+ try {
+ var entry = JSON.parse(chunk);
+ var levelMatch = /^(\w+)/.exec(entry.levelPath);
+ if (logStdout && levelMatch) {
+ var level = levelMatch[1];
+ if (parrot[level]) {
+ parrot[level](entry);
+ }
+ }
+ } catch (e) {
+ console.error('something went wrong trying to parrot a log entry', e,
chunk);
+ }
+
+ log.push(chunk);
+ }
+
+ // to implement the stream writer interface
+ function end(chunk, encoding, callback) {
+ }
+
+ function get() {
+ return log;
+ }
+
+ function slice() {
+
+ var begin = log.length;
+ var end = null;
+
+ function halt() {
+ if (end === null) {
+ end = log.length;
+ }
+ }
+
+ function get() {
+ return log.slice(begin, end);
+ }
+
+ return {
+ halt: halt,
+ get: get
+ };
+
+ }
+
+ return {
+ write: write,
+ end: end,
+ slice: slice,
+ get: get
+ };
+}
+
+module.exports = logStream;
diff --git a/test/utils/server.js b/test/utils/server.js
new file mode 100644
index 0000000..4ec8c19
--- /dev/null
+++ b/test/utils/server.js
@@ -0,0 +1,93 @@
+'use strict';
+
+
+// mocha defines to avoid JSHint breakage
+/* global describe, it, before, beforeEach, after, afterEach */
+
+
+var BBPromise = require('bluebird');
+var ServiceRunner = require('service-runner');
+var logStream = require('./logStream');
+var fs = require('fs');
+var assert = require('./assert');
+var yaml = require('js-yaml');
+var preq = require('preq');
+
+
+// set up the configuration
+var config = {
+ conf: yaml.safeLoad(fs.readFileSync(__dirname + '/../../config.yaml'))
+};
+// build the API endpoint URI by supposing the actual service
+// is the last one in the 'services' list in the config file
+var myService = config.conf.services[config.conf.services.length - 1];
+config.uri = 'http://localhost:' + myService.conf.port + '/';
+config.q_uri = config.uri + 'api';
+// no forking, run just one process when testing
+config.conf.num_workers = 0;
+// have a separate, in-memory logger only
+config.conf.logging = {
+ name: 'test-log',
+ level: 'trace',
+ stream: logStream()
+};
+
+var stop = function () {};
+var options = null;
+var runner = new ServiceRunner();
+
+
+function start(_options) {
+
+ _options = _options || {};
+
+ if (!assert.isDeepEqual(options, _options)) {
+ console.log('server options changed; restarting');
+ stop();
+ options = _options;
+ return runner.run(config.conf)
+ .then(function(servers) {
+ var server = servers[0];
+ stop = function () {
+ console.log('stopping test server');
+ server.close();
+ stop = function () {};
+ };
+ return true;
+ });
+ } else {
+ return BBPromise.resolve();
+ }
+
+}
+
+
+function query(search, format, language) {
+
+ if (!format) {
+ format = 'mediawiki';
+ }
+ if (!language) {
+ language = 'en';
+ }
+
+ return preq.get({
+ uri: config.q_uri,
+ query: {
+ format: format,
+ search: search
+ },
+ headers: {
+ 'accept-language': language
+ }
+ });
+
+}
+
+
+module.exports = {
+ config: config,
+ start: start,
+ query: query
+};
+
--
To view, visit https://gerrit.wikimedia.org/r/199250
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I7370c9a91e67c263a291aab4b8ae9014a23efa5f
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/citoid
Gerrit-Branch: master
Gerrit-Owner: Mobrovac <mobro...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
