Mobrovac has submitted this change and it was merged.
Change subject: Add ability to set requesting User-Agent
......................................................................
Add ability to set requesting User-Agent
Modify localsettings.js.sample to contain a
userAgent string.
Pass this string to the Scraper.js object
and use this string as the User-Agent in
all outbound requests.
Simplify Object parameters in
ZoteroService.js, CitoidService.js, and
Scraper.js.
Bug: T89757
Change-Id: I7dfa933181118527783da12cdb5b277329aef7dd
---
M lib/CitoidService.js
M lib/Scraper.js
M lib/ZoteroService.js
M localsettings.js.sample
4 files changed, 20 insertions(+), 14 deletions(-)
Approvals:
Mobrovac: Looks good to me, approved
diff --git a/lib/CitoidService.js b/lib/CitoidService.js
index 3f4deeb..7d5f666 100644
--- a/lib/CitoidService.js
+++ b/lib/CitoidService.js
@@ -3,8 +3,7 @@
*/
/* Import Modules */
-var urlParse = require('url'),
- util = require('util');
+var urlParse = require('url');
/* Import Local Modules */
var unshorten = require('./unshorten.js'),
@@ -17,13 +16,10 @@
* @param {Object} CitoidConfig configuration object
* @param {Object} logger bunyan logger object
*/
-function CitoidService(CitoidConfig, logger){
- this.CitoidConfig = CitoidConfig;
+function CitoidService(citoidConfig, logger){
this.log = logger;
- this.zoteroURL = util.format('http://%s:%s/',
- CitoidConfig.zoteroInterface,
CitoidConfig.zoteroPort.toString());
- this.zoteroService = new ZoteroService(this.zoteroURL, logger);
- this.scraper = new Scraper(CitoidConfig, logger);
+ this.zoteroService = new ZoteroService(citoidConfig, logger);
+ this.scraper = new Scraper(citoidConfig, logger);
}
/**
diff --git a/lib/Scraper.js b/lib/Scraper.js
index 34e8d8b..57edbab 100644
--- a/lib/Scraper.js
+++ b/lib/Scraper.js
@@ -16,6 +16,7 @@
function Scraper(citoidConfig, logger){
this.log = logger;
+ this.userAgent = citoidConfig.userAgent || 'Citoid/0.0.0';
}
/**
@@ -30,6 +31,7 @@
var chtml,
log = this.log,
scraper = this,
+ userAgent = this.userAgent,
citation = {url: url, title: url};
log.info("Using native scraper on " + url);
@@ -38,6 +40,9 @@
{
url: url,
followAllRedirects: true,
+ headers: {
+ 'User-Agent': userAgent
+ }
}, function(error, response, html){
if (error || !response || response.statusCode !== 200) {
@@ -174,4 +179,4 @@
}
/**Exports*/
-module.exports = Scraper;
\ No newline at end of file
+module.exports = Scraper;
diff --git a/lib/ZoteroService.js b/lib/ZoteroService.js
index 0cc4a0c..d6df076 100644
--- a/lib/ZoteroService.js
+++ b/lib/ZoteroService.js
@@ -9,13 +9,15 @@
var async = require('async'),
crypto = require('crypto'),
request = require('request'),
+ util = require('util'),
pubMedRequest = require('./pubMedRequest.js');
-function ZoteroService(zoteroURL, logger){
+function ZoteroService(citoidConfig, logger){
this.log = logger;
- this.baseURL = zoteroURL;
- this.webURL = zoteroURL + 'web';
- this.exportURL = zoteroURL + 'export';
+ var baseURL = util.format('http://%s:%s/',
+ citoidConfig.zoteroInterface,
citoidConfig.zoteroPort.toString());
+ this.webURL = baseURL + 'web';
+ this.exportURL = baseURL + 'export';
}
/**
diff --git a/localsettings.js.sample b/localsettings.js.sample
index f503415..23e60ba 100644
--- a/localsettings.js.sample
+++ b/localsettings.js.sample
@@ -21,7 +21,10 @@
citoidPort : 1970,
citoidInterface : 'localhost',
- //Settings for Zotero server
+ // userAgent string for lib/Scraper.js
+ userAgent : null,
+
+ // Settings for Zotero server
zoteroPort : 1969,
zoteroInterface : 'localhost',
};
--
To view, visit https://gerrit.wikimedia.org/r/193134
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I7dfa933181118527783da12cdb5b277329aef7dd
Gerrit-PatchSet: 10
Gerrit-Project: mediawiki/services/citoid
Gerrit-Branch: master
Gerrit-Owner: Mvolz <mv...@wikimedia.org>
Gerrit-Reviewer: Mobrovac <mobro...@wikimedia.org>
Gerrit-Reviewer: Mvolz <mv...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
