Mvolz has uploaded a new change for review.
https://gerrit.wikimedia.org/r/232259
Change subject: [WIP] Remove query parameters before getting doi
......................................................................
[WIP] Remove query parameters before getting doi
Bug: T108832
Change-Id: Ib1631eed1c2081de94623f1e163bddc308f5a741
---
M lib/CitoidService.js
M test/features/scraping/index.js
2 files changed, 49 insertions(+), 16 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/citoid
refs/changes/59/232259/1
diff --git a/lib/CitoidService.js b/lib/CitoidService.js
index 08f7e5b..e984db9 100644
--- a/lib/CitoidService.js
+++ b/lib/CitoidService.js
@@ -60,6 +60,8 @@
* @return {Object} BBPromise object for modified CitoidRequest
object from input
*/
CitoidService.prototype.addResponseFunction = BBPromise.method(function(cr){
+ var matchDOI; // filled by getDOI function
+ var self = this;
var search = cr.search;
var reHTTP = new RegExp('^((https?)://.+\\..+)'); // Assumes all
strings with http/s protocol are URLs
@@ -67,17 +69,12 @@
var reDOI = new RegExp('\\b10\\.?[0-9]{3,5}(?:[.][0-9]+)*/.*');
var matchHTTP = search.match(reHTTP);
var matchWWW = search.match(reWWW);
- var matchDOI = search.match(reDOI);
- // TODO: Resolve to make sure DOI scraped from inside URLs are valid
(except when dx.doi.org links)
- if (matchDOI && matchDOI[0]) { cr.doi = matchDOI[0].toLowerCase(); }
+ var url = addDOI(); // Returns a string with the http protocol appended
to search which may or may not be an actual url
if (matchHTTP || matchWWW){
- cr.idType = 'url';
cr.idValue = matchHTTP ? encodeURI(matchHTTP[0]) : 'http://' +
encodeURI(matchWWW[0]);
- cr.getResponse = this.requestFromURL.bind(this);
- this.stats.increment('input.' + cr.idType);
- return cr;
+ return foundURI();
}
if (matchDOI) {
@@ -118,16 +115,34 @@
return cr;
}
- // Assume url by default
- cr.idType = 'url';
- var parsedURL = urlParse.parse(search);
- if (!parsedURL.protocol){
- search = 'http://'+ encodeURI(search);
+ // Assume url if no other matches are made
+ cr.idValue = url;
+ return foundURI();
+
+ function addDOI(){
+ var url;
+ var parsedURL = urlParse.parse(search);
+
+ // url gets protocol if missing and re-parsed
+ if (!parsedURL.protocol){
+ url = 'http://'+ encodeURI(search);
+ parsedURL = urlParse.parse(url);
+ }
+
+ // Look for DOI in URL with query string removed
+ parsedURL.query = null;
+ parsedURL.search = null;
+ matchDOI = urlParse.format(parsedURL).match(reDOI);
+ if (matchDOI && matchDOI[0]) { cr.doi =
matchDOI[0].toLowerCase(); }
+ return url;
}
- cr.idValue = search;
- cr.getResponse = this.requestFromURL.bind(this);
- this.stats.increment('input.' + cr.idType);
- return cr;
+
+ function foundURI(){
+ cr.idType = 'url';
+ cr.getResponse = self.requestFromURL.bind(self);
+ self.stats.increment('input.' + cr.idType);
+ return cr;
+ }
});
diff --git a/test/features/scraping/index.js b/test/features/scraping/index.js
index cf12e11..80e4366 100644
--- a/test/features/scraping/index.js
+++ b/test/features/scraping/index.js
@@ -156,6 +156,15 @@
});
});
+ // Fake url but with info in cross ref that can be pulled from
doi in url - uses requestFromDOI & zotero
+ it('doi in url with query parameters', function() {
+ return
server.query('example.com/10.1086/378695?uid=3739832&uid=2&uid=4&uid=3739256&sid=21105503736473').then(function(res)
{
+ assert.status(res, 200);
+ assert.checkZotCitation(res, 'Salaries,
Turnover, and Performance in the Federal Criminal Justice System');
+ assert.deepEqual(res.body[0].DOI,
'10.1086/378695');
+ });
+ });
+
// Ensure html tags are stripped out of title
it('zotero gives us html tags in title', function() {
return
server.query('http://fr.wikipedia.org/w/index.php?title=Ninja_Turtles_(film)&oldid=115125238').then(function(res)
{
@@ -293,6 +302,15 @@
});
});
+ // Fake url but with info in cross ref that can be pulled from
doi in url - uses requestFromURL & crossRef
+ it('doi in url with query parameters', function() {
+ return
server.query('http://www.example.com/10.1086/378695?uid=3739832&uid=2&uid=4&uid=3739256&sid=21105503736473').then(function(res)
{
+ assert.status(res, 200);
+ assert.checkZotCitation(res, 'Salaries,
Turnover, and Performance in the Federal Criminal Justice System*');
+ assert.deepEqual(res.body[0].DOI,
'10.1086/378695');
+ });
+ });
+
});
});
--
To view, visit https://gerrit.wikimedia.org/r/232259
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib1631eed1c2081de94623f1e163bddc308f5a741
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/citoid
Gerrit-Branch: master
Gerrit-Owner: Mvolz <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
