[MediaWiki-commits] [Gerrit] T87520: Initial repo commit - change (mediawiki...RestBaseUpdateJobs)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/188618
Change subject: T87520: Initial repo commit
..
T87520: Initial repo commit
This commit also includes some work in progress started before the
repository was created.
Change-Id: Id9e4af9035adbbe2e45c90dfd7a3346fc685aa18
---
A AUTHORS.txt
A RestbaseUpdate.hooks.php
A RestbaseUpdate.i18n.php
A RestbaseUpdate.php
A RestbaseUpdateJob.php
A i18n/de.json
A i18n/en.json
A i18n/es.json
A i18n/fr.json
A i18n/it.json
A i18n/pt.json
11 files changed, 523 insertions(+), 0 deletions(-)
git pull
ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/RestBaseUpdateJobs
refs/changes/18/188618/1
diff --git a/AUTHORS.txt b/AUTHORS.txt
new file mode 100644
index 000..27b2432
--- /dev/null
+++ b/AUTHORS.txt
@@ -0,0 +1,5 @@
+The Services Team:
+* Gabriel Wicke gwi...@wikimedia.org
+* Marko Obrovac mobro...@wikimedia.org
+* James Earl Douglas jdoug...@wikimedia.org
+
diff --git a/RestbaseUpdate.hooks.php b/RestbaseUpdate.hooks.php
new file mode 100644
index 000..d1b6f51
--- /dev/null
+++ b/RestbaseUpdate.hooks.php
@@ -0,0 +1,98 @@
+?php
+
+/**
+ * Hooks for events that should trigger RESTBase updates.
+ */
+class RestbaseUpdateHooks {
+
+
+ /**
+* Schedule an async update job in the job queue.
+*
+* @param Title $title
+* @param string $type
+* @param array $extra_params
+*/
+ private static function schedule( $title, $type, $extra_params =
array() ) {
+
+ $params = array( 'type' = $type ) + $extra_params;
+ JobQueueGroup::singleton()-push( new RestbaseUpdateJob(
$title, $params ) );
+
+ }
+
+
+ /**
+* Callback for regular article edits
+*
+* @param $article WikiPage the modified wiki page object
+* @param $editInfo
+* @param bool $changed
+* @return bool
+*/
+ public static function onArticleEditUpdates( $article, $editInfo,
$changed ) {
+
+ if ( $changed ) {
+ self::schedule( $article-getTitle(), 'edit' );
+ }
+ return true;
+
+ }
+
+
+ /**
+* Callback for article deletions
+*
+* @param $article WikiPage the modified wiki page object
+* @param $user User the deleting user
+* @param string $reason
+* @param int $id the page id
+* @return bool
+*/
+ public static function onArticleDeleteComplete( $article, $user,
$reason, $id ) {
+
+ self::schedule( $article-getTitle(), 'delete' );
+ return true;
+
+ }
+
+
+ /**
+* Callback for article undeletion. See specials/SpecialUndelete.php.
+*/
+ public static function onArticleUndelete( Title $title, $created,
$comment ) {
+
+ self::schedule( $title, 'edit' );
+ return true;
+
+ }
+
+
+ /**
+* Callback for article revision changes. See
+* revisiondelete/RevDelRevisionList.php.
+*/
+ public static function onArticleRevisionVisibilitySet( $title, $revs ) {
+
+ // TODO complete here with more info / the hidden fields
perhaps ?
+ // XXX do not forget that rev IDs are not yet actually returned
+ self::schedule( $title, 'rev_delete', array( 'revs' = $revs )
);
+ return true;
+
+ }
+
+
+ /**
+* Title move callback. See Title.php.
+*/
+ public static function onTitleMoveComplete( $title, Title $newtitle,
$user, $oldid, $newid ) {
+
+ # Simply update both old and new title.
+ self::schedule( $title, 'delete', array( 'rev' = $oldid ) );
+ self::schedule( $newtitle, 'edit', array( 'rev' = $newid ) );
+ return true;
+
+ }
+
+
+}
+
diff --git a/RestbaseUpdate.i18n.php b/RestbaseUpdate.i18n.php
new file mode 100644
index 000..0c95e5c
--- /dev/null
+++ b/RestbaseUpdate.i18n.php
@@ -0,0 +1,35 @@
+?php
+/**
+ * This is a backwards-compatibility shim, generated by:
+ *
https://git.wikimedia.org/blob/mediawiki%2Fcore.git/HEAD/maintenance%2FgenerateJsonI18n.php
+ *
+ * Beginning with MediaWiki 1.23, translation strings are stored in json files,
+ * and the EXTENSION.i18n.php file only exists to provide compatibility with
+ * older releases of MediaWiki. For more information about this migration, see:
+ * https://www.mediawiki.org/wiki/Requests_for_comment/Localisation_format
+ *
+ * This shim maintains compatibility back to MediaWiki 1.17.
+ */
+$messages = array();
[MediaWiki-commits] [Gerrit] Code clean-up - change (mediawiki...RestBaseUpdateJobs)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/188677
Change subject: Code clean-up
..
Code clean-up
The basis of this extension was shamelessly copied over from the Parsoid
extension. This commit removes Parsoid relics we do not need and cleans
up the code a bit.
Bug: T87520
Change-Id: I780fa82d4df0aa66b01513673b37eee74888251f
---
M RestbaseUpdate.php
M RestbaseUpdateJob.php
2 files changed, 14 insertions(+), 204 deletions(-)
git pull
ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/RestBaseUpdateJobs
refs/changes/77/188677/1
diff --git a/RestbaseUpdate.php b/RestbaseUpdate.php
index a74cde2..3d6f0fd 100644
--- a/RestbaseUpdate.php
+++ b/RestbaseUpdate.php
@@ -1,7 +1,7 @@
?php
/**
- * Basic cache invalidation for RESTBase
+ * Basic content invalidation and re-rendering for RESTBase
*/
if ( !defined( 'MEDIAWIKI' ) ) {
echo RestbaseUpdateJobs extension\n;
@@ -27,31 +27,26 @@
# Set up class autoloading
$wgAutoloadClasses['RestbaseUpdateHooks'] =
$dir/RestbaseUpdate.hooks.php;
$wgAutoloadClasses['RestbaseUpdateJob'] =
$dir/RestbaseUpdateJob.php;
- $wgAutoloadClasses['CurlMultiClient'] =
$dir/CurlMultiClient.php;
- # Add the parsoid job types
- $wgJobClasses['RestbaseUpdateJobOnEdit'] = 'RestbaseUpdateJob';
- $wgJobClasses['RestbaseUpdateJobOnDependencyChange'] =
'RestbaseUpdateJob';
- # Old type for transition
- # @TODO: remove when old jobs are drained
- $wgJobClasses['RestabseUpdateJob'] = 'RestbaseUpdateJob';
+ # Add the Restbase job type
+ $wgJobClasses['RestbaseUpdateJob'] = 'RestbaseUpdateJob';
$wgExtensionCredits['other'][] = array(
'path' = __FILE__,
- 'name' = 'RestbaseUpdate',
+ 'name' = 'RestBaseUpdateJobs',
'author' = array(
'Gabriel Wicke',
'Marko Obrovac'
),
- 'version' = '0.2.0',
- 'url' =
'https://www.mediawiki.org/wiki/Extension:RestbaseUpdateJobs',
+ 'version' = '0.1.0',
+ 'url' =
'https://www.mediawiki.org/wiki/Extension:RestBaseUpdateJobs',
'descriptionmsg' = 'restbase-desc',
'license-name' = 'GPL-2.0+',
);
# Register localizations.
- $wgMessagesDirs['RestbaseUpdateJobs'] = __DIR__ . '/i18n';
- $wgExtensionMessagesFiles['RestbaseUpdateJobs'] = $dir .
'/RestbaseUpdate.i18n.php';
+ $wgMessagesDirs['RestBaseUpdateJobs'] = __DIR__ . '/i18n';
+ $wgExtensionMessagesFiles['RestBaseUpdateJobs'] = $dir .
'/RestbaseUpdate.i18n.php';
# Set up a default configuration
self::setupDefaultConfig();
diff --git a/RestbaseUpdateJob.php b/RestbaseUpdateJob.php
index 717f580..a59dcd5 100644
--- a/RestbaseUpdateJob.php
+++ b/RestbaseUpdateJob.php
@@ -1,217 +1,32 @@
?php
/**
- * HTML cache refreshing and -invalidation job for the Parsoid varnish caches.
- *
- * This job comes in a few variants:
- * - a) Recursive jobs to purge caches for backlink pages for a given title.
- *They have have
(type:OnDependencyChange,recursive:true,table:table) set.
- * - b) Jobs to purge caches for a set of titles (the job title is ignored).
- * They have have (type:OnDependencyChange,pages:(page
ID:(namespace,title),...) set.
- * - c) Jobs to purge caches for a single page (the job title)
- *They have (type:OnEdit) set.
- *
- * See
- *
http://www.mediawiki.org/wiki/Parsoid/Minimal_performance_strategy_for_July_release
+ * The JobQueue job definition for updating RESTBase content
*/
class RestbaseUpdateJob extends Job {
function __construct( $title, $params, $id = 0 ) {
- // Map old jobs to new 'OnEdit' jobs
+ // assume 'edit' jobs if not set
if ( !isset( $params['type'] ) ) {
- $params['type'] = 'OnEdit'; // b/c
+ $params['type'] = 'edit';
}
parent::__construct( 'RestbaseUpdateJob' . $params['type'],
$title, $params, $id );
- if ( $params['type'] == 'OnEdit' ) {
- // Simple duplicate removal for single-title jobs.
Other jobs are
- // deduplicated with root job parameters.
- $this-removeDuplicates = true;
- }
+ // we always want to remove duplicates
+ $this-removeDuplicates = true;
}
function run() {
- // for now we are capable of
[MediaWiki-commits] [Gerrit] Consistency tweak in preparation for adding extension to tra... - change (mediawiki...RestBaseUpdateJobs)
Mobrovac has submitted this change and it was merged.
Change subject: Consistency tweak in preparation for adding extension to
translatewiki
..
Consistency tweak in preparation for adding extension to translatewiki
* Use full extension name as description message key to avoid conflicts
with other extensions and for consistency
Change-Id: Ief963a535e15f2865ea41fcc51dccae6d8848e62
---
M RestbaseUpdate.php
M i18n/de.json
M i18n/en.json
M i18n/es.json
M i18n/fr.json
M i18n/it.json
M i18n/pt.json
A i18n/qqq.json
8 files changed, 15 insertions(+), 7 deletions(-)
Approvals:
Mobrovac: Verified; Looks good to me, approved
diff --git a/RestbaseUpdate.php b/RestbaseUpdate.php
index a74cde2..bed4007 100644
--- a/RestbaseUpdate.php
+++ b/RestbaseUpdate.php
@@ -45,7 +45,7 @@
),
'version' = '0.2.0',
'url' =
'https://www.mediawiki.org/wiki/Extension:RestbaseUpdateJobs',
- 'descriptionmsg' = 'restbase-desc',
+ 'descriptionmsg' = 'restbaseupdatejobs-desc',
'license-name' = 'GPL-2.0+',
);
diff --git a/i18n/de.json b/i18n/de.json
index 1024707..7badb8e 100644
--- a/i18n/de.json
+++ b/i18n/de.json
@@ -4,6 +4,6 @@
Marko Obrovac
]
},
- restbase-desc: Erweiterung für aktualisieren RESTBase
+ restbaseupdatejobs-desc: Erweiterung für aktualisieren RESTBase
}
diff --git a/i18n/en.json b/i18n/en.json
index 15e2f27..7e77d42 100644
--- a/i18n/en.json
+++ b/i18n/en.json
@@ -4,6 +4,6 @@
Marko Obrovac
]
},
- restbase-desc: RESTBase update job extension
+ restbaseupdatejobs-desc: RESTBase update job extension
}
diff --git a/i18n/es.json b/i18n/es.json
index 89e6422..25055ac 100644
--- a/i18n/es.json
+++ b/i18n/es.json
@@ -4,6 +4,6 @@
Marko Obrovac
]
},
- restbase-desc: Extensione para actualizar el contenido de RESTBase
+ restbaseupdatejobs-desc: Extensione para actualizar el contenido de
RESTBase
}
diff --git a/i18n/fr.json b/i18n/fr.json
index 94ba97a..43c2a31 100644
--- a/i18n/fr.json
+++ b/i18n/fr.json
@@ -4,6 +4,6 @@
Marko Obrovac
]
},
- restbase-desc: Extension pour la mise à jour du contenu de RESTBase
+ restbaseupdatejobs-desc: Extension pour la mise à jour du contenu de
RESTBase
}
diff --git a/i18n/it.json b/i18n/it.json
index 21ab5d1..244e79a 100644
--- a/i18n/it.json
+++ b/i18n/it.json
@@ -4,6 +4,6 @@
Marko Obrovac
]
},
- restbase-desc: Estonsione per l'aggiornamento del contenuto di
RESTBase
+ restbaseupdatejobs-desc: Estonsione per l'aggiornamento del
contenuto di RESTBase
}
diff --git a/i18n/pt.json b/i18n/pt.json
index 6ed36bf..b8fab92 100644
--- a/i18n/pt.json
+++ b/i18n/pt.json
@@ -4,6 +4,6 @@
Marko Obrovac
]
},
- parsoid-desc: Extensão para a atualização do conteúdo de RESTBase
+ restbaseupdatejobs-desc: Extensão para a atualização do conteúdo de
RESTBase
}
diff --git a/i18n/qqq.json b/i18n/qqq.json
new file mode 100644
index 000..fcf51da
--- /dev/null
+++ b/i18n/qqq.json
@@ -0,0 +1,8 @@
+{
+ @metadata: {
+ authors: [
+ Raimond Spekking
+ ]
+ },
+ restbaseupdatejobs-desc: {{desc|name=Restbase Update
Jobs|url=https://www.mediawiki.org/wiki/Extension:RestbaseUpdateJobs}};
+}
--
To view, visit https://gerrit.wikimedia.org/r/188775
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ief963a535e15f2865ea41fcc51dccae6d8848e62
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/RestBaseUpdateJobs
Gerrit-Branch: master
Gerrit-Owner: Raimond Spekking raimond.spekk...@gmail.com
Gerrit-Reviewer: Mobrovac mobro...@wikimedia.org
Gerrit-Reviewer: Siebrand siebr...@kitano.nl
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Adapt the extension to RESTBase's needs - change (mediawiki...RestBaseUpdateJobs)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/188952
Change subject: Adapt the extension to RESTBase's needs
..
Adapt the extension to RESTBase's needs
The extension monitors the following hooks:
- ArticleEditUpdates
- ArticleDeleteComplete
- ArticleUndelete
- ArticleRevisionVisibilitySet
- TitleMoveComplete
- FileUpload
For each case, a queue job is created invalidating RESTBase's version of
the page (by sending a request with the no-cache header). Additional
jobs are created for any dependent pages which transclude the edited or
deleted page.
Bug: T87520
Change-Id: I9814249f350e2278f3941d62255e4ee9bb89c4b1
---
A CurlMultiClient.php
M RestbaseUpdate.hooks.php
M RestbaseUpdate.php
M RestbaseUpdateJob.php
4 files changed, 339 insertions(+), 131 deletions(-)
git pull
ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/RestBaseUpdateJobs
refs/changes/52/188952/1
diff --git a/CurlMultiClient.php b/CurlMultiClient.php
new file mode 100644
index 000..61928a4
--- /dev/null
+++ b/CurlMultiClient.php
@@ -0,0 +1,113 @@
+?php
+
+/**
+ * A simple parallel CURL client helper class
+ */
+class CurlMultiClient {
+
+ /**
+* Get the default CURL options used for each request
+*
+* @static
+* @return array default options
+*/
+ public static function getDefaultOptions() {
+ return array(
+ CURLOPT_HEADER = 0,
+ CURLOPT_RETURNTRANSFER = 1
+ );
+ }
+
+ /**
+* Peform several CURL requests in parallel, and return the combined
+* results.
+*
+* @static
+* @param $requests array requests, each with an url and an optional
+* 'headers' member:
+*array(
+* 'url' = 'http://server.com/foo',
+* 'headers' = array( 'X-Foo: Bar' )
+*)
+* @param $options array curl options used for each request, default
+* {CurlMultiClient::getDefaultOptions}.
+* @return array An array of arrays containing 'error' and 'data'
+* members. If there are errors, data will be null. If there are no
+* errors, the error member will be null and data will contain the
+* response data as a string.
+*/
+ public static function request( $requests, array $options = null ) {
+ if ( !count( $requests ) ) {
+ return array();
+ }
+
+ $handles = array();
+
+ if ( $options === null ) { // add default options
+ $options = CurlMultiClient::getDefaultOptions();
+ }
+
+ // add curl options to each handle
+ foreach ( $requests as $k = $row ) {
+ $handle = curl_init();
+ $reqOptions = array(
+ CURLOPT_URL = $row['url'],
+ // https://github.com/guzzle/guzzle/issues/349
+ CURLOPT_FORBID_REUSE = true
+ ) + $options;
+ wfDebug( adding url: . $row['url'] );
+ if ( isset( $row['headers'] ) ) {
+ $reqOptions[CURLOPT_HTTPHEADER] =
$row['headers'];
+ }
+ curl_setopt_array( $handle, $reqOptions );
+
+ $handles[$k] = $handle;
+ }
+
+ $mh = curl_multi_init();
+
+ foreach ( $handles as $handle ) {
+ curl_multi_add_handle( $mh, $handle );
+ }
+
+ $active = null; // handles still being processed
+ //execute the handles
+ do {
+ do {
+ // perform work as long as there is any
+ $status_cme = curl_multi_exec( $mh, $active );
+ } while ( $status_cme == CURLM_CALL_MULTI_PERFORM );
+ if ( $active 0 $status_cme === CURLM_OK ) {
+ // wait for more work to become available
+ if ( curl_multi_select( $mh, 10 ) ) {
+ // Wait for 5 ms, somewhat similar to
the suggestion at
+ //
http://curl.haxx.se/libcurl/c/curl_multi_fdset.html
+ // We pick a smaller value as we are
typically hitting
+ // fast internal services so status
changes are more
+ // likely.
+ usleep(5000);
+ }
+ }
+ } while ( $active $status_cme ==
[MediaWiki-commits] [Gerrit] vbench: Fix minor bug in std() - change (operations/puppet)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/189477
Change subject: vbench: Fix minor bug in std()
..
vbench: Fix minor bug in std()
std(), introduced in https://gerrit.wikimedia.org/r/#/c/189305/ ,
calculates the standard deviation of a given data set. However, there
was a small bug using an undefined variable - 'c' - rendering the
calculation incorrect. This path fixes it by using the mean value
computed in the previous step.
Change-Id: I32ed2ab16da7b952e61f6d4059b46691b33509c8
---
M files/ve/vbench
1 file changed, 1 insertion(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/77/189477/1
diff --git a/files/ve/vbench b/files/ve/vbench
index ee0b764..88416fb 100755
--- a/files/ve/vbench
+++ b/files/ve/vbench
@@ -112,7 +112,7 @@
if n == 0:
raise ValueError('cannot compute stdev of empty set')
m = mean(data)
-ss = sum((x - c) ** 2 for x in data)
+ss = sum((x - m) ** 2 for x in data)
return (ss / n) ** 0.5
--
To view, visit https://gerrit.wikimedia.org/r/189477
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I32ed2ab16da7b952e61f6d4059b46691b33509c8
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Mobrovac mobro...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Update RestBaseUpdateJobs to send the correct HTTP headers - change (mediawiki/core)
Mobrovac has uploaded a new change for review. https://gerrit.wikimedia.org/r/197066 Change subject: Update RestBaseUpdateJobs to send the correct HTTP headers .. Update RestBaseUpdateJobs to send the correct HTTP headers For this change: 196865 Set HTTP headers as an associative array Change-Id: I1257095e166174e299d7708f5040a3867eb80397 --- M extensions/RestBaseUpdateJobs 1 file changed, 0 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core refs/changes/66/197066/1 diff --git a/extensions/RestBaseUpdateJobs b/extensions/RestBaseUpdateJobs index d52eee9..e791862 16 --- a/extensions/RestBaseUpdateJobs +++ b/extensions/RestBaseUpdateJobs -Subproject commit d52eee9c77f1ffd940390471b62c30aa130b18d0 +Subproject commit e79186206da8fe627e1ba845858cde5364e575b0 -- To view, visit https://gerrit.wikimedia.org/r/197066 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I1257095e166174e299d7708f5040a3867eb80397 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/core Gerrit-Branch: wmf/1.25wmf20 Gerrit-Owner: Mobrovac mobro...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Add field websiteTitle in itemType webpage - change (mediawiki...citoid)
Mobrovac has submitted this change and it was merged.
Change subject: Add field websiteTitle in itemType webpage
..
Add field websiteTitle in itemType webpage
Field publicationTitle was erroneously being
used instead of field websiteTitle in the native
scraper for items of type webpage. This is a
temporary change in a backwards compatible way
until the erroneous field can be removed safely.
* Fix translators/openGraph.js to have the correct
field websiteTitle instead of publicationTitle.
* Convert backup behaviour for publicationTitle
in lib/Scraper.js to create field websiteTitle
instead of publicationTitle
* Add temporary statement to copy value of field
websiteTitle to field publicationTitle for
backwards compatibility until TemplateData
can be updated on all wikis.
* Add test ensuring both fields websiteTitle
and publicationTitle are present in natively
scraped webpage citations.
Bug: T93513
Change-Id: I2c9d81cea144ddd3bc5bac917cbbeb629d5aa608
---
M lib/Scraper.js
M lib/translators/openGraph.js
M test/index.js
3 files changed, 39 insertions(+), 6 deletions(-)
Approvals:
Mobrovac: Verified; Looks good to me, approved
diff --git a/lib/Scraper.js b/lib/Scraper.js
index c4e7c55..4e6e43c 100644
--- a/lib/Scraper.js
+++ b/lib/Scraper.js
@@ -112,7 +112,7 @@
* @param {Function} callback callback(citation)
*/
Scraper.prototype.parseHTML = function(url, chtml, citation, callback){
- var metaData, typeTranslator;
+ var metaData, typeTranslator, parsedUrl;
parseMetaData(chtml, function(err, results){
metaData = results; //only use open graph here
@@ -150,13 +150,20 @@
// Access date - universal - format -MM-DD
citation.accessDate = (new Date()).toISOString().substring(0, 10);
- // Fall back publication title - webpage only
- if (!citation.publicationTitle citation.itemType === 'webpage'){
- var parsedUrl = urlParse.parse(url);
+ // Fall back websiteTitle - webpage only
+ if (citation.itemType === 'webpage' !citation.websiteTitle){
+ parsedUrl = urlParse.parse(url);
if (citation.title parsedUrl parsedUrl.hostname) {
- citation.publicationTitle = parsedUrl.hostname;
+ citation.websiteTitle = parsedUrl.hostname;
}
}
+
+ // Fall back publicationTitle - webpage only
+ // TODO: REMOVE BLOCK - temporarily kept in for backwards compatibility
+ if (citation.itemType === 'webpage' citation.websiteTitle){
+ citation.publicationTitle = citation.websiteTitle;
+ }
+
callback(citation);
};
diff --git a/lib/translators/openGraph.js b/lib/translators/openGraph.js
index 823ba80..b178ffc 100644
--- a/lib/translators/openGraph.js
+++ b/lib/translators/openGraph.js
@@ -43,7 +43,7 @@
* @type {Object}
*/
exports.webpage = {
- site_name: 'publicationTitle' // prefix og: general property, but
should only be assigned if type webpage is used
+ site_name: 'websiteTitle' // prefix og: general property, but should
only be assigned if type webpage is used
};
exports.videoRecording = {
diff --git a/test/index.js b/test/index.js
index 365f1f1..95ad698 100644
--- a/test/index.js
+++ b/test/index.js
@@ -206,6 +206,32 @@
});
});
+
+describe('websiteTitle', function() {
+
+ var opts = {
+ search :
'http://blog.woorank.com/2013/04/dublin-core-metadata-for-seo-and-usability/',
+ format : 'mediawiki',
+ acceptLanguage : 'en'
+ };
+
+ it('should contain a websiteTitle and a publicationTitle',
function(done) {
+ citoidService.request(opts, function(error, responseCode,
citation){
+ if (error) {throw error;}
+ if (responseCode !== 200){
+ throw new Error('Should respond 200: Response
code is ' + responseCode);
+ }
+ if (!citation) {throw new Error ('Empty body');}
+ if (!citation[0].publicationTitle){
+ throw new Error('Should contain field
publicationTitle');
+ }
+ if (!citation[0].websiteTitle){
+ throw new Error('Should contain field
websiteTitle');
+ }
+ done();
+ });
+ });
+});
describe('scrape open graph', function() {
var opts = {
--
To view, visit https://gerrit.wikimedia.org/r/198554
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I2c9d81cea144ddd3bc5bac917cbbeb629d5aa608
Gerrit-PatchSet: 3
Gerrit-Project: mediawiki/services/citoid
Gerrit-Branch: master
Gerrit-Owner: Mvolz mv...@wikimedia.org
Gerrit-Reviewer: Mobrovac
[MediaWiki-commits] [Gerrit] Update front page to use /api endpoint - change (mediawiki...citoid)
Mobrovac has submitted this change and it was merged.
Change subject: Update front page to use /api endpoint
..
Update front page to use /api endpoint
* Move front page html from server.js into
static/index.html and serve static/ folder.
* Modify html to use api/ GET endpoint
instead of url/ POST endpoint.
Bug: T93518
Change-Id: I9af748f120d8feea80e3011234afedde9605929c
---
M server.js
A static/index.html
2 files changed, 17 insertions(+), 22 deletions(-)
Approvals:
Mobrovac: Verified; Looks good to me, approved
diff --git a/server.js b/server.js
index d3913fc..7f8711f 100644
--- a/server.js
+++ b/server.js
@@ -52,28 +52,8 @@
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({extended: false}));
-app.use(express.static('api')); //cache api pages
-
-/* Landing Page */
-app.get('/', function(req, res){
- res.setHeader(Content-Type, text/html);
- res.send('!DOCTYPE html\
-html\
- head\
- meta charset=UTF-8\
- titleCitoid service/title\
-/head\
-body\
- h1Citoid/h1\
- h2a href=https://www.mediawiki.org/wiki/Citoid;
target=_blankDocumentation/a/h2\
- h2Test request/h2\
- form action=/url method=POST\
- input type=hidden name=format value=mediawiki /\
- pURL: input name=url size=100
value=http://link.springer.com/chapter/10.1007/11926078_68; / input
type=submit //p\
- /form\
-/body/html\
- ');
-});
+app.use(express.static('api')); // Cache api pages
+app.use(express.static(__dirname + '/static')); // Static HTML files
/* Endpoint for retrieving citations in JSON format from a URL */
app.post('/url', function(req, res){
diff --git a/static/index.html b/static/index.html
new file mode 100644
index 000..3b0f8db
--- /dev/null
+++ b/static/index.html
@@ -0,0 +1,15 @@
+!DOCTYPE html
+html
+ head
+ meta charset=UTF-8
+ titleCitoid service/title
+/head
+body
+ h1Citoid/h1
+ h2a href=https://www.mediawiki.org/wiki/Citoid;
target=_blankDocumentation/a/h2
+ h2Test request/h2
+ form action=/api method=GET
+ input type=hidden name=format value=mediawiki /
+ pURL: input name=search size=100
value=http://link.springer.com/chapter/10.1007/11926078_68; / input
type=submit //p
+ /form
+/body/html
\ No newline at end of file
--
To view, visit https://gerrit.wikimedia.org/r/198555
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I9af748f120d8feea80e3011234afedde9605929c
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/citoid
Gerrit-Branch: master
Gerrit-Owner: Mvolz mv...@wikimedia.org
Gerrit-Reviewer: Catrope roan.katt...@gmail.com
Gerrit-Reviewer: Mobrovac mobro...@wikimedia.org
Gerrit-Reviewer: jenkins-bot
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Update citoid submodule to master (b32713b) - change (mediawiki...deploy)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/198648
Change subject: Update citoid submodule to master (b32713b)
..
Update citoid submodule to master (b32713b)
New changes:
168578b Fix global memory leak when scraping OG data
0a8a9df Update front page to use /api endpoint
01d8f6a Add field websiteTitle in itemType webpage
26ca7db Don't add openGraph 'locale' to language field
b32713b Validate language codes in mediawiki format
Change-Id: I601f005da53d845194280d9026b21031ef8e938c
---
M node_modules/body-parser/package.json
M node_modules/cheerio/node_modules/htmlparser2/package.json
M node_modules/cheerio/package.json
M node_modules/express/node_modules/cookie/package.json
M node_modules/express/node_modules/proxy-addr/HISTORY.md
M
node_modules/express/node_modules/proxy-addr/node_modules/ipaddr.js/ipaddr.min.js
M
node_modules/express/node_modules/proxy-addr/node_modules/ipaddr.js/lib/ipaddr.js
M
node_modules/express/node_modules/proxy-addr/node_modules/ipaddr.js/package.json
M
node_modules/express/node_modules/proxy-addr/node_modules/ipaddr.js/src/ipaddr.coffee
M
node_modules/express/node_modules/proxy-addr/node_modules/ipaddr.js/test/ipaddr.test.coffee
M node_modules/express/node_modules/proxy-addr/package.json
M node_modules/express/node_modules/send/node_modules/mime/package.json
M
node_modules/grunt-contrib-jshint/node_modules/jshint/node_modules/htmlparser2/package.json
M
node_modules/grunt-simple-mocha/node_modules/mocha/node_modules/commander/package.json
M
node_modules/grunt-simple-mocha/node_modules/mocha/node_modules/mkdirp/package.json
M node_modules/grunt/node_modules/dateformat/package.json
M node_modules/grunt/node_modules/findup-sync/node_modules/lodash/package.json
M
node_modules/grunt/node_modules/grunt-legacy-log/node_modules/lodash/package.json
M node_modules/grunt/package.json
A node_modules/html-metadata/.jshintignore
A node_modules/html-metadata/.jshintrc
A node_modules/html-metadata/.npmignore
M node_modules/html-metadata/index.js
M
node_modules/html-metadata/node_modules/microdata-node/node_modules/htmlparser2/package.json
M node_modules/html-metadata/package.json
A node_modules/html-metadata/test/index.js
M node_modules/node-txstatsd/package.json
M node_modules/path/package.json
M
node_modules/request/node_modules/bl/node_modules/readable-stream/node_modules/inherits/package.json
M
node_modules/request/node_modules/combined-stream/node_modules/delayed-stream/package.json
M
node_modules/request/node_modules/http-signature/node_modules/asn1/package.json
M
node_modules/request/node_modules/http-signature/node_modules/ctype/package.json
M src
33 files changed, 150 insertions(+), 50 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/citoid/deploy
refs/changes/48/198648/1
diff --git a/node_modules/body-parser/package.json
b/node_modules/body-parser/package.json
index e602960..57d0158 100644
--- a/node_modules/body-parser/package.json
+++ b/node_modules/body-parser/package.json
@@ -92,5 +92,6 @@
tarball: http://registry.npmjs.org/body-parser/-/body-parser-1.10.0.tgz;
},
directories: {},
- _resolved:
https://registry.npmjs.org/body-parser/-/body-parser-1.10.0.tgz;
+ _resolved:
https://registry.npmjs.org/body-parser/-/body-parser-1.10.0.tgz;,
+ readme: ERROR: No README data found!
}
diff --git a/node_modules/cheerio/node_modules/htmlparser2/package.json
b/node_modules/cheerio/node_modules/htmlparser2/package.json
index c3374b0..7596b24 100644
--- a/node_modules/cheerio/node_modules/htmlparser2/package.json
+++ b/node_modules/cheerio/node_modules/htmlparser2/package.json
@@ -75,7 +75,7 @@
homepage: https://github.com/fb55/htmlparser2;,
_id: htmlparser2@3.8.2,
_shasum: 0d6bc3471d01e9766fc2c274cbac1d55b36c009c,
- _from: htmlparser2@=3.8.0 3.9.0,
+ _from: htmlparser2@=3.8.1 3.9.0,
_npmVersion: 2.1.5,
_nodeVersion: 0.10.32,
_npmUser: {
diff --git a/node_modules/cheerio/package.json
b/node_modules/cheerio/package.json
index 251cbf9..406cc0f 100644
--- a/node_modules/cheerio/package.json
+++ b/node_modules/cheerio/package.json
@@ -76,5 +76,6 @@
tarball: http://registry.npmjs.org/cheerio/-/cheerio-0.18.0.tgz;
},
directories: {},
- _resolved: https://registry.npmjs.org/cheerio/-/cheerio-0.18.0.tgz;
+ _resolved: https://registry.npmjs.org/cheerio/-/cheerio-0.18.0.tgz;,
+ readme: ERROR: No README data found!
}
diff --git a/node_modules/express/node_modules/cookie/package.json
b/node_modules/express/node_modules/cookie/package.json
index 80aecf4..7dbb395 100644
--- a/node_modules/express/node_modules/cookie/package.json
+++ b/node_modules/express/node_modules/cookie/package.json
@@ -49,5 +49,6 @@
],
directories: {},
_shasum: 72fec3d24e48a3432073d90c12642005061004b1,
- _resolved: https://registry.npmjs.org/cookie/-/cookie-0.1.2.tgz;
+ _resolved:
[MediaWiki-commits] [Gerrit] Update citoid submodule to master (b32713b) - change (mediawiki...deploy)
Mobrovac has submitted this change and it was merged.
Change subject: Update citoid submodule to master (b32713b)
..
Update citoid submodule to master (b32713b)
New changes:
168578b Fix global memory leak when scraping OG data
0a8a9df Update front page to use /api endpoint
01d8f6a Add field websiteTitle in itemType webpage
26ca7db Don't add openGraph 'locale' to language field
b32713b Validate language codes in mediawiki format
Change-Id: I601f005da53d845194280d9026b21031ef8e938c
---
M node_modules/body-parser/package.json
M node_modules/cheerio/node_modules/htmlparser2/package.json
M node_modules/cheerio/package.json
M node_modules/express/node_modules/cookie/package.json
M node_modules/express/node_modules/proxy-addr/HISTORY.md
M
node_modules/express/node_modules/proxy-addr/node_modules/ipaddr.js/ipaddr.min.js
M
node_modules/express/node_modules/proxy-addr/node_modules/ipaddr.js/lib/ipaddr.js
M
node_modules/express/node_modules/proxy-addr/node_modules/ipaddr.js/package.json
M
node_modules/express/node_modules/proxy-addr/node_modules/ipaddr.js/src/ipaddr.coffee
M
node_modules/express/node_modules/proxy-addr/node_modules/ipaddr.js/test/ipaddr.test.coffee
M node_modules/express/node_modules/proxy-addr/package.json
M node_modules/express/node_modules/send/node_modules/mime/package.json
M
node_modules/grunt-contrib-jshint/node_modules/jshint/node_modules/htmlparser2/package.json
M
node_modules/grunt-simple-mocha/node_modules/mocha/node_modules/commander/package.json
M
node_modules/grunt-simple-mocha/node_modules/mocha/node_modules/mkdirp/package.json
M node_modules/grunt/node_modules/dateformat/package.json
M node_modules/grunt/node_modules/findup-sync/node_modules/lodash/package.json
M
node_modules/grunt/node_modules/grunt-legacy-log/node_modules/lodash/package.json
M node_modules/grunt/package.json
A node_modules/html-metadata/.jshintignore
A node_modules/html-metadata/.jshintrc
A node_modules/html-metadata/.npmignore
M node_modules/html-metadata/index.js
M
node_modules/html-metadata/node_modules/microdata-node/node_modules/htmlparser2/package.json
M node_modules/html-metadata/package.json
A node_modules/html-metadata/test/index.js
M node_modules/node-txstatsd/package.json
M node_modules/path/package.json
M
node_modules/request/node_modules/bl/node_modules/readable-stream/node_modules/inherits/package.json
M
node_modules/request/node_modules/combined-stream/node_modules/delayed-stream/package.json
M
node_modules/request/node_modules/http-signature/node_modules/asn1/package.json
M
node_modules/request/node_modules/http-signature/node_modules/ctype/package.json
M src
33 files changed, 150 insertions(+), 50 deletions(-)
Approvals:
Mobrovac: Verified; Looks good to me, approved
diff --git a/node_modules/body-parser/package.json
b/node_modules/body-parser/package.json
index e602960..57d0158 100644
--- a/node_modules/body-parser/package.json
+++ b/node_modules/body-parser/package.json
@@ -92,5 +92,6 @@
tarball: http://registry.npmjs.org/body-parser/-/body-parser-1.10.0.tgz;
},
directories: {},
- _resolved:
https://registry.npmjs.org/body-parser/-/body-parser-1.10.0.tgz;
+ _resolved:
https://registry.npmjs.org/body-parser/-/body-parser-1.10.0.tgz;,
+ readme: ERROR: No README data found!
}
diff --git a/node_modules/cheerio/node_modules/htmlparser2/package.json
b/node_modules/cheerio/node_modules/htmlparser2/package.json
index c3374b0..7596b24 100644
--- a/node_modules/cheerio/node_modules/htmlparser2/package.json
+++ b/node_modules/cheerio/node_modules/htmlparser2/package.json
@@ -75,7 +75,7 @@
homepage: https://github.com/fb55/htmlparser2;,
_id: htmlparser2@3.8.2,
_shasum: 0d6bc3471d01e9766fc2c274cbac1d55b36c009c,
- _from: htmlparser2@=3.8.0 3.9.0,
+ _from: htmlparser2@=3.8.1 3.9.0,
_npmVersion: 2.1.5,
_nodeVersion: 0.10.32,
_npmUser: {
diff --git a/node_modules/cheerio/package.json
b/node_modules/cheerio/package.json
index 251cbf9..406cc0f 100644
--- a/node_modules/cheerio/package.json
+++ b/node_modules/cheerio/package.json
@@ -76,5 +76,6 @@
tarball: http://registry.npmjs.org/cheerio/-/cheerio-0.18.0.tgz;
},
directories: {},
- _resolved: https://registry.npmjs.org/cheerio/-/cheerio-0.18.0.tgz;
+ _resolved: https://registry.npmjs.org/cheerio/-/cheerio-0.18.0.tgz;,
+ readme: ERROR: No README data found!
}
diff --git a/node_modules/express/node_modules/cookie/package.json
b/node_modules/express/node_modules/cookie/package.json
index 80aecf4..7dbb395 100644
--- a/node_modules/express/node_modules/cookie/package.json
+++ b/node_modules/express/node_modules/cookie/package.json
@@ -49,5 +49,6 @@
],
directories: {},
_shasum: 72fec3d24e48a3432073d90c12642005061004b1,
- _resolved: https://registry.npmjs.org/cookie/-/cookie-0.1.2.tgz;
+ _resolved: https://registry.npmjs.org/cookie/-/cookie-0.1.2.tgz;,
+ readme: ERROR: No README data found!
}
diff --git
[MediaWiki-commits] [Gerrit] Validate language codes in mediawiki format - change (mediawiki...citoid)
Mobrovac has submitted this change and it was merged.
Change subject: Validate language codes in mediawiki format
..
Validate language codes in mediawiki format
* Add fixLang method to lib/ZoteroService which
uses regex to loosely fix and validate language
codes:
1. Replaces underscores with dashes.
2. Language codes must consist of two alpha
characters with an optional group starting with
'-' and consisting of any number of alpha
characters.
* Call fixLang from within
ZoteroService.prototype.convertToMediawiki
function.
* Change OG test to verify that bad language codes
are deleted from the citation.
Bug: T93337
Change-Id: I797cc30ebc278a6b8d8a310ab68665bf7341b353
---
M lib/ZoteroService.js
M test/index.js
2 files changed, 26 insertions(+), 10 deletions(-)
Approvals:
Mobrovac: Looks good to me, approved
diff --git a/lib/ZoteroService.js b/lib/ZoteroService.js
index d6df076..b15613a 100644
--- a/lib/ZoteroService.js
+++ b/lib/ZoteroService.js
@@ -173,7 +173,8 @@
replaceCreators,
addPubMedIdentifiers,
fixISBN,
- fixISSN
+ fixISSN,
+ fixLang
], function (err, citation) {
callback([citation]);
});
@@ -311,7 +312,7 @@
/**
* Replace Zotero output of CURRENT_TIMESTAMP with ISO time
* @param {Object} citation citation object
- * @param {Function} callback callback on citation object
+ * @param {Function} callback callback(error, citation)
*/
function fixAccessDate(citation, callback){
if (!citation.accessDate || (citation.accessDate ===
CURRENT_TIMESTAMP)){
@@ -323,7 +324,7 @@
/**
* Convert String of ISSNs into an Array of ISSNs
* @param {Object} citation citation object
- * @param {Function} callback callback on citation object
+ * @param {Function} callback callback(error, citation)
*/
function fixISSN(citation, callback){
var match, i, reISSN,
@@ -349,7 +350,7 @@
/**
* Convert String of ISBNs into an Array of ISBNs
* @param {Object} citation citation object
- * @param {Function} callback callback on citation object
+ * @param {Function} callback callback(error, citation)
*/
function fixISBN(citation, callback){
var match, i, reISBN,
@@ -372,6 +373,21 @@
callback(null, citation);
}
+/**
+ * Validate language codes
+ * @param {Object} citation citation object
+ * @param {Function} callback callback(error, citation)
+ */
+function fixLang(citation, callback){
+ if (citation.language) {
+ citation.language = citation.language.replace('_', '-');
+ if (!/^[a-z]{2}(?:-?[a-z]{2,})*$/i.test(citation.language)){
+ delete citation.language;
+ }
+ }
+ callback(null, citation);
+}
+
/* Exports */
module.exports = ZoteroService;
diff --git a/test/index.js b/test/index.js
index 95ad698..cfd87c5 100644
--- a/test/index.js
+++ b/test/index.js
@@ -206,7 +206,6 @@
});
});
-
describe('websiteTitle', function() {
var opts = {
@@ -232,23 +231,24 @@
});
});
});
-describe('scrape open graph', function() {
+
+describe('invalid language code', function() {
var opts = {
- search :
'http://www.pbs.org/newshour/making-sense/care-peoples-kids/',
+ search : 'http://www.ncbi.nlm.nih.gov/pubmed/23555203',
format : 'mediawiki',
acceptLanguage : 'en'
};
- it('should correctly scrape open graph data', function(done) {
+ it('should delete invalid language code', function(done) {
citoidService.request(opts, function(error, responseCode,
citation){
if (error) {throw error;}
if (responseCode !== 200){
throw new Error('Should respond 200: Response
code is ' + responseCode);
}
if (!citation) {throw new Error ('Empty body');}
- if (!citation[0].language){
- throw new Error('Should contain language code');
+ if (citation[0].language){
+ throw new Error('Should not contain language
code');
}
done();
});
--
To view, visit https://gerrit.wikimedia.org/r/198491
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I797cc30ebc278a6b8d8a310ab68665bf7341b353
Gerrit-PatchSet: 6
Gerrit-Project: mediawiki/services/citoid
Gerrit-Branch: master
Gerrit-Owner: Mvolz mv...@wikimedia.org
Gerrit-Reviewer: Mobrovac mobro...@wikimedia.org
Gerrit-Reviewer: Mvolz mv...@wikimedia.org
Gerrit-Reviewer: jenkins-bot
[MediaWiki-commits] [Gerrit] Set HTTP headers as an associative array - change (mediawiki...RestBaseUpdateJobs)
Mobrovac has submitted this change and it was merged. Change subject: Set HTTP headers as an associative array .. Set HTTP headers as an associative array MultiHttpClient expects headers in an associative array. The extension instead sent headers as an array of strings of the form 'Cache-control: no-cache', which silently produced HTTP headers of the form '0: Cache-control: no-cache'. Needless to say, that rendered those headers useless, which in turn caused template updates to not happen as the no-cache header was never seen. The same issue also affected revision deletion updates. Also: Fix the default server port to use RESTBase's default port. Bug: T92703 Change-Id: I57082560cb6b00d1bbe05b8f06c686475803f160 --- M RestbaseUpdate.php M RestbaseUpdateJob.php 2 files changed, 6 insertions(+), 6 deletions(-) Approvals: Alex Monk: Looks good to me, approved Mobrovac: Verified; Looks good to me, approved diff --git a/RestbaseUpdate.php b/RestbaseUpdate.php index ce17548..a8f0bf9 100644 --- a/RestbaseUpdate.php +++ b/RestbaseUpdate.php @@ -69,7 +69,7 @@ /** * The RESTBase server to inform of updates. */ - $wgRestbaseServer = 'http://localhost:7321'; + $wgRestbaseServer = 'http://localhost:7231'; /** * The RESTBase API version in use diff --git a/RestbaseUpdateJob.php b/RestbaseUpdateJob.php index 981881e..a873947 100644 --- a/RestbaseUpdateJob.php +++ b/RestbaseUpdateJob.php @@ -180,7 +180,7 @@ 'method' = 'GET', 'url' = self::getRevisionURL( $revid ), 'headers' = array( - 'Cache-control: no-cache' + 'Cache-control' = 'no-cache' ) ); } @@ -208,8 +208,8 @@ 'method' = 'GET', 'url' = self::getPageTitleURL( $title, $latest ), 'headers' = array( - 'X-Restbase-ParentRevision: ' . $previous, - 'Cache-control: no-cache' + 'X-Restbase-ParentRevision' = $previous, + 'Cache-control' = 'no-cache' ) ) ); ///wfDebug( RestbaseUpdateJob::invalidateTitle: . json_encode( $requests ) . \n ); @@ -239,8 +239,8 @@ 'method' = 'GET', 'url' = $url, 'headers' = array( - 'X-Restbase-Mode: ' . $mode, - 'Cache-control: no-cache' + 'X-Restbase-Mode' = $mode, + 'Cache-control' = 'no-cache' ) ); } -- To view, visit https://gerrit.wikimedia.org/r/196865 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I57082560cb6b00d1bbe05b8f06c686475803f160 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/extensions/RestBaseUpdateJobs Gerrit-Branch: master Gerrit-Owner: GWicke gwi...@wikimedia.org Gerrit-Reviewer: Alex Monk kren...@gmail.com Gerrit-Reviewer: Eevans eev...@wikimedia.org Gerrit-Reviewer: Krinkle krinklem...@gmail.com Gerrit-Reviewer: Mobrovac mobro...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Citoid: set the StatsD host to statsd.eqiad.wmnet - change (operations/puppet)
Mobrovac has uploaded a new change for review. https://gerrit.wikimedia.org/r/197310 Change subject: Citoid: set the StatsD host to statsd.eqiad.wmnet .. Citoid: set the StatsD host to statsd.eqiad.wmnet Bug: T87496 Change-Id: I55765038c98f0a67fa23e54c18632034b9b7714b --- M hieradata/common/citoid.yaml 1 file changed, 2 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/10/197310/1 diff --git a/hieradata/common/citoid.yaml b/hieradata/common/citoid.yaml index bde4591..f450cbb 100644 --- a/hieradata/common/citoid.yaml +++ b/hieradata/common/citoid.yaml @@ -2,3 +2,5 @@ http_proxy: url-downloader.wikimedia.org:8080 zotero_host: zotero.svc.eqiad.wmnet zotero_port: 1969 +statsd_host: statsd.eqiad.wmnet +statsd_port: 8125 -- To view, visit https://gerrit.wikimedia.org/r/197310 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I55765038c98f0a67fa23e54c18632034b9b7714b Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Mobrovac mobro...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Citoid: Set the protocol for the proxy - change (operations/puppet)
Mobrovac has uploaded a new change for review. https://gerrit.wikimedia.org/r/197884 Change subject: Citoid: Set the protocol for the proxy .. Citoid: Set the protocol for the proxy Bug: T93157 Change-Id: I4e832ee57becc60135ac0ce93cdd6eb917bf9a9d --- M hieradata/common/citoid.yaml 1 file changed, 1 insertion(+), 1 deletion(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/84/197884/1 diff --git a/hieradata/common/citoid.yaml b/hieradata/common/citoid.yaml index 80c5111..21353cb 100644 --- a/hieradata/common/citoid.yaml +++ b/hieradata/common/citoid.yaml @@ -1,3 +1,3 @@ -http_proxy: url-downloader.wikimedia.org:8080 +http_proxy: http://url-downloader.wikimedia.org:8080 zotero_host: zotero.svc.eqiad.wmnet statsd_host: statsd.eqiad.wmnet -- To view, visit https://gerrit.wikimedia.org/r/197884 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I4e832ee57becc60135ac0ce93cdd6eb917bf9a9d Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Mobrovac mobro...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Fix: Stop crashes when PMID and PCMIDs are entered - change (mediawiki...citoid)
Mobrovac has submitted this change and it was merged.
Change subject: Fix: Stop crashes when PMID and PCMIDs are entered
..
Fix: Stop crashes when PMID and PCMIDs are entered
* Add test for pmid input
* Fixed errors in CitoidService.prototype.requestFromPubMedID
which used the wrong instance of 'this'
Bug: T93335
Change-Id: I98205962845437d1f65d54fe62a9e28101665741
---
M lib/CitoidService.js
M test/index.js
2 files changed, 31 insertions(+), 2 deletions(-)
Approvals:
Mobrovac: Verified; Looks good to me, approved
diff --git a/lib/CitoidService.js b/lib/CitoidService.js
index adae651..25a6065 100644
--- a/lib/CitoidService.js
+++ b/lib/CitoidService.js
@@ -129,14 +129,15 @@
* @param {Function} callback callback (error, statusCode, body)
*/
CitoidService.prototype.requestFromPubMedID = function(opts, callback){
+var citoidService = this;
pubMedRequest(opts.search, function(error, obj){
if(error){
callback(error, null, null);
} else {
var doi = obj.records[0].doi;
- this.log.info(Got DOI + doi);
+ citoidService.log.info(Got DOI + doi);
opts.search = doi;
- this.requestFromDOI(opts, callback);
+ citoidService.requestFromDOI(opts, callback);
}
});
};
diff --git a/test/index.js b/test/index.js
index 97366ac..cc7985c 100644
--- a/test/index.js
+++ b/test/index.js
@@ -37,6 +37,34 @@
citoidService = new CitoidService(citoidConfig, log);
+describe('pmid', function() {
+
+ var opts = {
+ search : '23555203',
+ format : 'mediawiki',
+ acceptLanguage : 'en'
+ },
+ expectedTitle = 'Viral Phylodynamics';
+
+ it('should scrape info successfully', function(done) {
+ citoidService.request(opts, function(error, responseCode,
citation){
+ if (error) {throw error;}
+ if (responseCode !== 200){
+ throw new Error('Not successful: Response code
is' + responseCode);
+ }
+ if (!citation) {throw new Error ('Empty body');}
+ if (citation[0].title !== expectedTitle){
+ throw new Error('Expected title is: ' +
expectedTitle +
+ ;\nGot: + citation[0].title);
+ }
+ if (!citation[0].itemType){
+ throw new Error('Missing itemType');
+ }
+ done();
+ });
+ });
+});
+
describe('200', function() {
var opts = {
--
To view, visit https://gerrit.wikimedia.org/r/198204
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I98205962845437d1f65d54fe62a9e28101665741
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/citoid
Gerrit-Branch: master
Gerrit-Owner: Mvolz mv...@wikimedia.org
Gerrit-Reviewer: Mobrovac mobro...@wikimedia.org
Gerrit-Reviewer: jenkins-bot
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Bump src to 6152c80 - change (mediawiki...deploy)
Mobrovac has submitted this change and it was merged. Change subject: Bump src to 6152c80 .. Bump src to 6152c80 Change-Id: I5a3dc3374b2e633363f348e93de9bad3daf203b6 --- M src 1 file changed, 0 insertions(+), 0 deletions(-) Approvals: Mobrovac: Verified; Looks good to me, approved diff --git a/src b/src index fd026c6..6152c80 16 --- a/src +++ b/src -Subproject commit fd026c69c99f8f78878fb771920c582666e692d0 +Subproject commit 6152c80c5ca28022ac0e4aed0003139e6ebf6efe -- To view, visit https://gerrit.wikimedia.org/r/198206 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I5a3dc3374b2e633363f348e93de9bad3daf203b6 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/services/citoid/deploy Gerrit-Branch: master Gerrit-Owner: Mobrovac mobro...@wikimedia.org Gerrit-Reviewer: Mobrovac mobro...@wikimedia.org Gerrit-Reviewer: Mvolz mv...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Activate RESTBase in the Beta Cluster - change (operations/mediawiki-config)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/198221
Change subject: Activate RESTBase in the Beta Cluster
..
Activate RESTBase in the Beta Cluster
RESTBase has been set up in the Beta Cluster and has been verified to
work there. It is time to activate its revision update extension and the
VirtualRESTService configuration allowing users such as VE to use it
en lieu of Parsoid.
Bug: T91102
Change-Id: Ied3e9b8fa6b427e2d58e17d8170232f1bbdf7ac4
---
M wmf-config/CommonSettings-labs.php
M wmf-config/InitialiseSettings-labs.php
2 files changed, 28 insertions(+), 4 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/mediawiki-config
refs/changes/21/198221/1
diff --git a/wmf-config/CommonSettings-labs.php
b/wmf-config/CommonSettings-labs.php
index d5adac4..78c3314 100644
--- a/wmf-config/CommonSettings-labs.php
+++ b/wmf-config/CommonSettings-labs.php
@@ -74,6 +74,30 @@
$wgVectorBetaWinter = $wmgVectorBetaWinter;
}
+if ( $wmgUseRestbaseUpdateJobs ) {
+ require_once( $IP/extensions/RestBaseUpdateJobs/RestbaseUpdate.php );
+ $wgRestbaseServer = http://10.68.17.227:7231;; //
deployment-restbase01.eqiad.wmflabs
+}
+
+if ( $wmgUseRestbaseVRS ) {
+ if( !isset( $wgVirtualRestConfig ) ) {
+ $wgVirtualRestConfig = array(
+ 'modules' = array(),
+ 'global' = array(
+ 'timeout' = 360,
+ 'forwardCookies' = false,
+ 'HTTPProxy' = null
+ )
+ );
+ }
+ $wgVirtualRestConfig['modules']['restbase'] = array(
+ 'url' = 'http://10.68.17.189:7231', //
deployment-restbase02.eqiad.wmflabs
+ 'domain' = $wgCanonicalServer,
+ 'forwardCookies' = false,
+ 'parsoidCompat' = false
+ );
+}
+
if ( $wmgUseParsoid ) {
$wmgParsoidURL = 'http://10.68.16.145'; //
deployment-parsoidcache02.eqiad
$wgParsoidCacheServers = array ( 'http://10.68.16.145' ); //
deployment-parsoidcache01.eqiad
diff --git a/wmf-config/InitialiseSettings-labs.php
b/wmf-config/InitialiseSettings-labs.php
index c48419e..a457ae5 100644
--- a/wmf-config/InitialiseSettings-labs.php
+++ b/wmf-config/InitialiseSettings-labs.php
@@ -362,12 +362,12 @@
'default' = 1,
),
- '-wmgUseRestbaseUpdateJobs' = array(
- 'default' = false,
+ 'wmgUseRestbaseUpdateJobs' = array(
+ 'default' = true,
),
- '-wmgUseRestbaseVRS' = array(
- 'default' = false,
+ 'wmgUseRestbaseVRS' = array(
+ 'default' = true,
),
'wmgUseVectorBeta' = array(
--
To view, visit https://gerrit.wikimedia.org/r/198221
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ied3e9b8fa6b427e2d58e17d8170232f1bbdf7ac4
Gerrit-PatchSet: 1
Gerrit-Project: operations/mediawiki-config
Gerrit-Branch: master
Gerrit-Owner: Mobrovac mobro...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Update repo to fd026c6 and update node dependencies - change (mediawiki...deploy)
Mobrovac has submitted this change and it was merged. Change subject: Update repo to fd026c6 and update node dependencies .. Update repo to fd026c6 and update node dependencies Change-Id: I80b6ec8e9450bd1b66012a5e8abd52f37e648e9b --- D node_modules/bunyan/node_modules/dtrace-provider/.gitmodules D node_modules/cheerio/node_modules/htmlparser2/.gitattributes D node_modules/grunt-contrib-jshint/node_modules/jshint/node_modules/htmlparser2/.gitattributes D node_modules/grunt-simple-mocha/node_modules/mocha/node_modules/jade/node_modules/mkdirp/.gitignore.orig D node_modules/grunt-simple-mocha/node_modules/mocha/node_modules/jade/node_modules/mkdirp/.gitignore.rej D node_modules/grunt/node_modules/async/.gitmodules D node_modules/html-metadata/node_modules/microdata-node/node_modules/array-unique/.gitattributes D node_modules/html-metadata/node_modules/microdata-node/node_modules/htmlparser2/.gitattributes A node_modules/node-txstatsd/.jshintignore A node_modules/node-txstatsd/.jshintrc A node_modules/node-txstatsd/.travis.yml A node_modules/node-txstatsd/README.md A node_modules/node-txstatsd/index.js A node_modules/node-txstatsd/package.json M src 15 files changed, 299 insertions(+), 39 deletions(-) Approvals: Mobrovac: Verified; Looks good to me, approved diff --git a/node_modules/bunyan/node_modules/dtrace-provider/.gitmodules b/node_modules/bunyan/node_modules/dtrace-provider/.gitmodules deleted file mode 100644 index 933bc02..000 --- a/node_modules/bunyan/node_modules/dtrace-provider/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule libusdt] - path = libusdt - url = https://github.com/chrisa/libusdt diff --git a/node_modules/cheerio/node_modules/htmlparser2/.gitattributes b/node_modules/cheerio/node_modules/htmlparser2/.gitattributes deleted file mode 100644 index 4bb50dc..000 --- a/node_modules/cheerio/node_modules/htmlparser2/.gitattributes +++ /dev/null @@ -1,2 +0,0 @@ -# Auto detect text files and perform LF normalization -* text eol=lf \ No newline at end of file diff --git a/node_modules/grunt-contrib-jshint/node_modules/jshint/node_modules/htmlparser2/.gitattributes b/node_modules/grunt-contrib-jshint/node_modules/jshint/node_modules/htmlparser2/.gitattributes deleted file mode 100644 index 4bb50dc..000 --- a/node_modules/grunt-contrib-jshint/node_modules/jshint/node_modules/htmlparser2/.gitattributes +++ /dev/null @@ -1,2 +0,0 @@ -# Auto detect text files and perform LF normalization -* text eol=lf \ No newline at end of file diff --git a/node_modules/grunt-simple-mocha/node_modules/mocha/node_modules/jade/node_modules/mkdirp/.gitignore.orig b/node_modules/grunt-simple-mocha/node_modules/mocha/node_modules/jade/node_modules/mkdirp/.gitignore.orig deleted file mode 100644 index 9303c34..000 --- a/node_modules/grunt-simple-mocha/node_modules/mocha/node_modules/jade/node_modules/mkdirp/.gitignore.orig +++ /dev/null @@ -1,2 +0,0 @@ -node_modules/ -npm-debug.log \ No newline at end of file diff --git a/node_modules/grunt-simple-mocha/node_modules/mocha/node_modules/jade/node_modules/mkdirp/.gitignore.rej b/node_modules/grunt-simple-mocha/node_modules/mocha/node_modules/jade/node_modules/mkdirp/.gitignore.rej deleted file mode 100644 index 69244ff..000 --- a/node_modules/grunt-simple-mocha/node_modules/mocha/node_modules/jade/node_modules/mkdirp/.gitignore.rej +++ /dev/null @@ -1,5 +0,0 @@ /dev/null -+++ .gitignore -@@ -0,0 +1,2 @@ -+node_modules/ -+npm-debug.log \ No newline at end of file diff --git a/node_modules/grunt/node_modules/async/.gitmodules b/node_modules/grunt/node_modules/async/.gitmodules deleted file mode 100644 index a9aae98..000 --- a/node_modules/grunt/node_modules/async/.gitmodules +++ /dev/null @@ -1,9 +0,0 @@ -[submodule deps/nodeunit] - path = deps/nodeunit - url = git://github.com/caolan/nodeunit.git -[submodule deps/UglifyJS] - path = deps/UglifyJS - url = https://github.com/mishoo/UglifyJS.git -[submodule deps/nodelint] - path = deps/nodelint - url = https://github.com/tav/nodelint.git diff --git a/node_modules/html-metadata/node_modules/microdata-node/node_modules/array-unique/.gitattributes b/node_modules/html-metadata/node_modules/microdata-node/node_modules/array-unique/.gitattributes deleted file mode 100644 index 759c2c5..000 --- a/node_modules/html-metadata/node_modules/microdata-node/node_modules/array-unique/.gitattributes +++ /dev/null @@ -1,14 +0,0 @@ -# Enforce Unix newlines -*.* text eol=lf -*.css text eol=lf -*.html text eol=lf -*.jstext eol=lf -*.json text eol=lf -*.less text eol=lf -*.mdtext eol=lf -*.yml text eol=lf - -*.jpg binary -*.gif binary -*.png binary -*.jpeg binary \ No newline at end of file diff --git a/node_modules/html-metadata/node_modules/microdata-node/node_modules/htmlparser2/.gitattributes
[MediaWiki-commits] [Gerrit] Update repo to fd026c6 and update node dependencies - change (mediawiki...deploy)
Mobrovac has uploaded a new change for review. https://gerrit.wikimedia.org/r/197972 Change subject: Update repo to fd026c6 and update node dependencies .. Update repo to fd026c6 and update node dependencies Change-Id: I80b6ec8e9450bd1b66012a5e8abd52f37e648e9b --- D node_modules/bunyan/node_modules/dtrace-provider/.gitmodules D node_modules/cheerio/node_modules/htmlparser2/.gitattributes D node_modules/grunt-contrib-jshint/node_modules/jshint/node_modules/htmlparser2/.gitattributes D node_modules/grunt-simple-mocha/node_modules/mocha/node_modules/jade/node_modules/mkdirp/.gitignore.orig D node_modules/grunt-simple-mocha/node_modules/mocha/node_modules/jade/node_modules/mkdirp/.gitignore.rej D node_modules/grunt/node_modules/async/.gitmodules D node_modules/html-metadata/node_modules/microdata-node/node_modules/array-unique/.gitattributes D node_modules/html-metadata/node_modules/microdata-node/node_modules/htmlparser2/.gitattributes A node_modules/node-txstatsd/.jshintignore A node_modules/node-txstatsd/.jshintrc A node_modules/node-txstatsd/.travis.yml A node_modules/node-txstatsd/README.md A node_modules/node-txstatsd/index.js A node_modules/node-txstatsd/package.json M src 15 files changed, 299 insertions(+), 39 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/citoid/deploy refs/changes/72/197972/1 diff --git a/node_modules/bunyan/node_modules/dtrace-provider/.gitmodules b/node_modules/bunyan/node_modules/dtrace-provider/.gitmodules deleted file mode 100644 index 933bc02..000 --- a/node_modules/bunyan/node_modules/dtrace-provider/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule libusdt] - path = libusdt - url = https://github.com/chrisa/libusdt diff --git a/node_modules/cheerio/node_modules/htmlparser2/.gitattributes b/node_modules/cheerio/node_modules/htmlparser2/.gitattributes deleted file mode 100644 index 4bb50dc..000 --- a/node_modules/cheerio/node_modules/htmlparser2/.gitattributes +++ /dev/null @@ -1,2 +0,0 @@ -# Auto detect text files and perform LF normalization -* text eol=lf \ No newline at end of file diff --git a/node_modules/grunt-contrib-jshint/node_modules/jshint/node_modules/htmlparser2/.gitattributes b/node_modules/grunt-contrib-jshint/node_modules/jshint/node_modules/htmlparser2/.gitattributes deleted file mode 100644 index 4bb50dc..000 --- a/node_modules/grunt-contrib-jshint/node_modules/jshint/node_modules/htmlparser2/.gitattributes +++ /dev/null @@ -1,2 +0,0 @@ -# Auto detect text files and perform LF normalization -* text eol=lf \ No newline at end of file diff --git a/node_modules/grunt-simple-mocha/node_modules/mocha/node_modules/jade/node_modules/mkdirp/.gitignore.orig b/node_modules/grunt-simple-mocha/node_modules/mocha/node_modules/jade/node_modules/mkdirp/.gitignore.orig deleted file mode 100644 index 9303c34..000 --- a/node_modules/grunt-simple-mocha/node_modules/mocha/node_modules/jade/node_modules/mkdirp/.gitignore.orig +++ /dev/null @@ -1,2 +0,0 @@ -node_modules/ -npm-debug.log \ No newline at end of file diff --git a/node_modules/grunt-simple-mocha/node_modules/mocha/node_modules/jade/node_modules/mkdirp/.gitignore.rej b/node_modules/grunt-simple-mocha/node_modules/mocha/node_modules/jade/node_modules/mkdirp/.gitignore.rej deleted file mode 100644 index 69244ff..000 --- a/node_modules/grunt-simple-mocha/node_modules/mocha/node_modules/jade/node_modules/mkdirp/.gitignore.rej +++ /dev/null @@ -1,5 +0,0 @@ /dev/null -+++ .gitignore -@@ -0,0 +1,2 @@ -+node_modules/ -+npm-debug.log \ No newline at end of file diff --git a/node_modules/grunt/node_modules/async/.gitmodules b/node_modules/grunt/node_modules/async/.gitmodules deleted file mode 100644 index a9aae98..000 --- a/node_modules/grunt/node_modules/async/.gitmodules +++ /dev/null @@ -1,9 +0,0 @@ -[submodule deps/nodeunit] - path = deps/nodeunit - url = git://github.com/caolan/nodeunit.git -[submodule deps/UglifyJS] - path = deps/UglifyJS - url = https://github.com/mishoo/UglifyJS.git -[submodule deps/nodelint] - path = deps/nodelint - url = https://github.com/tav/nodelint.git diff --git a/node_modules/html-metadata/node_modules/microdata-node/node_modules/array-unique/.gitattributes b/node_modules/html-metadata/node_modules/microdata-node/node_modules/array-unique/.gitattributes deleted file mode 100644 index 759c2c5..000 --- a/node_modules/html-metadata/node_modules/microdata-node/node_modules/array-unique/.gitattributes +++ /dev/null @@ -1,14 +0,0 @@ -# Enforce Unix newlines -*.* text eol=lf -*.css text eol=lf -*.html text eol=lf -*.jstext eol=lf -*.json text eol=lf -*.less text eol=lf -*.mdtext eol=lf -*.yml text eol=lf - -*.jpg binary -*.gif binary -*.png binary -*.jpeg binary \ No newline at end of file diff --git
[MediaWiki-commits] [Gerrit] Bump src to 6152c80 - change (mediawiki...deploy)
Mobrovac has uploaded a new change for review. https://gerrit.wikimedia.org/r/198206 Change subject: Bump src to 6152c80 .. Bump src to 6152c80 Change-Id: I5a3dc3374b2e633363f348e93de9bad3daf203b6 --- M src 1 file changed, 0 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/citoid/deploy refs/changes/06/198206/1 diff --git a/src b/src index fd026c6..6152c80 16 --- a/src +++ b/src -Subproject commit fd026c69c99f8f78878fb771920c582666e692d0 +Subproject commit 6152c80c5ca28022ac0e4aed0003139e6ebf6efe -- To view, visit https://gerrit.wikimedia.org/r/198206 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I5a3dc3374b2e633363f348e93de9bad3daf203b6 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/services/citoid/deploy Gerrit-Branch: master Gerrit-Owner: Mobrovac mobro...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Quote the proxy setting - change (operations/puppet)
Mobrovac has uploaded a new change for review. https://gerrit.wikimedia.org/r/197052 Change subject: Quote the proxy setting .. Quote the proxy setting This is directly imported as JS script into Citoid, so it needs to be valid JS, hence, strings need to be quoted. Change-Id: I90eb8a0df1126dfa01bef286c9206800ef9d --- M modules/citoid/templates/localsettings.js.erb 1 file changed, 1 insertion(+), 1 deletion(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/52/197052/1 diff --git a/modules/citoid/templates/localsettings.js.erb b/modules/citoid/templates/localsettings.js.erb index a116cdb..ed8d040 100644 --- a/modules/citoid/templates/localsettings.js.erb +++ b/modules/citoid/templates/localsettings.js.erb @@ -16,7 +16,7 @@ //allowCORS : 'some.domain.org', // Set proxy for outgoing requests - proxy : %= @http_proxy %, + proxy : '%= @http_proxy %', // Allow override of port/interface: citoidPort : %= @port %, -- To view, visit https://gerrit.wikimedia.org/r/197052 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I90eb8a0df1126dfa01bef286c9206800ef9d Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Mobrovac mobro...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Set HTTP headers as an associative array - change (mediawiki...RestBaseUpdateJobs)
Mobrovac has uploaded a new change for review. https://gerrit.wikimedia.org/r/197042 Change subject: Set HTTP headers as an associative array .. Set HTTP headers as an associative array MultiHttpClient expects headers in an associative array. The extension instead sent headers as an array of strings of the form 'Cache-control: no-cache', which silently produced HTTP headers of the form '0: Cache-control: no-cache'. Needless to say, that rendered those headers useless, which in turn caused template updates to not happen as the no-cache header was never seen. The same issue also affected revision deletion updates. Also: Fix the default server port to use RESTBase's default port. Bug: T92703 Change-Id: I57082560cb6b00d1bbe05b8f06c686475803f160 --- M RestbaseUpdate.php M RestbaseUpdateJob.php 2 files changed, 6 insertions(+), 6 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/RestBaseUpdateJobs refs/changes/42/197042/1 diff --git a/RestbaseUpdate.php b/RestbaseUpdate.php index c523f20..399826b 100644 --- a/RestbaseUpdate.php +++ b/RestbaseUpdate.php @@ -69,7 +69,7 @@ /** * The RESTBase server to inform of updates. */ - $wgRestbaseServer = 'http://localhost:7321'; + $wgRestbaseServer = 'http://localhost:7231'; /** * The RESTBase API version in use diff --git a/RestbaseUpdateJob.php b/RestbaseUpdateJob.php index 2347207..dfaf331 100644 --- a/RestbaseUpdateJob.php +++ b/RestbaseUpdateJob.php @@ -180,7 +180,7 @@ 'method' = 'GET', 'url' = self::getRevisionURL( $revid ), 'headers' = array( - 'Cache-control: no-cache' + 'Cache-control' = 'no-cache' ) ); } @@ -208,8 +208,8 @@ 'method' = 'GET', 'url' = self::getPageTitleURL( $title, $latest ), 'headers' = array( - 'X-Restbase-ParentRevision: ' . $previous, - 'Cache-control: no-cache' + 'X-Restbase-ParentRevision' = $previous, + 'Cache-control' = 'no-cache' ) ) ); ///wfDebug( RestbaseUpdateJob::invalidateTitle: . json_encode( $requests ) . \n ); @@ -239,8 +239,8 @@ 'method' = 'GET', 'url' = $url, 'headers' = array( - 'X-Restbase-Mode: ' . $mode, - 'Cache-control: no-cache' + 'X-Restbase-Mode' = $mode, + 'Cache-control' = 'no-cache' ) ); } -- To view, visit https://gerrit.wikimedia.org/r/197042 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I57082560cb6b00d1bbe05b8f06c686475803f160 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/extensions/RestBaseUpdateJobs Gerrit-Branch: wmf/1.25wmf21 Gerrit-Owner: Mobrovac mobro...@wikimedia.org Gerrit-Reviewer: GWicke gwi...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Set HTTP headers as an associative array - change (mediawiki...RestBaseUpdateJobs)
Mobrovac has uploaded a new change for review. https://gerrit.wikimedia.org/r/197041 Change subject: Set HTTP headers as an associative array .. Set HTTP headers as an associative array MultiHttpClient expects headers in an associative array. The extension instead sent headers as an array of strings of the form 'Cache-control: no-cache', which silently produced HTTP headers of the form '0: Cache-control: no-cache'. Needless to say, that rendered those headers useless, which in turn caused template updates to not happen as the no-cache header was never seen. The same issue also affected revision deletion updates. Also: Fix the default server port to use RESTBase's default port. Bug: T92703 Change-Id: I57082560cb6b00d1bbe05b8f06c686475803f160 --- M RestbaseUpdate.php M RestbaseUpdateJob.php 2 files changed, 6 insertions(+), 6 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/RestBaseUpdateJobs refs/changes/41/197041/1 diff --git a/RestbaseUpdate.php b/RestbaseUpdate.php index ce17548..a8f0bf9 100644 --- a/RestbaseUpdate.php +++ b/RestbaseUpdate.php @@ -69,7 +69,7 @@ /** * The RESTBase server to inform of updates. */ - $wgRestbaseServer = 'http://localhost:7321'; + $wgRestbaseServer = 'http://localhost:7231'; /** * The RESTBase API version in use diff --git a/RestbaseUpdateJob.php b/RestbaseUpdateJob.php index 981881e..a873947 100644 --- a/RestbaseUpdateJob.php +++ b/RestbaseUpdateJob.php @@ -180,7 +180,7 @@ 'method' = 'GET', 'url' = self::getRevisionURL( $revid ), 'headers' = array( - 'Cache-control: no-cache' + 'Cache-control' = 'no-cache' ) ); } @@ -208,8 +208,8 @@ 'method' = 'GET', 'url' = self::getPageTitleURL( $title, $latest ), 'headers' = array( - 'X-Restbase-ParentRevision: ' . $previous, - 'Cache-control: no-cache' + 'X-Restbase-ParentRevision' = $previous, + 'Cache-control' = 'no-cache' ) ) ); ///wfDebug( RestbaseUpdateJob::invalidateTitle: . json_encode( $requests ) . \n ); @@ -239,8 +239,8 @@ 'method' = 'GET', 'url' = $url, 'headers' = array( - 'X-Restbase-Mode: ' . $mode, - 'Cache-control: no-cache' + 'X-Restbase-Mode' = $mode, + 'Cache-control' = 'no-cache' ) ); } -- To view, visit https://gerrit.wikimedia.org/r/197041 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I57082560cb6b00d1bbe05b8f06c686475803f160 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/extensions/RestBaseUpdateJobs Gerrit-Branch: wmf/1.25wmf20 Gerrit-Owner: Mobrovac mobro...@wikimedia.org Gerrit-Reviewer: GWicke gwi...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Enable edit updates for RESTBase on testwiki - change (operations/mediawiki-config)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/195348
Change subject: Enable edit updates for RESTBase on testwiki
..
Enable edit updates for RESTBase on testwiki
RESTBase needs to be updated whenever a new revision is created, or an
old one modified. To that end, we have the RestBaseUpdateJobs extension,
which has been tested in a local setting. With the advent of RESTBase's
public release, it is time to test it on the testwiki to make sure it
performs well.
Bug: T87520
Change-Id: Ie16d1c8718c614031808242c220bd5c793cc18c1
---
M wmf-config/CommonSettings.php
M wmf-config/InitialiseSettings.php
2 files changed, 16 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/mediawiki-config
refs/changes/48/195348/1
diff --git a/wmf-config/CommonSettings.php b/wmf-config/CommonSettings.php
index 4bd459b..b52701d 100644
--- a/wmf-config/CommonSettings.php
+++ b/wmf-config/CommonSettings.php
@@ -1928,6 +1928,11 @@
require_once( $IP/extensions/VectorBeta/VectorBeta.php );
}
+if ( $wmgUseRestbaseUpdateJobs ) {
+ require_once( $IP/extensions/RestBaseUpdateJobs/RestbaseUpdate.php );
+ $wgRestbaseServer = http://10.2.2.17;; // restbase.svc.eqiad.wmnet
+}
+
if ( $wmgUseParsoid ) {
require_once( $IP/extensions/Parsoid/Parsoid.php );
diff --git a/wmf-config/InitialiseSettings.php
b/wmf-config/InitialiseSettings.php
index 4060ca2..e7c81c0 100644
--- a/wmf-config/InitialiseSettings.php
+++ b/wmf-config/InitialiseSettings.php
@@ -12245,6 +12245,17 @@
'default' = false,
),
+// -- RESTBase start --
+
+// whether to activate the extension sending page edit
+// updates to RESTBase
+'wmgUseRestbaseUpdateJobs' = array(
+ 'default' = false,
+ 'testwiki' = true
+),
+
+// -- RESTBase end --
+
// -- Shared Parsoid start --
'wmgUseParsoid' = array(
--
To view, visit https://gerrit.wikimedia.org/r/195348
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ie16d1c8718c614031808242c220bd5c793cc18c1
Gerrit-PatchSet: 1
Gerrit-Project: operations/mediawiki-config
Gerrit-Branch: master
Gerrit-Owner: Mobrovac mobro...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Exclude slow edit dependency jobs from the default queue - change (operations/mediawiki-config)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/195366
Change subject: Exclude slow edit dependency jobs from the default queue
..
Exclude slow edit dependency jobs from the default queue
Change-Id: I3e129f1cc455d1ccc04e0b94f7b6a70001e5e39b
---
M wmf-config/CommonSettings.php
1 file changed, 2 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/mediawiki-config
refs/changes/66/195366/1
diff --git a/wmf-config/CommonSettings.php b/wmf-config/CommonSettings.php
index 99353a4..42a3541 100644
--- a/wmf-config/CommonSettings.php
+++ b/wmf-config/CommonSettings.php
@@ -1931,6 +1931,8 @@
if ( $wmgUseRestbaseUpdateJobs ) {
require_once( $IP/extensions/RestBaseUpdateJobs/RestbaseUpdate.php );
$wgRestbaseServer = http://10.2.2.17:7231;; // restbase.svc.eqiad.wmnet
+ # Slow Restbase jobs
+ $wgJobTypesExcludedFromDefaultQueue[] =
'RestbaseUpdateJobOnDependencyChange';
}
if ( $wmgUseParsoid ) {
--
To view, visit https://gerrit.wikimedia.org/r/195366
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I3e129f1cc455d1ccc04e0b94f7b6a70001e5e39b
Gerrit-PatchSet: 1
Gerrit-Project: operations/mediawiki-config
Gerrit-Branch: master
Gerrit-Owner: Mobrovac mobro...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Fix acceptLanguage commit a30d6be - change (mediawiki...citoid)
Mobrovac has submitted this change and it was merged.
Change subject: Fix acceptLanguage commit a30d6be
..
Fix acceptLanguage commit a30d6be
* Fixed errors to CitoidService
requestFromDOI method.
* Added test for requestFromDOI
'doi'
* Fixed test from acceptLanguage commit
'German Twitter' which checked for
incorrect responseCode
* Throw callback errors in all tests
* Check error in test 520 (now test
ENOTFOUND) and throw if the error is not
the expected error (getaddrinfo ENOTFOUND)
* Add test 404 for cases where a 520 error
is expected but no error should be thrown
(for when there is a resource at the server
but the HTTP response is 404).
* Increase timeout default for mocha
since the 'doi' test takes longer
than the previous timeout length.
Change-Id: I93f8c699e7f58375e4fa8dd7c48dd721969e3659
---
M Gruntfile.js
M lib/CitoidService.js
M test/index.js
3 files changed, 74 insertions(+), 14 deletions(-)
Approvals:
Mobrovac: Looks good to me, approved
diff --git a/Gruntfile.js b/Gruntfile.js
index 8c04814..7178f57 100644
--- a/Gruntfile.js
+++ b/Gruntfile.js
@@ -22,7 +22,7 @@
simplemocha: {
options: {
globals: ['describe', 'its'],
- timeout: 3000,
+ timeout: 2,
ignoreLeaks: false,
ui: 'bdd',
reporter: 'tap'
@@ -33,7 +33,7 @@
// Default task.
grunt.registerTask('test', ['jshint:all']);
- grunt.registerTask('withzotero', ['simplemocha']);
+ grunt.registerTask('all', ['jshint:all', 'simplemocha']);
grunt.registerTask('default', 'test');
};
diff --git a/lib/CitoidService.js b/lib/CitoidService.js
index e5d2f03..7550f5a 100644
--- a/lib/CitoidService.js
+++ b/lib/CitoidService.js
@@ -114,8 +114,9 @@
*/
CitoidService.prototype.requestFromDOI = function (opts, callback){
var doiLink = 'http://dx.doi.org/'+ opts.search;
+ opts.search = doiLink;
// TODO: optimise this (can skip some steps in requestFromURL)
- this.requestFromURL(doiLink, opts.format, callback);
+ this.requestFromURL(opts, callback);
};
/**
diff --git a/test/index.js b/test/index.js
index 8a5b414..97366ac 100644
--- a/test/index.js
+++ b/test/index.js
@@ -61,24 +61,31 @@
throw new Error('Missing itemType');
}
done();
- // TODO: Match retrieved citation to expected citation
});
});
});
-describe('520', function() {
+describe('ENOTFOUND', function() {
- var opts = {
- search : 'example./com',
+ var url = 'example./com',
+ opts = {
+ search : url,
format : 'mediawiki',
acceptLanguage : 'en'
- },
+ },
expectedTitle = 'http://example./com';
- it('should return 520 error and citation', function(done) {
+ it('should return a ENOTFOUND error, a 520 responseCode, and citation',
function(done) {
citoidService.request(opts, function(error, responseCode,
citation){
+ if (!error) {
+ throw new Error('No error');
+ }
+ // Throw errors except the expected error, ENOTFOUND
+ if (error.message !== 'getaddrinfo ENOTFOUND'){
+ throw error;
+ }
if (responseCode !== 520){
- throw new Error('Should throw 520: Response
code is' + responseCode);
+ throw new Error('Should throw 520: Response
code is ' + responseCode);
}
if (!citation) {throw new Error ('Empty body');}
if (citation[0].title !== expectedTitle){
@@ -88,11 +95,39 @@
if (!citation[0].itemType){
throw new Error('Missing itemType');
}
- // TODO: Match retrieved citation to expected citation
done();
});
});
});
+
+describe('404', function() {
+
+ var url = 'http://example.com/thisurldoesntexist',
+ opts = {
+ search : url,
+ format : 'mediawiki',
+ acceptLanguage : 'en'
+ },
+ expectedTitle = url;
+
+ it('should return a 520 responseCode and citation', function(done) {
+ citoidService.request(opts, function(error, responseCode,
citation){
+ if (responseCode !== 520){
+ throw new Error('Should throw 520: Response
[MediaWiki-commits] [Gerrit] Activate the RESTBase Virtual REST Service on test.wp - change (operations/mediawiki-config)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/196000
Change subject: Activate the RESTBase Virtual REST Service on test.wp
..
Activate the RESTBase Virtual REST Service on test.wp
Change-Id: Ia386b87da6d3ff20c071b8dba1d7323c681747de
---
M wmf-config/CommonSettings.php
M wmf-config/InitialiseSettings.php
2 files changed, 26 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/mediawiki-config
refs/changes/00/196000/1
diff --git a/wmf-config/CommonSettings.php b/wmf-config/CommonSettings.php
index 1d4af61..36b8a28 100644
--- a/wmf-config/CommonSettings.php
+++ b/wmf-config/CommonSettings.php
@@ -1930,6 +1930,25 @@
$wgRestbaseServer = http://10.2.2.17:7231;; // restbase.svc.eqiad.wmnet
}
+if ( $wmgUseRestbaseVRS ) {
+ if( !isset( $wgVirtualRestConfig ) ) {
+ $wgVirtualRestConfig = array(
+ 'modules' = array(),
+ 'global' = array(
+ 'timeout' = 360,
+ 'forwardCookies' = false,
+ 'HTTPProxy' = null
+ )
+ );
+ }
+ $wgVirtualRestConfig['modules']['restbase'] = array(
+ 'url' = 'http://10.2.2.17:7231', // restbase.svc.eqiad.wmnet
+ 'domain' = $wgCanonicalServer,
+ 'forwardCookies' = true,
+ 'parsoidCompat' = false
+ );
+}
+
if ( $wmgUseParsoid ) {
require_once( $IP/extensions/Parsoid/Parsoid.php );
diff --git a/wmf-config/InitialiseSettings.php
b/wmf-config/InitialiseSettings.php
index a35f774..2f2bad2 100644
--- a/wmf-config/InitialiseSettings.php
+++ b/wmf-config/InitialiseSettings.php
@@ -12236,6 +12236,13 @@
'testwiki' = true
),
+// whether to configure RESTBase as a Virtual REST Service
+// in MW Core
+'wmgUseRestbaseVRS' = array(
+ 'default' = false,
+ 'testwiki' = true
+),
+
// -- RESTBase end --
// -- Shared Parsoid start --
--
To view, visit https://gerrit.wikimedia.org/r/196000
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ia386b87da6d3ff20c071b8dba1d7323c681747de
Gerrit-PatchSet: 1
Gerrit-Project: operations/mediawiki-config
Gerrit-Branch: master
Gerrit-Owner: Mobrovac mobro...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Add the RESTBase and Cassandra roles to MW-Vagrant - change (mediawiki/vagrant)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/195106
Change subject: Add the RESTBase and Cassandra roles to MW-Vagrant
..
Add the RESTBase and Cassandra roles to MW-Vagrant
RESTBase is a REST API service serving content produced by Parsoid. It
uses Cassandra as its storage. This patch add roles for both of them.
Note that, as Cassandra is memory-hungry, its role ups the VM's settings
to use 2G of RAM.
Bug: T91661
Change-Id: If956146546c801d75e490ace1d2784558ce41176
---
M puppet/hieradata/common.yaml
M puppet/modules/mediawiki/templates/parsoid.localsettings.js.erb
A puppet/modules/restbase/manifests/init.pp
A puppet/modules/restbase/templates/config.yaml.erb
A puppet/modules/restbase/templates/upstart.conf.erb
A puppet/modules/restbase/templates/vrs.php.erb
A puppet/modules/role/manifests/cassandra.pp
A puppet/modules/role/manifests/restbase.pp
8 files changed, 329 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/vagrant
refs/changes/06/195106/1
diff --git a/puppet/hieradata/common.yaml b/puppet/hieradata/common.yaml
index d1f85a7..e95ca12 100644
--- a/puppet/hieradata/common.yaml
+++ b/puppet/hieradata/common.yaml
@@ -3,6 +3,10 @@
- '::role::mediawiki'
- '::puppet::agent'
+cassandra::max_heap: 64M
+cassandra::new_size: 10M
+cassandra::logdir: /vagrant/logs/cassandra
+
cdh::hue::secret_key: vagrant
cdh::hue::hive_server_host: %{::fqdn}
@@ -173,6 +177,12 @@
redis::persist: true
redis::max_memory: 256M
+restbase::dir: /srv/restbase
+restbase::port: 7231
+restbase::domain: localhost
+restbase::log_file: /vagrant/logs/restbase.log
+restbase::log_level: debug
+
role::elk::vhost_name: 'logstash.local.wmftest.net'
role::mediawiki::hostname: 127.0.0.1
diff --git a/puppet/modules/mediawiki/templates/parsoid.localsettings.js.erb
b/puppet/modules/mediawiki/templates/parsoid.localsettings.js.erb
index ec668d2..b1e005a 100644
--- a/puppet/modules/mediawiki/templates/parsoid.localsettings.js.erb
+++ b/puppet/modules/mediawiki/templates/parsoid.localsettings.js.erb
@@ -4,6 +4,7 @@
*/
exports.setup = function( parsoidConfig ) {
parsoidConfig.setInterwiki( 'localhost', '%=
scope[mediawiki::server_url] %/w/api.php' );
+parsoidConfig.setInterwiki( 'localhost', 'http://localhost/w/api.php' );
parsoidConfig.usePHPPreProcessor = %= @use_php_preprocessor %;
parsoidConfig.useSelser = %= @use_selser %;
%- if @allow_cors -%
diff --git a/puppet/modules/restbase/manifests/init.pp
b/puppet/modules/restbase/manifests/init.pp
new file mode 100644
index 000..c524b52
--- /dev/null
+++ b/puppet/modules/restbase/manifests/init.pp
@@ -0,0 +1,107 @@
+# == Class: restbase
+#
+# RESTBase is a REST API service serving MW content from
+# a Cassandra storage, proxying requests to Parsoid in
+# case of storage misses.
+#
+# [*dir*]
+# the directory where to clone the git repo
+#
+# [*port*]
+# the port RESTBase will be running on
+#
+# [*domain*]
+# the domain to serve
+#
+# [*log_file*]
+# where logs should be written to
+#
+# [*log_level*]
+# the lowest level to log (trace, debug, info, warn, error, fatal)
+#
+class restbase (
+$dir,
+$port,
+$domain,
+$log_file,
+$log_level
+) {
+
+## ##
+# PART 1 - RESTBase SERVICE CONFIGURATION #
+## ##
+
+# RESTBase is a node service
+require_package('nodejs')
+require_package('npm')
+
+# the repo
+git::clone { 'wikimedia/restbase':
+directory = $dir,
+owner = 'root',
+group = 'root',
+remote = 'https://github.com/wikimedia/restbase.git',
+ensure = 'latest',
+require = Package['nodejs', 'npm'],
+notify = Exec['restbase_npm_install']
+}
+
+# install dependencies if they're missing
+exec { 'restbase_npm_install':
+command = '/usr/bin/npm install',
+cwd = $dir,
+user = 'root',
+environment = ['HOME=/root/'],
+unless = test -e node_modules,
+before = Service['restbase'],
+}
+
+# the service's configuration file
+file { 'restbase_config_yaml':
+path = ${dir}/config.yaml,
+ensure = present,
+content = template('restbase/config.yaml.erb'),
+require = Git::Clone['wikimedia/restbase']
+}
+
+# set the right permissions to the log file
+file { $log_file:
+mode = 0666,
+ensure = present
+}
+
+# upstart config
+file { '/etc/init/restbase.conf':
+content = template('restbase/upstart.conf.erb')
+}
+
+# the service
+service { 'restbase':
+ensure = running,
+provider = 'upstart',
+subscribe = [
+File['restbase_config_yaml', '/etc/init/restbase.conf'],
+Exec['restbase_npm_install'],
+
[MediaWiki-commits] [Gerrit] Minor language fixes and version bump - change (mediawiki...RestBaseUpdateJobs)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/195881
Change subject: Minor language fixes and version bump
..
Minor language fixes and version bump
* update the English, French and Portuguese descriptions
* fix the extension's URL
* add the Croatian, Serbian and Bosnian descriptions
* bump the version a bit
Bug: T92250
Change-Id: I930c7058009bb2072844f51ad6fc92132af573b6
---
M RestbaseUpdate.php
A i18n/bs.json
M i18n/en.json
M i18n/fr.json
A i18n/hr.json
M i18n/it.json
M i18n/pt.json
M i18n/qqq.json
A i18n/sr.json
9 files changed, 33 insertions(+), 6 deletions(-)
git pull
ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/RestBaseUpdateJobs
refs/changes/81/195881/1
diff --git a/RestbaseUpdate.php b/RestbaseUpdate.php
index c523f20..ce17548 100644
--- a/RestbaseUpdate.php
+++ b/RestbaseUpdate.php
@@ -39,7 +39,7 @@
'Gabriel Wicke',
'Marko Obrovac'
),
- 'version' = '0.2.0',
+ 'version' = '0.2.1',
'url' =
'https://www.mediawiki.org/wiki/Extension:RestBaseUpdateJobs',
'descriptionmsg' = 'restbaseupdatejobs-desc',
'license-name' = 'GPL-2.0+',
diff --git a/i18n/bs.json b/i18n/bs.json
new file mode 100644
index 000..87c5d54
--- /dev/null
+++ b/i18n/bs.json
@@ -0,0 +1,9 @@
+{
+ @metadata: {
+ authors: [
+ Marko Obrovac
+ ]
+ },
+ restbaseupdatejobs-desc: Ekstenzija za ažuriranje sadržaja RESTBase
API servisa
+}
+
diff --git a/i18n/en.json b/i18n/en.json
index 7e77d42..10c2ae2 100644
--- a/i18n/en.json
+++ b/i18n/en.json
@@ -4,6 +4,6 @@
Marko Obrovac
]
},
- restbaseupdatejobs-desc: RESTBase update job extension
+ restbaseupdatejobs-desc: An extension for updating the RESTBase API
service with revision changes
}
diff --git a/i18n/fr.json b/i18n/fr.json
index 4495b3d..6d1d52e 100644
--- a/i18n/fr.json
+++ b/i18n/fr.json
@@ -4,5 +4,5 @@
Marko Obrovac
]
},
- restbaseupdatejobs-desc: Extension pour la mise à jour du contenu de
RESTBase
+ restbaseupdatejobs-desc: Extension pour la mise à jour du contenu de
RESTBase, le service API pour le contenu
}
diff --git a/i18n/hr.json b/i18n/hr.json
new file mode 100644
index 000..87c5d54
--- /dev/null
+++ b/i18n/hr.json
@@ -0,0 +1,9 @@
+{
+ @metadata: {
+ authors: [
+ Marko Obrovac
+ ]
+ },
+ restbaseupdatejobs-desc: Ekstenzija za ažuriranje sadržaja RESTBase
API servisa
+}
+
diff --git a/i18n/it.json b/i18n/it.json
index 809636a..a3c138c 100644
--- a/i18n/it.json
+++ b/i18n/it.json
@@ -4,5 +4,5 @@
Marko Obrovac
]
},
- restbaseupdatejobs-desc: Estonsione per l'aggiornamento del
contenuto di RESTBase
+ restbaseupdatejobs-desc: Estonsione per l'aggiornamento del
contenuto del servizio API RESTBase
}
diff --git a/i18n/pt.json b/i18n/pt.json
index f7477b1..6483ff8 100644
--- a/i18n/pt.json
+++ b/i18n/pt.json
@@ -4,5 +4,5 @@
Marko Obrovac
]
},
- restbaseupdatejobs-desc: Extensão para a atualização do conteúdo de
RESTBase
+ restbaseupdatejobs-desc: Extensão para a atualização do conteúdo do
serviço API RESTBase
}
diff --git a/i18n/qqq.json b/i18n/qqq.json
index fcf51da..396d461 100644
--- a/i18n/qqq.json
+++ b/i18n/qqq.json
@@ -4,5 +4,5 @@
Raimond Spekking
]
},
- restbaseupdatejobs-desc: {{desc|name=Restbase Update
Jobs|url=https://www.mediawiki.org/wiki/Extension:RestbaseUpdateJobs}};
+ restbaseupdatejobs-desc: {{desc|name=Restbase Update
Jobs|url=https://www.mediawiki.org/wiki/Extension:RestBaseUpdateJobs}};
}
diff --git a/i18n/sr.json b/i18n/sr.json
new file mode 100644
index 000..1e87521
--- /dev/null
+++ b/i18n/sr.json
@@ -0,0 +1,9 @@
+{
+ @metadata: {
+ authors: [
+ Marko Obrovac
+ ]
+ },
+ restbaseupdatejobs-desc: Екстензија за ажурирање садржаја РЕСТБасе
АПИ сервиса
+}
+
--
To view, visit https://gerrit.wikimedia.org/r/195881
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I930c7058009bb2072844f51ad6fc92132af573b6
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/RestBaseUpdateJobs
Gerrit-Branch: master
Gerrit-Owner: Mobrovac mobro...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Hit Parsoid directly - change (operations/puppet)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/196229
Change subject: Hit Parsoid directly
..
Hit Parsoid directly
Until now we have been calling Parsoid by passing through Varnish.
However, given that we use exclusively Parsoid's v2, which isn't cached
at all, it's better to hit the LVS directly.
Change-Id: Idce3d86ee6a9de1d029058e020467f9edf178f98
---
M hieradata/role/common/restbase.yaml
M modules/restbase/manifests/init.pp
M modules/restbase/templates/config.yaml.erb
3 files changed, 5 insertions(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/29/196229/1
diff --git a/hieradata/role/common/restbase.yaml
b/hieradata/role/common/restbase.yaml
index 0cb2926..a9dca47 100644
--- a/hieradata/role/common/restbase.yaml
+++ b/hieradata/role/common/restbase.yaml
@@ -23,6 +23,7 @@
restbase::cassandra_defaultConsistency: localQuorum
restbase::cassandra_localDc: %{::site}
restbase::statsd_host: statsd.eqiad.wmnet
+restbase::parsoid_uri: http://10.2.2.28:8000 # parsoid.svc.eqiad.wmnet
lvs::realserver::realserver_ips:
- '10.2.2.17' # restbase.svc.eqiad.wmnet
diff --git a/modules/restbase/manifests/init.pp
b/modules/restbase/manifests/init.pp
index 456b430..86be047 100644
--- a/modules/restbase/manifests/init.pp
+++ b/modules/restbase/manifests/init.pp
@@ -19,6 +19,8 @@
# Which DC should be considered local. Default: 'datacenter1'.
# [*port*]
# Port where to run the restbase service. Default: 7231
+# [*parsoid_uri*]
+# URI to reach Parsoid. Default: http://parsoid-lb.eqiad.wikimedia.org
# [*logstash_host*]
# GELF logging host. Default: localhost
# [*logstash_port*]
@@ -38,6 +40,7 @@
$cassandra_defaultConsistency = 'localQuorum',
$cassandra_localDc = 'datacenter1',
$port = 7231,
+$parsoid_uri= 'http://parsoid-lb.eqiad.wikimedia.org',
$logstash_host = 'localhost',
$logstash_port = 12201,
$logging_level = 'warn',
diff --git a/modules/restbase/templates/config.yaml.erb
b/modules/restbase/templates/config.yaml.erb
index 9892e5f..8298e72 100644
--- a/modules/restbase/templates/config.yaml.erb
+++ b/modules/restbase/templates/config.yaml.erb
@@ -115,7 +115,7 @@
version: 1.0.0
type: file
options:
- parsoidHost: http://parsoid-lb.eqiad.wikimedia.org
+ parsoidHost: %= @parsoid_uri %
/{module:action}:
x-modules:
--
To view, visit https://gerrit.wikimedia.org/r/196229
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Idce3d86ee6a9de1d029058e020467f9edf178f98
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Mobrovac mobro...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Use urlencode so slashes can be encoded properly - change (mediawiki...RestBaseUpdateJobs)
Mobrovac has uploaded a new change for review. https://gerrit.wikimedia.org/r/196444 Change subject: Use urlencode so slashes can be encoded properly .. Use urlencode so slashes can be encoded properly Change-Id: I9e567b33251564b5429eada32f399a5fd949a8a8 --- M RestbaseUpdateJob.php 1 file changed, 1 insertion(+), 1 deletion(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/RestBaseUpdateJobs refs/changes/44/196444/1 diff --git a/RestbaseUpdateJob.php b/RestbaseUpdateJob.php index 2347207..981881e 100644 --- a/RestbaseUpdateJob.php +++ b/RestbaseUpdateJob.php @@ -89,7 +89,7 @@ // construct the URL return implode( '/', array( self::getRestbasePrefix(), 'page', - 'html', wfUrlencode( $title-getPrefixedDBkey() ), $revid ) ); + 'html', urlencode( $title-getPrefixedDBkey() ), $revid ) ); } -- To view, visit https://gerrit.wikimedia.org/r/196444 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I9e567b33251564b5429eada32f399a5fd949a8a8 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/extensions/RestBaseUpdateJobs Gerrit-Branch: master Gerrit-Owner: Mobrovac mobro...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Fix function call - change (mediawiki...RestBaseUpdateJobs)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/196460
Change subject: Fix function call
..
Fix function call
Change-Id: I8ec45f036fb46e7dc2dda90b6f576d2c1a63cce2
---
M RestbaseUpdate.hooks.php
1 file changed, 1 insertion(+), 1 deletion(-)
git pull
ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/RestBaseUpdateJobs
refs/changes/60/196460/1
diff --git a/RestbaseUpdate.hooks.php b/RestbaseUpdate.hooks.php
index 5948457..3de2d33 100644
--- a/RestbaseUpdate.hooks.php
+++ b/RestbaseUpdate.hooks.php
@@ -148,7 +148,7 @@
*/
public static function onFileUpload( File $file ) {
- self::updateTitle( $file-getTitle(), 'file' );
+ self::schedule( $file-getTitle(), 'file' );
return true;
}
--
To view, visit https://gerrit.wikimedia.org/r/196460
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I8ec45f036fb46e7dc2dda90b6f576d2c1a63cce2
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/RestBaseUpdateJobs
Gerrit-Branch: master
Gerrit-Owner: Mobrovac mobro...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Puppetise Citoid's configuration - change (operations/puppet)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/195896
Change subject: Puppetise Citoid's configuration
..
Puppetise Citoid's configuration
localsettings.js is now controlled by puppet with the information from
the citoid class, as well as the LVS config, thus keeping it updated
with any eventual changes there. Also, because of this change, the
service now reads its config from /etc/citoid/localsettings.js rather
than the file deployed with the repository.
Bug: T89875
Change-Id: I3ceb235edbabbe48a00633c428c51b0588f98b9b
---
M modules/citoid/manifests/init.pp
A modules/citoid/templates/localsettings.js.erb
M modules/citoid/templates/upstart-citoid.erb
3 files changed, 88 insertions(+), 3 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/96/195896/1
diff --git a/modules/citoid/manifests/init.pp b/modules/citoid/manifests/init.pp
index e7bfc88..38531cf 100644
--- a/modules/citoid/manifests/init.pp
+++ b/modules/citoid/manifests/init.pp
@@ -5,9 +5,37 @@
# === Parameters
#
# [*port*]
-# Port where to run the citoid service. Defaults to 1970.
+# Port where to run the citoid service
#
-class citoid( $port = 1970 ) {
+# [*http_proxy*]
+# URL of the proxy to use, defaults to the one set for zotero
+#
+# [*zotero_host*]
+# Host of the zotero service
+#
+# [*zotero_port*]
+# Port of the zotero service
+#
+class citoid(
+$port= undef,
+$http_proxy = 'url-downloader.wikimedia.org:8080',
+$zotero_host = undef,
+$zotero_port = undef
+) {
+
+include lvs::configuration
+$lvs_services = $lvs::configuration::lvs_services
+
+unless $port {
+$port = $lvs_services['citoid']['port']
+}
+unless $zotero_host {
+$zotero_host = $lvs_services['zotero']['host']
+}
+unless $zotero_port {
+$zotero_port = $lvs_services['zotero']['port']
+}
+
require_package('nodejs')
package { 'citoid/deploy':
@@ -26,6 +54,24 @@
shell = '/bin/false',
system = true,
before = Service['citoid'],
+}
+
+file { '/etc/citoid':
+ensure = directory,
+owner = 'root',
+group = 'root',
+mode = '0755',
+before = Service['citoid'],
+}
+
+file { '/etc/citoid/localsettings.js':
+ensure = present,
+content = template('citoid/localsettings.js.erb'),
+owner = 'root',
+group = 'root',
+mode= '0444',
+require = Package['citoid/deploy'],
+before = Service['citoid'],
}
file { '/var/log/citoid':
@@ -56,5 +102,6 @@
hasstatus = true,
hasrestart = true,
provider = 'upstart',
+require= Package['nodejs'],
}
}
diff --git a/modules/citoid/templates/localsettings.js.erb
b/modules/citoid/templates/localsettings.js.erb
new file mode 100644
index 000..a116cdb
--- /dev/null
+++ b/modules/citoid/templates/localsettings.js.erb
@@ -0,0 +1,38 @@
+/*
+ * NOTE: This file is managed by puppet
+ *
+ * Citoud configuration file
+ */
+
+var CitoidConfig = {
+
+ // Allow cross-domain requests to the API (default '*')
+ // Sets Access-Control-Allow-Origin header
+ // enable:
+ allowCORS : '*',
+ // disable:
+ //allowCORS : false,
+ // restrict:
+ //allowCORS : 'some.domain.org',
+
+ // Set proxy for outgoing requests
+ proxy : %= @http_proxy %,
+
+ // Allow override of port/interface:
+ citoidPort : %= @port %,
+ citoidInterface : '0.0.0.0',
+
+ // userAgent string for lib/Scraper.js
+ userAgent : 'WikimediaBot',
+
+ // Settings for Zotero server
+ zoteroPort : %= @zotero_port %,
+ zoteroInterface : '%= @zotero_host %',
+ // Whether or not the outbound proxy should be used to access Zotero
+ zoteroUseProxy : false,
+};
+
+/*Exports*/
+module.exports = CitoidConfig;
+
+
diff --git a/modules/citoid/templates/upstart-citoid.erb
b/modules/citoid/templates/upstart-citoid.erb
index 7a4fa0e..3df55d9 100644
--- a/modules/citoid/templates/upstart-citoid.erb
+++ b/modules/citoid/templates/upstart-citoid.erb
@@ -20,4 +20,4 @@
kill timeout 60
chdir /srv/deployment/citoid/deploy
-exec /usr/bin/nodejs src/server.js -c localsettings.js
/var/log/citoid/main.log 21
+exec /usr/bin/nodejs src/server.js -c /etc/citoid/localsettings.js
/var/log/citoid/main.log 21
--
To view, visit https://gerrit.wikimedia.org/r/195896
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I3ceb235edbabbe48a00633c428c51b0588f98b9b
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Mobrovac mobro...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
[MediaWiki-commits] [Gerrit] Bump service-runner to 0.1.5 - change (mediawiki...mobileapps)
Mobrovac has submitted this change and it was merged.
Change subject: Bump service-runner to 0.1.5
..
Bump service-runner to 0.1.5
Change-Id: I8fd4c8f61ae9c1163b4f74baa9462f34cb05377e
---
M package.json
1 file changed, 1 insertion(+), 1 deletion(-)
Approvals:
Mobrovac: Verified; Looks good to me, approved
diff --git a/package.json b/package.json
index 5d0853f..4223f67 100644
--- a/package.json
+++ b/package.json
@@ -40,7 +40,7 @@
js-yaml: ^3.2.7,
node-uuid: ^1.4.3,
preq: ^0.3.12,
-service-runner: ^0.1.4
+service-runner: ^0.1.5
},
devDependencies: {
assert: ^1.3.0,
--
To view, visit https://gerrit.wikimedia.org/r/199259
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I8fd4c8f61ae9c1163b4f74baa9462f34cb05377e
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/mobileapps
Gerrit-Branch: master
Gerrit-Owner: Mobrovac mobro...@wikimedia.org
Gerrit-Reviewer: BearND bsitzm...@wikimedia.org
Gerrit-Reviewer: Mobrovac mobro...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Bump service-runner to 0.1.5 - change (mediawiki...mobileapps)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/199259
Change subject: Bump service-runner to 0.1.5
..
Bump service-runner to 0.1.5
Change-Id: I8fd4c8f61ae9c1163b4f74baa9462f34cb05377e
---
M package.json
1 file changed, 1 insertion(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/mobileapps
refs/changes/59/199259/1
diff --git a/package.json b/package.json
index 5d0853f..4223f67 100644
--- a/package.json
+++ b/package.json
@@ -40,7 +40,7 @@
js-yaml: ^3.2.7,
node-uuid: ^1.4.3,
preq: ^0.3.12,
-service-runner: ^0.1.4
+service-runner: ^0.1.5
},
devDependencies: {
assert: ^1.3.0,
--
To view, visit https://gerrit.wikimedia.org/r/199259
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I8fd4c8f61ae9c1163b4f74baa9462f34cb05377e
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/mobileapps
Gerrit-Branch: master
Gerrit-Owner: Mobrovac mobro...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] [BREAKING] Make Citoid use service-template-node and service... - change (mediawiki...citoid)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/199250
Change subject: [BREAKING] Make Citoid use service-template-node and
service-runner
..
[BREAKING] Make Citoid use service-template-node and service-runner
This patch converts Citoid into a service-template-node-based service,
thus easing its development and deployment. Amongst other things, this
new basis brings:
- unified command-line arguments
- unified configuration management; localsettings.js is gone, the
configuration is to be managed by altering config.dev.yaml
- support for process clustering; this should improve performance and
availability, as until now only one process per host has been executed
- support for process management (OOM prevents and the like)
- support for logging directly to logstash (no more grepping and tailing)
- support for metrics reporting, both to a StatsD server, as well as to
the logger when run locally during development
- support for auto-magic error handling (with or without promises)
- code coverage reporting
- support for API versioning (yet to be used in Citoid)
This patch also structures and adds some more tests. Grunt has been
removed in favour of mocha. Note that 'npm test' still reports only
jshint results as a compatibility requirement with Jenkins. To run all
of the tests locally, use the 'mocha' command instead.
Note that, due to the disruptive nature of the patch, the following
commands should be run:
rm localsettings.js
rm -rf node_modules
npm install
Bug: T75993
Change-Id: I7370c9a91e67c263a291aab4b8ae9014a23efa5f
---
M .gitignore
M .jshintignore
M .jshintrc
D Gruntfile.js
A app.js
A config.dev.yaml
A config.yaml
M lib/CitoidService.js
M lib/Scraper.js
M lib/ZoteroService.js
M lib/pubMedRequest.js
M lib/translators/general.js
M lib/translators/openGraph.js
M lib/unshorten.js
A lib/util.js
D localsettings.js.sample
M package.json
A routes/root.js
M server.js
A test/features/app/index.js
A test/features/errors/index.js
A test/features/scraping/index.js
A test/features/scraping/lang.js
M test/index.js
A test/mocha.opts
A test/utils/assert.js
A test/utils/logStream.js
A test/utils/server.js
28 files changed, 1,154 insertions(+), 543 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/citoid
refs/changes/50/199250/1
diff --git a/.gitignore b/.gitignore
index c88c5d5..ea81c00 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,5 @@
*~
.DS_Store
*.log
-
-localsettings.js
-
node_modules
+coverage
diff --git a/.jshintignore b/.jshintignore
index 79e6ea4..1c69eee 100644
--- a/.jshintignore
+++ b/.jshintignore
@@ -1,2 +1,3 @@
+coverage
node_modules
-test
\ No newline at end of file
+test
diff --git a/.jshintrc b/.jshintrc
index bae215e..6e8ef79 100644
--- a/.jshintrc
+++ b/.jshintrc
@@ -1,4 +1,10 @@
{
+
+ predef: [
+ Map,
+ Set
+ ],
+
// Enforcing
bitwise: true,
eqeqeq: true,
@@ -6,7 +12,6 @@
noarg: true,
nonew: true,
undef: true,
- unused: true,
curly: true,
newcap: true,
@@ -33,5 +38,7 @@
module: true,
global: true
},
+ node: true,
esnext: true
+
}
diff --git a/Gruntfile.js b/Gruntfile.js
deleted file mode 100644
index 7178f57..000
--- a/Gruntfile.js
+++ /dev/null
@@ -1,39 +0,0 @@
-module.exports = function( grunt ) {
-
- // These plugins provide necessary tasks.
- grunt.loadNpmTasks('grunt-contrib-jshint');
- grunt.loadNpmTasks('grunt-simple-mocha');
-
- // Project configuration.
- grunt.initConfig({
- // Task configuration.
- jshint: {
- options: {
- jshintrc: true
- },
- all: [
- '*.js',
- 'localsettings.js.sample',
- 'lib/*.js',
- 'lib/translators/*.js',
- 'test/*.js'
- ]
- },
- simplemocha: {
- options: {
- globals: ['describe', 'its'],
- timeout: 2,
- ignoreLeaks: false,
- ui: 'bdd',
- reporter: 'tap'
- },
- all: { src: ['test/*.js'] }
- }
- });
-
- // Default task.
- grunt.registerTask('test', ['jshint:all']);
- grunt.registerTask('all', ['jshint:all', 'simplemocha']);
- grunt.registerTask('default', 'test');
-
-};
diff --git a/app.js b/app.js
new file mode 100644
index 000..d636a6b
--- /dev/null
+++ b/app.js
@@ -0,0 +1,148 @@
+'use strict';
+
+
+var http = require('http');
[MediaWiki-commits] [Gerrit] Use standard If-Unmodified-Since header for conditional updates - change (mediawiki...RestBaseUpdateJobs)
Mobrovac has submitted this change and it was merged. Change subject: Use standard If-Unmodified-Since header for conditional updates .. Use standard If-Unmodified-Since header for conditional updates http://tools.ietf.org/html/draft-ietf-httpbis-p4-conditional-22#page-16 Change-Id: I99dbf6fc5369ca1523b266e711cdc5e0a553b28f --- M RestbaseUpdateJob.php 1 file changed, 2 insertions(+), 2 deletions(-) Approvals: Aaron Schulz: Looks good to me, but someone else must approve Mobrovac: Verified; Looks good to me, approved diff --git a/RestbaseUpdateJob.php b/RestbaseUpdateJob.php index e7a0a95..b72b029 100644 --- a/RestbaseUpdateJob.php +++ b/RestbaseUpdateJob.php @@ -246,7 +246,7 @@ 'url' = self::getPageTitleURL( $title, $latest ), 'headers' = array( 'X-Restbase-ParentRevision' = $previous, - 'X-Restbase-Timestamp' = $this-params['ts'], + 'If-Unmodified-Since' = date( 'r', intval( $this-params['ts'] ) ), 'Cache-control' = 'no-cache' ) ) ); @@ -278,7 +278,7 @@ 'url' = $url, 'headers' = array( 'X-Restbase-Mode' = $mode, - 'X-Restbase-Timestamp' = $this-params['ts'], + 'If-Unmodified-Since' = date( 'r', intval( $this-params['ts'] ) ), 'Cache-control' = 'no-cache' ) ); -- To view, visit https://gerrit.wikimedia.org/r/199286 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I99dbf6fc5369ca1523b266e711cdc5e0a553b28f Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/extensions/RestBaseUpdateJobs Gerrit-Branch: master Gerrit-Owner: GWicke gwi...@wikimedia.org Gerrit-Reviewer: Aaron Schulz asch...@wikimedia.org Gerrit-Reviewer: Eevans eev...@wikimedia.org Gerrit-Reviewer: Mobrovac mobro...@wikimedia.org Gerrit-Reviewer: jenkins-bot ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Fix Content-Type header and add test for bibtex exports - change (mediawiki...citoid)
Mobrovac has submitted this change and it was merged.
Change subject: Fix Content-Type header and add test for bibtex exports
..
Fix Content-Type header and add test for bibtex exports
* Add test for exporting bibtex format
* Fix the Content-Type header for bibtex entries,
as it's the only format not returning a JSON-
formatted response
Bug: T91168
Change-Id: Ie27289f2cb578e6d7b9e6f3c93d416e9c3c7cbe2
---
M package.json
M routes/root.js
A test/features/scraping/export.js
M test/utils/assert.js
4 files changed, 44 insertions(+), 2 deletions(-)
Approvals:
Mobrovac: Verified; Looks good to me, approved
diff --git a/package.json b/package.json
index f88ab74..3477cf1 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
name: citoid,
- version: 0.2.0,
+ version: 0.2.1,
description: Converts search terms such as URL or DOI into citations.,
scripts: {
start: service-runner,
diff --git a/routes/root.js b/routes/root.js
index b2e9684..58d94ee 100644
--- a/routes/root.js
+++ b/routes/root.js
@@ -95,7 +95,12 @@
};
app.citoid.request(opts, function(error, responseCode, body) {
- res.status(responseCode).type('application/json');
+ res.status(responseCode);
+ if(format === 'bibtex') {
+ res.type('application/x-bibtex');
+ } else {
+ res.type('application/json');
+ }
res.send(body);
});
diff --git a/test/features/scraping/export.js b/test/features/scraping/export.js
new file mode 100644
index 000..c467e87
--- /dev/null
+++ b/test/features/scraping/export.js
@@ -0,0 +1,23 @@
+'use strict';
+
+
+var preq = require('preq');
+var assert = require('../../utils/assert.js');
+var server = require('../../utils/server.js');
+
+
+describe('export', function() {
+
+ this.timeout(2);
+
+ before(function () { return server.start(); });
+
+ it('bibtex', function() {
+ return server.query('http://example.com',
'bibtex').then(function(res) {
+ assert.status(res, 200);
+ assert.checkBibtex(res, '\n@misc{_example_???');
+ });
+ });
+
+});
+
diff --git a/test/utils/assert.js b/test/utils/assert.js
index 2a65763..d65def4 100644
--- a/test/utils/assert.js
+++ b/test/utils/assert.js
@@ -110,6 +110,19 @@
}
+function checkBibtex(res, beginning) {
+
+ var cit = res.body;
+
+ if (cit instanceof String) {
+ throw new Error('Expected String, got: ' + JSON.stringify(cit));
+ }
+
+ assert.deepEqual(cit.substring(0, beginning.length), beginning,
Beginning of citation does not match);
+
+}
+
+
module.exports.ok = assert.ok;
module.exports.fails = fails;
module.exports.deepEqual = deepEqual;
@@ -118,4 +131,5 @@
module.exports.contentType= contentType;
module.exports.status = status;
module.exports.checkCitation = checkCitation;
+module.exports.checkBibtex= checkBibtex;
--
To view, visit https://gerrit.wikimedia.org/r/199623
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ie27289f2cb578e6d7b9e6f3c93d416e9c3c7cbe2
Gerrit-PatchSet: 5
Gerrit-Project: mediawiki/services/citoid
Gerrit-Branch: master
Gerrit-Owner: Mvolz mv...@wikimedia.org
Gerrit-Reviewer: Mobrovac mobro...@wikimedia.org
Gerrit-Reviewer: Mvolz mv...@wikimedia.org
Gerrit-Reviewer: jenkins-bot
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Stop pinning jobrunner version with Puppet - change (mediawiki/vagrant)
Mobrovac has submitted this change and it was merged.
Change subject: Stop pinning jobrunner version with Puppet
..
Stop pinning jobrunner version with Puppet
Allow jobrunner to clone at the latest HEAD and don't reset it to a fix
hash when Puppet is run. The `vagrant git-update` command already
updates jobrunner, but the pinned version made it toggle back and forth
between HEAD and 5c927f90.
Bug: T92345
Change-Id: Ia69f2135094871d8ebfcdb6df35fff53d4772b0d
---
M puppet/hieradata/common.yaml
M puppet/modules/git/manifests/clone.pp
M puppet/modules/mediawiki/manifests/jobrunner.pp
3 files changed, 15 insertions(+), 16 deletions(-)
Approvals:
Mobrovac: Looks good to me, approved
diff --git a/puppet/hieradata/common.yaml b/puppet/hieradata/common.yaml
index 84bc0b4..de507ab 100644
--- a/puppet/hieradata/common.yaml
+++ b/puppet/hieradata/common.yaml
@@ -150,8 +150,6 @@
mediawiki::apache::docroot: %{hiera('apache::docroot')}
-mediawiki::jobrunner::commit: 5c927f9091f446452b9fd7bcb69614c7a7fe6eff
-
mediawiki::multiwiki::base_domain: '.wiki.local.wmftest.net'
mediawiki::multiwiki::script_dir: %{hiera('mediawiki::apache::docroot')}/w
mediawiki::multiwiki::settings_root:
%{hiera('mediawiki::settings_dir')}/wikis
diff --git a/puppet/modules/git/manifests/clone.pp
b/puppet/modules/git/manifests/clone.pp
index 95c7a97..7c55b8d 100644
--- a/puppet/modules/git/manifests/clone.pp
+++ b/puppet/modules/git/manifests/clone.pp
@@ -19,10 +19,10 @@
# [*owner*]
# User that should own the checked out repository. Git commands will run as
# this user so the user must have the ability to create the target
-# directory. Default 'vagrant'.
+# directory. Default $::share_owner.
#
# [*group*]
-# Group that should own the checked out repostory. Default 'vagrant'.
+# Group that should own the checked out repostory. Default $::share_group.
#
# [*ensure*]
# What state the clone should be in. Valid values are `present` and
@@ -48,8 +48,8 @@
$directory,
$branch = undef,
$remote = undef,
-$owner = 'vagrant',
-$group = 'vagrant',
+$owner = $::share_owner,
+$group = $::share_group,
$ensure = 'present',
$depth = $::git::default_depth,
$recurse_submodules = true,
@@ -88,6 +88,15 @@
timeout = 0,
}
+if (!defined(File[$directory])) {
+file { $directory:
+ensure = 'directory',
+owner = $owner,
+group = $group,
+before = Exec[git_clone_${title}],
+}
+}
+
if $ensure == 'latest' {
exec { git_pull_${title}:
command = git pull ${arg_recurse} ${arg_depth},
diff --git a/puppet/modules/mediawiki/manifests/jobrunner.pp
b/puppet/modules/mediawiki/manifests/jobrunner.pp
index 7e9fef6..adbb105 100644
--- a/puppet/modules/mediawiki/manifests/jobrunner.pp
+++ b/puppet/modules/mediawiki/manifests/jobrunner.pp
@@ -3,20 +3,12 @@
# jobrunner continuously processes the MediaWiki job queue by dispatching
# workers to perform tasks and monitoring their success or failure.
#
-# === Parameters
-#
-# [*commit*]
-# Git commit to install.
-#
-class mediawiki::jobrunner(
-$commit,
-) {
+class mediawiki::jobrunner {
include ::mediawiki
require ::mediawiki::multiwiki
-git::install { 'mediawiki/services/jobrunner':
+git::clone { 'mediawiki/services/jobrunner':
directory = '/srv/jobrunner',
-commit= $commit,
before= Service['jobrunner'],
}
--
To view, visit https://gerrit.wikimedia.org/r/200498
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ia69f2135094871d8ebfcdb6df35fff53d4772b0d
Gerrit-PatchSet: 2
Gerrit-Project: mediawiki/vagrant
Gerrit-Branch: master
Gerrit-Owner: BryanDavis bda...@wikimedia.org
Gerrit-Reviewer: Aaron Schulz asch...@wikimedia.org
Gerrit-Reviewer: Dduvall dduv...@wikimedia.org
Gerrit-Reviewer: Mobrovac mobro...@wikimedia.org
Gerrit-Reviewer: Ori.livneh o...@wikimedia.org
Gerrit-Reviewer: jenkins-bot
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Fix I2756d11 to use request instead of http lib - change (mediawiki...citoid)
Mobrovac has submitted this change and it was merged.
Change subject: Fix I2756d11 to use request instead of http lib
..
Fix I2756d11 to use request instead of http lib
Change I2756d11 used the http library, which does
not respect environment proxy settings. Fixed to
use request library instead, which will respect
the proxy settings.
Change-Id: Id9925189fbe4ac9d2efeb05cfe248e477b65932b
---
M lib/CitoidService.js
M package.json
2 files changed, 19 insertions(+), 8 deletions(-)
Approvals:
Mobrovac: Looks good to me, approved
diff --git a/lib/CitoidService.js b/lib/CitoidService.js
index 9b3d79d..c8aadcc 100644
--- a/lib/CitoidService.js
+++ b/lib/CitoidService.js
@@ -5,7 +5,7 @@
*/
/* Import Modules */
-var http = require('http');
+var request = require('request');
var urlParse = require('url');
/* Import Local Modules */
@@ -122,16 +122,27 @@
CitoidService.prototype.requestFromDOI = function (doiOpts, callback){
var doiLink = 'http://dx.doi.org/'+ doiOpts.search;
var citoidService = this;
- var urlOpts = Object.assign({}, doiOpts); // Shallow clone doiOpts
+ // Shallow clone doiOpts for requestFromURL method
+ var urlOpts = Object.assign({}, doiOpts);
+ // Options for obtaining url the DOI resolves to
+ var resolveOpts = {
+ url : doiLink,
+ followRedirect : false,
+ };
- // Follow one redirect here from the DOI to the canonical url
- http.get(doiLink, function (res) {
- // Detect a redirect
- if (res res.statusCode 300 res.statusCode 400
res.headers.location) {
+ // Resolve canonical URL from DOI URL
+ request.head(resolveOpts, function (err, res) {
+ if (!err res res.statusCode 300 res.statusCode 400
+ res.headers.location) {
urlOpts.search = res.headers.location;
+ // Send canonical URL to requestFromURL
+ citoidService.logger.log('debug/DOI', Resolved DOI
+ + doiOpts.search + to URL + urlOpts.search +
+ ; Sending to requestFromURL);
citoidService.requestFromURL(urlOpts, callback);
} else {
- citoidService.logger.log('debug/DOI', Unable to
resolve DOI + doiOpts.search);
+ citoidService.logger.log('debug/DOI', Unable to
resolve DOI
+ + doiOpts.search);
var message = 'Unable to resolve DOI';
var error = new Error(message);
callback(error, 404, {Error: message});
diff --git a/package.json b/package.json
index 4bdfddc..1707ab2 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
name: citoid,
- version: 0.2.3,
+ version: 0.2.4,
description: Converts search terms such as URL or DOI into citations.,
scripts: {
start: service-runner,
--
To view, visit https://gerrit.wikimedia.org/r/200546
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Id9925189fbe4ac9d2efeb05cfe248e477b65932b
Gerrit-PatchSet: 2
Gerrit-Project: mediawiki/services/citoid
Gerrit-Branch: master
Gerrit-Owner: Mvolz mv...@wikimedia.org
Gerrit-Reviewer: Mobrovac mobro...@wikimedia.org
Gerrit-Reviewer: Mvolz mv...@wikimedia.org
Gerrit-Reviewer: jenkins-bot
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Use external request ID if supplied - change (mediawiki...parsoid)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/200829
Change subject: Use external request ID if supplied
..
Use external request ID if supplied
RESTBase sends out a X-Request-Id header with each request, allowing us
to track a particular request (and its sub-requests) throughout the
cluster. This patch adds the capability to Parsoid to honour and use it
if it has been supplied. Otherwise, a new request ID is generated.
Change-Id: I1e94dd7fe05fb0e5a5272a6c2129a9ee5c78d452
---
M api/ParsoidService.js
1 file changed, 6 insertions(+), 2 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/parsoid
refs/changes/29/200829/1
diff --git a/api/ParsoidService.js b/api/ParsoidService.js
index b35a3bc..1138291 100644
--- a/api/ParsoidService.js
+++ b/api/ParsoidService.js
@@ -51,8 +51,12 @@
// request ids
var buf = new Buffer(16);
app.use(function(req, res, next) {
- uuid(null, buf);
- res.local('reqId', buf.toString('hex'));
+ if(req.headers req.headers['x-request-id']) {
+ res.local('reqId', req.headers['x-request-id']);
+ } else {
+ uuid(null, buf);
+ res.local('reqId', buf.toString('hex'));
+ }
next();
});
--
To view, visit https://gerrit.wikimedia.org/r/200829
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I1e94dd7fe05fb0e5a5272a6c2129a9ee5c78d452
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/parsoid
Gerrit-Branch: master
Gerrit-Owner: Mobrovac mobro...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Fix mising phab/ dir introduced in 55deca3 - change (mediawiki/vagrant)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/200850
Change subject: Fix mising phab/ dir introduced in 55deca3
..
Fix mising phab/ dir introduced in 55deca3
55deca3 added phab/ to .gitignore, while at the same time requiring the
directory to be present in order to install arcanist (put as a
dependency of mediawiki). Consequently, the provisioning would fail for
fresh clones. This patch aims at quick-fixing this by forcing Puppet to
ensure the parent directory exists when either arcanist of phabricator
modules are being applied. Because they share a common parent directory,
the /phab directory is required to be created before any git clone is to
be executed.
Note that this is only a quick fix, we should find a better way to
organise things (as in general one does not need arcanist to be
installed in order to run mediawiki).
Bug: T94469
Bug: T94553
Change-Id: Icf333d03f2c7d0673c74a9ef01de72744ae6bb57
---
M puppet/modules/arcanist/manifests/init.pp
M puppet/modules/phabricator/manifests/init.pp
2 files changed, 18 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/vagrant
refs/changes/50/200850/1
diff --git a/puppet/modules/arcanist/manifests/init.pp
b/puppet/modules/arcanist/manifests/init.pp
index 6f62b7c..721a4b8 100644
--- a/puppet/modules/arcanist/manifests/init.pp
+++ b/puppet/modules/arcanist/manifests/init.pp
@@ -12,6 +12,15 @@
){
include ::php
+if (!defined(File[$deploy_dir])) {
+file { $deploy_dir:
+ensure = 'directory',
+owner = $::share_owner,
+group = $::share_group,
+}
+File[$deploy_dir] - Git::Clone | |
+}
+
git::clone { 'https://github.com/phacility/libphutil':
directory = ${deploy_dir}/libphutil,
remote= 'https://github.com/phacility/libphutil',
diff --git a/puppet/modules/phabricator/manifests/init.pp
b/puppet/modules/phabricator/manifests/init.pp
index a7079ad..277017e 100644
--- a/puppet/modules/phabricator/manifests/init.pp
+++ b/puppet/modules/phabricator/manifests/init.pp
@@ -24,6 +24,15 @@
ensure = present,
}
+if (!defined(File[$deploy_dir])) {
+file { $deploy_dir:
+ensure = 'directory',
+owner = $::share_owner,
+group = $::share_group,
+}
+File[$deploy_dir] - Git::Clone | |
+}
+
git::clone { 'https://github.com/phacility/phabricator':
directory = ${deploy_dir}/phabricator,
remote= 'https://github.com/phacility/phabricator',
--
To view, visit https://gerrit.wikimedia.org/r/200850
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Icf333d03f2c7d0673c74a9ef01de72744ae6bb57
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/vagrant
Gerrit-Branch: master
Gerrit-Owner: Mobrovac mobro...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Let zotero use our proxy for HTTPS requests as well - change (operations/puppet)
Mobrovac has uploaded a new change for review. https://gerrit.wikimedia.org/r/201198 Change subject: Let zotero use our proxy for HTTPS requests as well .. Let zotero use our proxy for HTTPS requests as well Change-Id: I40c870a28d103f83b07ad150741aaf70fad6802a --- M modules/zotero/templates/defaults.js.erb 1 file changed, 2 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/98/201198/1 diff --git a/modules/zotero/templates/defaults.js.erb b/modules/zotero/templates/defaults.js.erb index a12f034..bbb5b4d 100644 --- a/modules/zotero/templates/defaults.js.erb +++ b/modules/zotero/templates/defaults.js.erb @@ -9,6 +9,8 @@ pref(network.proxy.type, 1); pref(network.proxy.http, %= @http_proxy_host %); pref(network.proxy.http_port, %= @http_proxy_port %); +pref(network.proxy.https, %= @http_proxy_host %); +pref(network.proxy.https_port, %= @http_proxy_port %); %- end -% // Don't retrieve unrequested links when performing standalone translation -- To view, visit https://gerrit.wikimedia.org/r/201198 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I40c870a28d103f83b07ad150741aaf70fad6802a Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Mobrovac mobro...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Fix permissions in labs-vagrant - change (mediawiki/vagrant)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/200859
Change subject: Fix permissions in labs-vagrant
..
Fix permissions in labs-vagrant
The MediaWiki-Vagrant variant has been updated to use $::share_owner and
$::share_group facts as default uid/gid. However, these have been left
undefined in labs-vagrant, causing permission problems (everything was
effectively set to be owned by root). This patch sets these two facts in
labs-vagrant as well.
Bug: T94469
Change-Id: I75f5d46086ff3b836cbb1f8621ab0f2bf5e1f58d
---
M lib/labs-vagrant.rb
1 file changed, 3 insertions(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/vagrant
refs/changes/59/200859/1
diff --git a/lib/labs-vagrant.rb b/lib/labs-vagrant.rb
index 4a34f56..327f8bc 100755
--- a/lib/labs-vagrant.rb
+++ b/lib/labs-vagrant.rb
@@ -68,7 +68,9 @@
when 'provision'
puppet_path = '/vagrant/puppet'
- exec sudo env FACTER_environment=labs puppet apply \
+ exec sudo env FACTER_environment=labs \
+FACTER_share_owner=vagrant FACTER_share_group=wikidev \
+puppet apply \
--modulepath #{puppet_path}/modules \
--manifestdir #{puppet_path}/manifests \
--templatedir #{puppet_path}/templates \
--
To view, visit https://gerrit.wikimedia.org/r/200859
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I75f5d46086ff3b836cbb1f8621ab0f2bf5e1f58d
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/vagrant
Gerrit-Branch: master
Gerrit-Owner: Mobrovac mobro...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Citoid: switch from localsettings.js to config.yaml - change (operations/puppet)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/200356
Change subject: Citoid: switch from localsettings.js to config.yaml
..
Citoid: switch from localsettings.js to config.yaml
Citoid has been ported to use the service template/runner combo, which
means that as of now the configuration is read from config.yaml. This
patch ports the old-style configuration to the new one.
Additionally, production logs will be sent to logstash from now on.
Change-Id: I25a70352977102ee858b58cbee5825e526f7286c
---
M hieradata/common/citoid.yaml
M modules/citoid/manifests/init.pp
A modules/citoid/templates/config.yaml.erb
D modules/citoid/templates/localsettings.js.erb
M modules/citoid/templates/upstart-citoid.erb
5 files changed, 71 insertions(+), 59 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/56/200356/1
diff --git a/hieradata/common/citoid.yaml b/hieradata/common/citoid.yaml
index 21353cb..eabce7e 100644
--- a/hieradata/common/citoid.yaml
+++ b/hieradata/common/citoid.yaml
@@ -1,3 +1,4 @@
http_proxy: http://url-downloader.wikimedia.org:8080
zotero_host: zotero.svc.eqiad.wmnet
statsd_host: statsd.eqiad.wmnet
+logstash_host: logstash1001.eqiad.wmnet
diff --git a/modules/citoid/manifests/init.pp b/modules/citoid/manifests/init.pp
index 59a1732..169a235 100644
--- a/modules/citoid/manifests/init.pp
+++ b/modules/citoid/manifests/init.pp
@@ -22,24 +22,24 @@
# [*statsd_port*]
# StatsD port. Defaults to 8125.
#
+# [*logstash_host*]
+# GELF logging host. Default: localhost
+#
+# [*logstash_port*]
+# GELF logging port. Default: 12201
+#
class citoid(
-$port= 1970,
-$statsd_port = 8125,
-$zotero_port = 1969,
-$http_proxy,
-$zotero_host,
-$statsd_host,
+$port = 1970,
+$statsd_port= 8125,
+$zotero_port= 1969,
+$http_proxy = undef,
+$zotero_host= 'localhost',
+$statsd_host= 'localhost',
+$logstash_host = 'localhost',
+$logstash_port = 12201,
) {
require_package('nodejs')
-
-$statsd_config = $statsd_host ? {
-undef= 'false',
-default = ordered_json({
-host = $statsd_host,
-port = $statsd_port
-}),
-}
package { 'citoid/deploy':
provider = 'trebuchet',
@@ -66,9 +66,9 @@
mode = '0755',
}
-file { '/etc/citoid/localsettings.js':
+file { '/etc/citoid/config.yaml':
ensure = present,
-content = template('citoid/localsettings.js.erb'),
+content = template('citoid/config.yaml.erb'),
owner = 'root',
group = 'root',
mode= '0444',
diff --git a/modules/citoid/templates/config.yaml.erb
b/modules/citoid/templates/config.yaml.erb
new file mode 100644
index 000..266163c
--- /dev/null
+++ b/modules/citoid/templates/config.yaml.erb
@@ -0,0 +1,53 @@
+# Number of worker processes to spawn.
+# Set to 0 to run everything in a single process without clustering.
+# Use 'ncpu' to run as many workers as there are CPU units
+num_workers: ncpu
+
+# Log error messages and gracefully restart a worker if v8 reports that it
+# uses more heap (note: not RSS) than this many mb.
+worker_heap_limit_mb: 300
+
+# Logger info
+logging:
+ name: citoid
+ level: warn
+ streams:
+ # Use gelf-stream - logstash
+ - type: gelf
+host: %= @logstash_host %
+port: %= @logstash_port %
+
+# Statsd metrics reporter
+metrics:
+ type: txstatsd
+ host: %= @statsd_host %
+ port: %= @statsd_port %
+
+services:
+ - name: citoid
+# a relative path or the name of an npm package, if different from name
+module: ./src/app.js
+# optionally, a version constraint of the npm package
+# version: ^0.4.0
+# per-service config
+conf:
+ # the port to bind to
+ port: %= @port %
+ # IP address to bind to, all IPs by default
+ # interface: localhost # uncomment to only listen on localhost
+ # allow cross-domain requests to the API (default '*')
+ cors: '*'
+ # to disable use:
+ # cors: false
+ # to restrict to a particular domain, use:
+ # cors: restricted.domain.org
+ # URL of the outbound proxy to use (complete with protocol)
+ proxy: %= @http_proxy %
+ # User-Agent HTTP header to use for requests
+ userAgent: WikimediaBot
+ # URL where to contact Zotero
+ zoteroInterface: %= @zotero_host %
+ # zotero's server port
+ zoteroPort: %= @zotero_port %
+ # whether the proxy should be used to contact zotero
+ zoteroUseProxy: false
diff --git a/modules/citoid/templates/localsettings.js.erb
b/modules/citoid/templates/localsettings.js.erb
deleted file mode 100644
index 84da3a0..000
--- a/modules/citoid/templates/localsettings.js.erb
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * NOTE: This file is managed by puppet
- *
- * Citoud configuration file
- */
[MediaWiki-commits] [Gerrit] Add ability to set requesting User-Agent - change (mediawiki...citoid)
Mobrovac has submitted this change and it was merged.
Change subject: Add ability to set requesting User-Agent
..
Add ability to set requesting User-Agent
Modify localsettings.js.sample to contain a
userAgent string.
Pass this string to the Scraper.js object
and use this string as the User-Agent in
all outbound requests.
Simplify Object parameters in
ZoteroService.js, CitoidService.js, and
Scraper.js.
Bug: T89757
Change-Id: I7dfa933181118527783da12cdb5b277329aef7dd
---
M lib/CitoidService.js
M lib/Scraper.js
M lib/ZoteroService.js
M localsettings.js.sample
4 files changed, 20 insertions(+), 14 deletions(-)
Approvals:
Mobrovac: Looks good to me, approved
diff --git a/lib/CitoidService.js b/lib/CitoidService.js
index 3f4deeb..7d5f666 100644
--- a/lib/CitoidService.js
+++ b/lib/CitoidService.js
@@ -3,8 +3,7 @@
*/
/* Import Modules */
-var urlParse = require('url'),
- util = require('util');
+var urlParse = require('url');
/* Import Local Modules */
var unshorten = require('./unshorten.js'),
@@ -17,13 +16,10 @@
* @param {Object} CitoidConfig configuration object
* @param {Object} logger bunyan logger object
*/
-function CitoidService(CitoidConfig, logger){
- this.CitoidConfig = CitoidConfig;
+function CitoidService(citoidConfig, logger){
this.log = logger;
- this.zoteroURL = util.format('http://%s:%s/',
- CitoidConfig.zoteroInterface,
CitoidConfig.zoteroPort.toString());
- this.zoteroService = new ZoteroService(this.zoteroURL, logger);
- this.scraper = new Scraper(CitoidConfig, logger);
+ this.zoteroService = new ZoteroService(citoidConfig, logger);
+ this.scraper = new Scraper(citoidConfig, logger);
}
/**
diff --git a/lib/Scraper.js b/lib/Scraper.js
index 34e8d8b..57edbab 100644
--- a/lib/Scraper.js
+++ b/lib/Scraper.js
@@ -16,6 +16,7 @@
function Scraper(citoidConfig, logger){
this.log = logger;
+ this.userAgent = citoidConfig.userAgent || 'Citoid/0.0.0';
}
/**
@@ -30,6 +31,7 @@
var chtml,
log = this.log,
scraper = this,
+ userAgent = this.userAgent,
citation = {url: url, title: url};
log.info(Using native scraper on + url);
@@ -38,6 +40,9 @@
{
url: url,
followAllRedirects: true,
+ headers: {
+ 'User-Agent': userAgent
+ }
}, function(error, response, html){
if (error || !response || response.statusCode !== 200) {
@@ -174,4 +179,4 @@
}
/**Exports*/
-module.exports = Scraper;
\ No newline at end of file
+module.exports = Scraper;
diff --git a/lib/ZoteroService.js b/lib/ZoteroService.js
index 0cc4a0c..d6df076 100644
--- a/lib/ZoteroService.js
+++ b/lib/ZoteroService.js
@@ -9,13 +9,15 @@
varasync = require('async'),
crypto = require('crypto'),
request = require('request'),
+ util = require('util'),
pubMedRequest = require('./pubMedRequest.js');
-function ZoteroService(zoteroURL, logger){
+function ZoteroService(citoidConfig, logger){
this.log = logger;
- this.baseURL = zoteroURL;
- this.webURL = zoteroURL + 'web';
- this.exportURL = zoteroURL + 'export';
+ var baseURL = util.format('http://%s:%s/',
+ citoidConfig.zoteroInterface,
citoidConfig.zoteroPort.toString());
+ this.webURL = baseURL + 'web';
+ this.exportURL = baseURL + 'export';
}
/**
diff --git a/localsettings.js.sample b/localsettings.js.sample
index f503415..23e60ba 100644
--- a/localsettings.js.sample
+++ b/localsettings.js.sample
@@ -21,7 +21,10 @@
citoidPort : 1970,
citoidInterface : 'localhost',
- //Settings for Zotero server
+ // userAgent string for lib/Scraper.js
+ userAgent : null,
+
+ // Settings for Zotero server
zoteroPort : 1969,
zoteroInterface : 'localhost',
};
--
To view, visit https://gerrit.wikimedia.org/r/193134
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I7dfa933181118527783da12cdb5b277329aef7dd
Gerrit-PatchSet: 10
Gerrit-Project: mediawiki/services/citoid
Gerrit-Branch: master
Gerrit-Owner: Mvolz mv...@wikimedia.org
Gerrit-Reviewer: Mobrovac mobro...@wikimedia.org
Gerrit-Reviewer: Mvolz mv...@wikimedia.org
Gerrit-Reviewer: jenkins-bot
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Add mocha tests - change (mediawiki...citoid)
Mobrovac has submitted this change and it was merged.
Change subject: Add mocha tests
..
Add mocha tests
Add mocha tests to grunt.
Update .jshintignore files.
Add two simple tests in tests/index.js
Bug: T91168
Change-Id: Id26b25bb14039adf68a8cf66d356f0ad0957bf58
---
M .jshintignore
M Gruntfile.js
M package.json
A test/index.js
4 files changed, 133 insertions(+), 47 deletions(-)
Approvals:
Mobrovac: Looks good to me, approved
diff --git a/.jshintignore b/.jshintignore
index 3c3629e..79e6ea4 100644
--- a/.jshintignore
+++ b/.jshintignore
@@ -1 +1,2 @@
node_modules
+test
\ No newline at end of file
diff --git a/Gruntfile.js b/Gruntfile.js
index 4bf92a2..ee5289a 100644
--- a/Gruntfile.js
+++ b/Gruntfile.js
@@ -2,6 +2,7 @@
// These plugins provide necessary tasks.
grunt.loadNpmTasks('grunt-contrib-jshint');
+ grunt.loadNpmTasks('grunt-simple-mocha');
// Project configuration.
grunt.initConfig({
@@ -16,11 +17,22 @@
'lib/*.js',
'lib/translators/*.js'
]
+ },
+ simplemocha: {
+ options: {
+ globals: ['describe', 'its'],
+ timeout: 3000,
+ ignoreLeaks: false,
+ ui: 'bdd',
+ reporter: 'tap'
+ },
+ all: { src: ['test/*.js'] }
}
});
// Default task.
- grunt.registerTask( 'test', [ 'jshint:all' ] );
- grunt.registerTask( 'default', 'test' );
+ grunt.registerTask('test', ['jshint:all']);
+ grunt.registerTask('withzotero', ['simplemocha']);
+ grunt.registerTask('default', 'test');
};
diff --git a/package.json b/package.json
index fa0c739..2468a97 100644
--- a/package.json
+++ b/package.json
@@ -1,47 +1,49 @@
{
- name : citoid,
- version : 0.0.0,
- description : Converts search terms such as URL or DOI into
citations.,
- scripts: {
- test: grunt test
- },
- dependencies : {
- async : 0.9.0,
- bluebird : 2.3.11,
- body-parser : 1.10.0,
- bunyan : 1.2.3,
- cheerio : 0.18.0,
- express : 4.10.4,
- html-metadata: 0.1.0,
- path: 0.4.9,
- request: 2.49.0,
- xmldom: 0.1.19,
- xpath: 0.0.7,
- yargs: 1.3.3
- },
- devDependencies: {
- grunt: 0.4.5,
- grunt-contrib-jshint: 0.10.0
- },
- repository: {
- type: git,
- url:
https://gerrit.wikimedia.org/r/mediawiki/services/citoid;
- },
- contributors: [
- {
- name: Marielle Volz,
- email: marielle.v...@gmail.com
- },
- {
- name: Danny Wu,
- email: utf8snow...@gmail.com
- },
- {
- name: Geoffrey Mon,
- email: geof...@gmail.com
- },
- {
- name: Dan Michael O. Heggø,
- email: danmicha...@gmail.com
- }]
+ name: citoid,
+ version: 0.0.0,
+ description: Converts search terms such as URL or DOI into citations.,
+ scripts: {
+test: grunt test
+ },
+ dependencies: {
+async: 0.9.0,
+bluebird: 2.3.11,
+body-parser: 1.10.0,
+bunyan: 1.2.3,
+cheerio: 0.18.0,
+express: 4.10.4,
+html-metadata: 0.1.0,
+path: 0.4.9,
+request: 2.49.0,
+xmldom: 0.1.19,
+xpath: 0.0.7,
+yargs: 1.3.3
+ },
+ devDependencies: {
+grunt: 0.4.5,
+grunt-contrib-jshint: 0.10.0,
+grunt-simple-mocha: 0.4.0
+ },
+ repository: {
+type: git,
+url: https://gerrit.wikimedia.org/r/mediawiki/services/citoid;
+ },
+ contributors: [
+{
+ name: Marielle Volz,
+ email: marielle.v...@gmail.com
+},
+{
+ name: Danny Wu,
+ email: utf8snow...@gmail.com
+},
+{
+ name: Geoffrey Mon,
+ email: geof...@gmail.com
+},
+{
+ name: Dan Michael O. Heggø,
+ email: danmicha...@gmail.com
+}
+ ]
}
diff --git a/test/index.js b/test/index.js
new file mode 100644
index 000..15b80ec
--- /dev/null
+++ b/test/index.js
@@ -0,0 +1,71 @@
+#!/usr/bin/env node
+/**
+ * https://www.mediawiki.org/wiki/citoid
+ */
+
+// mocha defines to avoid JSHint errors
+/* global describe, it */
+
+var CitoidService = require('../lib/CitoidService.js'),
+ bunyan = require('bunyan'),
+ path = require('path'),
+ opts = require('yargs')
+ .usage('Usage: $0 [-c configfile|--config=configfile]')
+ .default({
+ c: __dirname + '/localsettings.js'
+ })
+ .alias( 'c', 'config' ),
+ argv = opts.argv,
+
[MediaWiki-commits] [Gerrit] Introduce the Restbase Virtual REST Service class - change (mediawiki/core)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/193826
Change subject: Introduce the Restbase Virtual REST Service class
..
Introduce the Restbase Virtual REST Service class
Restbase, the REST content API service, is to be queried instead of
Parsoid by current Parsoid users (most importantly VE). This patch
introduces the Restbase virtual REST service class and transparently
forwards current Parsoid calls to Restbase, without any need for
intervention in user code. The switch is based on the
$wgParsoidVRSUseRestbase global variable. If set to a true value, the
back-end will use a RestbaseVirtualRESTService object instead of the
current ParsoidVirtualRESTService and remap all paths to match
Restbase's. This is a safe replacement, given that (a) both Parsoid and
Restbase return the same result for the same request; and (b) enabling
the Restbase back-end is done with a switch and can, thus, be
selectively enabled on individual wikis (for now, the plan is to
activate it on the test wiki for testing/performance).
Bug: T89066
Change-Id: I4d4043e5052327bbd789331f1c05b607c45fe7cb
---
M autoload.php
M includes/libs/virtualrest/ParsoidVirtualRESTService.php
A includes/libs/virtualrest/RestbaseVirtualRESTService.php
3 files changed, 175 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core
refs/changes/26/193826/1
diff --git a/autoload.php b/autoload.php
index 90fab64..9d80aab 100644
--- a/autoload.php
+++ b/autoload.php
@@ -995,6 +995,7 @@
'ResourceLoaderUserOptionsModule' = __DIR__ .
'/includes/resourceloader/ResourceLoaderUserOptionsModule.php',
'ResourceLoaderUserTokensModule' = __DIR__ .
'/includes/resourceloader/ResourceLoaderUserTokensModule.php',
'ResourceLoaderWikiModule' = __DIR__ .
'/includes/resourceloader/ResourceLoaderWikiModule.php',
+ 'RestbaseVirtualRESTService' = __DIR__ .
'/includes/libs/virtualrest/RestbaseVirtualRESTService.php',
'ResultWrapper' = __DIR__ . '/includes/db/DatabaseUtility.php',
'RevDelArchiveItem' = __DIR__ .
'/includes/revisiondelete/RevDelArchiveItem.php',
'RevDelArchiveList' = __DIR__ .
'/includes/revisiondelete/RevDelArchiveList.php',
diff --git a/includes/libs/virtualrest/ParsoidVirtualRESTService.php
b/includes/libs/virtualrest/ParsoidVirtualRESTService.php
index 03bdf0d..2b65e88 100644
--- a/includes/libs/virtualrest/ParsoidVirtualRESTService.php
+++ b/includes/libs/virtualrest/ParsoidVirtualRESTService.php
@@ -41,10 +41,29 @@
* - HTTPProxy : Parsoid HTTP proxy (optional)
*/
public function __construct( array $params ) {
+ global $wgParsoidVRSUseRestbase, $wgRestbaseServer, $wgServer;
parent::__construct( $params );
+ // if the config says we should use Restbase, create
+ // a new RestbaseVirtualRESTService object
+ $this-params['use_restbase'] = isset( $wgParsoidVRSUseRestbase
)
+ ? $wgParsoidVRSUseRestbase : false;
+ if ( $this-params['use_restbase'] ) {
+ $rbParams = array(
+ 'url' = $wgRestbaseServer,
+ 'domain' = preg_replace(
'/^(https?:\/\/)?([^\/:]+?)(\/|:\d+\/?)?$/', '$2', $wgServer ),
+ 'timeout' = $this-params['timeout'],
+ 'forwardCookies' =
$this-params['forwardCookies'],
+ 'HTTPProxy' = $this-params['HTTPProxy']
+ );
+ $this-restbaseVRS = new RestbaseVirtualRESTService(
$rbParams );
+ }
}
public function onRequests( array $reqs, Closure $idGeneratorFunc ) {
+ // just forward the request to Restbase if it's used
+ if ( $this-params['use_restbase'] ) {
+ return $this-restbaseVRS-onParsoidRequests( $reqs,
$idGeneratorFunc );
+ }
$result = array();
foreach ( $reqs as $key = $req ) {
$parts = explode( '/', $req['url'] );
diff --git a/includes/libs/virtualrest/RestbaseVirtualRESTService.php
b/includes/libs/virtualrest/RestbaseVirtualRESTService.php
new file mode 100644
index 000..6ccd24a
--- /dev/null
+++ b/includes/libs/virtualrest/RestbaseVirtualRESTService.php
@@ -0,0 +1,155 @@
+?php
+/**
+ * Virtual HTTP service client for Restbase
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY
[MediaWiki-commits] [Gerrit] Minor fixes - change (mediawiki...RestBaseUpdateJobs)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/193829
Change subject: Minor fixes
..
Minor fixes
* better RegEx for extracting the domain from $wgServer - now accounts
for the possibility of $wgServer having the form host:port
* minor variable name fix in RestbaseUpdate.php
Bug: T87520
Change-Id: I4f4242e31aeb21cf7aaff9fed24e0f376a0da4d2
---
M RestbaseUpdate.php
M RestbaseUpdateJob.php
2 files changed, 2 insertions(+), 2 deletions(-)
git pull
ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/RestBaseUpdateJobs
refs/changes/29/193829/1
diff --git a/RestbaseUpdate.php b/RestbaseUpdate.php
index bce71ac..93f2d60 100644
--- a/RestbaseUpdate.php
+++ b/RestbaseUpdate.php
@@ -70,7 +70,7 @@
/**
* The RESTBase server to inform of updates.
*/
- $wgRestbaseServers = 'http://localhost:7321';
+ $wgRestbaseServer = 'http://localhost:7321';
/**
* The RESTBase API version in use
diff --git a/RestbaseUpdateJob.php b/RestbaseUpdateJob.php
index 1883f5f..4b08ed2 100644
--- a/RestbaseUpdateJob.php
+++ b/RestbaseUpdateJob.php
@@ -45,7 +45,7 @@
global $wgRestbaseServer, $wgRestbaseAPIVersion,
$wgRestbaseDomain, $wgServer;
if ( !isset( $wgRestbaseDomain ) || is_null(
$wgRestbaseDomain ) ) {
- $wgRestbaseDomain = preg_replace(
'/^(https?:\/\/)?(.+?)\/?$/', '$2', $wgServer );
+ $wgRestbaseDomain = preg_replace(
'/^(https?:\/\/)?([^\/:]+?)(\/|:\d+\/?)?$/', '$2', $wgServer );
}
$prefix = implode( '/', array(
$wgRestbaseServer,
--
To view, visit https://gerrit.wikimedia.org/r/193829
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I4f4242e31aeb21cf7aaff9fed24e0f376a0da4d2
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/RestBaseUpdateJobs
Gerrit-Branch: master
Gerrit-Owner: Mobrovac mobro...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Make zotero.js an object - change (mediawiki...citoid)
Mobrovac has submitted this change and it was merged.
Change subject: Make zotero.js an object
..
Make zotero.js an object
Convert zotero.js to ZoteroService.js
object.
Modify CitoidService.js to point to
an instance of a ZoteroService object.
Bug: T78389
Change-Id: I48158fc037277ee688ef35ab0db688a46aaf54d7
---
M lib/CitoidService.js
R lib/ZoteroService.js
M server.js
3 files changed, 90 insertions(+), 112 deletions(-)
Approvals:
Mobrovac: Looks good to me, approved
diff --git a/lib/CitoidService.js b/lib/CitoidService.js
index 72b9334..73711dd 100644
--- a/lib/CitoidService.js
+++ b/lib/CitoidService.js
@@ -3,15 +3,13 @@
*/
/* Import Modules */
-var crypto = require('crypto'),
- urlParse = require('url'),
+var urlParse = require('url'),
util = require('util');
/* Import Local Modules */
var unshorten = require('./unshorten.js'),
scrape = require('./scrape.js'),
- zoteroWebRequest = require('./zotero.js').zoteroWebRequest,
- zoteroExportRequest = require('./zotero.js').zoteroExportRequest,
+ ZoteroService = require('./ZoteroService.js'),
pubMedRequest = require('./pubMedRequest.js');
/**
@@ -22,28 +20,23 @@
function CitoidService(CitoidConfig, logger){
this.CitoidConfig = CitoidConfig;
this.log = logger;
- this.zoteroURL = util.format('http://%s:%s/%s',
+ this.zoteroURL = util.format('http://%s:%s/',
CitoidConfig.zoteroInterface,
CitoidConfig.zoteroPort.toString());
+ this.zoteroService = new ZoteroService(this.zoteroURL, logger);
}
/**
* Requests to the citoid service
* @param {String} searchTerm searchTerm metadata is being requested about
- * @param {Object} opts zoteroWebRequest options object
+ * @param {Object} format export format
* @param {Function} callback callback (error, statusCode, body)
*/
CitoidService.prototype.request = function(searchTerm, format, callback){
- var citoidService = this,
- sessionID = crypto.randomBytes(20).toString('hex'), //required
zotero- not terribly important for this to be secure
- opts = {
- zoteroURL:citoidService.zoteroURL,
- sessionID:sessionID,
- format:format
- };
+ var citoidService = this;
citoidService.distinguish(searchTerm, function(extractedID,
runnerFunction){
- runnerFunction(extractedID, opts, function(error, responseCode,
body){
+ runnerFunction(extractedID, format, function(error,
responseCode, body){
callback(error, responseCode, body);
});
});
@@ -52,17 +45,20 @@
/**
* Request citation metadata from a URL
* @param {String} requestedURL URL metadata is being requested about
- * @param {Object} opts zoteroWebRequest options object
+ * @param {Object} format requested export format
* @param {Function} callback callback (error, statusCode, body)
*/
-CitoidService.prototype.requestFromURL = function (requestedURL, opts,
callback){
- var log = this.log;
- zoteroWebRequest(requestedURL, opts, function(error, response, body){
+CitoidService.prototype.requestFromURL = function (requestedURL, format,
callback){
+ var self = this,
+ log = self.log,
+ zoteroWebRequest =
this.zoteroService.zoteroWebRequest.bind(this.zoteroService);
+
+ zoteroWebRequest(requestedURL, format, function(error, response, body){
log.info(Zotero request made for: + requestedURL);
if (error) {
log.error(error);
log.info(Falling back on native scraper.);
- scrapeHelper(requestedURL, opts, callback);
+ self.scrapeHelper(requestedURL, format, callback);
} else if (response) {
//501 indicates no translator available
//this is common- can indicate shortened url
@@ -76,18 +72,18 @@
unshorten(requestedURL, function(detected,
expandedURL) {
if (detected) {
log.info(Redirect detected to
+ expandedURL);
- zoteroWebRequest(expandedURL,
opts, function(error, response, body){
+ zoteroWebRequest(expandedURL,
format, function(error, response, body){
if (response !error
response.statusCode === 200){
log.info(Successfully retrieved and translated body from Zotero);
callback(null,
200,
[MediaWiki-commits] [Gerrit] [WIP] Convert scrape.js to Scraper.js - change (mediawiki...citoid)
Mobrovac has submitted this change and it was merged.
Change subject: [WIP] Convert scrape.js to Scraper.js
..
[WIP] Convert scrape.js to Scraper.js
Convert scrape.js to an object, Scraper.js
In CitoidService.js, create new Scraper. Convert
scrapeHelper function to CitoidService.prototype.scrape
and modify scrape appropriately.
In Scraper.js, pass through logging object and enrich
logging messages.
Change-Id: I9617e810e8126fe1a17d94c1a6cb25f1e9c5316b
---
M lib/CitoidService.js
R lib/Scraper.js
2 files changed, 71 insertions(+), 67 deletions(-)
Approvals:
Mobrovac: Looks good to me, approved
diff --git a/lib/CitoidService.js b/lib/CitoidService.js
index 73711dd..4d284ee 100644
--- a/lib/CitoidService.js
+++ b/lib/CitoidService.js
@@ -8,7 +8,7 @@
/* Import Local Modules */
var unshorten = require('./unshorten.js'),
- scrape = require('./scrape.js'),
+ Scraper = require('./Scraper.js'),
ZoteroService = require('./ZoteroService.js'),
pubMedRequest = require('./pubMedRequest.js');
@@ -23,6 +23,7 @@
this.zoteroURL = util.format('http://%s:%s/',
CitoidConfig.zoteroInterface,
CitoidConfig.zoteroPort.toString());
this.zoteroService = new ZoteroService(this.zoteroURL, logger);
+ this.scraper = new Scraper(CitoidConfig, logger);
}
/**
@@ -51,24 +52,23 @@
CitoidService.prototype.requestFromURL = function (requestedURL, format,
callback){
var self = this,
log = self.log,
- zoteroWebRequest =
this.zoteroService.zoteroWebRequest.bind(this.zoteroService);
+ zoteroWebRequest =
self.zoteroService.zoteroWebRequest.bind(self.zoteroService);
zoteroWebRequest(requestedURL, format, function(error, response, body){
log.info(Zotero request made for: + requestedURL);
if (error) {
log.error(error);
- log.info(Falling back on native scraper.);
- self.scrapeHelper(requestedURL, format, callback);
+ self.scrape(requestedURL, format, callback);
} else if (response) {
- //501 indicates no translator available
- //this is common- can indicate shortened url
- //or a website not specified in the translators
+ // 501 indicates no translator available
+ // This is common- can indicate shortened url,
+ // or a website not specified in the translators
if (response.statusCode === 501){
- log.info(Status Code from Zotero: +
response.statusCode);
+ log.info(No Zotero translator found.);
log.info(Looking for redirects...);
- //try again following all redirects
- //we don't do this initially because many sites
- //will redirect this fcn to a log-in screen
+ // Try again following all redirects-
+ // We don't do this initially because many sites
+ // will redirect to a log-in screen
unshorten(requestedURL, function(detected,
expandedURL) {
if (detected) {
log.info(Redirect detected to
+ expandedURL);
@@ -77,26 +77,32 @@
log.info(Successfully retrieved and translated body from Zotero);
callback(null,
200, body);
} else {
- log.info(No
Zotero response available; falling back on native scraper.);
-
self.scrapeHelper(requestedURL, format, callback);
+ log.info(No
Zotero response available.);
+ // Try scraping
original URL before expansion
+
self.scrape(requestedURL, format, function(error, responseCode, body){
+ if
(error || responseCode !== 200){
+
// Try scraping expanded URL
+
self.scrape(expandedURL, format, callback);
+ } else {
+
[MediaWiki-commits] [Gerrit] Revert [WIP] Convert scrape.js to Scraper.js - change (mediawiki...citoid)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/193873
Change subject: Revert [WIP] Convert scrape.js to Scraper.js
..
Revert [WIP] Convert scrape.js to Scraper.js
This reverts commit 522a3ae04d4decb1da975845ab03d4e9a0800d50.
Change-Id: I4e16854c2c7ca318d675ecba00a2e9debad90583
---
M lib/CitoidService.js
R lib/scrape.js
2 files changed, 67 insertions(+), 71 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/citoid
refs/changes/73/193873/1
diff --git a/lib/CitoidService.js b/lib/CitoidService.js
index 4d284ee..73711dd 100644
--- a/lib/CitoidService.js
+++ b/lib/CitoidService.js
@@ -8,7 +8,7 @@
/* Import Local Modules */
var unshorten = require('./unshorten.js'),
- Scraper = require('./Scraper.js'),
+ scrape = require('./scrape.js'),
ZoteroService = require('./ZoteroService.js'),
pubMedRequest = require('./pubMedRequest.js');
@@ -23,7 +23,6 @@
this.zoteroURL = util.format('http://%s:%s/',
CitoidConfig.zoteroInterface,
CitoidConfig.zoteroPort.toString());
this.zoteroService = new ZoteroService(this.zoteroURL, logger);
- this.scraper = new Scraper(CitoidConfig, logger);
}
/**
@@ -52,23 +51,24 @@
CitoidService.prototype.requestFromURL = function (requestedURL, format,
callback){
var self = this,
log = self.log,
- zoteroWebRequest =
self.zoteroService.zoteroWebRequest.bind(self.zoteroService);
+ zoteroWebRequest =
this.zoteroService.zoteroWebRequest.bind(this.zoteroService);
zoteroWebRequest(requestedURL, format, function(error, response, body){
log.info(Zotero request made for: + requestedURL);
if (error) {
log.error(error);
- self.scrape(requestedURL, format, callback);
+ log.info(Falling back on native scraper.);
+ self.scrapeHelper(requestedURL, format, callback);
} else if (response) {
- // 501 indicates no translator available
- // This is common- can indicate shortened url,
- // or a website not specified in the translators
+ //501 indicates no translator available
+ //this is common- can indicate shortened url
+ //or a website not specified in the translators
if (response.statusCode === 501){
- log.info(No Zotero translator found.);
+ log.info(Status Code from Zotero: +
response.statusCode);
log.info(Looking for redirects...);
- // Try again following all redirects-
- // We don't do this initially because many sites
- // will redirect to a log-in screen
+ //try again following all redirects
+ //we don't do this initially because many sites
+ //will redirect this fcn to a log-in screen
unshorten(requestedURL, function(detected,
expandedURL) {
if (detected) {
log.info(Redirect detected to
+ expandedURL);
@@ -77,32 +77,26 @@
log.info(Successfully retrieved and translated body from Zotero);
callback(null,
200, body);
} else {
- log.info(No
Zotero response available.);
- // Try scraping
original URL before expansion
-
self.scrape(requestedURL, format, function(error, responseCode, body){
- if
(error || responseCode !== 200){
-
// Try scraping expanded URL
-
self.scrape(expandedURL, format, callback);
- } else {
-
callback(error, responseCode, body);
- }
- });
+ log.info(No
Zotero response available; falling back on native
[MediaWiki-commits] [Gerrit] Update service git repos with vagrant git-update as well - change (mediawiki/vagrant)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/194556
Change subject: Update service git repos with vagrant git-update as well
..
Update service git repos with vagrant git-update as well
vagrant git-update is a great tool for automatically updating all of the
MediaWiki repositories checked out in the VM. However, it did not cover
services installed via roles (traditionally put in /srv). This patch
adds that functionality to it.
Bug: T91668
Change-Id: I1cc453eebe4bfc9a56bd3f0337f86bfdf0f325d4
---
M puppet/modules/mediawiki/files/run-git-update
1 file changed, 18 insertions(+), 6 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/vagrant
refs/changes/56/194556/1
diff --git a/puppet/modules/mediawiki/files/run-git-update
b/puppet/modules/mediawiki/files/run-git-update
index 747bead..52022e8 100644
--- a/puppet/modules/mediawiki/files/run-git-update
+++ b/puppet/modules/mediawiki/files/run-git-update
@@ -10,26 +10,33 @@
UPDATE_ERRORS=( )
function pull {
+ git_cmd=git
+ # /srv dirs are all root, so use sudo in that case
+ [[ -O . ]] || git_cmd=sudo git
echo -e \e[36m== Updating $(pwd) ...\e[0m
- branch=$(expr $(git symbolic-ref HEAD) : 'refs/heads/\(.*\)')
+ branch=$(expr $($git_cmd symbolic-ref HEAD) : 'refs/heads/\(.*\)')
+ if [[ -z $branch ]]; then
+$git_cmd checkout master
+branch=$(expr $($git_cmd symbolic-ref HEAD) : 'refs/heads/\(.*\)')
+ fi
err=''
if [[ -n $branch ]]; then
-remote=$(git config branch.${branch}.remote)
+remote=$($git_cmd config branch.${branch}.remote)
if [[ -n $remote ]]; then
-url=$(git config --get remote.${remote}.url)
+url=$($git_cmd config --get remote.${remote}.url)
if [[ $url == ssh://* ]]; then
# Convert remote git url from ssh:// to anonymous https://
tempurl=$(echo $url | sed -e 's!ssh://[^@]\+@!https://!g' -e
's!:29418!/r!g')
-git pull $tempurl
+$git_cmd pull $tempurl
else
-git pull
+$git_cmd pull
fi
if [[ $? -ne 0 ]]; then
# If we didn't successfully update (possibly because we're on
# a local branch), leave the submodules alone.
err=GIT PULL failed in $(pwd) for branch '$branch'
else
- git submodule update --init --recursive
+ $git_cmd submodule update --init --recursive
if [[ $? -ne 0 ]]; then
err=GIT SUBMODULE UPDATE failed in $(pwd) for branch '$branch'
fi
@@ -49,6 +56,11 @@
echo
}
+for srvdir in $(find /srv -maxdepth 2 -type d -name .git -printf %h\n); do
+ cd ${srvdir}
+ pull
+done
+
cd $MW_INSTALL_PATH
pull
--
To view, visit https://gerrit.wikimedia.org/r/194556
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I1cc453eebe4bfc9a56bd3f0337f86bfdf0f325d4
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/vagrant
Gerrit-Branch: master
Gerrit-Owner: Mobrovac mobro...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Add ability to request AcceptLanguage - change (mediawiki...citoid)
Mobrovac has submitted this change and it was merged.
Change subject: Add ability to request AcceptLanguage
..
Add ability to request AcceptLanguage
*Retrieves accept-language header in both api
and url endpoints
*Uses same value of accept-language to make
requests in lib/Scraper.js
*Add test for requesting twitter.com in German
*Modify existing tests to check more fields in
citation.
*Change first argument from url to opts, an
options argument, to include acceptLanguage
parameter, in many functions.
Change-Id: I9a20ffed68c2a7337031c444ee141b08b2ffb074
---
M Gruntfile.js
M lib/CitoidService.js
M lib/Scraper.js
M server.js
M test/index.js
5 files changed, 121 insertions(+), 54 deletions(-)
Approvals:
Mobrovac: Looks good to me, approved
diff --git a/Gruntfile.js b/Gruntfile.js
index ee5289a..8c04814 100644
--- a/Gruntfile.js
+++ b/Gruntfile.js
@@ -15,7 +15,8 @@
'*.js',
'localsettings.js.sample',
'lib/*.js',
- 'lib/translators/*.js'
+ 'lib/translators/*.js',
+ 'test/*.js'
]
},
simplemocha: {
diff --git a/lib/CitoidService.js b/lib/CitoidService.js
index 7d5f666..e5d2f03 100644
--- a/lib/CitoidService.js
+++ b/lib/CitoidService.js
@@ -13,7 +13,7 @@
/**
* Constructor for CitoidService object
- * @param {Object} CitoidConfig configuration object
+ * @param {Object} citoidConfig configuration object
* @param {Object} logger bunyan logger object
*/
function CitoidService(citoidConfig, logger){
@@ -24,37 +24,41 @@
/**
* Requests to the citoid service
- * @param {String} searchTerm searchTerm metadata is being requested about
- * @param {Object} format export format
- * @param {Function} callback callback (error, statusCode, body)
+ * @param {Object} opts options object containing request information
+ * @param {Function} callback callback (error, statusCode, body)
*/
-CitoidService.prototype.request = function(searchTerm, format, callback){
+CitoidService.prototype.request = function(opts, callback){
- var citoidService = this;
+ var runnerOpts,
+ citoidService = this;
- citoidService.distinguish(searchTerm, function(extractedID,
runnerFunction){
- runnerFunction(extractedID, format, function(error,
responseCode, body){
- callback(error, responseCode, body);
- });
+ citoidService.distinguish(opts.search, function(extractedID,
runnerFunction){
+ runnerOpts = {
+ format : opts.format,
+ search : extractedID,
+ acceptLanguage : opts.acceptLanguage
+ };
+ runnerFunction(runnerOpts, callback);
});
};
/**
* Request citation metadata from a URL
- * @param {String} requestedURL URL metadata is being requested about
- * @param {Object} format requested export format
+ * @param {Object} opts options object containing requested url
* @param {Function} callback callback (error, statusCode, body)
*/
-CitoidService.prototype.requestFromURL = function (requestedURL, format,
callback){
+CitoidService.prototype.requestFromURL = function (opts, callback){
var self = this,
log = self.log,
- zoteroWebRequest =
self.zoteroService.zoteroWebRequest.bind(self.zoteroService);
+ zoteroWebRequest =
self.zoteroService.zoteroWebRequest.bind(self.zoteroService),
+ requestedURL = opts.search,
+ format = opts.format;
zoteroWebRequest(requestedURL, format, function(error, response, body){
log.info(Zotero request made for: + requestedURL);
if (error) {
log.error(error);
- self.scrape(requestedURL, format, callback);
+ self.scrape(opts, callback);
} else if (response) {
// 501 indicates no translator available
// This is common- can indicate shortened url,
@@ -75,10 +79,10 @@
} else {
log.info(No
Zotero response available.);
// Try scraping
original URL before expansion
-
self.scrape(requestedURL, format, function(error, responseCode, body){
+
self.scrape(opts, function(error, responseCode, body){
if
[MediaWiki-commits] [Gerrit] Remove i18n shim for 1.22 and below - change (mediawiki...RestBaseUpdateJobs)
Mobrovac has submitted this change and it was merged.
Change subject: Remove i18n shim for 1.22 and below
..
Remove i18n shim for 1.22 and below
Change-Id: Icadf07644b015f26390ec9e55f124f6ebf68a5bd
---
D RestbaseUpdate.i18n.php
M RestbaseUpdate.php
2 files changed, 0 insertions(+), 36 deletions(-)
Approvals:
Mobrovac: Verified; Looks good to me, approved
diff --git a/RestbaseUpdate.i18n.php b/RestbaseUpdate.i18n.php
deleted file mode 100644
index 0c95e5c..000
--- a/RestbaseUpdate.i18n.php
+++ /dev/null
@@ -1,35 +0,0 @@
-?php
-/**
- * This is a backwards-compatibility shim, generated by:
- *
https://git.wikimedia.org/blob/mediawiki%2Fcore.git/HEAD/maintenance%2FgenerateJsonI18n.php
- *
- * Beginning with MediaWiki 1.23, translation strings are stored in json files,
- * and the EXTENSION.i18n.php file only exists to provide compatibility with
- * older releases of MediaWiki. For more information about this migration, see:
- * https://www.mediawiki.org/wiki/Requests_for_comment/Localisation_format
- *
- * This shim maintains compatibility back to MediaWiki 1.17.
- */
-$messages = array();
-if ( !function_exists( 'wfJsonI18nShim8d1c46917fb38c99' ) ) {
- function wfJsonI18nShim8d1c46917fb38c99( $cache, $code, $cachedData ) {
- $codeSequence = array_merge( array( $code ),
$cachedData['fallbackSequence'] );
- foreach ( $codeSequence as $csCode ) {
- $fileName = dirname( __FILE__ ) . /i18n/$csCode.json;
- if ( is_readable( $fileName ) ) {
- $data = FormatJson::decode( file_get_contents(
$fileName ), true );
- foreach ( array_keys( $data ) as $key ) {
- if ( $key === '' || $key[0] === '@' ) {
- unset( $data[$key] );
- }
- }
- $cachedData['messages'] = array_merge( $data,
$cachedData['messages'] );
- }
-
- $cachedData['deps'][] = new FileDependency( $fileName );
- }
- return true;
- }
-
- $GLOBALS['wgHooks']['LocalisationCacheRecache'][] =
'wfJsonI18nShim8d1c46917fb38c99';
-}
diff --git a/RestbaseUpdate.php b/RestbaseUpdate.php
index bce71ac..d6a50e2 100644
--- a/RestbaseUpdate.php
+++ b/RestbaseUpdate.php
@@ -47,7 +47,6 @@
# Register localizations.
$wgMessagesDirs['RestBaseUpdateJobs'] = __DIR__ . '/i18n';
- $wgExtensionMessagesFiles['RestBaseUpdateJobs'] = $dir .
'/RestbaseUpdate.i18n.php';
# Set up a default configuration
self::setupDefaultConfig();
--
To view, visit https://gerrit.wikimedia.org/r/192817
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Icadf07644b015f26390ec9e55f124f6ebf68a5bd
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/RestBaseUpdateJobs
Gerrit-Branch: master
Gerrit-Owner: Chad ch...@wikimedia.org
Gerrit-Reviewer: GWicke gwi...@wikimedia.org
Gerrit-Reviewer: Mobrovac mobro...@wikimedia.org
Gerrit-Reviewer: Siebrand siebr...@kitano.nl
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Ensure that hhvm is fully installed before mediawiki - change (mediawiki/vagrant)
Mobrovac has submitted this change and it was merged. Change subject: Ensure that hhvm is fully installed before mediawiki .. Ensure that hhvm is fully installed before mediawiki Make Puppet resource ordering slightly more defined by ensuring that all of the statements in the ::hhvm class have been applied before any of the statements in ::mediawiki attempt to use HHVM. Bug: T89390 Change-Id: Ib3755d94bfd936e78f112605099a4d1d14e7b5ec --- M puppet/modules/mediawiki/manifests/init.pp 1 file changed, 1 insertion(+), 1 deletion(-) Approvals: Mobrovac: Verified; Looks good to me, approved diff --git a/puppet/modules/mediawiki/manifests/init.pp b/puppet/modules/mediawiki/manifests/init.pp index c9b887c..a35d563 100644 --- a/puppet/modules/mediawiki/manifests/init.pp +++ b/puppet/modules/mediawiki/manifests/init.pp @@ -70,7 +70,7 @@ } require ::php -include ::hhvm +require ::hhvm include ::mediawiki::apache include ::mediawiki::jobrunner -- To view, visit https://gerrit.wikimedia.org/r/194147 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ib3755d94bfd936e78f112605099a4d1d14e7b5ec Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/vagrant Gerrit-Branch: master Gerrit-Owner: BryanDavis bda...@wikimedia.org Gerrit-Reviewer: BryanDavis bda...@wikimedia.org Gerrit-Reviewer: Mobrovac mobro...@wikimedia.org Gerrit-Reviewer: Ori.livneh o...@wikimedia.org Gerrit-Reviewer: jenkins-bot ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Use the RESTBase back-end if available - change (mediawiki...VisualEditor)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/194338
Change subject: Use the RESTBase back-end if available
..
Use the RESTBase back-end if available
Change I4d4043e5052327bbd789331f1c05b607c45fe7cb introduces RESTBase's
virtual REST service in the MW core, as well as $wgVRS, which represents
the first step towards central VRS configuration. This patch modifies VE
to use RESTBase instead of Parsoid, if available. RESTBase's virtual
REST service transparently maps parsoid URIs to RESTBase's, so there is
no need to change them in VE for now.
Note that the patch keeps full compatibility with systems/domains that
do not have $wgVRS declared (or even with those that do not include the
RESTBase virtual REST service class). This allows us to use RESTBase as
the back-end only on selected domains.
Bug: T89066
Change-Id: Ie7488f64868a41f28ec24ab1217c12c6249b5523
---
M ApiVisualEditor.php
1 file changed, 47 insertions(+), 11 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/VisualEditor
refs/changes/38/194338/1
diff --git a/ApiVisualEditor.php b/ApiVisualEditor.php
index b6a5498..4494611 100644
--- a/ApiVisualEditor.php
+++ b/ApiVisualEditor.php
@@ -23,18 +23,54 @@
public function __construct( ApiMain $main, $name, Config $config ) {
parent::__construct( $main, $name );
$this-veConfig = $config;
- $forwardCookies = false;
- if ( $config-get( 'VisualEditorParsoidForwardCookies' )
!User::isEveryoneAllowed( 'read' ) ) {
- $forwardCookies =
RequestContext::getMain()-getRequest()-getHeader( 'Cookie' );
- }
$this-serviceClient = new VirtualRESTServiceClient( new
MultiHttpClient( array() ) );
- $this-serviceClient-mount( '/parsoid/', new
ParsoidVirtualRESTService( array(
- 'URL' = $config-get( 'VisualEditorParsoidURL' ),
- 'prefix' = $config-get( 'VisualEditorParsoidPrefix' ),
- 'timeout' = $config-get( 'VisualEditorParsoidTimeout'
),
- 'HTTPProxy' = $config-get(
'VisualEditorParsoidHTTPProxy' ),
- 'forwardCookies' = $forwardCookies,
- ) ) );
+ $this-serviceClient-mount( '/parsoid/', $this-getVRSObject()
);
+ }
+
+ private function getVRSObject() {
+ // the params array to create the service object with
+ $params = array();
+ // the VRS class to use, defaults to Parsoid
+ $class = 'ParsoidVirtualRESTService';
+ $config = $this-veConfig;
+ // the global virtual rest service config object, if any
+ $vrs = array();
+ try {
+ $vrs = $config-get( 'VRS' );
+ } catch ( ConfigException $e ) {
+ // noop, just means $wgVRS is not defined
+ }
+ if ( isset( $vrs['modules']['restbase'] ) ) {
+ // if restbase is available, use it
+ $params = $vrs['modules']['restbase'];
+ $class = 'RestbaseVirtualRESTService';
+ // remove once VE generates restbase paths
+ $params['parsoidCompat'] = true;
+ } elseif ( isset( $vrs['modules']['parsoid'] ) ) {
+ // there's a global parsoid config, use it next
+ $params = $vrs['modules']['parsoid'];
+ } else {
+ // no global modules defined, fall back to old defaults
+ $params = array(
+ 'URL' = $config-get( 'VisualEditorParsoidURL'
),
+ 'prefix' = $config-get(
'VisualEditorParsoidPrefix' ),
+ 'timeout' = $config-get(
'VisualEditorParsoidTimeout' ),
+ 'HTTPProxy' = $config-get(
'VisualEditorParsoidHTTPProxy' ),
+ 'forwardCookies' = $config-get(
'VisualEditorParsoidForwardCookies' )
+ );
+ }
+ // merge the global and service-specific params
+ if ( isset( $vrs['global'] ) ) {
+ $params = array_merge($vrs['global'], $params);
+ }
+ // set up cookie forwarding
+ if ( $params['forwardCookies'] !User::isEveryoneAllowed(
'read' ) ) {
+ $params['forwardCookies'] =
RequestContext::getMain()-getRequest()-getHeader( 'Cookie' );
+ } else {
+ $params['forwardCookies'] = false;
+ }
+ // create the VRS object and return it
+ return new $class( $params );
}
private function
[MediaWiki-commits] [Gerrit] Use html-metadata library - change (mediawiki...citoid)
Mobrovac has submitted this change and it was merged.
Change subject: Use html-metadata library
..
Use html-metadata library
Use-html metadata library to extract additional
metadata from scraped pages.
Add translator folder to contain files to translate
between scraped metadata and interal format (Zotero).
Add translators for general metadata and for OpenGraph
metadata. Support for vertical types in OpenGraph.js
is incomplete.
Bug: T1069
Change-Id: I8d571305e853f41748ade494f52430a44e11bf75
---
M Gruntfile.js
M lib/requests.js
M lib/scrape.js
A lib/translators/README.md
A lib/translators/general.js
A lib/translators/openGraph.js
M package.json
7 files changed, 263 insertions(+), 58 deletions(-)
Approvals:
Mobrovac: Looks good to me, approved
GWicke: Looks good to me, but someone else must approve
diff --git a/Gruntfile.js b/Gruntfile.js
index 554661b..4bf92a2 100644
--- a/Gruntfile.js
+++ b/Gruntfile.js
@@ -13,7 +13,8 @@
all: [
'*.js',
'localsettings.js.sample',
- 'lib/*.js'
+ 'lib/*.js',
+ 'lib/translators/*.js'
]
}
});
diff --git a/lib/requests.js b/lib/requests.js
index a198730..73d2795 100644
--- a/lib/requests.js
+++ b/lib/requests.js
@@ -11,7 +11,7 @@
/* Import Local Modules */
var unshorten = require('./unshorten.js'),
- scrape = require('./scrape.js').scrape,
+ scrape = require('./scrape.js'),
zoteroWebRequest = require('./zotero.js').zoteroWebRequest,
zoteroExportRequest = require('./zotero.js').zoteroExportRequest,
pubMedRequest = require('./pubMedRequest.js');
diff --git a/lib/scrape.js b/lib/scrape.js
index 21e4c1b..38abf8e 100644
--- a/lib/scrape.js
+++ b/lib/scrape.js
@@ -3,82 +3,176 @@
* https://www.mediawiki.org/wiki/citoid
*/
-/* Import Modules */
+/*
+ Module dependencies
+*/
+
var request = require('request'),
urlParse = require('url'),
- cheerio = require('cheerio');
+ cheerio = require('cheerio'),
+ parseMetaData = require('html-metadata').parseAll,
+ bunyan = require('bunyan'),
+ og = require('./translators/openGraph.js'),
+ gen = require('./translators/general.js'),
+ log = bunyan.createLogger({name: citoid});
/**
- * Currently scrapes title only
+ * Scrapes, parses, and translates webpages to obtain Zotero format metadata
* callback runs on list of json objs (var body)
* @param {String} url url to scrape
* @param {Function} callback callback(error, statusCode, body)
*/
-var scrape = function(url, callback){
+exports = module.exports = function(url, callback){
- var $;
-
- function getTitle() {
-
- var title;
-
- // Try to get title from itemprop=heading
- title = $('*[itemprop~=headline]').first().text();
- if (title) { return title; }
-
- // Try to get title from title tag
- title = $('title').first().text();
- if (title) { return title; }
-
- // Default
- return url;
- }
+ var chtml,
+ citation = {url: url, title: url};
request(
{
url: url,
- followAllRedirects: true
+ followAllRedirects: true,
}, function(error, response, html){
-
- var citation = {itemType: 'webpage', url: url, title:
url};
if (error || !response || response.statusCode !== 200) {
callback(error, 520, [citation]);
return;
+ } else {
+ try {
+ chtml = cheerio.load(html);
+ citation.title = null;
+ exports.parseHTML(url, chtml, citation,
function(citation){
+ citation = citation;
+ callback(null, 200, [citation]);
+ });
+ } catch (e){
+ log.error(e);
+ callback(error, 520, [citation]);
+ }
}
-
- try{
- $ = cheerio.load(html);
- }
- catch (e){
- callback(error, 520, [citation]);
- }
-
- citation.title = getTitle();
-
- // Access date on format -MM-DD
-
[MediaWiki-commits] [Gerrit] Create CitoidService object - change (mediawiki...citoid)
Mobrovac has submitted this change and it was merged.
Change subject: Create CitoidService object
..
Create CitoidService object
In server.js, create new CitoidService
object and pass it the CitoidConfig objects
and the logger.
Move logic of citoid requests from server.js
into lib/CitoidService.js
Merge lib/requests.js into CitoidService.js
and convert functions into prototype functions
Merge lib/distinguish into CitoidService.js
and convert into prototype function.
Bug: T89968
Change-Id: I59f15059a83f8df2ed08ef3aab29ff5c65609b68
---
A lib/CitoidService.js
D lib/distinguish.js
D lib/requests.js
M server.js
4 files changed, 227 insertions(+), 294 deletions(-)
Approvals:
Mobrovac: Looks good to me, approved
diff --git a/lib/CitoidService.js b/lib/CitoidService.js
new file mode 100644
index 000..72b9334
--- /dev/null
+++ b/lib/CitoidService.js
@@ -0,0 +1,206 @@
+/**
+ * Handles requests to the citoid service
+ */
+
+/* Import Modules */
+var crypto = require('crypto'),
+ urlParse = require('url'),
+ util = require('util');
+
+/* Import Local Modules */
+var unshorten = require('./unshorten.js'),
+ scrape = require('./scrape.js'),
+ zoteroWebRequest = require('./zotero.js').zoteroWebRequest,
+ zoteroExportRequest = require('./zotero.js').zoteroExportRequest,
+ pubMedRequest = require('./pubMedRequest.js');
+
+/**
+ * Constructor for CitoidService object
+ * @param {Object} CitoidConfig configuration object
+ * @param {Object} logger bunyan logger object
+ */
+function CitoidService(CitoidConfig, logger){
+ this.CitoidConfig = CitoidConfig;
+ this.log = logger;
+ this.zoteroURL = util.format('http://%s:%s/%s',
+ CitoidConfig.zoteroInterface,
CitoidConfig.zoteroPort.toString());
+}
+
+/**
+ * Requests to the citoid service
+ * @param {String} searchTerm searchTerm metadata is being requested about
+ * @param {Object} opts zoteroWebRequest options object
+ * @param {Function} callback callback (error, statusCode, body)
+ */
+CitoidService.prototype.request = function(searchTerm, format, callback){
+
+ var citoidService = this,
+ sessionID = crypto.randomBytes(20).toString('hex'), //required
zotero- not terribly important for this to be secure
+ opts = {
+ zoteroURL:citoidService.zoteroURL,
+ sessionID:sessionID,
+ format:format
+ };
+
+ citoidService.distinguish(searchTerm, function(extractedID,
runnerFunction){
+ runnerFunction(extractedID, opts, function(error, responseCode,
body){
+ callback(error, responseCode, body);
+ });
+ });
+};
+
+/**
+ * Request citation metadata from a URL
+ * @param {String} requestedURL URL metadata is being requested about
+ * @param {Object} opts zoteroWebRequest options object
+ * @param {Function} callback callback (error, statusCode, body)
+ */
+CitoidService.prototype.requestFromURL = function (requestedURL, opts,
callback){
+ var log = this.log;
+ zoteroWebRequest(requestedURL, opts, function(error, response, body){
+ log.info(Zotero request made for: + requestedURL);
+ if (error) {
+ log.error(error);
+ log.info(Falling back on native scraper.);
+ scrapeHelper(requestedURL, opts, callback);
+ } else if (response) {
+ //501 indicates no translator available
+ //this is common- can indicate shortened url
+ //or a website not specified in the translators
+ if (response.statusCode === 501){
+ log.info(Status Code from Zotero: +
response.statusCode);
+ log.info(Looking for redirects...);
+ //try again following all redirects
+ //we don't do this initially because many sites
+ //will redirect this fcn to a log-in screen
+ unshorten(requestedURL, function(detected,
expandedURL) {
+ if (detected) {
+ log.info(Redirect detected to
+ expandedURL);
+ zoteroWebRequest(expandedURL,
opts, function(error, response, body){
+ if (response !error
response.statusCode === 200){
+
log.info(Successfully retrieved and translated body from Zotero);
+ callback(null,
200, body);
+
[MediaWiki-commits] [Gerrit] Update restbase ref - change (mediawiki...deploy)
Mobrovac has uploaded a new change for review. https://gerrit.wikimedia.org/r/191916 Change subject: Update restbase ref .. Update restbase ref Change-Id: I1867606ef135bc5d4fad4f4f50cd0f978b0dc820 --- M restbase 1 file changed, 0 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/restbase/deploy refs/changes/16/191916/1 diff --git a/restbase b/restbase index 2647407..bc8e1cd 16 --- a/restbase +++ b/restbase -Subproject commit 2647407a0661736683209025d3e57498d0220e9f +Subproject commit bc8e1cde0784fc9fb3dba4b4693216aee63ef484 -- To view, visit https://gerrit.wikimedia.org/r/191916 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I1867606ef135bc5d4fad4f4f50cd0f978b0dc820 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/services/restbase/deploy Gerrit-Branch: master Gerrit-Owner: Mobrovac mobro...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Update restbase ref - change (mediawiki...deploy)
Mobrovac has submitted this change and it was merged. Change subject: Update restbase ref .. Update restbase ref Change-Id: I1867606ef135bc5d4fad4f4f50cd0f978b0dc820 --- M restbase 1 file changed, 0 insertions(+), 0 deletions(-) Approvals: Mobrovac: Verified; Looks good to me, approved diff --git a/restbase b/restbase index 3d93e39..bc8e1cd 16 --- a/restbase +++ b/restbase -Subproject commit 3d93e39edbb400a176a4e4b3f83d29e37eb888f5 +Subproject commit bc8e1cde0784fc9fb3dba4b4693216aee63ef484 -- To view, visit https://gerrit.wikimedia.org/r/191916 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I1867606ef135bc5d4fad4f4f50cd0f978b0dc820 Gerrit-PatchSet: 2 Gerrit-Project: mediawiki/services/restbase/deploy Gerrit-Branch: master Gerrit-Owner: Mobrovac mobro...@wikimedia.org Gerrit-Reviewer: GWicke gwi...@wikimedia.org Gerrit-Reviewer: Mobrovac mobro...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Update RESTBase's configuration file - change (operations/puppet)
Mobrovac has uploaded a new change for review. https://gerrit.wikimedia.org/r/191922 Change subject: Update RESTBase's configuration file .. Update RESTBase's configuration file This commit includes only small description changes Change-Id: I0b53d61e96deccdede2329273bc7cbbe4eabeaec --- M modules/restbase/templates/config.yaml.erb 1 file changed, 3 insertions(+), 3 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/22/191922/1 diff --git a/modules/restbase/templates/config.yaml.erb b/modules/restbase/templates/config.yaml.erb index bd6a3db..a0f69d7 100644 --- a/modules/restbase/templates/config.yaml.erb +++ b/modules/restbase/templates/config.yaml.erb @@ -27,11 +27,11 @@ # swagger options, overriding the shared ones from the merged specs (?) info: version: 1.0.0-abcd - title: Standard Wikimedia content API - description: All the content for this domain. + title: Wikimedia REST API + description: A REST content API, currently strongly focused on page content. termsOfService: http://wikimedia.org/terms/ contact: -name: The project maintainers +name: the Wikimedia Services team url: http://mediawiki.org/wiki/RESTBase license: name: Creative Commons 4.0 International -- To view, visit https://gerrit.wikimedia.org/r/191922 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I0b53d61e96deccdede2329273bc7cbbe4eabeaec Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Mobrovac mobro...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Add 'v' option to Google Books translator - change (mediawiki...translators)
Mobrovac has submitted this change and it was merged. Change subject: Add 'v' option to Google Books translator .. Add 'v' option to Google Books translator Add 'v' option to Google Books translator so it use used by translation-server. (Translators without this option are ignored by translation-server). Bug: T94169 Change-Id: Ibf1614ec7fee348bbf981a2446d460cf7c1fb91a --- M Google Books.js 1 file changed, 2 insertions(+), 2 deletions(-) Approvals: Mobrovac: Verified; Looks good to me, approved diff --git a/Google Books.js b/Google Books.js index 188871c..642a69d 100644 --- a/Google Books.js +++ b/Google Books.js @@ -8,7 +8,7 @@ priority: 100, inRepository: true, translatorType: 4, - browserSupport: gcsb, + browserSupport: gcsbv, lastUpdated: 2014-12-11 17:17:45 } @@ -492,4 +492,4 @@ ] } ] -/** END TEST CASES **/ \ No newline at end of file +/** END TEST CASES **/ -- To view, visit https://gerrit.wikimedia.org/r/200157 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ibf1614ec7fee348bbf981a2446d460cf7c1fb91a Gerrit-PatchSet: 4 Gerrit-Project: mediawiki/services/zotero/translators Gerrit-Branch: master Gerrit-Owner: Mvolz mv...@wikimedia.org Gerrit-Reviewer: Alexandros Kosiaris akosia...@wikimedia.org Gerrit-Reviewer: Mobrovac mobro...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Fix: Handle case where Zotero gives us empty list - change (mediawiki...citoid)
Mobrovac has submitted this change and it was merged.
Change subject: Fix: Handle case where Zotero gives us empty list
..
Fix: Handle case where Zotero gives us empty list
Make citoid robust to the case where Zotero gives us
a 200 and a response of an empty list.
Bug: T94178
Change-Id: Id1d3e1fcfac03c92a6a1eb24e0746e6cd0a37ea1
---
M lib/CitoidService.js
M lib/ZoteroService.js
M package.json
M test/features/errors/index.js
4 files changed, 15 insertions(+), 3 deletions(-)
Approvals:
Mobrovac: Verified; Looks good to me, approved
Jforrester: Looks good to me, but someone else must approve
diff --git a/lib/CitoidService.js b/lib/CitoidService.js
index 6fb0875..3246e22 100644
--- a/lib/CitoidService.js
+++ b/lib/CitoidService.js
@@ -99,7 +99,8 @@
self.scrape(opts, callback);
}
});
- } else if (response.statusCode === 200){
+ // Need to check that response is a non-empty Array, as
occasionally this occurs
+ } else if (response.statusCode === 200
Array.isArray(response.body) response.body[0]){
logger.log('debug/zotero', Successfully
retrieved and translated body from Zotero);
callback (null, 200, body);
} else {
diff --git a/lib/ZoteroService.js b/lib/ZoteroService.js
index 7ab8683..6957b83 100644
--- a/lib/ZoteroService.js
+++ b/lib/ZoteroService.js
@@ -44,7 +44,7 @@
};
request(options, function (error, response, body) {
- if (!error response.statusCode === 200) {
+ if (!error response response.statusCode === 200
Array.isArray(response.body) response.body[0]) {
zoteroService.selectFormatFcn(format, function(convert){
convert(requestedURL, format, body,
function(modifiedBody){
callback(error, response, modifiedBody);
diff --git a/package.json b/package.json
index 3477cf1..51bb5b0 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
name: citoid,
- version: 0.2.1,
+ version: 0.2.2,
description: Converts search terms such as URL or DOI into citations.,
scripts: {
start: service-runner,
diff --git a/test/features/errors/index.js b/test/features/errors/index.js
index a2d1792..077bcec 100644
--- a/test/features/errors/index.js
+++ b/test/features/errors/index.js
@@ -59,5 +59,16 @@
});
});
+ it('faulty zotero results', function() {
+ var url = 'http://www.ncbi.nlm.nih.gov/pmc/articles/PMC99/';
+ return server.query(url, 'mediawiki', 'en')
+ .then(function(res) {
+ assert.status(res, 520);
+ }, function(err) {
+ assert.status(err, 520);
+ assert.checkCitation(err, url);
+ });
+ });
+
});
--
To view, visit https://gerrit.wikimedia.org/r/200163
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Id1d3e1fcfac03c92a6a1eb24e0746e6cd0a37ea1
Gerrit-PatchSet: 2
Gerrit-Project: mediawiki/services/citoid
Gerrit-Branch: master
Gerrit-Owner: Mvolz mv...@wikimedia.org
Gerrit-Reviewer: Jforrester jforres...@wikimedia.org
Gerrit-Reviewer: Mobrovac mobro...@wikimedia.org
Gerrit-Reviewer: jenkins-bot
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Restructure requestFromDOI + tests - change (mediawiki...citoid)
Mobrovac has submitted this change and it was merged.
Change subject: Restructure requestFromDOI + tests
..
Restructure requestFromDOI + tests
requestFromDOI:
* Follow exactly one redirect first before
sending to requestFromURL.
* Return 404s with error message instead
of 520s with citation.
tests:
* Add test for unresolvable DOIs confirming
they return 404s and an error message
* Add test for straight-forward DOI (where
resolved DOI points directly to resource)
and clarify previous DOI test as involving
multiple redirects.
Bug: T93876
Bug: T93785
Change-Id: I2756d11c30c41f88656b9d31ccd7d3e16553c7e2
---
M lib/CitoidService.js
M package.json
M test/features/errors/index.js
M test/features/scraping/index.js
M test/utils/assert.js
5 files changed, 65 insertions(+), 28 deletions(-)
Approvals:
Mobrovac: Verified; Looks good to me, approved
diff --git a/lib/CitoidService.js b/lib/CitoidService.js
index 3246e22..9b3d79d 100644
--- a/lib/CitoidService.js
+++ b/lib/CitoidService.js
@@ -5,19 +5,20 @@
*/
/* Import Modules */
+var http = require('http');
var urlParse = require('url');
/* Import Local Modules */
-var unshorten = require('./unshorten.js'),
- Scraper = require('./Scraper.js'),
- ZoteroService = require('./ZoteroService.js'),
- pubMedRequest = require('./pubMedRequest.js');
+var unshorten = require('./unshorten.js');
+var Scraper = require('./Scraper.js');
+var ZoteroService = require('./ZoteroService.js');
+var pubMedRequest = require('./pubMedRequest.js');
/**
* Constructor for CitoidService object
* @param {Object} citoidConfig configuration object
- * @param {Object} logger logger object, must have a log() method
- * @param {Object} statsd metrics object
+ * @param {Object} logger logger object, must have a log() method
+ * @param {Object} statsd metrics object
*/
function CitoidService(citoidConfig, logger, statsd) {
this.logger = logger;
@@ -28,13 +29,13 @@
/**
* Requests to the citoid service
- * @param {Object} opts options object containing request information
+ * @param {Object} opts options object containing request information
* @param {Function} callback callback (error, statusCode, body)
*/
CitoidService.prototype.request = function(opts, callback){
- var runnerOpts,
- citoidService = this;
+ var runnerOpts;
+ var citoidService = this;
citoidService.distinguish(opts.search, function(extractedID,
runnerFunction){
runnerOpts = {
@@ -48,8 +49,8 @@
/**
* Request citation metadata from a URL
- * @param {Object} opts options object containing requested url
- * @param {Function} callback callback (error, statusCode, body)
+ * @param {Object} opts options object containing requested url
+ * @param {Function} callback callback (error, statusCode, body)
*/
CitoidService.prototype.requestFromURL = function (opts, callback) {
var self = this,
@@ -115,14 +116,27 @@
/**
* Request citation metadata from a DOI
- * @param {Object} opts options object containing DOI and format
- * @param {Function} callback callback (error, statusCode, body)
+ * @param {Object} doiOptsoptions object containing DOI and format
+ * @param {Function} callback callback (error, statusCode, body)
*/
-CitoidService.prototype.requestFromDOI = function (opts, callback){
- var doiLink = 'http://dx.doi.org/'+ opts.search;
- opts.search = doiLink;
- // TODO: optimise this (can skip some steps in requestFromURL)
- this.requestFromURL(opts, callback);
+CitoidService.prototype.requestFromDOI = function (doiOpts, callback){
+ var doiLink = 'http://dx.doi.org/'+ doiOpts.search;
+ var citoidService = this;
+ var urlOpts = Object.assign({}, doiOpts); // Shallow clone doiOpts
+
+ // Follow one redirect here from the DOI to the canonical url
+ http.get(doiLink, function (res) {
+ // Detect a redirect
+ if (res res.statusCode 300 res.statusCode 400
res.headers.location) {
+ urlOpts.search = res.headers.location;
+ citoidService.requestFromURL(urlOpts, callback);
+ } else {
+ citoidService.logger.log('debug/DOI', Unable to
resolve DOI + doiOpts.search);
+ var message = 'Unable to resolve DOI';
+ var error = new Error(message);
+ callback(error, 404, {Error: message});
+ }
+ });
};
/**
@@ -146,8 +160,8 @@
/**
* Determine type of string (doi, url) and callback on correct handler
- * @param {String} rawSearchInput what the end user searched for
- * @param {Function} callback callback(extractedValue,
correctFunction)
+ * @param {String} rawSearchInput
[MediaWiki-commits] [Gerrit] Upgrade preq to 0.13.3 - change (mediawiki...citoid)
Mobrovac has submitted this change and it was merged. Change subject: Upgrade preq to 0.13.3 .. Upgrade preq to 0.13.3 The new version of preq avoids infinite error loops (and thus OOM) caused by extensive debug information. Now only certain, safe parts of the offending request are output. Change-Id: I98c03b374379ad1aa9d8a1500036a6b07cdf4038 --- M package.json 1 file changed, 1 insertion(+), 1 deletion(-) Approvals: Mobrovac: Verified; Looks good to me, approved diff --git a/package.json b/package.json index 64c496c..4bdfddc 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ html-metadata: 0.1.1, js-yaml: 3.2.7, node-uuid: 1.4.3, -preq: 0.3.12, +preq: 0.3.13, request: 2.49.0, service-runner: 0.1.5 }, -- To view, visit https://gerrit.wikimedia.org/r/200258 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I98c03b374379ad1aa9d8a1500036a6b07cdf4038 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/services/citoid Gerrit-Branch: master Gerrit-Owner: Mobrovac mobro...@wikimedia.org Gerrit-Reviewer: Mobrovac mobro...@wikimedia.org Gerrit-Reviewer: Mvolz mv...@wikimedia.org Gerrit-Reviewer: jenkins-bot ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Upgrade preq to 0.13.3 - change (mediawiki...citoid)
Mobrovac has uploaded a new change for review. https://gerrit.wikimedia.org/r/200258 Change subject: Upgrade preq to 0.13.3 .. Upgrade preq to 0.13.3 The new version of preq avoids infinite error loops (and thus OOM) caused by extensive debug information. Now only certain, safe parts of the offending request are output. Change-Id: I98c03b374379ad1aa9d8a1500036a6b07cdf4038 --- M package.json 1 file changed, 1 insertion(+), 1 deletion(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/citoid refs/changes/58/200258/1 diff --git a/package.json b/package.json index 64c496c..4bdfddc 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ html-metadata: 0.1.1, js-yaml: 3.2.7, node-uuid: 1.4.3, -preq: 0.3.12, +preq: 0.3.13, request: 2.49.0, service-runner: 0.1.5 }, -- To view, visit https://gerrit.wikimedia.org/r/200258 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I98c03b374379ad1aa9d8a1500036a6b07cdf4038 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/services/citoid Gerrit-Branch: master Gerrit-Owner: Mobrovac mobro...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Improve edit update job handling - change (mediawiki...RestBaseUpdateJobs)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/198804
Change subject: Improve edit update job handling
..
Improve edit update job handling
Regardless of the actual number of update jobs generated, they would be
sent out at a constant rate, causing overloads on both the Parsoid
cluster and Cassandra in cases where template updates would entail
millions of page re-renders. This patch mitigates that by:
* Providing better duplicate-job detection.
* Setting a threshold identifying 'small job clusters'. Currently set to
100, $wgRestbaseNoMinThrottle represents the minimum amount of update
jobs to be considered potentially harmful to performance. If the
actual number of linked jobs to execute, taken as the number of a
title's backlinks, is smaller than this threshold, all of the jobs are
batched together for execution.
* Lowering $wgRestbaseUpdateTitlesPerJob to 8, since this variable is
from now on used only in harmful situations. We favour update slowness
to overload.
* Sending a special 'X-Restbase-Timestamp' header which denotes the
original job enqueue time stamp, which is to be used by RESTBase to
determine whether a re-render is needed.
Change-Id: I73a08eadeb682fd514f2a9095bbab73208cb0e58
---
M RestbaseUpdate.hooks.php
M RestbaseUpdate.php
M RestbaseUpdateJob.php
3 files changed, 52 insertions(+), 11 deletions(-)
git pull
ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/RestBaseUpdateJobs
refs/changes/04/198804/1
diff --git a/RestbaseUpdate.hooks.php b/RestbaseUpdate.hooks.php
index 3de2d33..867da3d 100644
--- a/RestbaseUpdate.hooks.php
+++ b/RestbaseUpdate.hooks.php
@@ -18,7 +18,7 @@
*/
private static function getJobParams( Title $title, $type, $table =
null ) {
- $params = array( 'type' = $type );
+ $params = array( 'type' = $type, 'ts' = time() );
if ( $type == 'OnDependencyChange' ) {
$params['table'] = $table;
$params['recursive'] = true;
diff --git a/RestbaseUpdate.php b/RestbaseUpdate.php
index a8f0bf9..ea8deca 100644
--- a/RestbaseUpdate.php
+++ b/RestbaseUpdate.php
@@ -79,7 +79,14 @@
/**
* The number of recursive jobs to process in parallel
*/
- $wgRestbaseUpdateTitlesPerJob = 50;
+ $wgRestbaseUpdateTitlesPerJob = 8;
+
+ /**
+ * The minimum number of jobs considered to be a 'big update'.
If the number
+ * of actual jobs is lower than this threshold, all of the jobs
will be processed
+ * a a single job partition.
+ */
+ $wgRestbaseNoMinThrottle = 100;
}
diff --git a/RestbaseUpdateJob.php b/RestbaseUpdateJob.php
index a873947..fc15115 100644
--- a/RestbaseUpdateJob.php
+++ b/RestbaseUpdateJob.php
@@ -23,11 +23,8 @@
parent::__construct( 'RestbaseUpdateJob' . $params['type'],
$title, $params, $id );
- if ( $params['type'] == 'OnEdit' ) {
- // Simple duplicate removal for single-title jobs.
Other jobs are
- // deduplicated with root job parameters.
- $this-removeDuplicates = true;
- }
+ // enable duplication detection for all jobs
+ $this-removeDuplicates = true;
}
@@ -94,9 +91,31 @@
}
- function run() {
+ /**
+* Gets the info needed to determine equality between two jobs.
+*
+* @return array Map of key/values
+* @since 1.21
+*/
+ public function getDeduplicationInfo() {
- global $wgRestbaseUpdateTitlesPerJob, $wgUpdateRowsPerJob;
+ $info = parent::getDeduplicationInfo();
+
+ if ( is_array( $info['params'] ) ) {
+ // ignore the original job time stamp
+ unset( $info['params']['ts'] );
+ }
+
+ return $info;
+
+ }
+
+
+ public function run() {
+
+ global $wgRestbaseUpdateTitlesPerJob, $wgUpdateRowsPerJob,
$wgRestbaseNoMinThrottle;
+
+ $noJobs = $wgRestbaseUpdateTitlesPerJob;
if ( $this-params['type'] === 'OnEdit' ) {
// there are two cases here:
@@ -116,16 +135,29 @@
}
// Job to purge all (or a range of) backlink pages for
a page
if ( !empty( $this-params['recursive'] ) ) {
+ // if there is no range defined, we are about
to create
+ // a root partition job
+ if ( !isset( $this-params['range'] ) ) {
+ // how many backlinks does this title
have ?
+
[MediaWiki-commits] [Gerrit] Add a script to check for git changes and rebuild the deps i... - change (mediawiki...mobileapps)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/201429
Change subject: Add a script to check for git changes and rebuild the deps if
needed
..
Add a script to check for git changes and rebuild the deps if needed
The script checks for new, committed changes and pulls them in. If
package.json changes as well, the node dependencies are rebuilt and the
service is restarted.
The script is meant to be run by cron every couple of minutes, so keep
the git tree in labs clean!
Change-Id: Ic0897cce30aef1d450fc975d96389e5983756657
---
A scripts/puller.sh
1 file changed, 38 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/mobileapps
refs/changes/29/201429/1
diff --git a/scripts/puller.sh b/scripts/puller.sh
new file mode 100755
index 000..c8f496f
--- /dev/null
+++ b/scripts/puller.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+
+# ensure we are in the right dir
+cd $(dirname $0)/..;
+
+# check for a git dir
+if [[ ! -e .git ]]; then
+echo No .git directory here, exiting 2;
+exit 1;
+fi
+
+# be on master and get the updates
+git checkout master;
+git fetch origin;
+
+# inspect what has changed
+flist=$(git diff --name-only origin/master);
+
+if [[ -z ${flist} ]]; then
+# no changes, we are done
+exit 0;
+fi
+
+# there are updates, do them
+git pull;
+
+if echo -e ${flist} | grep -i package.json /dev/null; then
+# package.json has been changed, need to rebuild the modules
+rm -rf node_modules;
+npm install;
+fi
+
+# now, restart the service
+systemctl restart mobileapps;
+
+exit $?;
+
--
To view, visit https://gerrit.wikimedia.org/r/201429
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ic0897cce30aef1d450fc975d96389e5983756657
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/mobileapps
Gerrit-Branch: master
Gerrit-Owner: Mobrovac mobro...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Add a script to check for git changes and rebuild the deps i... - change (mediawiki...mobileapps)
Mobrovac has submitted this change and it was merged.
Change subject: Add a script to check for git changes and rebuild the deps if
needed
..
Add a script to check for git changes and rebuild the deps if needed
The script checks for new, committed changes and pulls them in. If
package.json changes as well, the node dependencies are rebuilt and the
service is restarted.
The script is meant to be run by cron every couple of minutes, so keep
the git tree in labs clean!
Change-Id: Ic0897cce30aef1d450fc975d96389e5983756657
---
A scripts/puller.sh
1 file changed, 38 insertions(+), 0 deletions(-)
Approvals:
Mobrovac: Verified; Looks good to me, approved
diff --git a/scripts/puller.sh b/scripts/puller.sh
new file mode 100755
index 000..c8f496f
--- /dev/null
+++ b/scripts/puller.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+
+# ensure we are in the right dir
+cd $(dirname $0)/..;
+
+# check for a git dir
+if [[ ! -e .git ]]; then
+echo No .git directory here, exiting 2;
+exit 1;
+fi
+
+# be on master and get the updates
+git checkout master;
+git fetch origin;
+
+# inspect what has changed
+flist=$(git diff --name-only origin/master);
+
+if [[ -z ${flist} ]]; then
+# no changes, we are done
+exit 0;
+fi
+
+# there are updates, do them
+git pull;
+
+if echo -e ${flist} | grep -i package.json /dev/null; then
+# package.json has been changed, need to rebuild the modules
+rm -rf node_modules;
+npm install;
+fi
+
+# now, restart the service
+systemctl restart mobileapps;
+
+exit $?;
+
--
To view, visit https://gerrit.wikimedia.org/r/201429
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ic0897cce30aef1d450fc975d96389e5983756657
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/mobileapps
Gerrit-Branch: master
Gerrit-Owner: Mobrovac mobro...@wikimedia.org
Gerrit-Reviewer: BearND bsitzm...@wikimedia.org
Gerrit-Reviewer: Mobrovac mobro...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] service::node: fix the look-up of undefined variables - change (operations/puppet)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/203886
Change subject: service::node: fix the look-up of undefined variables
..
service::node: fix the look-up of undefined variables
Some variables may not be defined when completing the template.
Unfortunately for us, scope#lookupvar does not return nil, but 'undef'
on such occasions (etiher as a string or symbol). This causes any
possible if-guards to be evaluated as true, and consequently we end up
with variables set to the string 'undef'. For instance, citoid's config
in the beta cluster does not have a proxy set, but the configuration
reads:
proxy: undef
which is clearly wrong. This patch translates such look-ups into empty
strings, which represents perfectly-valid YAML.
Bug: T95533
Change-Id: I9eab4f1627070e031c33ee4640ba146cdc9fd600
---
M modules/service/templates/node/config.yaml.erb
1 file changed, 18 insertions(+), 9 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/86/203886/1
diff --git a/modules/service/templates/node/config.yaml.erb
b/modules/service/templates/node/config.yaml.erb
index caa82cc..65bfe73 100644
--- a/modules/service/templates/node/config.yaml.erb
+++ b/modules/service/templates/node/config.yaml.erb
@@ -1,3 +1,16 @@
+%
+cvars = {
+ gelf_host = scope.lookupvar('service::configuration::logstash_host'),
+ gelf_port = scope.lookupvar('service::configuration::logstash_port'),
+ stat_host = scope.lookupvar('service::configuration::statsd_host'),
+ stat_port = scope.lookupvar('service::configuration::statsd_port'),
+ proxy = scope.lookupvar('service::configuration::http_proxy')
+}
+
+cvars.keys.each do |k|
+ cvars[k] = '' if !cvars[k] or ['undef', ''].include? cvars[k].to_s
+end
+%
# Number of worker processes to spawn.
# Set to 0 to run everything in a single process without clustering.
# Use 'ncpu' to run as many workers as there are CPU units
@@ -14,8 +27,8 @@
streams:
# Use gelf-stream - logstash
- type: gelf
-host: %= scope.lookupvar('service::configuration::logstash_host') %
-port: %= scope.lookupvar('service::configuration::logstash_port') %
+host: %= cvars['gelf_host'] %
+port: %= cvars['gel_port'] %
- type: file
path: %= @local_logfile %
level: info
@@ -23,8 +36,8 @@
# Statsd metrics reporter
metrics:
type: statsd
- host: %= scope.lookupvar('service::configuration::statsd_host') %
- port: %= scope.lookupvar('service::configuration::statsd_port') %
+ host: %= cvars['stat_host'] %
+ port: %= cvars['stat_port'] %
services:
- name: %= @title %
@@ -45,8 +58,4 @@
# to restrict to a particular domain, use:
# cors: restricted.domain.org
# URL of the outbound proxy to use (complete with protocol)
- % if scope.lookupvar('service::configuration::http_proxy') %
- proxy: %= scope.lookupvar('service::configuration::http_proxy') %
- % else %
- # proxy: http://my.proxy.org:8080
- % end %
+ proxy: %= cvars['proxy'] %
--
To view, visit https://gerrit.wikimedia.org/r/203886
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I9eab4f1627070e031c33ee4640ba146cdc9fd600
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Mobrovac mobro...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Update to latest service-template-node - change (mediawiki...graphoid)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/204027
Change subject: Update to latest service-template-node
..
Update to latest service-template-node
Includes:
- configurable CSP headers
- configurable HTTP(S) proxy
- configurable CORS headers
Change-Id: I82b270a27d01f42604ad98b518a98dba8e221bc9
---
M app.js
M lib/util.js
M package.json
M test/features/app/app.js
4 files changed, 52 insertions(+), 3 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/graphoid
refs/changes/27/204027/1
diff --git a/app.js b/app.js
index 88943b1..5f9e033 100644
--- a/app.js
+++ b/app.js
@@ -31,6 +31,31 @@
if(!app.conf.port) { app.conf.port = ; }
if(!app.conf.interface) { app.conf.interface = '0.0.0.0'; }
if(!app.conf.compression_level) { app.conf.compression_level = 3; }
+if(app.conf.cors === undefined) { app.conf.cors = '*'; }
+if(!app.conf.csp) {
+app.conf.csp =
+default-src 'self'; object-src 'none'; media-src *; img-src *;
style-src *; frame-ancestors 'self';
+}
+
+// set outgoing proxy
+if(app.conf.proxy) {
+process.env.HTTP_PROXY = app.conf.proxy;
+}
+
+// set the CORS and CSP headers
+app.all('*', function(req, res, next) {
+if(app.conf.cors !== false) {
+res.header('Access-Control-Allow-Origin', app.conf.cors);
+res.header('Access-Control-Allow-Headers', 'Accept,
X-Requested-With, Content-Type');
+}
+res.header('X-XSS-Protection', '1; mode=block');
+res.header('X-Content-Type-Options', 'nosniff');
+res.header('X-Frame-Options', 'SAMEORIGIN');
+res.header('Content-Security-Policy', app.conf.csp);
+res.header('X-Content-Security-Policy', app.conf.csp);
+res.header('X-WebKit-CSP', app.conf.csp);
+next();
+});
// disable the X-Powered-By header
app.set('x-powered-by', false);
diff --git a/lib/util.js b/lib/util.js
index 58ed1e3..fb83c9c 100644
--- a/lib/util.js
+++ b/lib/util.js
@@ -190,8 +190,8 @@
level = 'info';
}
// log the error
-req.logger.log(level +
-(errObj.component ? '/' + errObj.component : '/' +
errObj.status),
+req.logger.log(level + '/' +
+(errObj.component ? errObj.component : errObj.status),
errForLog(errObj));
// let through only non-sensitive info
var respBody = {
diff --git a/package.json b/package.json
index bbd3a0e..e352bf2 100644
--- a/package.json
+++ b/package.json
@@ -40,7 +40,7 @@
js-yaml: ^3.2.7,
node-uuid: ^1.4.3,
preq: ^0.3.12,
-service-runner: ^0.1.5
+service-runner: ^0.1.6
},
devDependencies: {
assert: ^1.3.0,
diff --git a/test/features/app/app.js b/test/features/app/app.js
index c7fb08c..6df85b4 100644
--- a/test/features/app/app.js
+++ b/test/features/app/app.js
@@ -25,6 +25,30 @@
});
});
+it('should set CORS headers', function() {
+return preq.get({
+uri: server.config.uri + 'robots.txt'
+}).then(function(res) {
+assert.deepEqual(res.status, 200);
+assert.deepEqual(res.headers['access-control-allow-origin'], '*');
+assert.notDeepEqual(res.headers['access-control-allow-headers'],
undefined);
+});
+});
+
+it('should set CSP headers', function() {
+return preq.get({
+uri: server.config.uri + 'robots.txt'
+}).then(function(res) {
+assert.deepEqual(res.status, 200);
+assert.deepEqual(res.headers['x-xss-protection'], '1; mode=block');
+assert.deepEqual(res.headers['x-content-type-options'], 'nosniff');
+assert.deepEqual(res.headers['x-frame-options'], 'SAMEORIGIN');
+assert.deepEqual(res.headers['content-security-policy'],
'default-src');
+assert.deepEqual(res.headers['x-content-security-policy'],
'default-src');
+assert.deepEqual(res.headers['x-webkit-csp'], 'default-src');
+});
+});
+
it('should get static content gzipped', function() {
return preq.get({
uri: server.config.uri + 'static/index.html',
--
To view, visit https://gerrit.wikimedia.org/r/204027
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I82b270a27d01f42604ad98b518a98dba8e221bc9
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/graphoid
Gerrit-Branch: master
Gerrit-Owner: Mobrovac mobro...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Explicitly define allowed formats - change (mediawiki...citoid)
Mobrovac has submitted this change and it was merged.
Change subject: Explicitly define allowed formats
..
Explicitly define allowed formats
* Specify allowed formats in app.js
* Respond with response type specified
for given format
* Change error responses to be json
Bug: T95308
Change-Id: I59059c1938a644f33ddf6bbeb998f1ad0bb46d7b
---
M app.js
M routes/root.js
M test/features/errors/index.js
3 files changed, 62 insertions(+), 27 deletions(-)
Approvals:
Mobrovac: Looks good to me, approved
jenkins-bot: Verified
diff --git a/app.js b/app.js
index d636a6b..cfb6a4d 100644
--- a/app.js
+++ b/app.js
@@ -64,6 +64,19 @@
// init the Citoid service object
app.citoid = new CitoidService(app.conf, app.logger, app.metrics);
+ // set allowed export formats and expected response type
+ var nativeFormats = {
+ 'mediawiki':'application/json',
+ 'zotero':'application/json',
+ 'mwDeprecated':'application/json'
+ };
+
+ var zoteroFormats = {
+ 'bibtex':'application/x-bibtex'
+ };
+
+ app.formats = Object.assign({}, nativeFormats, zoteroFormats);
+
return BBPromise.resolve(app);
}
diff --git a/routes/root.js b/routes/root.js
index 58d94ee..5744fca 100644
--- a/routes/root.js
+++ b/routes/root.js
@@ -14,7 +14,6 @@
*/
var app;
-
/**
* GET /robots.txt
* Instructs robots no indexing should occur on this domain.
@@ -47,8 +46,14 @@
}
if (!requestedURL) {
- res.status(400).type('text/plain');
- res.send('url is a required parameter');
+ res.status(400).type('application/json');
+ res.send({Error:No 'url' value specified});
+ return;
+ }
+
+ if (!app.formats[format]) {
+ res.status(400).type('application/json');
+ res.send({Error:'Invalid format requested ' + format});
return;
}
@@ -59,7 +64,7 @@
};
app.citoid.request(opts, function(error, responseCode, body){
- res.status(responseCode).type('application/json');
+ res.status(responseCode).type(app.formats[format]);
res.send(body);
});
@@ -78,12 +83,16 @@
search = req.query.search;
if (!search) {
- res.status(400).type('text/plain');
- res.send(No 'search' value specified\n);
+ res.status(400).type('application/json');
+ res.send({Error:No 'search' value specified});
return;
} else if(!format) {
- res.status(400).type('text/plain');
- res.send(No 'format' value specified\nOptions are
'mediawiki','zotero');
+ res.status(400).type('application/json');
+ res.send({Error:No 'format' value specified});
+ return;
+ } else if (!app.formats[format]) {
+ res.status(400).type('application/json');
+ res.send({Error:'Invalid format requested ' + format});
return;
}
@@ -95,12 +104,7 @@
};
app.citoid.request(opts, function(error, responseCode, body) {
- res.status(responseCode);
- if(format === 'bibtex') {
- res.type('application/x-bibtex');
- } else {
- res.type('application/json');
- }
+ res.status(responseCode).type(app.formats[format]);
res.send(body);
});
diff --git a/test/features/errors/index.js b/test/features/errors/index.js
index b56e120..87c3eaa 100644
--- a/test/features/errors/index.js
+++ b/test/features/errors/index.js
@@ -12,19 +12,6 @@
before(function () { return server.start(); });
- it('missing format in query', function() {
- return preq.get({
- uri: server.config.q_uri,
- query: {
- search: '123456'
- }
- }).then(function(res) {
- assert.status(res, 400);
- }, function(err) {
- assert.status(err, 400);
- });
- });
-
it('missing search in query', function() {
return preq.get({
uri: server.config.q_uri,
@@ -35,6 +22,37 @@
assert.status(res, 400);
}, function(err) {
assert.status(err, 400);
+ assert.deepEqual(err.body.Error, No 'search' value
specified);
+ });
+ });
+
+ it('missing format in query', function() {
+ return preq.get({
+ uri: server.config.q_uri,
+ query: {
+ search: '123456'
+
[MediaWiki-commits] [Gerrit] Move v1.js - graphoid-v1.js to avoid conflicts - change (mediawiki...graphoid)
Mobrovac has submitted this change and it was merged.
Change subject: Move v1.js - graphoid-v1.js to avoid conflicts
..
Move v1.js - graphoid-v1.js to avoid conflicts
When merging updates from service-template-node, v1.js keeps getting
conflicts due to the same name. This patch moves its contents to
graphoid-v1.js to avoid such situations in the future.
Change-Id: Ic1e0305e1cd34a580f310eb7c80cd8be6158394a
---
M package.json
R routes/graphoid-v1.js
2 files changed, 0 insertions(+), 2 deletions(-)
Approvals:
Mobrovac: Verified; Looks good to me, approved
diff --git a/package.json b/package.json
index 48fcbfa..1a51963 100644
--- a/package.json
+++ b/package.json
@@ -34,7 +34,6 @@
body-parser: ^1.12.3,
bunyan: ^1.3.5,
compression: ^1.4.3,
-domino: ^1.0.18,
express: ^4.12.3,
js-yaml: ^3.2.7,
node-uuid: ^1.4.3,
diff --git a/routes/v1.js b/routes/graphoid-v1.js
similarity index 99%
rename from routes/v1.js
rename to routes/graphoid-v1.js
index 134429d..47f595b 100644
--- a/routes/v1.js
+++ b/routes/graphoid-v1.js
@@ -2,7 +2,6 @@
var BBPromise = require('bluebird');
var preq = require('preq');
-var domino = require('domino');
var sUtil = require('../lib/util');
var urllib = require('url');
var vega = require('vega'); // Visualization grammar -
https://github.com/trifacta/vega
--
To view, visit https://gerrit.wikimedia.org/r/205568
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ic1e0305e1cd34a580f310eb7c80cd8be6158394a
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/graphoid
Gerrit-Branch: master
Gerrit-Owner: Mobrovac mobro...@wikimedia.org
Gerrit-Reviewer: Mobrovac mobro...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Update to service-template-node v0.1.3 - change (mediawiki...graphoid)
Mobrovac has submitted this change and it was merged.
Change subject: Update to service-template-node v0.1.3
..
Update to service-template-node v0.1.3
Change-Id: Icab18289fb220a3e5df033352faa3b3294d9d04f
---
M .gitignore
M doc/README.md
M doc/commands.md
A doc/deployment.md
M doc/template.md
M package.json
A targets.yaml
7 files changed, 194 insertions(+), 16 deletions(-)
Approvals:
Mobrovac: Verified; Looks good to me, approved
diff --git a/.gitignore b/.gitignore
index a359b3f..978f2d1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
.idea
graphoid.iml
+Dockerfile
coverage
config.yaml
node_modules
diff --git a/doc/README.md b/doc/README.md
index 26adc44..d837f0a 100644
--- a/doc/README.md
+++ b/doc/README.md
@@ -8,6 +8,7 @@
3. [Configuration](config.md)
4. [Useful Commands](commands.md)
5. [Coding Guide](coding.md)
+6. [Deployment](deployment.md)
Have fun while creating RESTful API services!
diff --git a/doc/commands.md b/doc/commands.md
index e1bc803..01b6c5a 100644
--- a/doc/commands.md
+++ b/doc/commands.md
@@ -95,3 +95,7 @@
After you log out completely and log back in, you should be able to run the
above scripts without resorting to `sudo`.
+## Deployment
+
+See [this document](deployment.md) for how to get ready to deploy your service.
+
diff --git a/doc/deployment.md b/doc/deployment.md
new file mode 100644
index 000..bb93f7d
--- /dev/null
+++ b/doc/deployment.md
@@ -0,0 +1,164 @@
+# Deployment
+
+Getting your service ready to be deployed on WMF production machines involves
+several tasks. This document explains the steps needed to get started and how
to
+keep your deployable copy up-to-date.
+
+## Repositories
+
+Because Node.js services use npm dependencies which can be binary, these need
to
+be pre-built. Therefore, two repositories are needed; one for the source code
of
+your service, and the other, so-called *deploy* repository. Both should be
+available as WM's Gerrit repositories with the paths
+*mediawiki/services/your-service-name* and
+*mediawiki/services/your-service-name/deploy*. When [requesting
+them](https://www.mediawiki.org/wiki/Git/New_repositories/Requests) ask for the
+former to be a clone of [the service
+template](https://github.com/wikimedia/service-template-node) and the latter to
+be empty.
+
+It is important to note that the deploy repository is only to be updated
+directly before (re-)deploying the service, and not on each patch merge
entering
+the *master* branch of the regular repository. In other words, **the deploy
+repository mirrors the code deployed in production at all times**.
+
+The remainder of the document assumes these two repositories have been created
+and that you have cloned them using your Gerrit account, i.e. not anonymously,
+with the following outline:
+
+```
+~/code/
+ |- your-service
+ -- deploy
+```
+
+Furthermore, it is assumed that you have initialised the deploy repository:
+
+```bash
+$ cd ~/code/deploy
+$ git review -s
+$ touch README.md
+$ git add README.md
+$ git commit -m Initial commit
+$ git push -u origin master # or git review -R if this fails
+# go to Gerrit and +2 your change, if needed and then:
+$ git pull
+```
+
+Finally, if you haven't yet done so, do [basic service
+configuration](config.md).
+
+The remainder of the document refers to these two repositories as the *source
+repository* and the *deploy repository*, respectively.
+
+## Configuration
+
+The service template includes an automation script which updates the deploy
+repository, but it needs to be configured properly in order to work.
+
+### package.json
+
+The first part of the configuration involves keeping your source repository's
+`package.json` updated. Look for its [deploy stanza](../package.json#L49).
+Depending on the exact machine on which your service will be deployed, you may
+need to set `target` to either `ubuntu` or `debian`.
+
+The important thing is keeping the `dependencies` field up to date at all
times.
+There you should list all of the extra packages that are needed in order to
+build the npm module dependencies. The `_all` field denotes packages which
+should be installed regardless of the target distribution, but you can add
+other, distribution-specific package lists, e.g.:
+
+```javascript
+deploy: {
+ target: ubuntu,
+ dependencies: {
+ubuntu: [pkg1, pkg2],
+debian: [pkgA, pkgB],
+_all: [pkgOne, pkgTwo]
+ }
+}
+```
+
+In this example, with the current configuration, packages *pkg1*, *pkg2*,
+*pkgOne* and *pkgTwo* are going to be installed before building the
+dependencies. If, instead, the target is changed to `debian`, then *pkgA*,
+*pkgB*, *pkgOne* and *pkgTwo* are selected.
+
+As a rule of thumb, **whenever you need to install extra packages into your
+development environment for satisfying node module dependencies, add them to
+*deploy.dependencies* to ensure the successful build and update of the deploy
[MediaWiki-commits] [Gerrit] Update to service-template-node v0.1.3 - change (mediawiki...graphoid)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/205563
Change subject: Update to service-template-node v0.1.3
..
Update to service-template-node v0.1.3
Change-Id: Icab18289fb220a3e5df033352faa3b3294d9d04f
---
M .gitignore
M doc/README.md
M doc/commands.md
A doc/deployment.md
M doc/template.md
M package.json
A targets.yaml
7 files changed, 194 insertions(+), 16 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/graphoid
refs/changes/63/205563/1
diff --git a/.gitignore b/.gitignore
index a359b3f..978f2d1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
.idea
graphoid.iml
+Dockerfile
coverage
config.yaml
node_modules
diff --git a/doc/README.md b/doc/README.md
index 26adc44..d837f0a 100644
--- a/doc/README.md
+++ b/doc/README.md
@@ -8,6 +8,7 @@
3. [Configuration](config.md)
4. [Useful Commands](commands.md)
5. [Coding Guide](coding.md)
+6. [Deployment](deployment.md)
Have fun while creating RESTful API services!
diff --git a/doc/commands.md b/doc/commands.md
index e1bc803..01b6c5a 100644
--- a/doc/commands.md
+++ b/doc/commands.md
@@ -95,3 +95,7 @@
After you log out completely and log back in, you should be able to run the
above scripts without resorting to `sudo`.
+## Deployment
+
+See [this document](deployment.md) for how to get ready to deploy your service.
+
diff --git a/doc/deployment.md b/doc/deployment.md
new file mode 100644
index 000..bb93f7d
--- /dev/null
+++ b/doc/deployment.md
@@ -0,0 +1,164 @@
+# Deployment
+
+Getting your service ready to be deployed on WMF production machines involves
+several tasks. This document explains the steps needed to get started and how
to
+keep your deployable copy up-to-date.
+
+## Repositories
+
+Because Node.js services use npm dependencies which can be binary, these need
to
+be pre-built. Therefore, two repositories are needed; one for the source code
of
+your service, and the other, so-called *deploy* repository. Both should be
+available as WM's Gerrit repositories with the paths
+*mediawiki/services/your-service-name* and
+*mediawiki/services/your-service-name/deploy*. When [requesting
+them](https://www.mediawiki.org/wiki/Git/New_repositories/Requests) ask for the
+former to be a clone of [the service
+template](https://github.com/wikimedia/service-template-node) and the latter to
+be empty.
+
+It is important to note that the deploy repository is only to be updated
+directly before (re-)deploying the service, and not on each patch merge
entering
+the *master* branch of the regular repository. In other words, **the deploy
+repository mirrors the code deployed in production at all times**.
+
+The remainder of the document assumes these two repositories have been created
+and that you have cloned them using your Gerrit account, i.e. not anonymously,
+with the following outline:
+
+```
+~/code/
+ |- your-service
+ -- deploy
+```
+
+Furthermore, it is assumed that you have initialised the deploy repository:
+
+```bash
+$ cd ~/code/deploy
+$ git review -s
+$ touch README.md
+$ git add README.md
+$ git commit -m Initial commit
+$ git push -u origin master # or git review -R if this fails
+# go to Gerrit and +2 your change, if needed and then:
+$ git pull
+```
+
+Finally, if you haven't yet done so, do [basic service
+configuration](config.md).
+
+The remainder of the document refers to these two repositories as the *source
+repository* and the *deploy repository*, respectively.
+
+## Configuration
+
+The service template includes an automation script which updates the deploy
+repository, but it needs to be configured properly in order to work.
+
+### package.json
+
+The first part of the configuration involves keeping your source repository's
+`package.json` updated. Look for its [deploy stanza](../package.json#L49).
+Depending on the exact machine on which your service will be deployed, you may
+need to set `target` to either `ubuntu` or `debian`.
+
+The important thing is keeping the `dependencies` field up to date at all
times.
+There you should list all of the extra packages that are needed in order to
+build the npm module dependencies. The `_all` field denotes packages which
+should be installed regardless of the target distribution, but you can add
+other, distribution-specific package lists, e.g.:
+
+```javascript
+deploy: {
+ target: ubuntu,
+ dependencies: {
+ubuntu: [pkg1, pkg2],
+debian: [pkgA, pkgB],
+_all: [pkgOne, pkgTwo]
+ }
+}
+```
+
+In this example, with the current configuration, packages *pkg1*, *pkg2*,
+*pkgOne* and *pkgTwo* are going to be installed before building the
+dependencies. If, instead, the target is changed to `debian`, then *pkgA*,
+*pkgB*, *pkgOne* and *pkgTwo* are selected.
+
+As a rule of thumb, **whenever you need to install extra packages into your
+development environment for satisfying node module dependencies, add them to
[MediaWiki-commits] [Gerrit] Declare the dependency on libcairo for building modules - change (mediawiki...graphoid)
Mobrovac has submitted this change and it was merged.
Change subject: Declare the dependency on libcairo for building modules
..
Declare the dependency on libcairo for building modules
Change-Id: I64d66697b834485d001bfb8944ee1c6c5c33b47d
---
M package.json
1 file changed, 1 insertion(+), 1 deletion(-)
Approvals:
Mobrovac: Verified; Looks good to me, approved
diff --git a/package.json b/package.json
index 1a51963..eb17425 100644
--- a/package.json
+++ b/package.json
@@ -50,7 +50,7 @@
deploy: {
target: ubuntu,
dependencies: {
- _all: []
+ _all: [libcairo-dev]
}
}
}
--
To view, visit https://gerrit.wikimedia.org/r/205569
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I64d66697b834485d001bfb8944ee1c6c5c33b47d
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/graphoid
Gerrit-Branch: master
Gerrit-Owner: Mobrovac mobro...@wikimedia.org
Gerrit-Reviewer: Mobrovac mobro...@wikimedia.org
Gerrit-Reviewer: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Declare the dependency on libcairo for building modules - change (mediawiki...graphoid)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/205569
Change subject: Declare the dependency on libcairo for building modules
..
Declare the dependency on libcairo for building modules
Change-Id: I64d66697b834485d001bfb8944ee1c6c5c33b47d
---
M package.json
1 file changed, 1 insertion(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/graphoid
refs/changes/69/205569/1
diff --git a/package.json b/package.json
index 1a51963..eb17425 100644
--- a/package.json
+++ b/package.json
@@ -50,7 +50,7 @@
deploy: {
target: ubuntu,
dependencies: {
- _all: []
+ _all: [libcairo-dev]
}
}
}
--
To view, visit https://gerrit.wikimedia.org/r/205569
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I64d66697b834485d001bfb8944ee1c6c5c33b47d
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/graphoid
Gerrit-Branch: master
Gerrit-Owner: Mobrovac mobro...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Update to service-template-node v0.1.3 #2 - change (mediawiki...graphoid)
Mobrovac has uploaded a new change for review. https://gerrit.wikimedia.org/r/205570 Change subject: Update to service-template-node v0.1.3 #2 .. Update to service-template-node v0.1.3 #2 Change-Id: Ie035103d4fbf08587ec81dfdfb832e0278dcb8fe --- M config.prod.yaml M lib/util.js 2 files changed, 2 insertions(+), 2 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/graphoid refs/changes/70/205570/1 diff --git a/config.prod.yaml b/config.prod.yaml index a759c13..9ce5970 100644 --- a/config.prod.yaml +++ b/config.prod.yaml @@ -18,7 +18,7 @@ # Statsd metrics reporter metrics: - type: txstatsd + type: statsd host: statsd.eqiad.wmnet port: 8125 diff --git a/lib/util.js b/lib/util.js index fb83c9c..9db1055 100644 --- a/lib/util.js +++ b/lib/util.js @@ -190,7 +190,7 @@ level = 'info'; } // log the error -req.logger.log(level + '/' + +(req.logger || app.logger).log(level + '/' + (errObj.component ? errObj.component : errObj.status), errForLog(errObj)); // let through only non-sensitive info -- To view, visit https://gerrit.wikimedia.org/r/205570 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ie035103d4fbf08587ec81dfdfb832e0278dcb8fe Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/services/graphoid Gerrit-Branch: master Gerrit-Owner: Mobrovac mobro...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Move v1.js - graphoid-v1.js to avoid conflicts - change (mediawiki...graphoid)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/205568
Change subject: Move v1.js - graphoid-v1.js to avoid conflicts
..
Move v1.js - graphoid-v1.js to avoid conflicts
When merging updates from service-template-node, v1.js keeps getting
conflicts due to the same name. This patch moves its contents to
graphoid-v1.js to avoid such situations in the future.
Change-Id: Ic1e0305e1cd34a580f310eb7c80cd8be6158394a
---
M package.json
R routes/graphoid-v1.js
2 files changed, 0 insertions(+), 2 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/graphoid
refs/changes/68/205568/1
diff --git a/package.json b/package.json
index 48fcbfa..1a51963 100644
--- a/package.json
+++ b/package.json
@@ -34,7 +34,6 @@
body-parser: ^1.12.3,
bunyan: ^1.3.5,
compression: ^1.4.3,
-domino: ^1.0.18,
express: ^4.12.3,
js-yaml: ^3.2.7,
node-uuid: ^1.4.3,
diff --git a/routes/v1.js b/routes/graphoid-v1.js
similarity index 99%
rename from routes/v1.js
rename to routes/graphoid-v1.js
index 134429d..47f595b 100644
--- a/routes/v1.js
+++ b/routes/graphoid-v1.js
@@ -2,7 +2,6 @@
var BBPromise = require('bluebird');
var preq = require('preq');
-var domino = require('domino');
var sUtil = require('../lib/util');
var urllib = require('url');
var vega = require('vega'); // Visualization grammar -
https://github.com/trifacta/vega
--
To view, visit https://gerrit.wikimedia.org/r/205568
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ic1e0305e1cd34a580f310eb7c80cd8be6158394a
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/graphoid
Gerrit-Branch: master
Gerrit-Owner: Mobrovac mobro...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Update to service-template-node v0.1.3 #2 - change (mediawiki...graphoid)
Mobrovac has submitted this change and it was merged. Change subject: Update to service-template-node v0.1.3 #2 .. Update to service-template-node v0.1.3 #2 Change-Id: Ie035103d4fbf08587ec81dfdfb832e0278dcb8fe --- M config.prod.yaml M lib/util.js 2 files changed, 2 insertions(+), 2 deletions(-) Approvals: Mobrovac: Verified; Looks good to me, approved diff --git a/config.prod.yaml b/config.prod.yaml index a759c13..9ce5970 100644 --- a/config.prod.yaml +++ b/config.prod.yaml @@ -18,7 +18,7 @@ # Statsd metrics reporter metrics: - type: txstatsd + type: statsd host: statsd.eqiad.wmnet port: 8125 diff --git a/lib/util.js b/lib/util.js index fb83c9c..9db1055 100644 --- a/lib/util.js +++ b/lib/util.js @@ -190,7 +190,7 @@ level = 'info'; } // log the error -req.logger.log(level + '/' + +(req.logger || app.logger).log(level + '/' + (errObj.component ? errObj.component : errObj.status), errForLog(errObj)); // let through only non-sensitive info -- To view, visit https://gerrit.wikimedia.org/r/205570 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ie035103d4fbf08587ec81dfdfb832e0278dcb8fe Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/services/graphoid Gerrit-Branch: master Gerrit-Owner: Mobrovac mobro...@wikimedia.org Gerrit-Reviewer: Mobrovac mobro...@wikimedia.org Gerrit-Reviewer: Yurik yu...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Add .dockerignore listing .git, node_modules/ and coverage/ - change (mediawiki...citoid)
Mobrovac has submitted this change and it was merged. Change subject: Add .dockerignore listing .git, node_modules/ and coverage/ .. Add .dockerignore listing .git, node_modules/ and coverage/ Change-Id: I37161b50e69f2be39551510af14a0853907147b8 --- A .dockerignore 1 file changed, 3 insertions(+), 0 deletions(-) Approvals: Mobrovac: Verified; Looks good to me, approved diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000..563df11 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +.git +coverage +node_modules -- To view, visit https://gerrit.wikimedia.org/r/205858 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I37161b50e69f2be39551510af14a0853907147b8 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/services/citoid Gerrit-Branch: master Gerrit-Owner: Mobrovac mobro...@wikimedia.org Gerrit-Reviewer: Mobrovac mobro...@wikimedia.org Gerrit-Reviewer: Mvolz mv...@wikimedia.org Gerrit-Reviewer: jenkins-bot ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Add .dockerignore listing .git, node_modules/ and coverage/ - change (mediawiki...citoid)
Mobrovac has uploaded a new change for review. https://gerrit.wikimedia.org/r/205858 Change subject: Add .dockerignore listing .git, node_modules/ and coverage/ .. Add .dockerignore listing .git, node_modules/ and coverage/ Change-Id: I37161b50e69f2be39551510af14a0853907147b8 --- A .dockerignore 1 file changed, 3 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/citoid refs/changes/58/205858/1 diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000..563df11 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +.git +coverage +node_modules -- To view, visit https://gerrit.wikimedia.org/r/205858 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I37161b50e69f2be39551510af14a0853907147b8 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/services/citoid Gerrit-Branch: master Gerrit-Owner: Mobrovac mobro...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Update to service-template-node 0.1.3 - change (mediawiki...mobileapps)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/205863
Change subject: Update to service-template-node 0.1.3
..
Update to service-template-node 0.1.3
Changes:
- send CORS and CSP headers
- update dependencies
- use app.loger when req.logger is not available
- various small fixes and updates
Please test this release, mainly due to rather restrictive CSP headers.
Change-Id: Ibbe14ffec488b5791ab3ea4ff5923aee9e2c0ae1
---
M .gitignore
M .jshintignore
D Dockerfile
M app.js
M config.prod.yaml
A dist/init-scripts/systemd.erb
A dist/init-scripts/sysvinit.erb
A dist/init-scripts/upstart.erb
M doc/README.md
M doc/commands.md
A doc/deployment.md
M doc/template.md
M lib/util.js
M package.json
D scripts/docker.js
A scripts/gen-init-scripts.rb
A targets.yaml
M test/features/app/app.js
18 files changed, 551 insertions(+), 205 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/mobileapps
refs/changes/63/205863/1
diff --git a/.gitignore b/.gitignore
index 5b6fe76..bc02473 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
+Dockerfile
coverage
config.yaml
node_modules
diff --git a/.jshintignore b/.jshintignore
index 1c69eee..c052634 100644
--- a/.jshintignore
+++ b/.jshintignore
@@ -1,3 +1,5 @@
coverage
node_modules
test
+www
+static
diff --git a/Dockerfile b/Dockerfile
deleted file mode 100644
index f8f2496..000
--- a/Dockerfile
+++ /dev/null
@@ -1,17 +0,0 @@
-FROM buildpack-deps:jessie
-
-# install node npm
-RUN apt-get update apt-get install -y nodejs npm rm -rf
/var/lib/apt/lists/*
-# link /usr/bin/node to /usr/bin/nodejs
-RUN ln -s /usr/bin/nodejs /usr/bin/node
-
-# copy the repo files over
-RUN mkdir -p /opt/service
-ADD . /opt/service
-# install the dependencies
-WORKDIR /opt/service
-RUN npm install
-
-# start the server
-CMD [/usr/bin/npm, start]
-
diff --git a/app.js b/app.js
index 88943b1..5f9e033 100644
--- a/app.js
+++ b/app.js
@@ -31,6 +31,31 @@
if(!app.conf.port) { app.conf.port = ; }
if(!app.conf.interface) { app.conf.interface = '0.0.0.0'; }
if(!app.conf.compression_level) { app.conf.compression_level = 3; }
+if(app.conf.cors === undefined) { app.conf.cors = '*'; }
+if(!app.conf.csp) {
+app.conf.csp =
+default-src 'self'; object-src 'none'; media-src *; img-src *;
style-src *; frame-ancestors 'self';
+}
+
+// set outgoing proxy
+if(app.conf.proxy) {
+process.env.HTTP_PROXY = app.conf.proxy;
+}
+
+// set the CORS and CSP headers
+app.all('*', function(req, res, next) {
+if(app.conf.cors !== false) {
+res.header('Access-Control-Allow-Origin', app.conf.cors);
+res.header('Access-Control-Allow-Headers', 'Accept,
X-Requested-With, Content-Type');
+}
+res.header('X-XSS-Protection', '1; mode=block');
+res.header('X-Content-Type-Options', 'nosniff');
+res.header('X-Frame-Options', 'SAMEORIGIN');
+res.header('Content-Security-Policy', app.conf.csp);
+res.header('X-Content-Security-Policy', app.conf.csp);
+res.header('X-WebKit-CSP', app.conf.csp);
+next();
+});
// disable the X-Powered-By header
app.set('x-powered-by', false);
diff --git a/config.prod.yaml b/config.prod.yaml
index aad96fd..0b2e676 100644
--- a/config.prod.yaml
+++ b/config.prod.yaml
@@ -18,7 +18,7 @@
# Statsd metrics reporter
metrics:
- type: txstatsd
+ type: statsd
host: statsd.eqiad.wmnet
port: 8125
diff --git a/dist/init-scripts/systemd.erb b/dist/init-scripts/systemd.erb
new file mode 100644
index 000..78b5be3
--- /dev/null
+++ b/dist/init-scripts/systemd.erb
@@ -0,0 +1,25 @@
+[Unit]
+Description=%= @description ? @service_name + ' - ' + @description :
@service_name %
+Documentation=%= @homepage %
+After=network.target local-fs.target
+
+[Service]
+Type=simple
+LimitNOFILE=%= @no_file %
+PIDFile=%t/%= @service_name %.pid
+User=%= @service_name %
+Group=%= @service_name %
+WorkingDirectory=/srv/deployment/%= @service_name %/deploy
+Environment=NODE_PATH='/srv/deployment/%= @service_name
%/deploy/node_modules' %= @service_name.gsub(/[^a-z0-9_]/, '_').upcase
%_PORT=%= @port %
+ExecStart=/usr/bin/nodejs src/server.js -c /etc/%= @service_name
%/config.yaml
+Restart=always
+RestartSec=5
+StandardOutput=syslog
+StandardError=syslog
+SyslogIdentifier=%= @service_name %
+TimeoutStartSec=5
+TimeoutStopSec=60
+
+[Install]
+WantedBy=multi-user.target
+
diff --git a/dist/init-scripts/sysvinit.erb b/dist/init-scripts/sysvinit.erb
new file mode 100644
index 000..f889596
--- /dev/null
+++ b/dist/init-scripts/sysvinit.erb
@@ -0,0 +1,172 @@
+#!/bin/sh
+### BEGIN INIT INFO
+# Provides: %= @service_name %
+# Required-Start:$local_fs $network $remote_fs $syslog
+# Required-Stop: $local_fs $network $remote_fs $syslog
+# Default-Start: 2 3 4 5
+# Default-Stop: 0 1 6
+#
[MediaWiki-commits] [Gerrit] Change route to /png instead of .png - change (mediawiki...graphoid)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/205861
Change subject: Change route to /png instead of .png
..
Change route to /png instead of .png
Having .png in a route is not really RESTful, especially not when the
same path component contains a variable, which might lead to problems.
As we envision other formats to be delivered as well, this makes an
ideal way to split the ID and the format. Alternatively, the preference
for PNG (over other, future formats) might be indicated in the Accept
header.
Change-Id: I093f1e0a4aad07477f1bfe0cc2b5cd8d7b9efe57
---
D Dockerfile
M routes/graphoid-v1.js
2 files changed, 1 insertion(+), 18 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/graphoid
refs/changes/61/205861/1
diff --git a/Dockerfile b/Dockerfile
deleted file mode 100644
index f8f2496..000
--- a/Dockerfile
+++ /dev/null
@@ -1,17 +0,0 @@
-FROM buildpack-deps:jessie
-
-# install node npm
-RUN apt-get update apt-get install -y nodejs npm rm -rf
/var/lib/apt/lists/*
-# link /usr/bin/node to /usr/bin/nodejs
-RUN ln -s /usr/bin/nodejs /usr/bin/node
-
-# copy the repo files over
-RUN mkdir -p /opt/service
-ADD . /opt/service
-# install the dependencies
-WORKDIR /opt/service
-RUN npm install
-
-# start the server
-CMD [/usr/bin/npm, start]
-
diff --git a/routes/graphoid-v1.js b/routes/graphoid-v1.js
index 47f595b..d25015f 100644
--- a/routes/graphoid-v1.js
+++ b/routes/graphoid-v1.js
@@ -331,7 +331,7 @@
/**
* Main entry point for graphoid
*/
-router.get('/:title/:revid/:id.png', function(req, res) {
+router.get('/:title/:revid/:id/png', function(req, res) {
var start = Date.now();
var state = {request: req, response: res};
--
To view, visit https://gerrit.wikimedia.org/r/205861
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I093f1e0a4aad07477f1bfe0cc2b5cd8d7b9efe57
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/graphoid
Gerrit-Branch: master
Gerrit-Owner: Mobrovac mobro...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Graphoid: LVS configuration - change (operations/puppet)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/206106
Change subject: Graphoid: LVS configuration
..
Graphoid: LVS configuration
This patch configures LVS for graphoid on 10.2.2.15 and adds simple
monitoring for it.
Bug: T90487
Change-Id: I97d2b53ebd3fce12c5b9443e2206ff30f9eaa3a7
---
M hieradata/role/common/sca.yaml
M manifests/role/lvs.pp
M modules/lvs/manifests/configuration.pp
M modules/lvs/manifests/monitor.pp
4 files changed, 20 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/06/206106/1
diff --git a/hieradata/role/common/sca.yaml b/hieradata/role/common/sca.yaml
index f6bb36d..115b300 100644
--- a/hieradata/role/common/sca.yaml
+++ b/hieradata/role/common/sca.yaml
@@ -9,6 +9,7 @@
- zotero-admin
lvs::realserver::realserver_ips:
+ - '10.2.2.15' # graphoid.svc.eqiad.wmnet
- '10.2.2.16' # zotero.svc.eqiad.wmnet
- '10.2.2.18' # cxserver.svc.eqiad.wmnet
- '10.2.2.19' # citoid.svc.eqiad.wmnet
diff --git a/manifests/role/lvs.pp b/manifests/role/lvs.pp
index efe2782..1d902fe 100644
--- a/manifests/role/lvs.pp
+++ b/manifests/role/lvs.pp
@@ -46,6 +46,7 @@
$sip['search'][$::site],
$sip['restbase'][$::site],
$sip['zotero'][$::site],
+$sip['graphoid'][$::site],
],
# codfw (should mirror eqiad above, eventually, and become merged with
it via regex
diff --git a/modules/lvs/manifests/configuration.pp
b/modules/lvs/manifests/configuration.pp
index f47249a6..ea03d11 100644
--- a/modules/lvs/manifests/configuration.pp
+++ b/modules/lvs/manifests/configuration.pp
@@ -177,6 +177,9 @@
'cxserver' = {
'eqiad' = 10.2.2.18,
},
+'graphoid' = {
+'eqiad' = 10.2.2.15,
+},
'restbase' = {
'eqiad' = 10.2.2.17,
},
@@ -200,6 +203,7 @@
'mathoid' = {},
'citoid' = {},
'cxserver' = {},
+'graphoid' = {},
'misc_web' = {},
'mobile' = {},
'ocg' = {},
@@ -710,6 +714,19 @@
'IdleConnection' = $idleconnection_monitor_options,
}
},
+'graphoid' = {
+'description' = 'Graph-rendering service,
graphoid.svc.eqiad.wmnet',
+'class' = 'low-traffic',
+'sites' = [ 'eqiad' ],
+'ip' = $service_ips['graphoid'][$::site],
+'port' = 19000,
+'bgp' = 'yes',
+'depool-threshold' = '.5',
+'monitors' = {
+'ProxyFetch' = { 'url' =
['http://graphoid.svc.eqiad.wmnet/_info' ] },
+'IdleConnection' = $idleconnection_monitor_options,
+}
+},
'restbase' = {
'description' = 'RESTBase, restbase.svc.eqiad.wmnet',
'class' = 'low-traffic',
diff --git a/modules/lvs/manifests/monitor.pp b/modules/lvs/manifests/monitor.pp
index 377ecf7..cc59907 100644
--- a/modules/lvs/manifests/monitor.pp
+++ b/modules/lvs/manifests/monitor.pp
@@ -17,6 +17,7 @@
lvs::monitor_service_http { 'mathoid.svc.eqiad.wmnet': ip_address =
$ip['mathoid']['eqiad'], check_command =
check_http_lvs_on_port!mathoid.svc.eqiad.wmnet!10042!/ }
lvs::monitor_service_http { 'citoid.svc.eqiad.wmnet': ip_address =
$ip['citoid']['eqiad'], check_command =
check_http_lvs_on_port!citoid.svc.eqiad.wmnet!1970!/, contact_group =
admins,parsoid }
lvs::monitor_service_http { 'cxserver.svc.eqiad.wmnet': ip_address =
$ip['cxserver']['eqiad'], check_command =
check_http_lvs_on_port!citoid.svc.eqiad.wmnet!8080!/ }
+lvs::monitor_service_http { 'graphoid.svc.eqiad.wmnet': ip_address =
$ip['graphoid']['eqiad'], check_command =
check_http_lvs_on_port!graphoid.svc.eqiad.wmnet!19000!/, contact_group =
admins,parsoid }
lvs::monitor_service_http { 'restbase.svc.eqiad.wmnet': ip_address =
$ip['restbase']['eqiad'], check_command =
check_http_lvs_on_port!restbase.svc.eqiad.wmnet!7231!/ }
lvs::monitor_service_http { 'zotero.svc.eqiad.wmnet': ip_address =
$ip['zotero']['eqiad'], check_command =
check_http_zotero_lvs_on_port!zotero.svc.eqiad.wmnet!1969!/export?format=wikipedia
}
--
To view, visit https://gerrit.wikimedia.org/r/206106
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I97d2b53ebd3fce12c5b9443e2206ff30f9eaa3a7
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Mobrovac mobro...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Graphoid: Varnish configuration - change (operations/puppet)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/206108
Change subject: Graphoid: Varnish configuration
..
Graphoid: Varnish configuration
This patch enables graphoid requests to pass through the parsoidcache
Varnish layer to graphoid.svc.eqiad.wmnet
Bug: T90487
Change-Id: I85da31d57ae02f07051203590581df68632d315a
---
M modules/role/manifests/cache/configuration.pp
M modules/role/manifests/cache/parsoid.pp
M templates/varnish/parsoid-common.inc.vcl.erb
M templates/varnish/parsoid-frontend.inc.vcl.erb
4 files changed, 22 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/08/206108/1
diff --git a/modules/role/manifests/cache/configuration.pp
b/modules/role/manifests/cache/configuration.pp
index 49e8b2d..a567f2d 100644
--- a/modules/role/manifests/cache/configuration.pp
+++ b/modules/role/manifests/cache/configuration.pp
@@ -24,6 +24,9 @@
'citoid' = {
'eqiad' = 'citoid.svc.eqiad.wmnet',
},
+'graphoid' = {
+'eqiad' = 'graphoid.svc.eqiad.wmnet',
+},
'restbase' = {
'eqiad' = 'restbase.svc.eqiad.wmnet',
},
diff --git a/modules/role/manifests/cache/parsoid.pp
b/modules/role/manifests/cache/parsoid.pp
index 8aa9910..851b02b 100644
--- a/modules/role/manifests/cache/parsoid.pp
+++ b/modules/role/manifests/cache/parsoid.pp
@@ -27,6 +27,7 @@
'backend' =
$::role::cache::configuration::backends[$::realm]['parsoid'][$::mw_primary],
'cxserver_backend' =
$::role::cache::configuration::backends[$::realm]['cxserver'][$::site],
'citoid_backend' =
$::role::cache::configuration::backends[$::realm]['citoid'][$::site],
+'graphoid_backend' =
$::role::cache::configuration::backends[$::realm]['graphoid'][$::site],
'restbase_backend' =
$::role::cache::configuration::backends[$::realm]['restbase'][$::site],
},
director_options = {
@@ -46,6 +47,11 @@
{
'backend_match' = '^citoid',
'port' = 1970,
+'probe' = false,
+},
+{
+'backend_match' = '^graphoid',
+'port' = 19000,
'probe' = false,
},
{
@@ -73,6 +79,7 @@
'backend' = $site_parsoid_nodes,
'cxserver_backend' =
$::role::cache::configuration::backends[$::realm]['cxserver'][$::site],
'citoid_backend' =
$::role::cache::configuration::backends[$::realm]['citoid'][$::site],
+'graphoid_backend' =
$::role::cache::configuration::backends[$::realm]['graphoid'][$::site],
'restbase_backend' =
$::role::cache::configuration::backends[$::realm]['restbase'][$::site],
},
director_type = 'chash',
@@ -95,6 +102,11 @@
'probe' = false,
},
{
+'backend_match' = '^graphoid',
+'port' = 19000,
+'probe' = false,
+},
+{
'backend_match' = '^restbase',
'port' = 7231,
'probe' = false, # TODO: Need probe here
diff --git a/templates/varnish/parsoid-common.inc.vcl.erb
b/templates/varnish/parsoid-common.inc.vcl.erb
index a42b0a1..14acef9 100644
--- a/templates/varnish/parsoid-common.inc.vcl.erb
+++ b/templates/varnish/parsoid-common.inc.vcl.erb
@@ -9,6 +9,10 @@
set req.backend = citoid_backend;
return (pass);
}
+ if (req.http.Host ~ graphoid ) {
+ set req.backend = graphoid_backend;
+ return (pass);
+ }
if (req.http.Host ~ restbase || req.http.Host ~ rest\. ) {
set req.backend = restbase_backend;
return (pass);
diff --git a/templates/varnish/parsoid-frontend.inc.vcl.erb
b/templates/varnish/parsoid-frontend.inc.vcl.erb
index df212c5..0b66734 100644
--- a/templates/varnish/parsoid-frontend.inc.vcl.erb
+++ b/templates/varnish/parsoid-frontend.inc.vcl.erb
@@ -25,6 +25,9 @@
if (req.http.Host ~ citoid) {
return (hit_for_pass);
}
+ if (req.http.Host ~ graphoid) {
+ return (hit_for_pass);
+ }
if (req.http.Host ~ restbase || req.http.Host ~ rest\.) {
// Dummy endpoint for eager DNS resolution / TLS handshake.
if (req.url ~ ^/_preconnect) {
--
To view, visit https://gerrit.wikimedia.org/r/206108
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id:
[MediaWiki-commits] [Gerrit] Graphoid: service deployment on SCA - change (operations/puppet)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/206105
Change subject: Graphoid: service deployment on SCA
..
Graphoid: service deployment on SCA
This patch introduces the role::graphoid and graphoid classes,
responsable for deploying graphoid on the SCA cluster, together with its
configuration. Concretely, the graphoid class installs the package
dependencies and uses service::node to create the rest. Hiera is
configured so that graphoid works both in production and
deployment-prep.
Bug: T90487
Change-Id: I1d3edad9f5df5b3a9ea9443411f228fd24e74ed0
---
M hieradata/labs/deployment-prep/common.yaml
M hieradata/role/common/sca.yaml
M manifests/role/deployment.pp
A manifests/role/graphoid.pp
M manifests/role/sca.pp
A modules/graphoid/manifests/init.pp
A modules/graphoid/templates/config.yaml.erb
7 files changed, 116 insertions(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/05/206105/1
diff --git a/hieradata/labs/deployment-prep/common.yaml
b/hieradata/labs/deployment-prep/common.yaml
index b40d40e..6a3042c 100644
--- a/hieradata/labs/deployment-prep/common.yaml
+++ b/hieradata/labs/deployment-prep/common.yaml
@@ -9,6 +9,20 @@
puppetmaster::scripts::keep_reports_minutes: 360
citoid::zotero_host: deployment-zotero01.eqiad.wmflabs
citoid::zotero_port: 1969
+graphoid::domains:
+ - wmflabs.org
+ - mediawiki.org
+ - wikibooks.org
+ - wikidata.org
+ - wikimedia.org
+ - wikimediafoundation.org
+ - wikinews.org
+ - wikipedia.org
+ - wikiquote.org
+ - wikisource.org
+ - wikiversity.org
+ - wikivoyage.org
+ - wiktionary.org
cassandra::seeds:
- 10.68.17.227
- 10.68.17.189
diff --git a/hieradata/role/common/sca.yaml b/hieradata/role/common/sca.yaml
index 358c35c..f6bb36d 100644
--- a/hieradata/role/common/sca.yaml
+++ b/hieradata/role/common/sca.yaml
@@ -17,3 +17,18 @@
service::configuration::http_proxy: http://url-downloader.wikimedia.org:8080
service::configuration::statsd_host: statsd.eqiad.wmnet
service::configuration::logstash_host: logstash1001.eqiad.wmnet
+
+graphoid::domains:
+ - mediawiki.org
+ - wikibooks.org
+ - wikidata.org
+ - wikimedia.org
+ - wikimediafoundation.org
+ - wikinews.org
+ - wikipedia.org
+ - wikiquote.org
+ - wikisource.org
+ - wikiversity.org
+ - wikivoyage.org
+ - wiktionary.org
+
diff --git a/manifests/role/deployment.pp b/manifests/role/deployment.pp
index b08e85f..7d77d88 100644
--- a/manifests/role/deployment.pp
+++ b/manifests/role/deployment.pp
@@ -58,6 +58,11 @@
'zotero/translators' = {
'upstream' =
'https://gerrit.wikimedia.org/r/mediawiki/services/zotero/translators',
},
+'graphoid/deploy' = {
+'upstream' =
'https://gerrit.wikimedia.org/r/mediawiki/services/graphoid/deploy',
+'service_name' = 'graphoid',
+'checkout_submodules' = true,
+},
'rcstream/rcstream' = {
'upstream' =
'https://gerrit.wikimedia.org/r/mediawiki/services/rcstream',
'service_name' = 'rcstream',
diff --git a/manifests/role/graphoid.pp b/manifests/role/graphoid.pp
new file mode 100644
index 000..22ccdb3
--- /dev/null
+++ b/manifests/role/graphoid.pp
@@ -0,0 +1,15 @@
+# vim: set ts=4 et sw=4:
+class role::graphoid {
+
+system::role { 'role::graphoid':
+description = 'node.js service converting graph definitions into PNG'
+}
+
+ferm::service { 'graphoid':
+proto = 'tcp',
+port = 19000,
+}
+
+include ::graphoid
+
+}
diff --git a/manifests/role/sca.pp b/manifests/role/sca.pp
index be0f3e0..3ccf665 100644
--- a/manifests/role/sca.pp
+++ b/manifests/role/sca.pp
@@ -1,7 +1,13 @@
# Compendium class for nodes supporting various *oid services
# This class is an intermediate step to better design
class role::sca {
-include role::apertium, role::citoid, role::cxserver, role::mathoid,
role::zotero
+include role::apertium
+include role::citoid
+include role::cxserver
+include role::mathoid
+include role::zotero
+include role::graphoid
+
include standard
include base::firewall
if $::realm == 'production' {
diff --git a/modules/graphoid/manifests/init.pp
b/modules/graphoid/manifests/init.pp
new file mode 100644
index 000..2d61149
--- /dev/null
+++ b/modules/graphoid/manifests/init.pp
@@ -0,0 +1,37 @@
+# == Class: graphoid
+#
+# This class installs and configures graphoid, a node.js service that
+# converts a graph definition into a PNG image
+#
+# === Parameters
+#
+# [*domains*]
+# The list of enabled domains. Default: []
+#
+# [*domain_map*]
+# The domain-to-domain alias map. Default: {}
+#
+# [*timeout*]
+# The timeout (in ms) for requests. Default: 5000
+#
+class graphoid(
+$domains = [],
+$domain_map = {},
+$timeout =
[MediaWiki-commits] [Gerrit] service::node: Deps fix + deployment-prep hieradata - change (operations/puppet)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/206115
Change subject: service::node: Deps fix + deployment-prep hieradata
..
service::node: Deps fix + deployment-prep hieradata
This patch addresses two minor issues in service::node - service
dependency on nodejs-legacy and Hiera data for deployment-prep.
For the former, the problem (not encountered thus far) is the suble case
where nodejs-legacy does not get installed prior to starting the service.
This package provides the /usr/bin/node - /usr/bin/nodejs symlink on
which our Node.JS services actually count on. As it itself lists nodejs
as a dependency, we can safely swap them in the service requirement
section.
The patch also adds missing configuration Hiera items for
deployment-prep, where no proxy is needed and the StatsD and Logstash
hosts differ from production (for obvious reasons).
Change-Id: I1848b670d8ee0371527ccf20961d613029c7e1b1
---
M hieradata/labs/deployment-prep/common.yaml
M modules/service/manifests/node.pp
2 files changed, 7 insertions(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/15/206115/1
diff --git a/hieradata/labs/deployment-prep/common.yaml
b/hieradata/labs/deployment-prep/common.yaml
index a385af4..6d4e3da 100644
--- a/hieradata/labs/deployment-prep/common.yaml
+++ b/hieradata/labs/deployment-prep/common.yaml
@@ -7,6 +7,9 @@
has_ganglia: false
has_lvs: false
puppetmaster::scripts::keep_reports_minutes: 360
+service::configuration::http_proxy:
+service::configuration::statsd_host: labmon1001.eqiad.wmnet
+service::configuration::logstash_host: deployment-logstash1.eqiad.wmflabs
citoid::zotero_host: deployment-zotero01.eqiad.wmflabs
citoid::zotero_port: 1969
cassandra::seeds:
diff --git a/modules/service/manifests/node.pp
b/modules/service/manifests/node.pp
index 9129880..6fd6aaa 100644
--- a/modules/service/manifests/node.pp
+++ b/modules/service/manifests/node.pp
@@ -139,7 +139,10 @@
hasstatus = true,
hasrestart = true,
provider = 'upstart',
-require= [Package[${title}/deploy], Class['packages::nodejs']],
+require= [
+Package[${title}/deploy],
+Class['packages::nodejs_legacy']
+],
}
# Basic monitoring
--
To view, visit https://gerrit.wikimedia.org/r/206115
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I1848b670d8ee0371527ccf20961d613029c7e1b1
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Mobrovac mobro...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] WIP: Graphoid: Puppet bits - change (operations/puppet)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/205350
Change subject: WIP: Graphoid: Puppet bits
..
WIP: Graphoid: Puppet bits
This patch introduces the role::graphoid and graphoid classes,
responsable for deploying graphoid on the SCA cluster, together with its
configuration. Concretely, the graphoid class installs the package
dependencies and uses service::node to create the rest.
Additionally, this patch:
- adds graphoid to LVS in svc.eqiad.wmnet (exact IP TBD)
- includes various monitoring bits for it
- adds exception rules for the parsoid cache layer
Bug: T90487
Change-Id: If9449c3af66bfe1e5e82f6f9c6594667b44069fe
---
M hieradata/role/common/sca.yaml
M manifests/role/deployment.pp
A manifests/role/graphoid.pp
M manifests/role/lvs.pp
M manifests/role/sca.pp
A modules/graphoid/manifests/init.pp
A modules/graphoid/templates/config.yaml.erb
M modules/lvs/manifests/configuration.pp
M modules/lvs/manifests/monitor.pp
M modules/role/manifests/cache/configuration.pp
M modules/role/manifests/cache/parsoid.pp
M templates/varnish/parsoid-common.inc.vcl.erb
M templates/varnish/parsoid-frontend.inc.vcl.erb
13 files changed, 138 insertions(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/50/205350/1
diff --git a/hieradata/role/common/sca.yaml b/hieradata/role/common/sca.yaml
index 358c35c..7bb01b4 100644
--- a/hieradata/role/common/sca.yaml
+++ b/hieradata/role/common/sca.yaml
@@ -13,7 +13,24 @@
- '10.2.2.18' # cxserver.svc.eqiad.wmnet
- '10.2.2.19' # citoid.svc.eqiad.wmnet
- '10.2.2.20' # mathoid.svc.eqiad.wmnet
+# uncomment once graphoid's IP becomes known
+# - '10.2.2.XX' # graphoid.svc.eqiad.wmnet
service::configuration::http_proxy: http://url-downloader.wikimedia.org:8080
service::configuration::statsd_host: statsd.eqiad.wmnet
service::configuration::logstash_host: logstash1001.eqiad.wmnet
+
+graphoid::domains:
+ - mediawiki.org
+ - wikibooks.org
+ - wikidata.org
+ - wikimedia.org
+ - wikimediafoundation.org
+ - wikinews.org
+ - wikipedia.org
+ - wikiquote.org
+ - wikisource.org
+ - wikiversity.org
+ - wikivoyage.org
+ - wiktionary.org
+
diff --git a/manifests/role/deployment.pp b/manifests/role/deployment.pp
index e84a8f2..50d5464 100644
--- a/manifests/role/deployment.pp
+++ b/manifests/role/deployment.pp
@@ -58,6 +58,11 @@
'zotero/translators' = {
'upstream' =
'https://gerrit.wikimedia.org/r/mediawiki/services/zotero/translators',
},
+'graphoid/deploy' = {
+'upstream' =
'https://gerrit.wikimedia.org/r/mediawiki/services/graphoid/deploy',
+'service_name' = 'graphoid',
+'checkout_submodules' = true,
+},
'rcstream/rcstream' = {
'upstream' =
'https://gerrit.wikimedia.org/r/mediawiki/services/rcstream',
'service_name' = 'rcstream',
diff --git a/manifests/role/graphoid.pp b/manifests/role/graphoid.pp
new file mode 100644
index 000..e8212ec
--- /dev/null
+++ b/manifests/role/graphoid.pp
@@ -0,0 +1,13 @@
+# vim: set ts=4 et sw=4:
+class role::graphoid {
+
+system::role { 'role::graphoid': }
+
+ferm::service { 'graphoid':
+proto = 'tcp',
+port = '19000',
+}
+
+include ::graphoid
+
+}
diff --git a/manifests/role/lvs.pp b/manifests/role/lvs.pp
index efe2782..1d902fe 100644
--- a/manifests/role/lvs.pp
+++ b/manifests/role/lvs.pp
@@ -46,6 +46,7 @@
$sip['search'][$::site],
$sip['restbase'][$::site],
$sip['zotero'][$::site],
+$sip['graphoid'][$::site],
],
# codfw (should mirror eqiad above, eventually, and become merged with
it via regex
diff --git a/manifests/role/sca.pp b/manifests/role/sca.pp
index be0f3e0..f1c4d02 100644
--- a/manifests/role/sca.pp
+++ b/manifests/role/sca.pp
@@ -1,7 +1,7 @@
# Compendium class for nodes supporting various *oid services
# This class is an intermediate step to better design
class role::sca {
-include role::apertium, role::citoid, role::cxserver, role::mathoid,
role::zotero
+include role::apertium, role::citoid, role::cxserver, role::mathoid,
role::zotero, role::graphoid
include standard
include base::firewall
if $::realm == 'production' {
diff --git a/modules/graphoid/manifests/init.pp
b/modules/graphoid/manifests/init.pp
new file mode 100644
index 000..22d598f
--- /dev/null
+++ b/modules/graphoid/manifests/init.pp
@@ -0,0 +1,37 @@
+# == Class: graphoid
+#
+# This class installs and configures graphoid, a node.js service that
+# converts a graph definition into a PNG image
+#
+# === Parameters
+#
+# [*domains*]
+# The list of enabled domains. Default: []
+#
+# [*domain_map*]
+# The domain-to-domain alias map. Default: {}
+#
+# [*timeout*]
+# The timeout (in ms) for requests.
[MediaWiki-commits] [Gerrit] Remove app.js link - change (mediawiki...deploy)
Mobrovac has uploaded a new change for review. https://gerrit.wikimedia.org/r/205576 Change subject: Remove app.js link .. Remove app.js link Instead of linking app.js, deploy repos should link package.json from src/ in order to have the proper node modules installed. Change-Id: I25fc841e394eb00240814b35aabb9fd476efa328 --- D app.js 1 file changed, 0 insertions(+), 1 deletion(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/graphoid/deploy refs/changes/76/205576/1 diff --git a/app.js b/app.js deleted file mode 12 index 948bb13..000 --- a/app.js +++ /dev/null @@ -1 +0,0 @@ -src/app.js \ No newline at end of file -- To view, visit https://gerrit.wikimedia.org/r/205576 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I25fc841e394eb00240814b35aabb9fd476efa328 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/services/graphoid/deploy Gerrit-Branch: master Gerrit-Owner: Mobrovac mobro...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Update to service-template-node 0.1.3 - change (mediawiki...mobileapps)
Mobrovac has submitted this change and it was merged.
Change subject: Update to service-template-node 0.1.3
..
Update to service-template-node 0.1.3
Changes:
- send CORS and CSP headers
- update dependencies
- use app.loger when req.logger is not available
- various small fixes and updates
Please test this release, mainly due to rather restrictive CSP headers.
Bug: T96126
Change-Id: Ibbe14ffec488b5791ab3ea4ff5923aee9e2c0ae1
---
M .gitignore
M .jshintignore
D Dockerfile
M app.js
M config.prod.yaml
A dist/init-scripts/systemd.erb
A dist/init-scripts/sysvinit.erb
A dist/init-scripts/upstart.erb
M doc/README.md
M doc/commands.md
A doc/deployment.md
M doc/template.md
M lib/util.js
M package.json
D scripts/docker.js
A scripts/gen-init-scripts.rb
A targets.yaml
M test/features/app/app.js
18 files changed, 551 insertions(+), 205 deletions(-)
Approvals:
BearND: Looks good to me, approved
Mobrovac: Verified; Looks good to me, approved
diff --git a/.gitignore b/.gitignore
index 5b6fe76..bc02473 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
+Dockerfile
coverage
config.yaml
node_modules
diff --git a/.jshintignore b/.jshintignore
index 1c69eee..c052634 100644
--- a/.jshintignore
+++ b/.jshintignore
@@ -1,3 +1,5 @@
coverage
node_modules
test
+www
+static
diff --git a/Dockerfile b/Dockerfile
deleted file mode 100644
index f8f2496..000
--- a/Dockerfile
+++ /dev/null
@@ -1,17 +0,0 @@
-FROM buildpack-deps:jessie
-
-# install node npm
-RUN apt-get update apt-get install -y nodejs npm rm -rf
/var/lib/apt/lists/*
-# link /usr/bin/node to /usr/bin/nodejs
-RUN ln -s /usr/bin/nodejs /usr/bin/node
-
-# copy the repo files over
-RUN mkdir -p /opt/service
-ADD . /opt/service
-# install the dependencies
-WORKDIR /opt/service
-RUN npm install
-
-# start the server
-CMD [/usr/bin/npm, start]
-
diff --git a/app.js b/app.js
index 88943b1..5f9e033 100644
--- a/app.js
+++ b/app.js
@@ -31,6 +31,31 @@
if(!app.conf.port) { app.conf.port = ; }
if(!app.conf.interface) { app.conf.interface = '0.0.0.0'; }
if(!app.conf.compression_level) { app.conf.compression_level = 3; }
+if(app.conf.cors === undefined) { app.conf.cors = '*'; }
+if(!app.conf.csp) {
+app.conf.csp =
+default-src 'self'; object-src 'none'; media-src *; img-src *;
style-src *; frame-ancestors 'self';
+}
+
+// set outgoing proxy
+if(app.conf.proxy) {
+process.env.HTTP_PROXY = app.conf.proxy;
+}
+
+// set the CORS and CSP headers
+app.all('*', function(req, res, next) {
+if(app.conf.cors !== false) {
+res.header('Access-Control-Allow-Origin', app.conf.cors);
+res.header('Access-Control-Allow-Headers', 'Accept,
X-Requested-With, Content-Type');
+}
+res.header('X-XSS-Protection', '1; mode=block');
+res.header('X-Content-Type-Options', 'nosniff');
+res.header('X-Frame-Options', 'SAMEORIGIN');
+res.header('Content-Security-Policy', app.conf.csp);
+res.header('X-Content-Security-Policy', app.conf.csp);
+res.header('X-WebKit-CSP', app.conf.csp);
+next();
+});
// disable the X-Powered-By header
app.set('x-powered-by', false);
diff --git a/config.prod.yaml b/config.prod.yaml
index aad96fd..0b2e676 100644
--- a/config.prod.yaml
+++ b/config.prod.yaml
@@ -18,7 +18,7 @@
# Statsd metrics reporter
metrics:
- type: txstatsd
+ type: statsd
host: statsd.eqiad.wmnet
port: 8125
diff --git a/dist/init-scripts/systemd.erb b/dist/init-scripts/systemd.erb
new file mode 100644
index 000..78b5be3
--- /dev/null
+++ b/dist/init-scripts/systemd.erb
@@ -0,0 +1,25 @@
+[Unit]
+Description=%= @description ? @service_name + ' - ' + @description :
@service_name %
+Documentation=%= @homepage %
+After=network.target local-fs.target
+
+[Service]
+Type=simple
+LimitNOFILE=%= @no_file %
+PIDFile=%t/%= @service_name %.pid
+User=%= @service_name %
+Group=%= @service_name %
+WorkingDirectory=/srv/deployment/%= @service_name %/deploy
+Environment=NODE_PATH='/srv/deployment/%= @service_name
%/deploy/node_modules' %= @service_name.gsub(/[^a-z0-9_]/, '_').upcase
%_PORT=%= @port %
+ExecStart=/usr/bin/nodejs src/server.js -c /etc/%= @service_name
%/config.yaml
+Restart=always
+RestartSec=5
+StandardOutput=syslog
+StandardError=syslog
+SyslogIdentifier=%= @service_name %
+TimeoutStartSec=5
+TimeoutStopSec=60
+
+[Install]
+WantedBy=multi-user.target
+
diff --git a/dist/init-scripts/sysvinit.erb b/dist/init-scripts/sysvinit.erb
new file mode 100644
index 000..f889596
--- /dev/null
+++ b/dist/init-scripts/sysvinit.erb
@@ -0,0 +1,172 @@
+#!/bin/sh
+### BEGIN INIT INFO
+# Provides: %= @service_name %
+# Required-Start:$local_fs $network $remote_fs $syslog
+# Required-Stop: $local_fs $network $remote_fs $syslog
+# Default-Start: 2 3 4 5
+# Default-Stop: 0 1 6
+# Short-Description: %=
[MediaWiki-commits] [Gerrit] Add the /_info routes and tests for it - change (mediawiki...mobileapps)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/207108
Change subject: Add the /_info routes and tests for it
..
Add the /_info routes and tests for it
Services based on the service template should preserve the /_info end
points since the health monitoring utility in production is using that
as the default end point which should respond with a 200 OK.
Change-Id: I0daa15be21e86219289fd0dac3aba7ed7ec80b57
---
A routes/info.js
A test/features/info/info.js
2 files changed, 164 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/mobileapps
refs/changes/08/207108/1
diff --git a/routes/info.js b/routes/info.js
new file mode 100644
index 000..5dd2590
--- /dev/null
+++ b/routes/info.js
@@ -0,0 +1,90 @@
+'use strict';
+
+
+var sUtil = require('../lib/util');
+
+
+/**
+ * The main router object
+ */
+var router = sUtil.router();
+
+/**
+ * The main application object reported when this module is require()d
+ */
+var app;
+
+
+/**
+ * GET /
+ * Gets some basic info about this service
+ */
+router.get('/', function(req, res) {
+
+// simple sync return
+res.json({
+name: app.info.name,
+version: app.info.version,
+description: app.info.description,
+home: app.info.homepage
+});
+
+});
+
+
+/**
+ * GET /name
+ * Gets the service's name as defined in package.json
+ */
+router.get('/name', function(req, res) {
+
+// simple return
+res.json({ name: app.info.name });
+
+});
+
+
+/**
+ * GET /version
+ * Gets the service's version as defined in package.json
+ */
+router.get('/version', function(req, res) {
+
+// simple return
+res.json({ version: app.info.version });
+
+});
+
+
+/**
+ * ALL /home
+ * Redirects to the service's home page if one is given,
+ * returns a 404 otherwise
+ */
+router.all('/home', function(req, res) {
+
+var home = app.info.homepage;
+if(home /^http/.test(home)) {
+// we have a home page URI defined, so send it
+res.redirect(301, home);
+return;
+} else {
+// no URI defined for the home page, error out
+res.status(404).end('No home page URL defined for ' + app.info.name);
+}
+
+});
+
+
+module.exports = function(appObj) {
+
+app = appObj;
+
+return {
+path: '/_info',
+skip_domain: true,
+router: router
+};
+
+};
+
diff --git a/test/features/info/info.js b/test/features/info/info.js
new file mode 100644
index 000..3450aff
--- /dev/null
+++ b/test/features/info/info.js
@@ -0,0 +1,74 @@
+'use strict';
+
+
+// mocha defines to avoid JSHint breakage
+/* global describe, it, before, beforeEach, after, afterEach */
+
+
+var preq = require('preq');
+var assert = require('../../utils/assert.js');
+var server = require('../../utils/server.js');
+
+
+describe('service information', function() {
+
+this.timeout(2);
+
+before(function () { return server.start(); });
+
+// common URI prefix for info tests
+var infoUri = server.config.uri + '_info/';
+
+// common function used for generating requests
+// and checking their return values
+function checkRet(fieldName) {
+return preq.get({
+uri: infoUri + fieldName
+}).then(function(res) {
+// check the returned Content-Type header
+assert.contentType(res, 'application/json');
+// the status as well
+assert.status(res, 200);
+// finally, check the body has the specified field
+assert.notDeepEqual(res.body, undefined, 'No body returned!');
+assert.notDeepEqual(res.body[fieldName], undefined, 'No ' +
fieldName + ' field returned!');
+});
+}
+
+it('should get the service name', function() {
+return checkRet('name');
+});
+
+it('should get the service version', function() {
+return checkRet('version');
+});
+
+it('should redirect to the service home page', function() {
+return preq.get({
+uri: infoUri + 'home',
+followRedirect: false
+}).then(function(res) {
+// check the status
+assert.status(res, 301);
+});
+});
+
+it('should get the service info', function() {
+return preq.get({
+uri: infoUri
+}).then(function(res) {
+// check the status
+assert.status(res, 200);
+// check the returned Content-Type header
+assert.contentType(res, 'application/json');
+// inspect the body
+assert.notDeepEqual(res.body, undefined, 'No body returned!');
+assert.notDeepEqual(res.body.name, undefined, 'No name field
returned!');
+assert.notDeepEqual(res.body.version, undefined, 'No version field
returned!');
+assert.notDeepEqual(res.body.description, undefined, 'No
description
[MediaWiki-commits] [Gerrit] Update citoid to 0b37b2c - change (mediawiki...deploy)
Mobrovac has submitted this change and it was merged. Change subject: Update citoid to 0b37b2c .. Update citoid to 0b37b2c List of changes: 0b37b2c Add the /_info routes and some tests xxx Update node module dependencies Change-Id: I54f992ba45bc4a1e5a045d6d49e21106cc0764da --- M node_modules/body-parser/node_modules/on-finished/index.js M node_modules/body-parser/node_modules/on-finished/package.json M node_modules/cheerio/node_modules/lodash/dist/lodash.compat.js M node_modules/cheerio/node_modules/lodash/dist/lodash.compat.min.js M node_modules/cheerio/node_modules/lodash/dist/lodash.js M node_modules/cheerio/node_modules/lodash/dist/lodash.min.js M node_modules/cheerio/node_modules/lodash/dist/lodash.underscore.js M node_modules/cheerio/node_modules/lodash/dist/lodash.underscore.min.js M node_modules/cheerio/node_modules/lodash/lodash.js M node_modules/cheerio/node_modules/lodash/package.json M node_modules/express/node_modules/debug/node_modules/ms/package.json M node_modules/express/node_modules/on-finished/index.js M node_modules/express/node_modules/on-finished/package.json M node_modules/request/node_modules/tough-cookie/lib/cookie.js M node_modules/request/node_modules/tough-cookie/package.json M node_modules/request/node_modules/tough-cookie/test/api_test.js M node_modules/request/node_modules/tough-cookie/test/parsing_test.js M node_modules/request/node_modules/tough-cookie/test/regression_test.js M node_modules/service-runner/node_modules/yargs/index.js M node_modules/service-runner/node_modules/yargs/lib/usage.js A node_modules/service-runner/node_modules/yargs/node_modules/cliui/.coveralls.yml A node_modules/service-runner/node_modules/yargs/node_modules/cliui/.npmignore A node_modules/service-runner/node_modules/yargs/node_modules/cliui/.travis.yml A node_modules/service-runner/node_modules/yargs/node_modules/cliui/index.js A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/center-align/index.js A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/center-align/node_modules/align-text/index.js A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/center-align/node_modules/align-text/node_modules/kind-of/index.js A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/center-align/node_modules/align-text/node_modules/kind-of/package.json A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/center-align/node_modules/align-text/node_modules/longest/index.js A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/center-align/node_modules/align-text/node_modules/longest/package.json A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/center-align/node_modules/align-text/node_modules/repeat-string/index.js A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/center-align/node_modules/align-text/node_modules/repeat-string/package.json A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/center-align/node_modules/align-text/package.json A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/center-align/package.json A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/right-align/index.js A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/right-align/node_modules/align-text/index.js A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/right-align/node_modules/align-text/node_modules/kind-of/index.js A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/right-align/node_modules/align-text/node_modules/kind-of/package.json A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/right-align/node_modules/align-text/node_modules/longest/index.js A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/right-align/node_modules/align-text/node_modules/longest/package.json A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/right-align/node_modules/align-text/node_modules/repeat-string/index.js A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/right-align/node_modules/align-text/node_modules/repeat-string/package.json A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/right-align/node_modules/align-text/package.json A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/right-align/package.json R node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/wordwrap/.npmignore R node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/wordwrap/README.markdown R
[MediaWiki-commits] [Gerrit] Update service-mobileapp-node to 7075d2e - change (mediawiki...deploy)
Mobrovac has uploaded a new change for review. https://gerrit.wikimedia.org/r/207110 Change subject: Update service-mobileapp-node to 7075d2e .. Update service-mobileapp-node to 7075d2e List of changes: 7075d2e Add the /_info routes and tests for it Change-Id: I0db17213a7ef6df17a8c84e2c7d78a4a1d3a6af7 --- M src 1 file changed, 0 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/mobileapps/deploy refs/changes/10/207110/1 diff --git a/src b/src index 0268250..7075d2e 16 --- a/src +++ b/src -Subproject commit 0268250208127c8e1613c5f1221c0f67d47d2697 +Subproject commit 7075d2ecaa3769cddc3a0b7bf79f433813fe1e0f -- To view, visit https://gerrit.wikimedia.org/r/207110 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I0db17213a7ef6df17a8c84e2c7d78a4a1d3a6af7 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/services/mobileapps/deploy Gerrit-Branch: master Gerrit-Owner: Mobrovac mobro...@wikimedia.org ___ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
[MediaWiki-commits] [Gerrit] Update citoid to 0b37b2c - change (mediawiki...deploy)
Mobrovac has uploaded a new change for review. https://gerrit.wikimedia.org/r/207095 Change subject: Update citoid to 0b37b2c .. Update citoid to 0b37b2c List of changes: 0b37b2c Add the /_info routes and some tests xxx Update node module dependencies Change-Id: I54f992ba45bc4a1e5a045d6d49e21106cc0764da --- M node_modules/body-parser/node_modules/on-finished/index.js M node_modules/body-parser/node_modules/on-finished/package.json M node_modules/cheerio/node_modules/lodash/dist/lodash.compat.js M node_modules/cheerio/node_modules/lodash/dist/lodash.compat.min.js M node_modules/cheerio/node_modules/lodash/dist/lodash.js M node_modules/cheerio/node_modules/lodash/dist/lodash.min.js M node_modules/cheerio/node_modules/lodash/dist/lodash.underscore.js M node_modules/cheerio/node_modules/lodash/dist/lodash.underscore.min.js M node_modules/cheerio/node_modules/lodash/lodash.js M node_modules/cheerio/node_modules/lodash/package.json M node_modules/express/node_modules/debug/node_modules/ms/package.json M node_modules/express/node_modules/on-finished/index.js M node_modules/express/node_modules/on-finished/package.json M node_modules/request/node_modules/tough-cookie/lib/cookie.js M node_modules/request/node_modules/tough-cookie/package.json M node_modules/request/node_modules/tough-cookie/test/api_test.js M node_modules/request/node_modules/tough-cookie/test/parsing_test.js M node_modules/request/node_modules/tough-cookie/test/regression_test.js M node_modules/service-runner/node_modules/yargs/index.js M node_modules/service-runner/node_modules/yargs/lib/usage.js A node_modules/service-runner/node_modules/yargs/node_modules/cliui/.coveralls.yml A node_modules/service-runner/node_modules/yargs/node_modules/cliui/.npmignore A node_modules/service-runner/node_modules/yargs/node_modules/cliui/.travis.yml A node_modules/service-runner/node_modules/yargs/node_modules/cliui/index.js A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/center-align/index.js A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/center-align/node_modules/align-text/index.js A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/center-align/node_modules/align-text/node_modules/kind-of/index.js A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/center-align/node_modules/align-text/node_modules/kind-of/package.json A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/center-align/node_modules/align-text/node_modules/longest/index.js A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/center-align/node_modules/align-text/node_modules/longest/package.json A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/center-align/node_modules/align-text/node_modules/repeat-string/index.js A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/center-align/node_modules/align-text/node_modules/repeat-string/package.json A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/center-align/node_modules/align-text/package.json A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/center-align/package.json A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/right-align/index.js A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/right-align/node_modules/align-text/index.js A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/right-align/node_modules/align-text/node_modules/kind-of/index.js A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/right-align/node_modules/align-text/node_modules/kind-of/package.json A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/right-align/node_modules/align-text/node_modules/longest/index.js A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/right-align/node_modules/align-text/node_modules/longest/package.json A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/right-align/node_modules/align-text/node_modules/repeat-string/index.js A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/right-align/node_modules/align-text/node_modules/repeat-string/package.json A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/right-align/node_modules/align-text/package.json A node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/right-align/package.json R node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/wordwrap/.npmignore R node_modules/service-runner/node_modules/yargs/node_modules/cliui/node_modules/wordwrap/README.markdown R
[MediaWiki-commits] [Gerrit] Add the /_info routes and some tests - change (mediawiki...citoid)
Mobrovac has submitted this change and it was merged.
Change subject: Add the /_info routes and some tests
..
Add the /_info routes and some tests
All services based on the service template expose the /_info routes,
which are also used by the health monitoring utilities in production to
check on them. This commit adds these routes to Citoid.
Also, some more tests have been added checking for correct CORS and CSP
headers as well as tests for /_info routes.
Change-Id: I9ad49cfe13278c07bef10bee9e736de9f2636706
---
M package.json
A routes/info.js
M test/features/app/index.js
A test/features/app/info.js
4 files changed, 189 insertions(+), 0 deletions(-)
Approvals:
Mobrovac: Verified; Looks good to me, approved
diff --git a/package.json b/package.json
index 5023a2c..b03422f 100644
--- a/package.json
+++ b/package.json
@@ -2,6 +2,7 @@
name: citoid,
version: 0.2.6,
description: Converts search terms such as URL or DOI into citations.,
+ homepage: https://www.mediawiki.org/wiki/Citoid;,
scripts: {
start: service-runner,
test: mocha test/index.js,
diff --git a/routes/info.js b/routes/info.js
new file mode 100644
index 000..e9b06fe
--- /dev/null
+++ b/routes/info.js
@@ -0,0 +1,90 @@
+'use strict';
+
+
+var sUtil = require('../lib/util');
+
+
+/**
+ * The main router object
+ */
+var router = sUtil.router();
+
+/**
+ * The main application object reported when this module is require()d
+ */
+var app;
+
+
+/**
+ * GET /
+ * Gets some basic info about this service
+ */
+router.get('/', function(req, res) {
+
+ // simple sync return
+ res.json({
+ name: app.info.name,
+ version: app.info.version,
+ description: app.info.description,
+ home: app.info.homepage
+ });
+
+});
+
+
+/**
+ * GET /name
+ * Gets the service's name as defined in package.json
+ */
+router.get('/name', function(req, res) {
+
+ // simple return
+ res.json({ name: app.info.name });
+
+});
+
+
+/**
+ * GET /version
+ * Gets the service's version as defined in package.json
+ */
+router.get('/version', function(req, res) {
+
+ // simple return
+ res.json({ version: app.info.version });
+
+});
+
+
+/**
+ * ALL /home
+ * Redirects to the service's home page if one is given,
+ * returns a 404 otherwise
+ */
+router.all('/home', function(req, res) {
+
+ var home = app.info.homepage;
+ if(home /^http/.test(home)) {
+ // we have a home page URI defined, so send it
+ res.redirect(301, home);
+ return;
+ } else {
+ // no URI defined for the home page, error out
+ res.status(404).end('No home page URL defined for ' +
app.info.name);
+ }
+
+});
+
+
+module.exports = function(appObj) {
+
+ app = appObj;
+
+ return {
+ path: '/_info',
+ skip_domain: true,
+ router: router
+ };
+
+};
+
diff --git a/test/features/app/index.js b/test/features/app/index.js
index ac590d6..ed74cd0 100644
--- a/test/features/app/index.js
+++ b/test/features/app/index.js
@@ -32,5 +32,29 @@
});
});
+ it('should set CORS headers', function() {
+ return preq.get({
+ uri: server.config.uri + 'robots.txt'
+ }).then(function(res) {
+ assert.deepEqual(res.status, 200);
+
assert.deepEqual(res.headers['access-control-allow-origin'], '*');
+
assert.notDeepEqual(res.headers['access-control-allow-headers'], undefined);
+ });
+ });
+
+ it('should set CSP headers', function() {
+ return preq.get({
+ uri: server.config.uri + 'robots.txt'
+ }).then(function(res) {
+ assert.deepEqual(res.status, 200);
+ assert.deepEqual(res.headers['x-xss-protection'], '1;
mode=block');
+ assert.deepEqual(res.headers['x-content-type-options'],
'nosniff');
+ assert.deepEqual(res.headers['x-frame-options'],
'SAMEORIGIN');
+
assert.deepEqual(res.headers['content-security-policy'], 'default-src');
+
assert.deepEqual(res.headers['x-content-security-policy'], 'default-src');
+ assert.deepEqual(res.headers['x-webkit-csp'],
'default-src');
+ });
+ });
+
});
diff --git a/test/features/app/info.js b/test/features/app/info.js
new file mode 100644
index 000..3394c75
--- /dev/null
+++ b/test/features/app/info.js
@@ -0,0 +1,74 @@
+'use strict';
+
+
+// mocha defines to avoid JSHint breakage
+/* global describe, it, before, beforeEach, after, afterEach */
+
+
+var preq = require('preq');
+var assert = require('../../utils/assert.js');
+var server =
[MediaWiki-commits] [Gerrit] Add the /_info routes and some tests - change (mediawiki...citoid)
Mobrovac has uploaded a new change for review.
https://gerrit.wikimedia.org/r/207091
Change subject: Add the /_info routes and some tests
..
Add the /_info routes and some tests
All services based on the service template expose the /_info routes,
which are also used by the health monitoring utilities in production to
check on them. This commit adds these routes to Citoid.
Also, some more tests have been added checking for correct CORS and CSP
headers as well as tests for /_info routes.
Change-Id: I9ad49cfe13278c07bef10bee9e736de9f2636706
---
M package.json
A routes/info.js
M test/features/app/index.js
A test/features/app/info.js
4 files changed, 189 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/citoid
refs/changes/91/207091/1
diff --git a/package.json b/package.json
index 5023a2c..b03422f 100644
--- a/package.json
+++ b/package.json
@@ -2,6 +2,7 @@
name: citoid,
version: 0.2.6,
description: Converts search terms such as URL or DOI into citations.,
+ homepage: https://www.mediawiki.org/wiki/Citoid;,
scripts: {
start: service-runner,
test: mocha test/index.js,
diff --git a/routes/info.js b/routes/info.js
new file mode 100644
index 000..e9b06fe
--- /dev/null
+++ b/routes/info.js
@@ -0,0 +1,90 @@
+'use strict';
+
+
+var sUtil = require('../lib/util');
+
+
+/**
+ * The main router object
+ */
+var router = sUtil.router();
+
+/**
+ * The main application object reported when this module is require()d
+ */
+var app;
+
+
+/**
+ * GET /
+ * Gets some basic info about this service
+ */
+router.get('/', function(req, res) {
+
+ // simple sync return
+ res.json({
+ name: app.info.name,
+ version: app.info.version,
+ description: app.info.description,
+ home: app.info.homepage
+ });
+
+});
+
+
+/**
+ * GET /name
+ * Gets the service's name as defined in package.json
+ */
+router.get('/name', function(req, res) {
+
+ // simple return
+ res.json({ name: app.info.name });
+
+});
+
+
+/**
+ * GET /version
+ * Gets the service's version as defined in package.json
+ */
+router.get('/version', function(req, res) {
+
+ // simple return
+ res.json({ version: app.info.version });
+
+});
+
+
+/**
+ * ALL /home
+ * Redirects to the service's home page if one is given,
+ * returns a 404 otherwise
+ */
+router.all('/home', function(req, res) {
+
+ var home = app.info.homepage;
+ if(home /^http/.test(home)) {
+ // we have a home page URI defined, so send it
+ res.redirect(301, home);
+ return;
+ } else {
+ // no URI defined for the home page, error out
+ res.status(404).end('No home page URL defined for ' +
app.info.name);
+ }
+
+});
+
+
+module.exports = function(appObj) {
+
+ app = appObj;
+
+ return {
+ path: '/_info',
+ skip_domain: true,
+ router: router
+ };
+
+};
+
diff --git a/test/features/app/index.js b/test/features/app/index.js
index ac590d6..ed74cd0 100644
--- a/test/features/app/index.js
+++ b/test/features/app/index.js
@@ -32,5 +32,29 @@
});
});
+ it('should set CORS headers', function() {
+ return preq.get({
+ uri: server.config.uri + 'robots.txt'
+ }).then(function(res) {
+ assert.deepEqual(res.status, 200);
+
assert.deepEqual(res.headers['access-control-allow-origin'], '*');
+
assert.notDeepEqual(res.headers['access-control-allow-headers'], undefined);
+ });
+ });
+
+ it('should set CSP headers', function() {
+ return preq.get({
+ uri: server.config.uri + 'robots.txt'
+ }).then(function(res) {
+ assert.deepEqual(res.status, 200);
+ assert.deepEqual(res.headers['x-xss-protection'], '1;
mode=block');
+ assert.deepEqual(res.headers['x-content-type-options'],
'nosniff');
+ assert.deepEqual(res.headers['x-frame-options'],
'SAMEORIGIN');
+
assert.deepEqual(res.headers['content-security-policy'], 'default-src');
+
assert.deepEqual(res.headers['x-content-security-policy'], 'default-src');
+ assert.deepEqual(res.headers['x-webkit-csp'],
'default-src');
+ });
+ });
+
});
diff --git a/test/features/app/info.js b/test/features/app/info.js
new file mode 100644
index 000..3394c75
--- /dev/null
+++ b/test/features/app/info.js
@@ -0,0 +1,74 @@
+'use strict';
+
+
+// mocha defines to avoid JSHint breakage
+/* global describe, it, before, beforeEach, after, afterEach */
+
+
+var preq = require('preq');
+var assert =
[MediaWiki-commits] [Gerrit] Add the /_info routes and tests for it - change (mediawiki...mobileapps)
Mobrovac has submitted this change and it was merged.
Change subject: Add the /_info routes and tests for it
..
Add the /_info routes and tests for it
Services based on the service template should preserve the /_info end
points since the health monitoring utility in production is using that
as the default end point which should respond with a 200 OK.
Change-Id: I0daa15be21e86219289fd0dac3aba7ed7ec80b57
---
A routes/info.js
A test/features/info/info.js
2 files changed, 164 insertions(+), 0 deletions(-)
Approvals:
Mobrovac: Verified; Looks good to me, approved
diff --git a/routes/info.js b/routes/info.js
new file mode 100644
index 000..5dd2590
--- /dev/null
+++ b/routes/info.js
@@ -0,0 +1,90 @@
+'use strict';
+
+
+var sUtil = require('../lib/util');
+
+
+/**
+ * The main router object
+ */
+var router = sUtil.router();
+
+/**
+ * The main application object reported when this module is require()d
+ */
+var app;
+
+
+/**
+ * GET /
+ * Gets some basic info about this service
+ */
+router.get('/', function(req, res) {
+
+// simple sync return
+res.json({
+name: app.info.name,
+version: app.info.version,
+description: app.info.description,
+home: app.info.homepage
+});
+
+});
+
+
+/**
+ * GET /name
+ * Gets the service's name as defined in package.json
+ */
+router.get('/name', function(req, res) {
+
+// simple return
+res.json({ name: app.info.name });
+
+});
+
+
+/**
+ * GET /version
+ * Gets the service's version as defined in package.json
+ */
+router.get('/version', function(req, res) {
+
+// simple return
+res.json({ version: app.info.version });
+
+});
+
+
+/**
+ * ALL /home
+ * Redirects to the service's home page if one is given,
+ * returns a 404 otherwise
+ */
+router.all('/home', function(req, res) {
+
+var home = app.info.homepage;
+if(home /^http/.test(home)) {
+// we have a home page URI defined, so send it
+res.redirect(301, home);
+return;
+} else {
+// no URI defined for the home page, error out
+res.status(404).end('No home page URL defined for ' + app.info.name);
+}
+
+});
+
+
+module.exports = function(appObj) {
+
+app = appObj;
+
+return {
+path: '/_info',
+skip_domain: true,
+router: router
+};
+
+};
+
diff --git a/test/features/info/info.js b/test/features/info/info.js
new file mode 100644
index 000..3450aff
--- /dev/null
+++ b/test/features/info/info.js
@@ -0,0 +1,74 @@
+'use strict';
+
+
+// mocha defines to avoid JSHint breakage
+/* global describe, it, before, beforeEach, after, afterEach */
+
+
+var preq = require('preq');
+var assert = require('../../utils/assert.js');
+var server = require('../../utils/server.js');
+
+
+describe('service information', function() {
+
+this.timeout(2);
+
+before(function () { return server.start(); });
+
+// common URI prefix for info tests
+var infoUri = server.config.uri + '_info/';
+
+// common function used for generating requests
+// and checking their return values
+function checkRet(fieldName) {
+return preq.get({
+uri: infoUri + fieldName
+}).then(function(res) {
+// check the returned Content-Type header
+assert.contentType(res, 'application/json');
+// the status as well
+assert.status(res, 200);
+// finally, check the body has the specified field
+assert.notDeepEqual(res.body, undefined, 'No body returned!');
+assert.notDeepEqual(res.body[fieldName], undefined, 'No ' +
fieldName + ' field returned!');
+});
+}
+
+it('should get the service name', function() {
+return checkRet('name');
+});
+
+it('should get the service version', function() {
+return checkRet('version');
+});
+
+it('should redirect to the service home page', function() {
+return preq.get({
+uri: infoUri + 'home',
+followRedirect: false
+}).then(function(res) {
+// check the status
+assert.status(res, 301);
+});
+});
+
+it('should get the service info', function() {
+return preq.get({
+uri: infoUri
+}).then(function(res) {
+// check the status
+assert.status(res, 200);
+// check the returned Content-Type header
+assert.contentType(res, 'application/json');
+// inspect the body
+assert.notDeepEqual(res.body, undefined, 'No body returned!');
+assert.notDeepEqual(res.body.name, undefined, 'No name field
returned!');
+assert.notDeepEqual(res.body.version, undefined, 'No version field
returned!');
+assert.notDeepEqual(res.body.description, undefined, 'No
description field returned!');
+assert.notDeepEqual(res.body.home,
[MediaWiki-commits] [Gerrit] Change route to include the format as well - change (mediawiki...graphoid)
Mobrovac has submitted this change and it was merged.
Change subject: Change route to include the format as well
..
Change route to include the format as well
This patch alters the exposed route to comprise the sought format as
well. The new route is /:format/:title/:revid/:id , where :id is allowed
to contain the trailing file extension as well. If it is provided, it
must match the given format. A few tests checking this as well as the
format itself have been added.
Bug: T97123
Change-Id: I093f1e0a4aad07477f1bfe0cc2b5cd8d7b9efe57
---
M routes/graphoid-v1.js
M test/features/v1/graph.js
2 files changed, 37 insertions(+), 4 deletions(-)
Approvals:
Mobrovac: Verified; Looks good to me, approved
diff --git a/routes/graphoid-v1.js b/routes/graphoid-v1.js
index 47f595b..cfa1419 100644
--- a/routes/graphoid-v1.js
+++ b/routes/graphoid-v1.js
@@ -149,10 +149,13 @@
var start = Date.now();
var p = state.request.params,
+format = p.format,
domain = p.domain,
title = p.title,
revid = p.revid,
-id = p.id;
+id_ext = p.id.split('.', 2),
+id = id_ext[0],
+ext = id_ext[1];
state.log = p; // log all parameters of the request
@@ -163,6 +166,14 @@
ppprop: 'graph_specs',
continue: ''
};
+
+// check the format / extension
+if (ext ext !== format) {
+throw new Err('info/param-ext', 'req.id');
+}
+if (format !== 'png') {
+throw new Err('info/param-format', 'req.format');
+}
if (revid) {
if (!/^[0-9]+$/.test(revid)) {
@@ -331,7 +342,7 @@
/**
* Main entry point for graphoid
*/
-router.get('/:title/:revid/:id.png', function(req, res) {
+router.get('/:format/:title/:revid/:id', function(req, res) {
var start = Date.now();
var state = {request: req, response: res};
diff --git a/test/features/v1/graph.js b/test/features/v1/graph.js
index 150d196..1379049 100644
--- a/test/features/v1/graph.js
+++ b/test/features/v1/graph.js
@@ -19,10 +19,10 @@
// common URI prefix for v1
var uri = function(domain, title, revId, graphId) {
return server.config.uri +
-( domain !== null ? domain : 'mediawiki.org' ) + '/v1/' +
+( domain !== null ? domain : 'mediawiki.org' ) + '/v1/png/' +
( title !== null ? title : 'Extension:Graph%2FDemo' ) + '/' +
( revId !== null ? revId : '1508976' ) + '/' +
-( graphId !== null ? graphId :
'597fd63eb884b45edcd7f71a2788bf01ce52ce9b' ) + '.png';
+( graphId !== null ? graphId :
'597fd63eb884b45edcd7f71a2788bf01ce52ce9b' );
};
it('should get a PNG image from the Extension:Graph/Demo page without
revision ID', function() {
@@ -100,5 +100,27 @@
});
});
+it('format - extension mismatch', function() {
+return preq.get({
+uri: uri(null, null, null, null) + '.jpg'
+}).then(function(res) {
+throw new Error('Expected an error to be thrown, got status: ' +
res.status);
+}, function(err) {
+assert.deepEqual(err.status, 400);
+assert.deepEqual(err.body, 'info/param-ext');
+});
+});
+
+it('wrong format', function() {
+return preq.get({
+uri: server.config.uri + 'bla/v1/foo/bar/1234/5678'
+}).then(function(res) {
+throw new Error('Expected an error to be thrown, got status: ' +
res.status);
+}, function(err) {
+assert.deepEqual(err.status, 400);
+assert.deepEqual(err.body, 'info/param-format');
+});
+});
+
});
--
To view, visit https://gerrit.wikimedia.org/r/205861
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I093f1e0a4aad07477f1bfe0cc2b5cd8d7b9efe57
Gerrit-PatchSet: 5
Gerrit-Project: mediawiki/services/graphoid
Gerrit-Branch: master
Gerrit-Owner: Mobrovac mobro...@wikimedia.org
Gerrit-Reviewer: Mobrovac mobro...@wikimedia.org
Gerrit-Reviewer: Yurik yu...@wikimedia.org
___
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
