Cscott has uploaded a new change for review.
https://gerrit.wikimedia.org/r/177120
Change subject: Add script to allow host decommissioning.
......................................................................
Add script to allow host decommissioning.
Change-Id: I099004f1a98d2fa2ff4e5db1806205f35b195f77
---
M lib/cli.js
M lib/threads/gc.js
A scripts/clear-host-cache.js
3 files changed, 127 insertions(+), 4 deletions(-)
git pull
ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Collection/OfflineContentGenerator
refs/changes/20/177120/1
diff --git a/lib/cli.js b/lib/cli.js
index 539bdfc..9bb6564 100644
--- a/lib/cli.js
+++ b/lib/cli.js
@@ -58,7 +58,7 @@
};
// Set up logging.
-var setupLogging = exports.setupLogging = function( config ) {
+var setupLogging = exports.setupLogging = function( config, forceStdout ) {
var bunyan = require( 'bunyan' );
var mkPrettyStream = function() {
try {
@@ -71,8 +71,16 @@
return { stream: process.stdout };
}
};
+ var checkStdout = function(streams) {
+ if (forceStdout && !streams.some(function(s) {
+ return s.stream === process.stdout;
+ })) {
+ streams.push( mkPrettyStream() );
+ }
+ return streams;
+ };
var streams = (config.logging && Array.isArray(config.logging.streams))
?
- config.logging.streams : [ mkPrettyStream() ];
+ checkStdout(config.logging.streams) : [ mkPrettyStream() ];
var serializers = (config.logging && config.logging.serializers) ||
bunyan.stdSerializers;
var logger = bunyan.createLogger({
diff --git a/lib/threads/gc.js b/lib/threads/gc.js
index 5896e07..cbcbc79 100644
--- a/lib/threads/gc.js
+++ b/lib/threads/gc.js
@@ -81,7 +81,7 @@
redisClient.connect();
}
-function doSingleRun() {
+function doSingleRun( what ) {
redisClient.on( 'closed', function () {
if ( running ) {
console.error( 'Garbage collector connection to redis
died, killing thread.', {
@@ -93,7 +93,7 @@
redisClient.connect();
return new Promise( function( resolve, reject ) {
redisClient.on('opened', function() {
- doGCRun().then( resolve ).catch( reject );
+ Promise.resolve().then(what || doGCRun).then( resolve,
reject );
} );
} );
}
@@ -316,3 +316,6 @@
exports.start = startThread;
exports.stop = stopThread;
exports.singleRun = doSingleRun;
+
+exports.cleanDir = cleanDir;
+exports.cleanJobStatusObjects = cleanJobStatusObjects;
diff --git a/scripts/clear-host-cache.js b/scripts/clear-host-cache.js
new file mode 100755
index 0000000..31abe1e
--- /dev/null
+++ b/scripts/clear-host-cache.js
@@ -0,0 +1,112 @@
+#!/usr/bin/env node
+"use strict";
+
+/**
+ * Collection Extension host decommission script.
+ *
+ * This script will remove all cached job entries which refer to a
+ * specific host, named on the command-line. This is intended for use
+ * when removing a host for maintenance. First, you'd remove the host
+ * from the round-robin DNS name specified in the Collection extension
+ * configuration, so it no longer accepted new jobs. Once the DNS change
+ * propagated and any existing jobs on that host were complete, you
+ * would run something like:
+ * $ sudo -u ocg -g ocg nodejs-ocg scripts/clear-host-cache.js -c
/etc/ocg/mw-ocg-service.js ocg1002
+ * where `ocg1002` is the name of the host you want to decommission.
+ *
+ * If the hostname is omitted, the script will use the name of the
+ * host on which the script is running.
+ *
+ * The script will not remove job status entries for pending jobs
+ * (unless you use the `--force` flag). It will complain on console
+ * if it finds pending jobs, and exit with a non-zero exit code.
+ * In that case, the operator should wait longer (say, 15 minutes)
+ * for the pending job to complete and the user to collect the
+ * results, before re-running the clear-host-cache script.
+ *
+ * @section LICENSE
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ */
+
+require( 'es6-shim' );
+require( 'prfun' );
+
+var cli = require( '../lib/cli.js' );
+var commander = require( 'commander' );
+var os = require( 'os' );
+
+// parse command-line options (with a possible additional config file override)
+commander
+ .version( cli.version )
+ .usage('[options] <hostname ...>')
+ .option( '-c, --config <path>', 'Path to the local configuration file' )
+ .option( '-f, --force', 'Remove even pending jobs' )
+ .option( '--quiet', "Don't add stdout to configured loggers")
+ .parse( process.argv );
+
+var config = cli.parseConfig( commander.config );
+cli.setupLogging( config, !commander.quiet );
+cli.setupStatsD( config );
+
+var hosts = commander.args.length ? commander.args :
+ [ config.coordinator.hostname || os.hostname() ];
+
+/* === Do the deed ========================================================= */
+var gc = require( '../lib/threads/gc.js' );
+gc.init( config );
+gc.singleRun(function() {
+ var startTime = Date.now();
+ var pending = 0;
+
+ console.info(
+ 'Clearing cache for hosts: %s', hosts.join(', '),
+ { channel: 'gc' }
+ );
+ var hostre = /^https?:\/\/([^:\/]*)/;
+
+ return gc.cleanJobStatusObjects(function(job) {
+ if (!job.host) {
+ return false; /* not picked up yet, keep */
+ }
+ if (!hosts.some(function(h) { return h === job.host; })) {
+ return false; /* not on the specified host(s) */
+ }
+ if (/^(finished|failed)$/.test(job.state)) {
+ return true; /* delete this cache entry */
+ }
+ pending += 1;
+ if (commander.force) {
+ return true; /* force-remove this pending entry */
+ }
+ return false; /* don't remove it, it's still pending */
+ }).spread(function(total, deleted) {
+ console.info(
+ 'Cleared %d (of %d total) entries from cache in %s
seconds',
+ deleted, total, (Date.now() - startTime) / 1000,
+ { channel: 'gc' }
+ );
+ return pending;
+ });
+}).tap( function() {
+ return new Promise(function(resolve) { gc.stop( resolve ); });
+}).then( function(pending) {
+ if (pending) {
+ console.info(' XXX' );
+ process.exit(1);
+ }
+}).done();
--
To view, visit https://gerrit.wikimedia.org/r/177120
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I099004f1a98d2fa2ff4e5db1806205f35b195f77
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/Collection/OfflineContentGenerator
Gerrit-Branch: master
Gerrit-Owner: Cscott <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
