Cscott has uploaded a new change for review.
https://gerrit.wikimedia.org/r/177115
Change subject: Refactor `cleanJobStatusObjects` in gc thread.
......................................................................
Refactor `cleanJobStatusObjects` in gc thread.
This allows us to more easily make variants which (for example) remove only
jobs from a specific host (to take that host down for maintenance, for
example).
Change-Id: I0319debff8b3a224ba840fa92156ef961ca99bd2
---
M lib/threads/gc.js
1 file changed, 40 insertions(+), 21 deletions(-)
git pull
ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Collection/OfflineContentGenerator
refs/changes/15/177115/1
diff --git a/lib/threads/gc.js b/lib/threads/gc.js
index 9503833..411ab0d 100644
--- a/lib/threads/gc.js
+++ b/lib/threads/gc.js
@@ -114,7 +114,7 @@
var startTime = Date.now();
return Promise.resolve()
- .then(cleanJobStatusObjects)
+ .then(cleanExpiredJobStatusObjects)
.then(cleanOutputDir)
.then(cleanTempDir)
.then(cleanPostmortemDir)
@@ -129,29 +129,58 @@
* Iterate through all JobStatus objects in Redis and clear those
* that are too old.
*/
-function cleanJobStatusObjects() {
+function cleanExpiredJobStatusObjects() {
var clearedFailedJobs = 0,
clearedNonFailedJobs = 0,
// job lifetimes are in seconds.
fjl = Date.now() -
(config.garbage_collection.failed_job_lifetime * 1000),
ajl = Date.now() - (config.garbage_collection.job_lifetime *
1000);
- console.info( "Starting run to clean job status objects", { channel:
'gc' } );
+ console.info(
+ "Starting run to clean job status objects",
+ { channel: 'gc' }
+ );
+ return cleanJobStatusObjects(function( job ) {
+ if ( job.status === 'failed' && ( job.timestamp < fjl ) ) {
+ clearedFailedJobs += 1;
+ return true; // delete this job
+ } else if ( job.timestamp < ajl ) {
+ clearedNonFailedJobs += 1;
+ return true; // delete this job
+ }
+ return false; // keep this job
+ }).spread(function( total, deleted ) {
+ console.info(
+ "Got %d status keys to iterate through",
+ total,
+ { channel: 'gc' }
+ );
+ console.info(
+ "Cleared %d non-failed jobs and %d failed jobs",
+ clearedNonFailedJobs,
+ clearedFailedJobs,
+ { channel: 'gc' }
+ );
+ });
+}
+
+/**
+ * Iterate through all JobStatus objects in Redis and clear those
+ * for which the `shouldDeleteJob` function returns `true`.
+ */
+function cleanJobStatusObjects(shouldDeleteJob) {
+ var total = 0, deleted = 0;
+
var scrubKey = function( key ) {
return redisClient.hget( config.redis.status_set_name, key
).then( function( jdjson ) {
var job = jd.fromJson( jdjson );
- if ( job.status === 'failed' && ( job.timestamp < fjl )
) {
- clearedFailedJobs += 1;
- return redisClient.hdel(
config.redis.status_set_name, job.collectionId );
- } else if ( job.timestamp < ajl ) {
- clearedNonFailedJobs += 1;
+ if ( shouldDeleteJob( job ) ) {
+ deleted += 1;
return redisClient.hdel(
config.redis.status_set_name, job.collectionId );
}
} );
};
var scrubKeyGuarded = Promise.guard( Promise.guard.n( 5 ), scrubKey );
-
- var total = 0;
var scanSome = function( cursor ) {
return redisClient.hscan( config.redis.status_set_name, cursor
).
@@ -167,17 +196,7 @@
};
return scanSome( 0 ).then( function() {
- console.info(
- "Got %d status keys to iterate through",
- total,
- { channel: 'gc' }
- );
- console.info(
- "Cleared %d non-failed jobs and %d failed jobs",
- clearedNonFailedJobs,
- clearedFailedJobs,
- { channel: 'gc' }
- );
+ return [ total, deleted ];
} );
}
--
To view, visit https://gerrit.wikimedia.org/r/177115
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I0319debff8b3a224ba840fa92156ef961ca99bd2
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/Collection/OfflineContentGenerator
Gerrit-Branch: master
Gerrit-Owner: Cscott <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
