Cscott has uploaded a new change for review.
https://gerrit.wikimedia.org/r/177241
Change subject: Redis hscan command returns keys and values interleaved.
......................................................................
Redis hscan command returns keys and values interleaved.
Our emulated `hscan` only returned keys. When running on redis > 2.6,
we'd use the real `hscan` and get values interspersed with our keys.
Fix the emulation to behave the same as the real hscan, and fix the
users to expect interleaves keys and values.
Change-Id: Ie796ba8ff3fde8926aa719adc019efbc6c52f9a4
---
M lib/RedisWrapper.js
M lib/threads/gc.js
2 files changed, 25 insertions(+), 13 deletions(-)
git pull
ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Collection/OfflineContentGenerator
refs/changes/41/177241/1
diff --git a/lib/RedisWrapper.js b/lib/RedisWrapper.js
index 474738c..34215f9 100644
--- a/lib/RedisWrapper.js
+++ b/lib/RedisWrapper.js
@@ -146,13 +146,22 @@
}
// emulate hscan -- the hkeys uses more memory, but still return
// keys in chunks so that user is throttled.
- return Promise.promisify( this.client.hkeys, false, this.client
)( key ).then( function( keys ) {
+ var hkeys = Promise.promisify( this.client.hkeys, false,
this.client );
+ var hmget = Promise.promisify( this.client.hmget, false,
this.client );
+ return hkeys( key ).then( function( keys ) {
var i = 0, COUNT = 50;
// return keys 50 at a time.
var some = function () {
var next = keys.slice(i, i+COUNT);
i += COUNT;
- return Promise.resolve([ i < keys.length ? some
: 0, next ]);
+ return hmget( key, next ).then( function(
values ) {
+ var j, result = [];
+ for (j=0; j<next.length; j++) {
+ result.push(next[j]);
+ result.push(values[j]);
+ }
+ return [ i < keys.length ? some : 0,
result ];
+ });
};
return some();
});
diff --git a/lib/threads/gc.js b/lib/threads/gc.js
index 5896e07..66130fd 100644
--- a/lib/threads/gc.js
+++ b/lib/threads/gc.js
@@ -171,22 +171,25 @@
function cleanJobStatusObjects(shouldDeleteJob) {
var total = 0, deleted = 0;
- var scrubKey = function( key ) {
- return redisClient.hget( config.redis.status_set_name, key
).then( function( jdjson ) {
- var job = jd.fromJson( jdjson );
- if ( shouldDeleteJob( job ) ) {
- deleted += 1;
- return redisClient.hdel(
config.redis.status_set_name, job.collectionId );
- }
- } );
+ var scrubKey = function( key, jdjson ) {
+ var job = jd.fromJson( jdjson );
+ if ( shouldDeleteJob( job ) ) {
+ deleted += 1;
+ return redisClient.hdel( config.redis.status_set_name,
job.collectionId );
+ }
};
var scrubKeyGuarded = Promise.guard( Promise.guard.n( 5 ), scrubKey );
var scanSome = function( cursor ) {
return redisClient.hscan( config.redis.status_set_name, cursor
).
- spread( function( cursor, keys ) {
- total += keys.length;
- return Promise.map(keys, scrubKeyGuarded).then(
function() {
+ spread( function( cursor, kvs ) {
+ var tasks = [], i;
+ for (i = 0; i < kvs.length; i += 2) {
+ var k = kvs[ i ], v = kvs[ i+1 ];
+ tasks.push( scrubKeyGuarded( k, v ) );
+ total++;
+ }
+ return Promise.all(tasks).then( function() {
if ( (+cursor) === 0) {
return; // done!
}
--
To view, visit https://gerrit.wikimedia.org/r/177241
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ie796ba8ff3fde8926aa719adc019efbc6c52f9a4
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/Collection/OfflineContentGenerator
Gerrit-Branch: master
Gerrit-Owner: Cscott <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
