Aaron Schulz has uploaded a new change for review.
https://gerrit.wikimedia.org/r/61743
Change subject: [JobQueue] Uses gzip on larger jobs in JobQueueRedis.
......................................................................
[JobQueue] Uses gzip on larger jobs in JobQueueRedis.
Change-Id: I74ebdcfa0d3f2bcdc429394f7873fa25b7eb0f58
---
M includes/job/JobQueueRedis.php
1 file changed, 34 insertions(+), 3 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core
refs/changes/43/61743/1
diff --git a/includes/job/JobQueueRedis.php b/includes/job/JobQueueRedis.php
index 249ba27..dd12a75 100644
--- a/includes/job/JobQueueRedis.php
+++ b/includes/job/JobQueueRedis.php
@@ -238,7 +238,7 @@
$args[] = (string)$item['uuid'];
$args[] = (string)$item['sha1'];
$args[] = (string)$item['rtimestamp'];
- $args[] = (string)serialize( $item );
+ $args[] = (string)$this->serialize( $item );
}
static $script =
<<<LUA
@@ -310,7 +310,7 @@
}
JobQueue::incrStats( 'job-pop', $this->type );
- $item = unserialize( $blob );
+ $item = $this->unserialize( $blob );
if ( $item === false ) {
wfDebugLog( 'JobQueueRedis', "Could not
unserialize {$this->type} job." );
continue;
@@ -532,7 +532,7 @@
*/
public function getJobFromUidInternal( $uid, RedisConnRef $conn ) {
try {
- $item = unserialize( $conn->hGet( $this->getQueueKey(
'h-data' ), $uid ) );
+ $item = $this->unserialize( $conn->hGet(
$this->getQueueKey( 'h-data' ), $uid ) );
if ( !is_array( $item ) ) { // this shouldn't happen
throw new MWException( "Could not find job with
ID '$uid'." );
}
@@ -716,6 +716,37 @@
}
/**
+ * @param array $fields
+ * @return string Serialized and possibly compressed version of $fields
+ */
+ protected function serialize( array $fields ) {
+ $blob = serialize( $fields );
+ if ( strlen( $blob ) >= 1024 && function_exists( 'gzdeflate' )
) {
+ $object = (object)array( 'blob' => gzdeflate( $blob ),
'enc' => 'gzip' );
+ $blobz = serialize( $object );
+ return ( strlen( $blobz ) < strlen( $blob ) ) ? $blobz
: $blob;
+ } else {
+ return $blob;
+ }
+ }
+
+ /**
+ * @param string $blob
+ * @return array|bool Unserialized version of $blob or false
+ */
+ protected function unserialize( $blob ) {
+ $fields = unserialize( $blob );
+ if ( is_object( $fields ) ) {
+ if ( $fields->enc === 'gzip' && function_exists(
'gzinflate' ) ) {
+ $fields = unserialize( gzinflate( $fields->blob
) );
+ } else {
+ $fields = false;
+ }
+ }
+ return is_array( $fields ) ? $fields : false;
+ }
+
+ /**
* Get a connection to the server that handles all sub-queues for this
queue
*
* @return Array (server name, Redis instance)
--
To view, visit https://gerrit.wikimedia.org/r/61743
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I74ebdcfa0d3f2bcdc429394f7873fa25b7eb0f58
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/core
Gerrit-Branch: master
Gerrit-Owner: Aaron Schulz <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
