Ejegg has uploaded a new change for review.

  https://gerrit.wikimedia.org/r/315754

Change subject: Update PopulateQueueFromDump for PHPQueue
......................................................................

Update PopulateQueueFromDump for PHPQueue

This can now import old 'raw' dumps and dumps with just one json
message per line. TODO: drop old file support after migrating
misplaced messages, support an actual json array of messages.

Bug: T147982
Change-Id: I9e30c7aa58ccedcb50e17fbfea091e68814752ee
---
M Maintenance/PopulateQueueFromDump.php
1 file changed, 26 insertions(+), 13 deletions(-)


  git pull ssh://gerrit.wikimedia.org:29418/wikimedia/fundraising/SmashPig 
refs/changes/54/315754/1

diff --git a/Maintenance/PopulateQueueFromDump.php 
b/Maintenance/PopulateQueueFromDump.php
index 4e78f96..4dd8c7a 100644
--- a/Maintenance/PopulateQueueFromDump.php
+++ b/Maintenance/PopulateQueueFromDump.php
@@ -3,12 +3,14 @@
 require ( 'MaintenanceBase.php' );
 
 use SmashPig\Core\Logging\Logger;
-use SmashPig\Core\DataStores\StompDataStore;
+use SmashPig\Core\QueueConsumers\BaseQueueConsumer;
 
 $maintClass = '\SmashPig\Maintenance\PopulateQueueFromDump';
 
 /**
- * Script to import a file created by EmptyQueueToDump back into a backing 
STOMP queue.
+ * Script to import a file created by EmptyQueueToDump back into a PHPQueue 
backend.
+ * Handles both 'raw' dumps from old Stomp queues and normal dumps with one 
JSON-
+ * encoded message body per line.
  *
  * @package SmashPig\Maintenance
  */
@@ -27,7 +29,9 @@
         * Do the actual work of the script.
         */
        public function execute() {
-               $this->datastore = new StompDataStore( $this->getOption( 
'queue' ) );
+               $this->datastore = BaseQueueConsumer::getQueue(
+                       $this->getOption( 'queue' )
+               );
 
                $startTime = time();
                $messageCount = 0;
@@ -41,19 +45,17 @@
 
                // Do the loop!
                while ( ( $line = fgets( $f ) ) !== false ) {
-                       $parts = explode( '=', $line, 2 );
-                       if ( count( $parts ) !== 2 ) {
-                               $this->error( "Improperly formatted line: 
$line" );
+                       if ( substr( $line, 0, 4 ) === 'raw=' ) {
+                               $message = $this->decodeLegacyMessage( $line );
+                       } else {
+                               $message = json_decode( $line, true );
+                       }
+                       if ( $message === null ) {
+                               Logger::error( "Invalid line: $line" );
                                continue;
                        }
 
-                       $obj = json_decode( $parts[ 1 ], true );
-                       if ( !array_key_exists( 'headers', $obj ) || 
!array_key_exists( 'body', $obj ) ) {
-                               $this->error( "Decoded line does not have 
headers and body elements: $line" );
-                               continue;
-                       }
-
-                       $this->datastore->addObjectRaw( $obj[ 'body' ], $obj[ 
'headers' ] );
+                       $this->datastore->push( $message );
 
                        $messageCount++;
                        if ( $messageCount % 1000 == 0 ) {
@@ -68,6 +70,17 @@
                );
        }
 
+       protected function decodeLegacyMessage( $line ) {
+               $parts = explode( '=', $line, 2 );
+
+               $obj = json_decode( $parts[ 1 ], true );
+               if ( !array_key_exists( 'headers', $obj ) || !array_key_exists( 
'body', $obj ) ) {
+                       return null;
+               }
+               // JSON-fried JSON
+               return json_decode( $obj['body'], true );
+       }
+
 }
 
 require ( RUN_MAINTENANCE_IF_MAIN );

-- 
To view, visit https://gerrit.wikimedia.org/r/315754
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I9e30c7aa58ccedcb50e17fbfea091e68814752ee
Gerrit-PatchSet: 1
Gerrit-Project: wikimedia/fundraising/SmashPig
Gerrit-Branch: master
Gerrit-Owner: Ejegg <eeggles...@wikimedia.org>

_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits

Reply via email to