jenkins-bot has submitted this change and it was merged.
Change subject: Update PopulateQueueFromDump for PHPQueue
......................................................................
Update PopulateQueueFromDump for PHPQueue
This can now import old 'raw' dumps and dumps with just one json
message per line. TODO: drop old file support after migrating
misplaced messages, support an actual json array of messages.
Bug: T147982
Change-Id: I9e30c7aa58ccedcb50e17fbfea091e68814752ee
---
M Maintenance/PopulateQueueFromDump.php
1 file changed, 26 insertions(+), 13 deletions(-)
Approvals:
XenoRyet: Looks good to me, approved
jenkins-bot: Verified
diff --git a/Maintenance/PopulateQueueFromDump.php
b/Maintenance/PopulateQueueFromDump.php
index 4e78f96..4dd8c7a 100644
--- a/Maintenance/PopulateQueueFromDump.php
+++ b/Maintenance/PopulateQueueFromDump.php
@@ -3,12 +3,14 @@
require ( 'MaintenanceBase.php' );
use SmashPig\Core\Logging\Logger;
-use SmashPig\Core\DataStores\StompDataStore;
+use SmashPig\Core\QueueConsumers\BaseQueueConsumer;
$maintClass = '\SmashPig\Maintenance\PopulateQueueFromDump';
/**
- * Script to import a file created by EmptyQueueToDump back into a backing
STOMP queue.
+ * Script to import a file created by EmptyQueueToDump back into a PHPQueue
backend.
+ * Handles both 'raw' dumps from old Stomp queues and normal dumps with one
JSON-
+ * encoded message body per line.
*
* @package SmashPig\Maintenance
*/
@@ -27,7 +29,9 @@
* Do the actual work of the script.
*/
public function execute() {
- $this->datastore = new StompDataStore( $this->getOption(
'queue' ) );
+ $this->datastore = BaseQueueConsumer::getQueue(
+ $this->getOption( 'queue' )
+ );
$startTime = time();
$messageCount = 0;
@@ -41,19 +45,17 @@
// Do the loop!
while ( ( $line = fgets( $f ) ) !== false ) {
- $parts = explode( '=', $line, 2 );
- if ( count( $parts ) !== 2 ) {
- $this->error( "Improperly formatted line:
$line" );
+ if ( substr( $line, 0, 4 ) === 'raw=' ) {
+ $message = $this->decodeLegacyMessage( $line );
+ } else {
+ $message = json_decode( $line, true );
+ }
+ if ( $message === null ) {
+ Logger::error( "Invalid line: $line" );
continue;
}
- $obj = json_decode( $parts[ 1 ], true );
- if ( !array_key_exists( 'headers', $obj ) ||
!array_key_exists( 'body', $obj ) ) {
- $this->error( "Decoded line does not have
headers and body elements: $line" );
- continue;
- }
-
- $this->datastore->addObjectRaw( $obj[ 'body' ], $obj[
'headers' ] );
+ $this->datastore->push( $message );
$messageCount++;
if ( $messageCount % 1000 == 0 ) {
@@ -68,6 +70,17 @@
);
}
+ protected function decodeLegacyMessage( $line ) {
+ $parts = explode( '=', $line, 2 );
+
+ $obj = json_decode( $parts[ 1 ], true );
+ if ( !array_key_exists( 'headers', $obj ) || !array_key_exists(
'body', $obj ) ) {
+ return null;
+ }
+ // JSON-fried JSON
+ return json_decode( $obj['body'], true );
+ }
+
}
require ( RUN_MAINTENANCE_IF_MAIN );
--
To view, visit https://gerrit.wikimedia.org/r/315754
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I9e30c7aa58ccedcb50e17fbfea091e68814752ee
Gerrit-PatchSet: 1
Gerrit-Project: wikimedia/fundraising/SmashPig
Gerrit-Branch: master
Gerrit-Owner: Ejegg <eeggles...@wikimedia.org>
Gerrit-Reviewer: Awight <awi...@wikimedia.org>
Gerrit-Reviewer: Cdentinger <cdentin...@wikimedia.org>
Gerrit-Reviewer: XenoRyet <dkozlow...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
