Matthias Mullie has uploaded a new change for review.
https://gerrit.wikimedia.org/r/245501
Change subject: [WIP] Import Flow data dump
......................................................................
[WIP] Import Flow data dump
Meanwhile also changes some Models to allow empty values as well
for nullable columns.
Bug: T114703
Change-Id: Ieae618d4aaa4b21a4eb8fe8d1f0627ee8bd1ff8d
---
M autoload.php
A includes/Dump/Importer.php
M includes/Model/AbstractRevision.php
M includes/Model/PostRevision.php
A maintenance/importDump.php
5 files changed, 351 insertions(+), 5 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Flow
refs/changes/01/245501/1
diff --git a/autoload.php b/autoload.php
index 9094e3c..988dc93 100644
--- a/autoload.php
+++ b/autoload.php
@@ -114,6 +114,7 @@
'Flow\\Data\\Utils\\UserMerger' => __DIR__ .
'/includes/Data/Utils/UserMerger.php',
'Flow\\DbFactory' => __DIR__ . '/includes/DbFactory.php',
'Flow\\Dump\\Exporter' => __DIR__ . '/includes/Dump/Exporter.php',
+ 'Flow\\Dump\\Importer' => __DIR__ . '/includes/Dump/Importer.php',
'Flow\\Exception\\CatchableFatalErrorException' => __DIR__ .
'/includes/Exception/CatchableFatalErrorException.php',
'Flow\\Exception\\CrossWikiException' => __DIR__ .
'/includes/Exception/ExceptionHandling.php',
'Flow\\Exception\\DataModelException' => __DIR__ .
'/includes/Exception/ExceptionHandling.php',
diff --git a/includes/Dump/Importer.php b/includes/Dump/Importer.php
new file mode 100644
index 0000000..8c3d30e
--- /dev/null
+++ b/includes/Dump/Importer.php
@@ -0,0 +1,243 @@
+<?php
+
+namespace Flow\Dump;
+
+use Exception;
+use Flow\Container;
+use Flow\Data\ManagerGroup;
+use Flow\Model\AbstractRevision;
+use Flow\Model\Header;
+use Flow\Model\PostRevision;
+use Flow\Model\TopicListEntry;
+use Flow\Model\UUID;
+use Flow\Model\Workflow;
+use Flow\OccupationController;
+use MWException;
+use WikiImporter;
+use XMLReader;
+
+class Importer extends WikiImporter {
+ /**
+ * @var ManagerGroup|null
+ */
+ protected $storage;
+
+ /**
+ * @var array
+ */
+ protected $metadata = array();
+
+ /**
+ * Copied from parent::doImport, but with different handle* calls
+ * for the different nodes we deal with.
+ *
+ * @return bool
+ * @throws Exception
+ * @throws MWException
+ * @throws null
+ */
+ public function doImport() {
+ // Calls to reader->read need to be wrapped in calls to
+ // libxml_disable_entity_loader() to avoid local file
+ // inclusion attacks (bug 46932).
+ $oldDisable = libxml_disable_entity_loader( true );
+ $this->reader->read();
+
+ if ( $this->reader->localName != 'mediawiki' ) {
+ libxml_disable_entity_loader( $oldDisable );
+ throw new MWException( "Expected <mediawiki> tag, got "
.
+ $this->reader->localName );
+ }
+ $this->debug( "<mediawiki> tag is correct." );
+
+ $this->debug( "Starting primary dump processing loop." );
+
+ $keepReading = $this->reader->read();
+ $skip = false;
+ $rethrow = null;
+ try {
+ while ( $keepReading ) {
+ $tag = $this->reader->localName;
+ $type = $this->reader->nodeType;
+
+ if ( $tag == 'mediawiki' && $type ===
XMLReader::END_ELEMENT ) {
+ break;
+ } elseif ( $tag == 'board' && $type ===
XMLReader::ELEMENT ) {
+ // prevent memory from being filled up
+ /** @var ManagerGroup $storage */
+ $storage = Container::get( 'storage' );
+ $storage->clear();
+
+ $this->handleBoard();
+ } elseif ( $tag == 'description' && $type ===
XMLReader::ELEMENT ) {
+ $this->handleHeader();
+ } elseif ( $tag == 'topic' && $type ===
XMLReader::ELEMENT ) {
+ $this->handleTopic();
+ } elseif ( $tag == 'post' && $type ===
XMLReader::ELEMENT ) {
+ $this->handlePost();
+ } elseif ( $tag == 'summary' && $type ===
XMLReader::ELEMENT ) {
+ $this->handleSummary();
+ } elseif ( $tag != '#text' ) {
+ $this->warn( "Unhandled top-level XML
tag $tag" );
+
+ $skip = true;
+ }
+
+ if ( $skip ) {
+ $keepReading = $this->reader->next();
+ $skip = false;
+ $this->debug( "Skip" );
+ } else {
+ $keepReading = $this->reader->read();
+ }
+ }
+ } catch ( Exception $ex ) {
+ $rethrow = $ex;
+ }
+
+ // finally
+ libxml_disable_entity_loader( $oldDisable );
+ $this->reader->close();
+
+ if ( $rethrow ) {
+ throw $rethrow;
+ }
+
+ return true;
+ }
+
+ /**
+ * @param ManagerGroup $storage
+ */
+ public function setStorage( ManagerGroup $storage ) {
+ $this->storage = $storage;
+ }
+
+ /**
+ * @param object $object
+ */
+ protected function put( $object ) {
+ if ( $this->storage ) {
+ $this->storage->put( $object, $this->metadata );
+ }
+ }
+
+ protected function handleBoard() {
+ $this->debug( 'Enter board handler.' );
+
+ global $wgFlowDefaultWorkflow;
+
+ $uuid = UUID::create( $this->nodeAttribute( 'id' ) );
+ $title = \Title::newFromDBkey( $this->nodeAttribute( 'title' )
);
+
+ $workflow = Workflow::fromStorageRow( array(
+ 'workflow_id' => $uuid->getAlphadecimal(),
+ 'workflow_type' => $wgFlowDefaultWorkflow,
+ 'workflow_wiki' => wfWikiID(),
+ 'workflow_page_id' => $title->getArticleID(),
+ 'workflow_namespace' => $title->getNamespace(),
+ 'workflow_title_text' => $title->getDBkey(),
+ 'workflow_last_update_timestamp' =>
$uuid->getTimestamp( TS_MW ),
+ ) );
+
+ // create page if it does not yet exist
+ /** @var OccupationController $occupationController */
+ $occupationController = Container::get( 'occupation_controller'
);
+ $occupationController->allowCreation( $title,
$occupationController->getTalkpageManager() );
+ $occupationController->ensureFlowRevision( new \Article( $title
), $workflow );
+
+ $this->metadata = array();
+ $this->metadata['board-workflow'] = $workflow;
+
+ $this->put( $workflow );
+ }
+
+ protected function handleHeader() {
+ $this->debug( 'Enter description handler.' );
+
+ $revisions = $this->getRevisions( array( 'Flow\\Model\\Header',
'fromStorageRow' ) );
+ foreach ( $revisions as $revision ) {
+ $this->put( $revision );
+ }
+ }
+
+ protected function handleTopic() {
+ $this->debug( 'Enter topic handler.' );
+
+ $uuid = UUID::create( $this->nodeAttribute( 'id' ) );
+ /** @var Workflow $boardWorkflow */
+ $boardWorkflow = $this->metadata['board-workflow'];
+ $title = $boardWorkflow->getArticleTitle();
+
+ $topicWorkflow = Workflow::fromStorageRow( array(
+ 'workflow_id' => $uuid->getAlphadecimal(),
+ 'workflow_type' => 'topic',
+ 'workflow_wiki' => wfWikiID(),
+ 'workflow_page_id' => $title->getArticleID(),
+ 'workflow_namespace' => $title->getNamespace(),
+ 'workflow_title_text' => $title->getDBkey(),
+ 'workflow_last_update_timestamp' =>
$uuid->getTimestamp( TS_MW ),
+ ) );
+ $topicListEntry = TopicListEntry::create( $boardWorkflow,
$topicWorkflow );
+
+ $this->metadata['workflow'] = $topicWorkflow;
+
+ $this->put( $topicWorkflow );
+ $this->put( $topicListEntry );
+ }
+
+ protected function handlePost() {
+ $this->debug( 'Enter post handler.' );
+
+ $revisions = $this->getRevisions( array(
'Flow\\Model\\PostRevision', 'fromStorageRow' ) );
+ foreach ( $revisions as $revision ) {
+ $this->put( $revision );
+ }
+ }
+
+ protected function handleSummary() {
+ $this->debug( 'Enter summary handler.' );
+
+ $revisions = $this->getRevisions( array(
'Flow\\Model\\PostSummary', 'fromStorageRow' ) );
+ foreach ( $revisions as $revision ) {
+ $this->put( $revision );
+ }
+ }
+
+ /**
+ * @param callable $callback The relevant fromStorageRow callback
+ * @return AbstractRevision[]
+ */
+ protected function getRevisions( $callback ) {
+ $revisions = array();
+
+ // keep processing <revision> nodes until </revisions>
+ while ( $this->reader->localName !== 'revisions' ||
$this->reader->nodeType !== XMLReader::END_ELEMENT ) {
+ if ( $this->reader->localName === 'revision' ) {
+ $revisions[] = $this->getRevision( $callback );
+ }
+ $this->reader->read();
+ }
+
+ return $revisions;
+ }
+
+ /**
+ * @param callable $callback The relevant fromStorageRow callback
+ * @return AbstractRevision
+ */
+ protected function getRevision( $callback ) {
+ $this->debug( 'Enter revision handler.' );
+
+ $data = array();
+
+ $this->reader->moveToFirstAttribute();
+ do {
+ $data[$this->reader->name] = $this->reader->value;
+ } while ( $this->reader->moveToNextAttribute() );
+
+ $data['rev_content'] = $this->nodeContents();
+
+ return call_user_func( $callback, $data );
+ }
+}
diff --git a/includes/Model/AbstractRevision.php
b/includes/Model/AbstractRevision.php
index fdb7ea2..6583a14 100644
--- a/includes/Model/AbstractRevision.php
+++ b/includes/Model/AbstractRevision.php
@@ -164,7 +164,7 @@
if ( $obj->user === null ) {
throw new DataModelException( 'Could not load UserTuple
for rev_user_' );
}
- $obj->prevRevision = UUID::create( $row['rev_parent_id'] );
+ $obj->prevRevision = $row['rev_parent_id'] ? UUID::create(
$row['rev_parent_id'] ) : null;
$obj->changeType = $row['rev_change_type'];
$obj->flags = array_filter( explode( ',', $row['rev_flags'] ) );
$obj->content = $row['rev_content'];
@@ -174,8 +174,8 @@
$obj->moderationState = $row['rev_mod_state'];
$obj->moderatedBy = UserTuple::newFromArray( $row,
'rev_mod_user_' );
- $obj->moderationTimestamp = $row['rev_mod_timestamp'];
- $obj->moderatedReason = isset( $row['rev_mod_reason'] ) ?
$row['rev_mod_reason'] : null;
+ $obj->moderationTimestamp = $row['rev_mod_timestamp'] ?: null;
+ $obj->moderatedReason = isset( $row['rev_mod_reason'] ) &&
$row['rev_mod_reason'] ? $row['rev_mod_reason'] : null;
// BC: 'suppress' used to be called 'censor' & 'lock' was
'close'
$bc = array(
@@ -185,7 +185,7 @@
$obj->moderationState = str_replace( array_keys( $bc ),
array_values( $bc ), $obj->moderationState );
// isset required because there is a possible db migration,
cached data will not have it
- $obj->lastEditId = isset( $row['rev_last_edit_id'] ) ?
UUID::create( $row['rev_last_edit_id'] ) : null;
+ $obj->lastEditId = isset( $row['rev_last_edit_id'] ) &&
$row['rev_last_edit_id'] ? UUID::create( $row['rev_last_edit_id'] ) : null;
$obj->lastEditUser = UserTuple::newFromArray( $row,
'rev_edit_user_' );
$obj->contentLength = isset( $row['rev_content_length'] ) ?
$row['rev_content_length'] : 0;
diff --git a/includes/Model/PostRevision.php b/includes/Model/PostRevision.php
index cac5c9c..f078192 100644
--- a/includes/Model/PostRevision.php
+++ b/includes/Model/PostRevision.php
@@ -132,7 +132,7 @@
'process-data'
);
}
- $obj->replyToId = UUID::create( $row['tree_parent_id'] );
+ $obj->replyToId = $row['tree_parent_id'] ? UUID::create(
$row['tree_parent_id'] ) : null;
$obj->postId = UUID::create( $row['rev_type_id'] );
$obj->origUser = UserTuple::newFromArray( $row,
'tree_orig_user_' );
if ( !$obj->origUser ) {
diff --git a/maintenance/importDump.php b/maintenance/importDump.php
new file mode 100644
index 0000000..831d65c
--- /dev/null
+++ b/maintenance/importDump.php
@@ -0,0 +1,102 @@
+<?php
+
+use Flow\Container;
+use Flow\Dump\Importer;
+
+require_once ( getenv( 'MW_INSTALL_PATH' ) !== false
+ ? getenv( 'MW_INSTALL_PATH' ) . '/maintenance/Maintenance.php'
+ : dirname( __FILE__ ) . '/../../../maintenance/Maintenance.php' );
+
+/**
+ * FlowBackupReader is mostly copied from core's importDump.php.
+ * importFromHandle will call a different Importer, but other than that,
+ * this class is mostly the same - it just has some options stripped.
+ * I just couldn't extend the original BackupReader class: including that
+ * file automatically launches the script.
+ */
+class FlowBackupReader extends Maintenance {
+ protected $dryRun = false;
+
+ public function __construct() {
+ parent::__construct();
+ $gz = in_array( 'compress.zlib', stream_get_wrappers() )
+ ? 'ok'
+ : '(disabled; requires PHP zlib module)';
+ $bz2 = in_array( 'compress.bzip2', stream_get_wrappers() )
+ ? 'ok'
+ : '(disabled; requires PHP bzip2 module)';
+
+ $this->mDescription = <<<TEXT
+This script reads pages from an XML file as produced from Flow's
+dumpBackup.php, and saves them into the current wiki.
+
+Compressed XML files may be read directly:
+ .gz $gz
+ .bz2 $bz2
+ .7z (if 7za executable is in PATH)
+
+Note that for very large data sets, importDump.php may be slow.
+TEXT;
+ $this->stderr = fopen( 'php://stderr', 'wt' );
+ $this->addOption( 'dry-run', 'Parse dump without actually
importing pages' );
+ $this->addOption( 'debug', 'Output extra verbose debug
information' );
+ $this->addArg( 'file', 'Dump file to import [else use stdin]',
false );
+ }
+
+ public function execute() {
+ if ( wfReadOnly() ) {
+ $this->error( "Wiki is in read-only mode; you'll need
to disable it for import to work.", true );
+ }
+
+ $this->dryRun = $this->hasOption( 'dry-run' );
+
+ if ( $this->hasArg() ) {
+ $this->importFromFile( $this->getArg() );
+ } else {
+ $this->importFromStdin();
+ }
+
+ $this->output( "Done!\n" );
+ }
+
+ protected function importFromFile( $filename ) {
+ if ( preg_match( '/\.gz$/', $filename ) ) {
+ $filename = 'compress.zlib://' . $filename;
+ } elseif ( preg_match( '/\.bz2$/', $filename ) ) {
+ $filename = 'compress.bzip2://' . $filename;
+ } elseif ( preg_match( '/\.7z$/', $filename ) ) {
+ $filename = 'mediawiki.compress.7z://' . $filename;
+ }
+
+ $file = fopen( $filename, 'rt' );
+
+ return $this->importFromHandle( $file );
+ }
+
+ protected function importFromStdin() {
+ $file = fopen( 'php://stdin', 'rt' );
+ if ( self::posix_isatty( $file ) ) {
+ $this->maybeHelp( true );
+ }
+
+ return $this->importFromHandle( $file );
+ }
+
+ protected function importFromHandle( $handle ) {
+ $source = new ImportStreamSource( $handle );
+ $importer = new Importer( $source, $this->getConfig() );
+
+ if ( $this->hasOption( 'debug' ) ) {
+ $importer->setDebug( true );
+ }
+
+ if ( !$this->dryRun ) {
+ $importer->setStorage( Container::get( 'storage' ) );
+ }
+
+ return $importer->doImport();
+ }
+}
+
+$maintClass = 'FlowBackupReader';
+require_once RUN_MAINTENANCE_IF_MAIN;
--
To view, visit https://gerrit.wikimedia.org/r/245501
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ieae618d4aaa4b21a4eb8fe8d1f0627ee8bd1ff8d
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/Flow
Gerrit-Branch: master
Gerrit-Owner: Matthias Mullie <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
