Daniel Kinzler has uploaded a new change for review.
https://gerrit.wikimedia.org/r/71336
Change subject: (bug 49742) Rebuild script for property info table.
......................................................................
(bug 49742) Rebuild script for property info table.
Change-Id: I131537e34bee71397918956e2e6652b9de739d75
---
M repo/Wikibase.classes.php
A repo/includes/store/sql/PropertyInfoTableBuilder.php
A repo/maintenance/rebuildPropertyInfo.php
A repo/tests/phpunit/includes/store/sql/PropertyInfoTableBuilderTest.php
4 files changed, 545 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Wikibase
refs/changes/36/71336/1
diff --git a/repo/Wikibase.classes.php b/repo/Wikibase.classes.php
index 63f62e1..9a3706c 100644
--- a/repo/Wikibase.classes.php
+++ b/repo/Wikibase.classes.php
@@ -143,6 +143,7 @@
'Wikibase\EntityPerPageTable' =>
'includes/store/sql/EntityPerPageTable.php',
'Wikibase\DispatchStats' =>
'includes/store/sql/DispatchStats.php',
'Wikibase\TermSearchKeyBuilder' =>
'includes/store/sql/TermSearchKeyBuilder.php',
+ 'Wikibase\PropertyInfoTableBuilder' =>
'includes/store/sql/PropertyInfoTableBuilder.php',
// includes/updates
'Wikibase\EntityDeletionUpdate' =>
'includes/updates/EntityDeletionUpdate.php',
@@ -158,6 +159,7 @@
// maintenance
'Wikibase\RebuildTermsSearchKey' =>
'maintenance/rebuildTermsSearchKey.php',
'Wikibase\RebuildEntityPerPage' =>
'maintenance/rebuildEntityPerPage.php',
+ 'Wikibase\RebuildPropertyInfo' =>
'maintenance/rebuildPropertyInfo.php',
// tests
'Wikibase\Test\TestItemContents' =>
'tests/phpunit/TestItemContents.php',
diff --git a/repo/includes/store/sql/PropertyInfoTableBuilder.php
b/repo/includes/store/sql/PropertyInfoTableBuilder.php
new file mode 100644
index 0000000..0602a74
--- /dev/null
+++ b/repo/includes/store/sql/PropertyInfoTableBuilder.php
@@ -0,0 +1,286 @@
+<?php
+
+namespace Wikibase;
+use DatabaseBase;
+use MessageReporter;
+
+/**
+ * Utility class for rebuilding the wb_property_info table.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @since 0.4
+ *
+ * @file
+ * @ingroup WikibaseRepo
+ *
+ * @licence GNU GPL v2+
+ * @author Daniel Kinzler
+ */
+class PropertyInfoTableBuilder {
+
+ /**
+ * @since 0.4
+ *
+ * @var PropertyInfoTable $table
+ */
+ protected $table;
+
+ /**
+ * @var EntityLookup
+ */
+ protected $entityLookup;
+
+ /**
+ * @since 0.4
+ *
+ * @var MessageReporter $reporter
+ */
+ protected $reporter;
+
+ /**
+ * Whether all keys should be updated, or only missing keys
+ *
+ * @var bool
+ */
+ protected $all = true;
+
+ /**
+ * Whether all keys should be updated, or only missing keys
+ *
+ * @var bool
+ */
+ protected $fromId = 1;
+
+ /**
+ * The batch size, giving the number of rows to be updated in each
database transaction.
+ *
+ * @var int
+ */
+ protected $batchSize = 100;
+
+ /**
+ * Constructor.
+ *
+ * @since 0.4
+ *
+ * @param PropertyInfoTable $table
+ */
+ public function __construct( PropertyInfoTable $table, EntityLookup
$entityLookup ) {
+ $this->table = $table;
+ $this->entityLookup = $entityLookup;
+ }
+
+ /**
+ * @return boolean
+ */
+ public function getRebuildAll() {
+ return $this->all;
+ }
+
+ /**
+ * @return int
+ */
+ public function getBatchSize() {
+ return $this->batchSize;
+ }
+
+ /**
+ * @return boolean
+ */
+ public function getFromId() {
+ return $this->fromId;
+ }
+
+ /**
+ * @param boolean $all
+ */
+ public function setRebuildAll( $all ) {
+ $this->all = $all;
+ }
+
+ /**
+ * @param int $batchSize
+ */
+ public function setBatchSize( $batchSize ) {
+ $this->batchSize = $batchSize;
+ }
+
+ /**
+ * @param boolean $fromId
+ */
+ public function setFromId( $fromId ) {
+ $this->fromId = $fromId;
+ }
+
+ /**
+ * Sets the reporter to use for reporting progress.
+ *
+ * @param \MessageReporter $reporter
+ */
+ public function setReporter( \MessageReporter $reporter ) {
+ $this->reporter = $reporter;
+ }
+
+ /**
+ * Rebuild the property info entries.
+ * Use the rebuildPropertyInfo.php maintenance script to invoke this
from the command line.
+ *
+ * Database updates a batched into multiple transactions. Do not call
this
+ * method within an (explicit) database transaction.
+ *
+ * @since 0.4
+ */
+ public function rebuildPropertyInfo() {
+ $dbw = $this->table->getWriteConnection();
+
+ $rowId = $this->fromId -1;
+
+ $total = 0;
+
+ $join = array();
+ $tables = array( 'wb_entity_per_page' );
+
+ if ( !$this->all ) {
+ // Find properties in wb_entity_per_page with no
corresponding
+ // entry in wb_property_info.
+
+ $piTable = $this->table->getTableName();
+
+ $tables[] = $piTable;
+ $join[$piTable] = array( 'LEFT JOIN',
+ array(
+ 'pi_property_id = epp_entity_id',
+ )
+ );
+ }
+
+ while ( true ) {
+ // Make sure we are not running too far ahead of the
slaves,
+ // as that would cause the site to be rendered read
only.
+ $this->waitForSlaves( $dbw );
+
+ $dbw->begin();
+
+ $props = $dbw->select(
+ $tables,
+ array(
+ 'epp_entity_id',
+ ),
+ array(
+ 'epp_entity_type = ' . $dbw->addQuotes(
Property::ENTITY_TYPE ),
+ 'epp_entity_id > ' . (int) $rowId,
+ $this->all ? '1' : 'pi_property_id IS
NULL', // if not $all, only add missing entries
+ ),
+ __METHOD__,
+ array(
+ 'LIMIT' => $this->batchSize,
+ // XXX: We currently have a unique key
defined as `wb_epp_entity` (`epp_entity_id`,`epp_entity_type`).
+ // This SHOULD be the other way
around: `wb_epp_entity` (`epp_entity_type`, `epp_entity_id`).
+ // Once this is fixed, the below
should probable be changed to:
+ // 'ORDER BY' => 'epp_entity_type
ASC, epp_entity_id ASC'
+ 'ORDER BY' => 'epp_entity_id ASC',
+ 'FOR UPDATE'
+ ),
+ $join
+ );
+
+ $c = 0;
+
+ foreach ( $props as $row ) {
+ $id = new EntityId( Property::ENTITY_TYPE,
(int)$row->epp_entity_id );
+ $this->updatePropertyInfo( $dbw, $id );
+
+ $rowId = $row->epp_entity_id;
+ $c+= 1;
+ }
+
+ $dbw->commit();
+
+ $this->report( "Updated $c properties, up to ID
$rowId." );
+ $total += $c;
+
+ if ( $c < $this->batchSize ) {
+ // we are done.
+ break;
+ }
+ }
+
+ return $total;
+ }
+
+ /**
+ * Wait for slaves (quietly)
+ *
+ * @todo: this should be in the Database class.
+ * @todo: thresholds should be configurable
+ *
+ * @author Tim Starling (stolen from recompressTracked.php)
+ */
+ protected function waitForSlaves() {
+ $lb = wfGetLB(); //TODO: allow foreign DB, get from $this->table
+
+ while ( true ) {
+ list( $host, $maxLag ) = $lb->getMaxLag();
+ if ( $maxLag < 2 ) {
+ break;
+ }
+
+ $this->report( "Slaves are lagged by $maxLag seconds,
sleeping..." );
+ sleep( 5 );
+ $this->report( "Resuming..." );
+ }
+ }
+
+ /**
+ * Updates the property info entry for the given property.
+ * The property is loaded in full using the EntityLookup
+ * provide to the constructor.
+ *
+ * @see Wikibase\PropertyInfoUpdate
+ *
+ * @since 0.4
+ *
+ * @param \DatabaseBase $dbw the database connection to use
+ * @param EntityId $id the Property to process
+ */
+ protected function updatePropertyInfo( \DatabaseBase $dbw, EntityId $id
) {
+ if ( $id->getEntityType() !== Property::ENTITY_TYPE ) {
+ throw new \InvalidArgumentException( 'Property ID
expected! ' . $id );
+ }
+
+ $property = $this->entityLookup->getEntity( $id );
+
+ assert( $property instanceof Property );
+
+ $update = new PropertyInfoUpdate( $property, $this->table );
+ $update->doUpdate();
+ }
+
+ /**
+ * reports a message
+ *
+ * @since 0.4
+ *
+ * @param $msg
+ */
+ protected function report( $msg ) {
+ if ( $this->reporter ) {
+ $this->reporter->reportMessage( $msg );
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/repo/maintenance/rebuildPropertyInfo.php
b/repo/maintenance/rebuildPropertyInfo.php
new file mode 100644
index 0000000..47903cd
--- /dev/null
+++ b/repo/maintenance/rebuildPropertyInfo.php
@@ -0,0 +1,104 @@
+<?php
+
+namespace Wikibase;
+use LoggedUpdateMaintenance;
+
+$basePath = getenv( 'MW_INSTALL_PATH' ) !== false ? getenv( 'MW_INSTALL_PATH'
) : __DIR__ . '/../../../..';
+
+require_once $basePath . '/maintenance/Maintenance.php';
+
+/**
+ * Maintenance script for rebuilding the property info table.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @since 0.2
+ *
+ * @file
+ * @ingroup WikibaseRepo
+ *
+ * @licence GNU GPL v2+
+ * @author Daniel Kinzler
+ */
+class RebuildPropertyInfo extends LoggedUpdateMaintenance {
+
+ public function __construct() {
+ parent::__construct();
+
+ $this->mDescription = 'Rebuild the property info table.';
+
+ $this->addOption( 'only-missing', "Update only missing keys
(per default, all entries are updated)" );
+ $this->addOption( 'start-row', "The ID of the first row to
update (useful for continuing aborted runs)", false, true );
+ $this->addOption( 'batch-size', "Number of rows to update per
database transaction (100 per default)", false, true );
+ }
+
+ /**
+ * @see LoggedUpdateMaintenance::doDBUpdates
+ *
+ * @return boolean
+ */
+ public function doDBUpdates() {
+ if ( !defined( 'WB_VERSION' ) ) {
+ $this->output( "You need to have Wikibase enabled in
order to use this maintenance script!\n\n" );
+ exit;
+ }
+
+ $reporter = new \ObservableMessageReporter();
+ $reporter->registerReporterCallback(
+ array( $this, 'report' )
+ );
+
+ $table = new PropertyInfoTable( false );
+ $entityLookup = new WikiPageEntityLookup( false );
+
+ $builder = new PropertyInfoTableBuilder( $table, $entityLookup
);
+ $builder->setReporter( $reporter );
+
+ $builder->setBatchSize( intval( $this->getOption( 'batch-size',
100 ) ) );
+ $builder->setRebuildAll( !$this->getOption( 'only-missing',
false ) );
+ $builder->setFromId( intval( $this->getOption( 'start-row', 1 )
) );
+
+ $n = $builder->rebuildPropertyInfo();
+
+ $this->output( "Done. Updated $n property info entries.\n" );
+
+ return true;
+ }
+
+ /**
+ * @see LoggedUpdateMaintenance::getUpdateKey
+ *
+ * @return string
+ */
+ public function getUpdateKey() {
+ return 'Wikibase\RebuildPropertyInfo';
+ }
+
+ /**
+ * Outputs a message vis the output() method.
+ *
+ * @since 0.4
+ *
+ * @param $msg
+ */
+ public function report( $msg ) {
+ $this->output( "$msg\n" );
+ }
+
+}
+
+$maintClass = 'Wikibase\RebuildPropertyInfo';
+require_once( RUN_MAINTENANCE_IF_MAIN );
diff --git
a/repo/tests/phpunit/includes/store/sql/PropertyInfoTableBuilderTest.php
b/repo/tests/phpunit/includes/store/sql/PropertyInfoTableBuilderTest.php
new file mode 100644
index 0000000..fbc12d3
--- /dev/null
+++ b/repo/tests/phpunit/includes/store/sql/PropertyInfoTableBuilderTest.php
@@ -0,0 +1,153 @@
+<?php
+ /**
+ *
+ * Copyright © 01.07.13 by the authors listed below.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @license GPL 2+
+ * @file
+ *
+ *
+ * @ingroup WikibaseRepoTest
+ * @ingroup Test
+ *
+ * @group Wikibase
+ * @group WikibaseStore
+ * @group WikibasePropertyInfo
+ * @group Database
+ *
+ * @author Daniel Kinzler
+ */
+
+
+namespace Wikibase\Test;
+
+
+use RuntimeException;
+use Wikibase\EntityId;
+use Wikibase\Property;
+use Wikibase\PropertyContent;
+use Wikibase\PropertyInfoStore;
+use Wikibase\PropertyInfoTable;
+use Wikibase\PropertyInfoTableBuilder;
+use Wikibase\WikiPageEntityLookup;
+
+/**
+ * Class PropertyInfoTableBuilderTest
+ *
+ * @covers PropertyInfoTableBuilder
+ *
+ * @package Wikibase\Test
+ */
+class PropertyInfoTableBuilderTest extends \MediaWikiTestCase {
+
+ protected static function initProperties() {
+ static $properties = null;
+
+ if ( $properties === null ) {
+ $infos = array(
+ array( PropertyInfoStore::KEY_DATA_TYPE =>
'string', 'test' => 'one' ),
+ array( PropertyInfoStore::KEY_DATA_TYPE =>
'string', 'test' => 'two' ),
+ array( PropertyInfoStore::KEY_DATA_TYPE =>
'time', 'test' => 'three' ),
+ array( PropertyInfoStore::KEY_DATA_TYPE =>
'time', 'test' => 'four' ),
+ array( PropertyInfoStore::KEY_DATA_TYPE =>
'string', 'test' => 'five' ),
+ );
+
+ foreach ( $infos as $info ) {
+ $dataType = $info[
PropertyInfoStore::KEY_DATA_TYPE ];
+ $label = $info[ 'test' ];
+
+ $content = PropertyContent::newEmpty();
+ $content->getProperty()->setDataTypeId(
$dataType );
+ $content->getProperty()->setDescription( 'en',
$label );
+
+ $status = $content->save( "test", null,
EDIT_NEW );
+
+ if ( !$status->isOK() ) {
+ throw new RuntimeException( "could not
save property: " . $status->getWikiText() );
+ }
+
+ $id =
$content->getProperty()->getId()->getNumericId();
+ $properties[$id] = $info;
+ }
+ }
+
+ return $properties;
+ }
+
+ public function testRebuildPropertyInfo() {
+ $properties = self::initProperties();
+ $propertyIds = array_keys( $properties );
+
+ $entityLookup = new WikiPageEntityLookup( false, false );
+ $table = new PropertyInfoTable( false );
+ $builder = new PropertyInfoTableBuilder( $table, $entityLookup
);
+ $builder->setBatchSize( 3 );
+
+ // rebuild all ----
+ $builder->setFromId( 0 );
+ $builder->setRebuildAll( true );
+
+ $builder->rebuildPropertyInfo();
+
+ foreach ( $properties as $id => $expected ) {
+ $info = $table->getPropertyInfo( new EntityId(
Property::ENTITY_TYPE, $id ) );
+ $this->assertEquals(
$expected[PropertyInfoStore::KEY_DATA_TYPE],
$info[PropertyInfoStore::KEY_DATA_TYPE], "Property $id" );
+ }
+
+ // make table incomplete ----
+ $propId1 = new EntityId( Property::ENTITY_TYPE, $propertyIds[0]
);
+ $table->removePropertyInfo( $propId1 );
+
+ // rebuild from offset, with no effect ----
+ $builder->setFromId( $propId1->getNumericId() +1 );
+ $builder->setRebuildAll( false );
+
+ $builder->rebuildPropertyInfo();
+
+ $info = $table->getPropertyInfo( $propId1 );
+ $this->assertNull( $info, "rebuild missing from offset should
have skipped this" );
+
+ // rebuild all from offset, with no effect ----
+ $builder->setFromId( $propId1->getNumericId() +1 );
+ $builder->setRebuildAll( false );
+
+ $builder->rebuildPropertyInfo();
+
+ $info = $table->getPropertyInfo( $propId1 );
+ $this->assertNull( $info, "rebuild all from offset should have
skipped this" );
+
+ // rebuild missing ----
+ $builder->setFromId( 0 );
+ $builder->setRebuildAll( false );
+
+ $builder->rebuildPropertyInfo();
+
+ foreach ( $properties as $propId => $expected ) {
+ $info = $table->getPropertyInfo( new EntityId(
Property::ENTITY_TYPE, $propId ) );
+ $this->assertEquals(
$expected[PropertyInfoStore::KEY_DATA_TYPE],
$info[PropertyInfoStore::KEY_DATA_TYPE], "Property $id" );
+ }
+
+ // rebuild again ----
+ $builder->setFromId( 0 );
+ $builder->setRebuildAll( false );
+
+ $c = $builder->rebuildPropertyInfo();
+ $this->assertEquals( 0, $c, "Thre should be nothing left to
rebuild" );
+ }
+
+}
\ No newline at end of file
--
To view, visit https://gerrit.wikimedia.org/r/71336
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I131537e34bee71397918956e2e6652b9de739d75
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/Wikibase
Gerrit-Branch: master
Gerrit-Owner: Daniel Kinzler <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
