jenkins-bot has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/386463 )
Change subject: Use very large insert batches and wait in EntityUsageTable
......................................................................
Use very large insert batches and wait in EntityUsageTable
Per Jaime's suggestion.
Change-Id: I6089ff695a50a284db59f09ff4889c9cb31664e6
---
M client/includes/Changes/InjectRCRecordsJob.php
M client/includes/Usage/Sql/EntityUsageTable.php
M repo/includes/Store/Sql/SqlChangeDispatchCoordinator.php
3 files changed, 28 insertions(+), 7 deletions(-)
Approvals:
jenkins-bot: Verified
Thiemo Mättig (WMDE): Looks good to me, approved
diff --git a/client/includes/Changes/InjectRCRecordsJob.php
b/client/includes/Changes/InjectRCRecordsJob.php
index dc5ec94..545fdd2 100644
--- a/client/includes/Changes/InjectRCRecordsJob.php
+++ b/client/includes/Changes/InjectRCRecordsJob.php
@@ -274,6 +274,8 @@
}
}
+ // Wait for all database replicas to be updated, but only for
the affected client wiki. The
+ // "domain" argument is documented at
ILBFactory::waitForReplication.
$this->lbFactory->commitAndWaitForReplication( __METHOD__,
$trxToken, [ 'domain' => wfWikiID() ] );
$this->incrementStats( 'InjectRCRecords.run.titles', count(
$titles ) );
diff --git a/client/includes/Usage/Sql/EntityUsageTable.php
b/client/includes/Usage/Sql/EntityUsageTable.php
index c555d8e..59a80df 100644
--- a/client/includes/Usage/Sql/EntityUsageTable.php
+++ b/client/includes/Usage/Sql/EntityUsageTable.php
@@ -13,6 +13,7 @@
use Wikibase\DataModel\Entity\EntityIdParser;
use Wikimedia\Rdbms\Database;
use Wikimedia\Rdbms\DBUnexpectedError;
+use Wikimedia\Rdbms\LBFactory;
/**
* Helper class for updating the wbc_entity_usage table.
@@ -26,6 +27,13 @@
class EntityUsageTable {
const DEFAULT_TABLE_NAME = 'wbc_entity_usage';
+
+ /**
+ * INSERTs are supposed to be done in much larger batches than SELECTs
or DELETEs, per the DBA.
+ * About 1000 was suggested. Given the default batch size is 100, a
factor of 5 seems to be a
+ * good compromise.
+ */
+ const INSERT_BATCH_SIZE_FACTOR = 5;
/**
* @var EntityIdParser
@@ -43,9 +51,9 @@
private $readConnection;
/**
- * @var string
+ * @var LBFactory
*/
- private $tableName;
+ private $loadBalancerFactory;
/**
* @var int
@@ -53,9 +61,15 @@
private $batchSize;
/**
+ * @var string
+ */
+ private $tableName;
+
+ /**
* @param EntityIdParser $idParser
* @param Database $writeConnection
- * @param int $batchSize defaults to 100
+ * @param int $batchSize Batch size for database queries on the entity
usage table, including
+ * INSERTs, SELECTs, and DELETEs. Defaults to 100.
* @param string|null $tableName defaults to wbc_entity_usage
*
* @throws InvalidArgumentException
@@ -80,8 +94,8 @@
$this->tableName = $tableName ?: self::DEFAULT_TABLE_NAME;
//TODO: Inject
- $this->readConnection =
MediaWikiServices::getInstance()->getDBLoadBalancer()
- ->getConnection( DB_REPLICA );
+ $this->loadBalancerFactory =
MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
+ $this->readConnection =
$this->loadBalancerFactory->getMainLB()->getConnection( DB_REPLICA );
}
/**
@@ -160,7 +174,7 @@
$batches = array_chunk(
$this->makeUsageRows( $pageId, $usages ),
- $this->batchSize
+ $this->batchSize * self::INSERT_BATCH_SIZE_FACTOR
);
$c = 0;
@@ -174,6 +188,10 @@
$this->writeConnection->endAtomic( __METHOD__ );
+ // Wait for all database replicas to be updated, but only for
the affected client wiki. The
+ // "domain" argument is documented at
ILBFactory::waitForReplication.
+ $this->loadBalancerFactory->waitForReplication( [ 'domain' =>
wfWikiID() ] );
+
return $c;
}
diff --git a/repo/includes/Store/Sql/SqlChangeDispatchCoordinator.php
b/repo/includes/Store/Sql/SqlChangeDispatchCoordinator.php
index 713b8c3..8497b3d 100644
--- a/repo/includes/Store/Sql/SqlChangeDispatchCoordinator.php
+++ b/repo/includes/Store/Sql/SqlChangeDispatchCoordinator.php
@@ -527,7 +527,8 @@
}
$db->commit( __METHOD__ );
- # wait for replication to finish
+ // Wait for all database replicas to be updated, but only for
the affected client wiki. The
+ // "domain" argument is documented at
ILBFactory::waitForReplication.
$this->LBFactory->waitForReplication( [ 'domain' =>
$this->repoDB ] );
$this->releaseRepoDb( $db );
--
To view, visit https://gerrit.wikimedia.org/r/386463
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I6089ff695a50a284db59f09ff4889c9cb31664e6
Gerrit-PatchSet: 3
Gerrit-Project: mediawiki/extensions/Wikibase
Gerrit-Branch: master
Gerrit-Owner: Hoo man <[email protected]>
Gerrit-Reviewer: Daniel Kinzler <[email protected]>
Gerrit-Reviewer: Thiemo Mättig (WMDE) <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
