[MediaWiki-commits] [Gerrit] mediawiki/core[wmf/1.29.0-wmf.14]: resourceloader: Add purgeModuleDeps.php maintenance script
jenkins-bot has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/340860 )
Change subject: resourceloader: Add purgeModuleDeps.php maintenance script
..
resourceloader: Add purgeModuleDeps.php maintenance script
Based on cleanupRemovedModules.php. Update both to use safer batching,
based on known selection instead of recursing until the table is empty
(which may end up deleting new rows from live traffic).
Bug: T158105
Change-Id: I05f650a0cfa8ca647f143a40e1087338adbef6da
---
M autoload.php
M maintenance/cleanupRemovedModules.php
A maintenance/purgeModuleDeps.php
3 files changed, 98 insertions(+), 13 deletions(-)
Approvals:
Mattflaschen: Looks good to me, approved
jenkins-bot: Verified
diff --git a/autoload.php b/autoload.php
index aa4e544..61a9caf 100644
--- a/autoload.php
+++ b/autoload.php
@@ -1129,6 +1129,7 @@
'PurgeChangedPages' => __DIR__ . '/maintenance/purgeChangedPages.php',
'PurgeJobUtils' => __DIR__ .
'/includes/jobqueue/utils/PurgeJobUtils.php',
'PurgeList' => __DIR__ . '/maintenance/purgeList.php',
+ 'PurgeModuleDeps' => __DIR__ . '/maintenance/purgeModuleDeps.php',
'PurgeOldText' => __DIR__ . '/maintenance/purgeOldText.php',
'PurgeParserCache' => __DIR__ . '/maintenance/purgeParserCache.php',
'QueryPage' => __DIR__ . '/includes/specialpage/QueryPage.php',
diff --git a/maintenance/cleanupRemovedModules.php
b/maintenance/cleanupRemovedModules.php
index 863d74a..83ab35c 100644
--- a/maintenance/cleanupRemovedModules.php
+++ b/maintenance/cleanupRemovedModules.php
@@ -38,31 +38,43 @@
parent::__construct();
$this->addDescription(
'Remove cache entries for removed ResourceLoader
modules from the database' );
- $this->addOption( 'batchsize', 'Delete rows in batches of this
size. Default: 500', false, true );
+ $this->setBatchSize( 500 );
}
public function execute() {
+ $this->output( "Cleaning up module_deps table...\n" );
+
$dbw = $this->getDB( DB_MASTER );
$rl = new ResourceLoader(
MediaWikiServices::getInstance()->getMainConfig() );
$moduleNames = $rl->getModuleNames();
- $moduleList = implode( ', ', array_map( [ $dbw, 'addQuotes' ],
$moduleNames ) );
- $limit = max( 1, intval( $this->getOption( 'batchsize', 500 ) )
);
+ $res = $dbw->select( 'module_deps',
+ [ 'md_module', 'md_skin' ],
+ $moduleNames ? 'md_module NOT IN (' . $dbw->makeList(
$moduleNames ) . ')' : '1=1',
+ __METHOD__
+ );
+ $rows = iterator_to_array( $res, false );
- $this->output( "Cleaning up module_deps table...\n" );
- $i = 1;
$modDeps = $dbw->tableName( 'module_deps' );
- do {
- // $dbw->delete() doesn't support LIMIT :(
- $where = $moduleList ? "md_module NOT IN ($moduleList)"
: '1=1';
- $dbw->query( "DELETE FROM $modDeps WHERE $where LIMIT
$limit", __METHOD__ );
+ $i = 1;
+ foreach ( array_chunk( $rows, $this->mBatchSize ) as $chunk ) {
+ // WHERE ( mod=A AND skin=A ) OR ( mod=A AND skin=B) ..
+ $conds = array_map( function ( stdClass $row ) use (
$dbw ) {
+ return $dbw->makeList( (array)$row,
IDatabase::LIST_AND );
+ }, $chunk );
+ $conds = $dbw->makeList( $conds, IDatabase::LIST_OR );
+
+ $this->beginTransaction( $dbw, __METHOD__ );
+ $dbw->query( "DELETE FROM $modDeps WHERE $conds",
__METHOD__ );
$numRows = $dbw->affectedRows();
$this->output( "Batch $i: $numRows rows\n" );
+ $this->commitTransaction( $dbw, __METHOD__ );
+
$i++;
- wfWaitForSlaves();
- } while ( $numRows > 0 );
- $this->output( "done\n" );
+ }
+
+ $this->output( "Done\n" );
}
}
-$maintClass = "CleanupRemovedModules";
+$maintClass = 'CleanupRemovedModules';
require_once RUN_MAINTENANCE_IF_MAIN;
diff --git a/maintenance/purgeModuleDeps.php b/maintenance/purgeModuleDeps.php
new file mode 100644
index 000..3088baa
--- /dev/null
+++ b/maintenance/purgeModuleDeps.php
@@ -0,0 +1,72 @@
+http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup Maintenance
+ * @author Timo Tijhof
+ */
+
+use MediaWiki\MediaWikiServices;
+
+require_once __DIR__ . '/Maintenance.php';
+
+/**
+ * Maintenance script to purge the module_deps database cache table.
+ *
+ * @ingroup Maintenance
+ */
+class PurgeModuleDeps
[MediaWiki-commits] [Gerrit] mediawiki/core[wmf/1.29.0-wmf.14]: resourceloader: Add purgeModuleDeps.php maintenance script
Krinkle has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/340860 )
Change subject: resourceloader: Add purgeModuleDeps.php maintenance script
..
resourceloader: Add purgeModuleDeps.php maintenance script
Based on cleanupRemovedModules.php. Update both to use safer batching,
based on known selection instead of recursing until the table is empty
(which may end up deleting new rows from live traffic).
Bug: T158105
Change-Id: I05f650a0cfa8ca647f143a40e1087338adbef6da
---
M autoload.php
M maintenance/cleanupRemovedModules.php
A maintenance/purgeModuleDeps.php
3 files changed, 98 insertions(+), 13 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core
refs/changes/60/340860/1
diff --git a/autoload.php b/autoload.php
index aa4e544..61a9caf 100644
--- a/autoload.php
+++ b/autoload.php
@@ -1129,6 +1129,7 @@
'PurgeChangedPages' => __DIR__ . '/maintenance/purgeChangedPages.php',
'PurgeJobUtils' => __DIR__ .
'/includes/jobqueue/utils/PurgeJobUtils.php',
'PurgeList' => __DIR__ . '/maintenance/purgeList.php',
+ 'PurgeModuleDeps' => __DIR__ . '/maintenance/purgeModuleDeps.php',
'PurgeOldText' => __DIR__ . '/maintenance/purgeOldText.php',
'PurgeParserCache' => __DIR__ . '/maintenance/purgeParserCache.php',
'QueryPage' => __DIR__ . '/includes/specialpage/QueryPage.php',
diff --git a/maintenance/cleanupRemovedModules.php
b/maintenance/cleanupRemovedModules.php
index 863d74a..83ab35c 100644
--- a/maintenance/cleanupRemovedModules.php
+++ b/maintenance/cleanupRemovedModules.php
@@ -38,31 +38,43 @@
parent::__construct();
$this->addDescription(
'Remove cache entries for removed ResourceLoader
modules from the database' );
- $this->addOption( 'batchsize', 'Delete rows in batches of this
size. Default: 500', false, true );
+ $this->setBatchSize( 500 );
}
public function execute() {
+ $this->output( "Cleaning up module_deps table...\n" );
+
$dbw = $this->getDB( DB_MASTER );
$rl = new ResourceLoader(
MediaWikiServices::getInstance()->getMainConfig() );
$moduleNames = $rl->getModuleNames();
- $moduleList = implode( ', ', array_map( [ $dbw, 'addQuotes' ],
$moduleNames ) );
- $limit = max( 1, intval( $this->getOption( 'batchsize', 500 ) )
);
+ $res = $dbw->select( 'module_deps',
+ [ 'md_module', 'md_skin' ],
+ $moduleNames ? 'md_module NOT IN (' . $dbw->makeList(
$moduleNames ) . ')' : '1=1',
+ __METHOD__
+ );
+ $rows = iterator_to_array( $res, false );
- $this->output( "Cleaning up module_deps table...\n" );
- $i = 1;
$modDeps = $dbw->tableName( 'module_deps' );
- do {
- // $dbw->delete() doesn't support LIMIT :(
- $where = $moduleList ? "md_module NOT IN ($moduleList)"
: '1=1';
- $dbw->query( "DELETE FROM $modDeps WHERE $where LIMIT
$limit", __METHOD__ );
+ $i = 1;
+ foreach ( array_chunk( $rows, $this->mBatchSize ) as $chunk ) {
+ // WHERE ( mod=A AND skin=A ) OR ( mod=A AND skin=B) ..
+ $conds = array_map( function ( stdClass $row ) use (
$dbw ) {
+ return $dbw->makeList( (array)$row,
IDatabase::LIST_AND );
+ }, $chunk );
+ $conds = $dbw->makeList( $conds, IDatabase::LIST_OR );
+
+ $this->beginTransaction( $dbw, __METHOD__ );
+ $dbw->query( "DELETE FROM $modDeps WHERE $conds",
__METHOD__ );
$numRows = $dbw->affectedRows();
$this->output( "Batch $i: $numRows rows\n" );
+ $this->commitTransaction( $dbw, __METHOD__ );
+
$i++;
- wfWaitForSlaves();
- } while ( $numRows > 0 );
- $this->output( "done\n" );
+ }
+
+ $this->output( "Done\n" );
}
}
-$maintClass = "CleanupRemovedModules";
+$maintClass = 'CleanupRemovedModules';
require_once RUN_MAINTENANCE_IF_MAIN;
diff --git a/maintenance/purgeModuleDeps.php b/maintenance/purgeModuleDeps.php
new file mode 100644
index 000..3088baa
--- /dev/null
+++ b/maintenance/purgeModuleDeps.php
@@ -0,0 +1,72 @@
+http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup Maintenance
+ * @author Timo Tijhof
+ */
+
+use MediaWiki\MediaWikiServices;
+
+require_once __DIR__ . '/Maintenance.php';
+
+/**
+ * Maintenance script to purge the module_deps database cache table.
+ *
+ * @ingroup Maintenance
+ */
+class PurgeModuleDeps extends
