Robert Vogel has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/350820 )
Change subject: Fix for archive images + maintenance scripts
......................................................................
Fix for archive images + maintenance scripts
When moving files withing one NS (renaming) archive images would get
broken, because name with NS prefix would be looked for on FS
Maintence scripts:
Database table oldimage might be corrupted, oi_archive_names might not
have NS prefix eventhough they should.
fixOldImages.php fixes this, and also fixes wrong files (with NS in name)
on the FS
checkFiles.php checks if every file in DB has its counterpart on FS
removeDuplicateNS.php fixes oi_archive_name if it contains NS prefix
multiple times
These bugs were identified on EWP, ERM: #4275
Change-Id: I3658ab9d807ff318aee14d9654350fb51dd53b17
---
M NSFileRepo_body.php
A maintenance/checkFiles.php
A maintenance/fixOldImage.php
A maintenance/removeDuplicateNS.php
4 files changed, 284 insertions(+), 3 deletions(-)
Approvals:
Robert Vogel: Verified; Looks good to me, approved
diff --git a/NSFileRepo_body.php b/NSFileRepo_body.php
index 112dddb..c7e7adf 100644
--- a/NSFileRepo_body.php
+++ b/NSFileRepo_body.php
@@ -483,7 +483,7 @@
* @return array List of archive names from old versions
*/
function addOlds() {
-/* This is the part that changed from LocalFile */
+/* This is the part that changed from LocalFileMoveBatch*/
$newName = $this->getFileNameStripped( $this->newName );
/* End of changes */
$archiveBase = 'archive';
@@ -513,6 +513,11 @@
wfDebug( "Old file name doesn't match:
'$oldName' \n" );
continue;
}
+/* This is the part that changed from LocalFileMoveBatch */
+ #When file is moved within a namespace we do not want it
+ #looking to NS:Name format in FS
+ $strippedOldName = $this->file->getFileNameStripped(
$oldName );
+/* End of changes */
$this->oldCount++;
@@ -520,9 +525,9 @@
if ( $row->oi_deleted & File::DELETED_FILE ) {
continue;
}
-/* This is the part that changed from LocalFile */
+/* This is the part that changed from LocalFileMoveBatch */
$this->olds[] = array(
- "{$archiveBase}/{$this->oldHash}{$oldName}",
+
"{$archiveBase}/{$this->oldHash}{$strippedOldName}",
"{$archiveBase}/{$this->newHash}{$timestamp}!{$newName}"
);
/* End of changes */
@@ -531,6 +536,61 @@
return $archiveNames;
}
+ /**
+ * Do the database updates and return a new FileRepoStatus indicating
how
+ * many rows where updated.
+ *
+ * @return FileRepoStatus
+ */
+ function doDBUpdates() {
+ $repo = $this->file->repo;
+ $status = $repo->newGood();
+ $dbw = $this->db;
+
+ // Update current image
+ $dbw->update(
+ 'image',
+ array( 'img_name' => $this->newName ),
+ array( 'img_name' => $this->oldName ),
+ __METHOD__
+ );
+
+ if ( $dbw->affectedRows() ) {
+ $status->successCount++;
+ } else {
+ $status->failCount++;
+ $status->fatal( 'imageinvalidfilename' );
+
+ return $status;
+ }
+/* Part that is changed from LocalFileMoveBatch' */
+ // Update old images
+ $dbw->update(
+ 'oldimage',
+ array(
+ 'oi_name' => $this->newName,
+ 'oi_archive_name = ' . $dbw->strreplace(
'oi_archive_name',
+ $dbw->addQuotes( $this->oldName ),
$dbw->addQuotes( $this->newName ) ),
+ ),
+ array( 'oi_name' => $this->oldName ),
+ __METHOD__
+ );
+/* End of changes */
+
+ $affected = $dbw->affectedRows();
+ $total = $this->oldCount;
+ $status->successCount += $affected;
+ // Bug 34934: $total is based on files that actually exist.
+ // There may be more DB rows than such files, in which case
$affected
+ // can be greater than $total. We use max() to avoid negatives
here.
+ $status->failCount += max( 0, $total - $affected );
+ if ( $status->failCount ) {
+ $status->error( 'imageinvalidfilename' );
+ }
+
+ return $status;
+ }
+
function getFileNameStripped( $suffix ) {
return(NSLocalFile::getFileNameStripped($suffix));
}
diff --git a/maintenance/checkFiles.php b/maintenance/checkFiles.php
new file mode 100644
index 0000000..7d44947
--- /dev/null
+++ b/maintenance/checkFiles.php
@@ -0,0 +1,73 @@
+<?php
+
+require_once( dirname(dirname(dirname(dirname(__DIR__)))) .
'/maintenance/Maintenance.php' );
+
+/**
+ * This script checks if all files in DB have their
+ * counterpart on FileSystem
+ */
+class CheckFiles extends Maintenance {
+
+ function __construct() {
+ parent::__construct();
+ }
+
+ function execute() {
+ $dbr = wfGetDB( DB_SLAVE );
+ print( "Using DB: " . $dbr->getDBName() ) . PHP_EOL;
+
+ $aImgNames = array();
+ $res = $dbr->select('image',
+ array( 'img_name' ),
+ array(),
+ __METHOD__
+ );
+
+ foreach( $res as $row ) {
+ $sName = preg_replace( '/_/', ' ', $row->img_name );
+ $oTitle = Title::makeTitle( NS_FILE, $sName );
+
+ if( !$oTitle->exists() ) {
+ print ( "Title for " . $sName . " does not
exist" . PHP_EOL );
+ continue;
+ }
+
+ if( $oTitle->getNamespace() !== NS_FILE ) {
+ print ( "Title for " . $sName . " is not in
NS_FILE" . PHP_EOL );
+ continue;
+ }
+
+ $oFile = wfFindFile( $sName );
+ if( !$oFile || !$oFile->exists() ) {
+ print( "File " . $sName . " does not exist!" .
PHP_EOL );
+ }
+ $sFileLocalPath = $oFile->getLocalRefPath();
+ if( !$sFileLocalPath || !file_exists( $sFileLocalPath )
) {
+ print( "Image " . $sName . " not found!" .
PHP_EOL );
+ }
+ }
+
+ $res = $dbr->select('oldimage',
+ array( 'oi_name', 'oi_archive_name' ),
+ array(),
+ __METHOD__
+ );
+
+ foreach( $res as $row ) {
+ $oTitle = Title::makeTitle( NS_FILE, $row->oi_name );
+ $repo = RepoGroup::singleton()->getRepo( 'local' );
+ $strippedName = NSLocalFile::getFilenameStripped(
$row->oi_archive_name );
+ $file = OldLocalFile::newFromArchiveName( $oTitle,
$repo, $strippedName );
+ if( !$file->getLocalRefPath() || !file_exists(
$file->getLocalRefPath() ) ) {
+ $file = OldLocalFile::newFromArchiveName(
$oTitle, $repo, $row->oi_archive_name );
+ print( "Archive file: " . $row->oi_archive_name
. " not found" . PHP_EOL );
+ if( $file->getLocalRefPath() && file_exists(
$file->getLocalRefPath() ) ) {
+ print( "\t...but wrong version of this
file exists: " . $file->getLocalRefPath() . PHP_EOL );
+ }
+ }
+ }
+ }
+}
+
+$maintClass = 'CheckFiles';
+require_once( RUN_MAINTENANCE_IF_MAIN );
diff --git a/maintenance/fixOldImage.php b/maintenance/fixOldImage.php
new file mode 100644
index 0000000..987ebb6
--- /dev/null
+++ b/maintenance/fixOldImage.php
@@ -0,0 +1,96 @@
+<?php
+
+require_once( dirname(dirname(dirname(dirname(__DIR__)))) .
'/maintenance/Maintenance.php' );
+
+class FixOldImage extends Maintenance {
+ protected $mDBName;
+
+ function __construct() {
+ parent::__construct();
+ }
+
+ function execute() {
+ $dbw = wfGetDB( DB_MASTER );
+ print( "USING DB: " . $dbw->getDBName() . "\n" );
+ $images = $dbw->select( 'oldimage',
+ array( 'oi_name', 'oi_archive_name' ),
+ array(),
+ __METHOD__
+ );
+
+ $count = 0;
+ $log = '';
+ foreach( $images as $image ) {
+ $nameBits = explode( ':', $image->oi_name );
+ $nameNS = '';
+ $nameName = $image->oi_name;
+ if( count( $nameBits ) == 2 ) {
+ $nameNS = $nameBits[0];
+ $nameName = $nameBits[1];
+ }
+
+ $archiveName = explode( '!', $image->oi_archive_name
)[1];
+ $archiveBits = explode( ':', $archiveName );
+ $archiveNS = '';
+ if( count( $archiveBits ) == 2 ) {
+ $archiveNS = $archiveBits[0];
+ $archiveName = $archiveBits[1];
+ }
+
+ $title = Title::makeTitle( NS_FILE, $image->oi_name );
+ if( !$title || !$title->exists() ) {
+ print( "Title wrong " . $image->oi_name .
PHP_EOL );
+ continue;
+ }
+
+ $repo = RepoGroup::singleton()->getRepo( 'local' );
+ $strippedName = NSLocalFile::getFilenameStripped(
$image->oi_archive_name );
+
+ $file = OldLocalFile::newFromArchiveName( $title,
$repo, $strippedName );
+ if( !$file || !$file->exists() ) {
+ $file = OldLocalFile::newFromArchiveName(
$title, $repo, $image->oi_archive_name );
+ if( $file && $file->exists() && file_exists(
$file->getLocalRefPath() ) ) {
+ $path = $file->getLocalRefPath();
+ print( "Found wrong file on FS: " .
$path . "..." );
+ $baseDir = dirname( $path );
+ $renamed = rename( $path, $baseDir .
'/' . $strippedName );
+ if( $renamed ) {
+ print("renamed" . PHP_EOL );
+ $log .= "Renamed " . $path . "
to " . $baseDir . '/' . $strippedName . "\n";
+ } else {
+ print("failed to rename" .
PHP_EOL );
+ $log .= "Failed to rename " .
$path . " to " . $baseDir . '/' . $strippedName . "\n";
+ }
+
+ }
+ }
+
+ if( $nameNS === $archiveNS && $nameName == $archiveName
) {
+ continue;
+ }
+
+ if( $nameNS != $archiveNS && $nameName != $archiveName
) {
+ continue;
+ }
+
+ print( "NAME: ns-" . $nameNS . ' name-' . $nameName . '
ARCHIVE: ns-' . $archiveNS . ' name-' . $archiveName . "\n" );
+ $log .= $nameNS . ";" . $nameName . ";" . $archiveNS .
";" . $archiveName . "\n";
+ $dbw->update(
+ 'oldimage',
+ array(
+ 'oi_archive_name = ' .
$dbw->strreplace( 'oi_archive_name',
+ $dbw->addQuotes( $archiveName
), $dbw->addQuotes( $image->oi_name ) )
+ ),
+ array( 'oi_archive_name' =>
$image->oi_archive_name ),
+ __METHOD__
+ );
+
+ $count++;
+ }
+ file_put_contents( '/tmp/fixOldImagesLog.log', $log );
+ print( "Total " . $count . " images altered\n" );
+ }
+}
+
+$maintClass = 'FixOldImage';
+require_once( RUN_MAINTENANCE_IF_MAIN );
diff --git a/maintenance/removeDuplicateNS.php
b/maintenance/removeDuplicateNS.php
new file mode 100644
index 0000000..7a197ff
--- /dev/null
+++ b/maintenance/removeDuplicateNS.php
@@ -0,0 +1,52 @@
+<?php
+
+require_once( dirname(dirname(dirname(dirname(__DIR__)))) .
'/maintenance/Maintenance.php' );
+
+/**
+ * If in table oldimage oi_archive_name contains
+ * multiple NS prefixes, this script fixes it
+ */
+class RemoveDuplicateNS extends Maintenance {
+
+ function __construct() {
+ parent::__construct();
+ }
+
+ function execute() {
+ $dbw = wfGetDB( DB_MASTER );
+ print( "USING DB: " . $dbw->getDBName() . "\n" );
+ $images = $dbw->select( 'oldimage',
+ array( 'oi_name', 'oi_archive_name' ),
+ array(),
+ __METHOD__
+ );
+
+ foreach( $images as $image ) {
+ $archiveTS = explode( '!', $image->oi_archive_name )[0];
+ $archiveName = explode( '!', $image->oi_archive_name
)[1];
+ $archiveBits = explode( ':', $archiveName );
+ if( count($archiveBits) > 2 ) {
+ $archiveNS = $archiveBits[0];
+ $archiveFName =
$archiveBits[count($archiveBits)-1];
+ unset($archiveBits[count($archiveBits)-1] );
+ $archiveFill = implode( ':', $archiveBits );
+ print( "Removing: " . $archiveFill . PHP_EOL );
+ $dbw->update(
+ 'oldimage',
+ array(
+ 'oi_archive_name = ' .
$dbw->strreplace( 'oi_archive_name',
+ $dbw->addQuotes(
$archiveFill ), $dbw->addQuotes( $archiveNS ) )
+ ),
+ array( 'oi_name' => $image->oi_name ),
+ __METHOD__
+ );
+ } else {
+ continue;
+ }
+ }
+
+ }
+}
+
+$maintClass = 'RemoveDuplicateNS';
+require_once( RUN_MAINTENANCE_IF_MAIN );
--
To view, visit https://gerrit.wikimedia.org/r/350820
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I3658ab9d807ff318aee14d9654350fb51dd53b17
Gerrit-PatchSet: 2
Gerrit-Project: mediawiki/extensions/NSFileRepo
Gerrit-Branch: REL1_23
Gerrit-Owner: ItSpiderman <[email protected]>
Gerrit-Reviewer: Robert Vogel <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
