Aaron Schulz has uploaded a new change for review.
https://gerrit.wikimedia.org/r/249494
Change subject: Add script to find orphaned LocalRepo files
......................................................................
Add script to find orphaned LocalRepo files
Bug: T111838
Change-Id: I60ac7f3451c8240a0716933dcfff413669458206
---
M autoload.php
A maintenance/findOrphanedFiles.php
2 files changed, 138 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core
refs/changes/94/249494/1
diff --git a/autoload.php b/autoload.php
index 2bcf6b5..2daa572 100644
--- a/autoload.php
+++ b/autoload.php
@@ -446,6 +446,7 @@
'FileRepoStatus' => __DIR__ . '/includes/filerepo/FileRepoStatus.php',
'FindHooks' => __DIR__ . '/maintenance/findHooks.php',
'FindMissingFiles' => __DIR__ . '/maintenance/findMissingFiles.php',
+ 'FindOrphanedFiles' => __DIR__ . '/maintenance/findOrphanedFiles.php',
'FixBug20757' => __DIR__ . '/maintenance/storage/fixBug20757.php',
'FixDoubleRedirects' => __DIR__ . '/maintenance/fixDoubleRedirects.php',
'FixExtLinksProtocolRelative' => __DIR__ .
'/maintenance/fixExtLinksProtocolRelative.php',
diff --git a/maintenance/findOrphanedFiles.php
b/maintenance/findOrphanedFiles.php
new file mode 100644
index 0000000..5ab5159
--- /dev/null
+++ b/maintenance/findOrphanedFiles.php
@@ -0,0 +1,137 @@
+<?php
+/**
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @author Aaron Schulz
+ */
+
+require_once __DIR__ . '/Maintenance.php';
+
+class FindOrphanedFiles extends Maintenance {
+ function __construct() {
+ parent::__construct();
+
+ $this->mDescription = "Find unregistered files in the 'public'
repo zone.";
+ $this->addOption( 'subdir',
+ 'Only scan files in this subdirectory (e.g. "a/a0")',
false, true );
+ $this->addOption( 'verbose', "Mention file paths checked" );
+ $this->setBatchSize( 500 );
+ }
+
+ function execute() {
+ $subdir = $this->getOption( 'subdir', '' );
+ $verbose = $this->hasOption( 'verbose' );
+
+ $repo = RepoGroup::singleton()->getLocalRepo();
+ if ( $repo->hasSha1Storage() ) {
+ $this->error( "Local repo uses SHA-1 file storage
names; aborting.", 1 );
+ }
+
+ $directory = $repo->getZonePath( 'public' );
+ if ( $subdir != '' ) {
+ $directory .= "/$subdir/";
+ }
+
+ if ( $verbose ) {
+ $this->output( "Scanning files under $directory:\n" );
+ }
+
+ $list = $repo->getBackend()->getFileList( array( 'dir' =>
$directory ) );
+ if ( $list === null ) {
+ $this->error( "Could not get file listing.", 1 );
+ }
+
+ $nameBatch = array();
+ foreach ( $list as $path ) {
+ $nameBatch[] = basename( $path );
+ if ( count( $nameBatch ) >= $this->mBatchSize ) {
+ $this->checkFiles( $repo, $nameBatch, $verbose
);
+ $nameBatch = array();
+ }
+ }
+ $this->checkFiles( $repo, $nameBatch, $verbose );
+ }
+
+ protected function checkFiles( LocalRepo $repo, array $names, $verbose
) {
+ if ( !count( $names ) ) {
+ return;
+ }
+
+ $dbr = $repo->getSlaveDB();
+
+ $imgIN = array();
+ $oiWheres = array();
+ foreach ( $names as $name ) {
+ if ( strpos( $name, '!' ) !== false ) {
+ if ( $verbose ) {
+ $this->output( "Checking old file
$name\n" );
+ }
+
+ list( , $base ) = explode( '!', $name ); //
<TS_MW>!<img_name>
+ $oiWheres[] = $dbr->makeList(
+ array( 'oi_name' => $base,
'oi_archive_name' => $name ),
+ LIST_AND
+ );
+ } else {
+ if ( $verbose ) {
+ $this->output( "Checking current file
$name\n" );
+ }
+
+ $imgIN[] = $name;
+ }
+ }
+
+ $res = $dbr->query(
+ $dbr->unionQueries(
+ array(
+ $dbr->selectSQLText(
+ 'image',
+ array( 'name' => 'img_name' ),
+ array( 'img_name' => $imgIN )
+ ),
+ $dbr->selectSQLText(
+ 'oldimage',
+ array( 'name' =>
'oi_archive_name' ),
+ $dbr->makeList( $oiWheres,
LIST_OR )
+ )
+ ),
+ true // UNION ALL (performance)
+ ),
+ __METHOD__
+ );
+
+ $namesFound = array();
+ foreach ( $res as $row ) {
+ $namesFound[] = $row->name;
+ }
+
+ $namesOrphans = array_diff( $names, $namesFound );
+ foreach ( $namesOrphans as $name ) {
+ // Print name and public URL to ease recovery
+ if ( strpos( $name, '!' ) !== false ) {
+ list( , $base ) = explode( '!', $name ); //
<TS_MW>!<img_name>
+ $file = $repo->newFromArchiveName(
Title::makeTitle( NS_FILE, $base ), $name );
+ } else {
+ $file = $repo->newFile( $name );
+ }
+ $this->output( $name . "\n" . $file->getUrl() . "\n\n"
);
+ }
+ }
+}
+
+$maintClass = 'FindOrphanedFiles';
+require_once RUN_MAINTENANCE_IF_MAIN;
--
To view, visit https://gerrit.wikimedia.org/r/249494
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I60ac7f3451c8240a0716933dcfff413669458206
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/core
Gerrit-Branch: master
Gerrit-Owner: Aaron Schulz <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
