Addshore has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/342845 )
Change subject: WIP Handle duplicate files gracefully
......................................................................
WIP Handle duplicate files gracefully
We need to decide if we want to look at all file revisions
or just current file revisions.
How about deleted files?
How do we want to act in all of these cases?
Bug: T160166
Change-Id: I351049b934f23817f41903292b8c75e69e352286
---
M extension.json
M i18n/en.json
M i18n/qqq.json
A src/Generic/Services/DuplicateFileRevisionChecker.php
M src/MediaWiki/ApiDetailRetriever.php
M src/ServiceWiring.php
M src/SpecialImportFile.php
7 files changed, 109 insertions(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/FileImporter
refs/changes/45/342845/1
diff --git a/extension.json b/extension.json
index bc83f2c..bffd6f4 100644
--- a/extension.json
+++ b/extension.json
@@ -28,6 +28,7 @@
"FileImporter\\Generic\\Services\\Importer":
"src/Generic/Services/Importer.php",
"FileImporter\\Generic\\Services\\HttpRequestExecutor":
"src/Generic/Services/HttpRequestExecutor.php",
"FileImporter\\Generic\\Services\\RevisionModifier":
"src/Generic/Services/RevisionModifier.php",
+
"FileImporter\\Generic\\Services\\DuplicateFileRevisionChecker":
"src/Generic/Services/DuplicateFileRevisionChecker.php",
"FileImporter\\Generic\\Data\\ImportTransformations":
"src/Generic/Data/ImportTransformations.php",
"FileImporter\\Generic\\Data\\ImportDetails":
"src/Generic/Data/ImportDetails.php",
"FileImporter\\Generic\\Data\\TargetUrl":
"src/Generic/Data/TargetUrl.php",
diff --git a/i18n/en.json b/i18n/en.json
index bb18904..8eae1dc 100644
--- a/i18n/en.json
+++ b/i18n/en.json
@@ -9,6 +9,7 @@
"fileimporter-specialpage": "Import file",
"fileimporter-cantparseurl": "Can't parse the given URL",
"fileimporter-cantimporturl": "Can't import the given URL",
+ "fileimporter-duplicatefilesdetected" : "The file you are currently
trying to import already exists on this wiki",
"fileimporter-exampleprefix": "Example",
"fileimporter-textrevisionsprefix": "Text Revisions",
"fileimporter-filerevisionsprefix": "File Revisions",
diff --git a/i18n/qqq.json b/i18n/qqq.json
index 64731b6..70f455e 100644
--- a/i18n/qqq.json
+++ b/i18n/qqq.json
@@ -10,6 +10,7 @@
"fileimporter-specialpage": "Title for the File Import special page.",
"fileimporter-cantparseurl": "Error message shown on the special page
when the URL entered can not be parsed.",
"fileimporter-cantimporturl": "Error message shown on the special page
when the URL entered can not be imported from.",
+ "fileimporter-duplicatefilesdetected" : "Error message shown on the
special page when the file to be imported has been detected as already existing
on the local wiki.",
"fileimporter-exampleprefix": "Prefix for the example URL contained
within the URL text box on the special page.\n{{Identical|Example}}",
"fileimporter-textrevisionsprefix" : "Prefix for the number of text
revisions to be imported on the special page.",
"fileimporter-filerevisionsprefix" : "Prefix for the number of file
revisions to be imported on the special page.",
diff --git a/src/Generic/Services/DuplicateFileRevisionChecker.php
b/src/Generic/Services/DuplicateFileRevisionChecker.php
new file mode 100644
index 0000000..c2408ea
--- /dev/null
+++ b/src/Generic/Services/DuplicateFileRevisionChecker.php
@@ -0,0 +1,71 @@
+<?php
+
+namespace FileImporter\Generic\Services;
+
+use File;
+use FileImporter\Generic\Data\FileRevision;
+use LocalRepo;
+use Wikimedia\Assert\Assert;
+
+/**
+ * Class that can be used to check if a FileRevision or collection of
FileRevisions already
+ * exists on the current wiki.
+ */
+class DuplicateFileRevisionChecker {
+
+ /**
+ * @var LocalRepo
+ */
+ private $localRepo;
+
+ public function __construct( LocalRepo $localRepo ) {
+ $this->localRepo = $localRepo;
+ }
+
+ /**
+ * @param FileRevision[] $fileRevisions
+ *
+ * @return File[][] 2D array, any matches for a given FileRevision will
be returned with the
+ * same key pointing to File objects that were found.
+ * If no files were found for a key then the key will not be present.
+ */
+ public function revisionsHaveDuplicate( array $fileRevisions ) {
+ Assert::parameterElementType( FileRevision::class,
$fileRevisions, '$fileRevisions' );
+ $hashes = $this->getHashesFromFileRevisions( $fileRevisions );
+ $filesByHash = $this->localRepo->findBySha1s( $hashes );
+ $filesByKey = $this->getFilesByKey( $hashes, $filesByHash );
+ return $filesByKey;
+ }
+
+ /**
+ * @param FileRevision[] $fileRevisions
+ *
+ * @return string[]
+ */
+ private function getHashesFromFileRevisions( array $fileRevisions ) {
+ $hashes = [];
+ foreach ( $fileRevisions as $key => $fileRevision ) {
+ $hashes[$key] = $fileRevision->getField( 'sha1' );
+ }
+ return $hashes;
+ }
+
+ /**
+ * @param string[] $hashes
+ * @param File[] $filesByHash
+ *
+ * @return File[]
+ */
+ private function getFilesByKey( array $hashes, array $filesByHash ) {
+ $filesByKey = [];
+ foreach ( $filesByHash as $hash => $files ) {
+ if ( empty( $files ) ) {
+ continue;
+ }
+ $key = array_search( $hash, $hashes );
+ $filesByKey[$key] = $files;
+ }
+ return $filesByKey;
+ }
+
+}
diff --git a/src/MediaWiki/ApiDetailRetriever.php
b/src/MediaWiki/ApiDetailRetriever.php
index db178b0..91a8b26 100644
--- a/src/MediaWiki/ApiDetailRetriever.php
+++ b/src/MediaWiki/ApiDetailRetriever.php
@@ -142,6 +142,13 @@
private function getFileRevisionsFromImageInfo( array $imageInfo ) {
$revisions = [];
foreach ( $imageInfo as $revisionInfo ) {
+ /**
+ * Convert from API sha1 format to DB sha1 format.
+ * The conversion can be se inside ApiQueryImageInfo.
+ * - API sha1 format is base 16 padded to 40 chars
+ * - DB sha1 format is base 36 padded to 31 chars
+ */
+ $revisionInfo['sha1'] = \Wikimedia\base_convert(
$revisionInfo['sha1'], 16, 36, 31 );
$revisions[] = new FileRevision( $revisionInfo );
}
return $revisions;
diff --git a/src/ServiceWiring.php b/src/ServiceWiring.php
index 38a1e02..4030ea5 100644
--- a/src/ServiceWiring.php
+++ b/src/ServiceWiring.php
@@ -3,10 +3,12 @@
namespace FileImporter;
use FileImporter\Generic\Services\DispatchingDetailRetriever;
+use FileImporter\Generic\Services\DuplicateFileRevisionChecker;
use FileImporter\Generic\Services\HttpRequestExecutor;
use MediaWiki\Logger\LoggerFactory;
use MediaWiki\MediaWikiServices;
use Psr\Log\LoggerInterface;
+use RepoGroup;
return [
@@ -29,6 +31,11 @@
return $service;
},
+ 'FileImporterDuplicateFileRevisionChecker' => function(
MediaWikiServices $services ) {
+ $localRepo = RepoGroup::singleton()->getLocalRepo();
+ return new DuplicateFileRevisionChecker( $localRepo );
+ },
+
// MediaWiki
'FileImporterMediaWikiHttpApiLookup' => function( MediaWikiServices
$services ) {
diff --git a/src/SpecialImportFile.php b/src/SpecialImportFile.php
index d0e82ff..e11020c 100644
--- a/src/SpecialImportFile.php
+++ b/src/SpecialImportFile.php
@@ -2,9 +2,11 @@
namespace FileImporter;
+use File;
use FileImporter\Generic\Data\ImportTransformations;
use FileImporter\Generic\Data\ImportDetails;
use FileImporter\Generic\Services\DetailRetriever;
+use FileImporter\Generic\Services\DuplicateFileRevisionChecker;
use FileImporter\Generic\Services\Importer;
use FileImporter\Generic\Data\TargetUrl;
use Html;
@@ -49,7 +51,15 @@
$this->showUrlEntryPage();
} else {
$importDetails = $detailRetriever->getImportDetails(
$targetUrl );
- if ( $wasPosted ) {
+ /** @var DuplicateFileRevisionChecker
$duplicateFileChecker */
+ $duplicateFileChecker = MediaWikiServices::getInstance()
+ ->getService(
'FileImporterDuplicateFileRevisionChecker' );
+ $duplicateFileMap =
$duplicateFileChecker->revisionsHaveDuplicate(
+ $importDetails->getFileRevisions()
+ );
+ if ( !empty( $duplicateFileMap ) ) {
+ $this->showDuplicateFilesDetectedMessage(
$duplicateFileMap );
+ } elseif ( $wasPosted ) {
$this->doImport( $importDetails );
} else {
$this->showImportPage( $importDetails );
@@ -97,6 +107,16 @@
$this->showWarningMessage( ( new Message(
'fileimporter-cantimporturl' ) )->plain() );
}
+ /**
+ * @param File[][] $duplicateFileMap
+ */
+ private function showDuplicateFilesDetectedMessage( array
$duplicateFileMap ) {
+ $this->showWarningMessage(
+ ( new Message( 'fileimporter-duplicatefilesdetected' )
)->plain()
+ );
+ // TODO show details of what files have been found
+ }
+
private function showWarningMessage( $message ) {
$this->getOutput()->addHTML(
Html::rawElement(
--
To view, visit https://gerrit.wikimedia.org/r/342845
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I351049b934f23817f41903292b8c75e69e352286
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/FileImporter
Gerrit-Branch: master
Gerrit-Owner: Addshore <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
