Aaron Schulz has submitted this change and it was merged. Change subject: Update the extension to work with MediaWiki 1.21 ......................................................................
Update the extension to work with MediaWiki 1.21 - Also updated to work with new Azure SDK Change-Id: I919fb8e29777e766b478a69167bd6e0c0b1f361f --- D README.txt A WindowsAzureFileBackend.php M WindowsAzureStorage.php D includes/filerepo/backend/WindowsAzureFileBackend.php 4 files changed, 892 insertions(+), 460 deletions(-) Approvals: Aaron Schulz: Verified; Looks good to me, approved diff --git a/README.txt b/README.txt deleted file mode 100644 index eb51baf..0000000 --- a/README.txt +++ /dev/null @@ -1,80 +0,0 @@ -==Prerequisites== -You will need to have all classes from the "PHPAzure - Windows Azure SDK for -PHP" by REALDOLMEN available. - -Download it at http://phpazure.codeplex.com/ - -You can use the WindowsAzureSDK extension for MediaWiki to register the SDK with -your environment. - - -==Installation== -Copy the WindowsAzureStorage extension to your <mediawiki>/extensions directory -and add the following line to your LocalSettings.php: - -include_once( "$IP/../extensions/WindowsAzureStorage/WindowsAzureStorage.php" ); - - -==Configuration== -Add an entry for the Azure File Backend to the $wgFileBackends array in your -LocalSettings.php. Also configure your repo (i.e. the LocalFileRepo) to use the -Azure File Backend. - -The following configuration is suitable for a development environment running -Microsoft Windows Azure Service Emulators from the Azure SDK. - -$wgFileBackends[] = array( - 'name' => 'azure-backend', - 'class' => 'WindowsAzureFileBackend', - 'lockManager' => 'nullLockManager', - 'azureHost' => 'http://127.0.0.1:10000', - 'azureAccount' => 'devstoreaccount1', - 'azureKey' => 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', - - //IMPORTANT: Mind the container naming conventions! http://msdn.microsoft.com/en-us/library/dd135715.aspx - 'containerPaths' => array( - 'media-public' => 'media-public', - 'media-thumb' => 'media-thumb', - 'media-deleted' => 'media-deleted', - 'media-temp' => 'media-temp', - - ) -); - -$wgLocalFileRepo = array ( - 'class' => 'LocalRepo', - 'name' => 'local', - 'scriptDirUrl' => '/php/mediawiki-filebackend-azure', - 'scriptExtension' => '.php', - 'url' => $wgScriptPath.'/img_auth.php', // It is important to set this to img_auth. Basically, there is no alternative. - 'hashLevels' => 2, - 'thumbScriptUrl' => false, - 'transformVia404' => false, - 'deletedHashLevels' => 3, - 'backend' => 'azure-backend', - 'zones' => - array ( - 'public' => - array ( - 'container' => 'local-public', - 'directory' => '', - ), - 'thumb' => - array( - 'container' => 'local-public', - 'directory' => 'thumb', - ), - 'deleted' => - array ( - 'container' => 'local-public', - 'directory' => 'deleted', - ), - 'temp' => - array( - 'container' => 'local-public', - 'directory' => 'temp', - ) - ) -); - -$wgImgAuthPublicTest = false; \ No newline at end of file diff --git a/WindowsAzureFileBackend.php b/WindowsAzureFileBackend.php new file mode 100644 index 0000000..7f3fe7c --- /dev/null +++ b/WindowsAzureFileBackend.php @@ -0,0 +1,888 @@ +<?php +/** + * Windows Azure based file backend. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup FileBackend + * @author Aaron Schulz + * @author Markus Glaser + * @author Robert Vogel + * @author Thai Phan + */ + +use WindowsAzure\Blob\Models\ContainerACL; +use WindowsAzure\Blob\Models\CreateBlobOptions; +use WindowsAzure\Blob\Models\ListBlobsOptions; +use WindowsAzure\Blob\Models\PublicAccessType; +use WindowsAzure\Common\ServiceException; +use WindowsAzure\Common\ServicesBuilder; + +/** + * @brief Class for a Windows Azure based file backend + * + * This requires the WindowAzureSDK extension in order to work. Information on + * how to install and set up this extension are all located at + * http://www.mediawiki.org/wiki/Extension:WindowsAzureSDK. + * + * @ingroup FileBackend + * @since 1.22 + */ +class WindowsAzureFileBackend extends FileBackendStore { + /** @var IBlob */ + private $proxy; + + /** @var string */ + private $connectionString; + + /** + * @see FileBackendStore::__construct() + * Additional $config params include: + * - azureAccount : Windows Azure storage account + * - azureKey : Windows Azure storage account key + */ + public function __construct( array $config ) { + parent::__construct( $config ); + + // Generate connection string to Windows Azure storage account + $this->connectionString = 'DefaultEndpointsProtocol=http;' + . 'AccountName=' . $config['azureAccount'] . ';' + . 'AccountKey=' . $config['azureKey']; + + $this->proxy = ServicesBuilder::getInstance()->createBlobService( $this->connectionString ); + } + + /** + * @see FileBackendStore::resolveContainerName() + * @return string|null + */ + protected function resolveContainerName( $container ) { + $container = strtolower( $container ); + + $container = preg_replace( '#[^a-z0-9\-]#', '', $container ); + $container = preg_replace( '#^-#', '', $container ); + $container = preg_replace( '#-$#', '', $container ); + $container = preg_replace( '#-{2,}#', '-', $container ); + + return $container; + } + + /** + * @see FileBackendStore::resolveContainerPath() + * @return null + */ + protected function resolveContainerPath( $container, $relStoragePath ) { + if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) { + return null; + } elseif ( strlen( urlencode( $relStoragePath ) ) > 1024 ) { + return null; + } + return $relStoragePath; + } + + /** + * @see FileBackendStore::isPathUsableInternal() + * @return bool + */ + public function isPathUsableInternal( $storagePath ) { + list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath ); + if ( $rel === null ) { + return false; // invalid + } + + try { + $this->proxy->getContainerProperties( $container ); + return true; // container exists + } catch ( ServiceException $e ) { + switch ( $e->getCode() ) { + case 404: + break; + + default: // some other exception? + $this->handleException( $e, null, __METHOD__, array( 'path' => $storagePath ) ); + } + + return false; + } + } + + /** + * @see FileBackendStore::doCreateInternal() + * @return Status + */ + protected function doCreateInternal( array $params ) { + $status = Status::newGood(); + + list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] ); + if ( $dstRel === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); + return $status; + } + + // (a) Get a SHA-1 hash of the object + $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 ); + + // (b) Actually create the object + try { + $options = new CreateBlobOptions(); + $options->setMetadata( array( 'sha1base36' => $sha1Hash ) ); + $this->proxy->createBlockBlob( $dstCont, $dstRel, (string)$params['content'], $options ); + } catch ( ServiceException $e ) { + switch ( $e->getCode() ) { + case 404: + $status->fatal( 'backend-fail-create', $params['dst'] ); + break; + + default: // some other exception? + $this->handleException( $e, $status, __METHOD__, $params ); + } + } + + return $status; + } + + /** + * @see FileBackendStore::doStoreInternal() + * @return Status + */ + protected function doStoreInternal( array $params ) { + $status = Status::newGood(); + + list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] ); + if ( $dstRel === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); + return $status; + } + + // (a) Get a SHA-1 hash of the object + wfSuppressWarnings(); + $sha1Hash = sha1_file( $params['src'] ); + wfRestoreWarnings(); + if ( $sha1Hash === false ) { // source doesn't exist? + $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] ); + return $status; + } + $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 ); + + // (b) Actually store the object + try { + $options = new CreateBlobOptions(); + $options->setMetadata( array( 'sha1base36' => $sha1Hash ) ); + wfSuppressWarnings(); + $fp = fopen( $params['src'], 'rb' ); + wfRestoreWarnings(); + if ( !$fp ) { + $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] ); + } else { + $this->proxy->createBlockBlob( $dstCont, $dstRel, $fp, $options ); + fclose( $fp ); + } + } catch ( ServiceException $e ) { + switch ( $e->getCode() ) { + case 404: + $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] ); + break; + + default: // some other exception? + $this->handleException( $e, $status, __METHOD__, $params ); + } + } + + return $status; + } + + /** + * @see FileBackendStore::doCopyInternal() + * @return Status + */ + protected function doCopyInternal( array $params ) { + $status = Status::newGood(); + + list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); + if ( $srcRel === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['src'] ); + return $status; + } + + list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] ); + if ( $dstRel === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); + return $status; + } + + try { + $this->proxy->copyBlob( $dstCont, $dstRel, $srcCont, $srcRel ); + } catch ( ServiceException $e ) { + switch ( $e->getCode() ) { + case 404: + if ( empty( $params['ignoreMissingSource'] ) ) { + $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); + } + break; + + default: // some other exception? + $this->handleException( $e, $status, __METHOD__, $params ); + } + } + + return $status; + } + + /** + * @see FileBackendStore::doDeleteInternal() + * @return Status + */ + protected function doDeleteInternal( array $params ) { + $status = Status::newGood(); + + list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); + if ( $srcRel === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['src'] ); + return $status; + } + + try { + $this->proxy->deleteBlob( $srcCont, $srcRel ); + } catch ( ServiceException $e ) { + switch ( $e->getCode() ) { + case 404: + if ( empty( $params['ignoreMissingSource'] ) ) { + $status->fatal( 'backend-fail-delete', $params['src'] ); + } + break; + + default: // some other exception? + $this->handleException( $e, $status, __METHOD__, $params ); + } + } + + return $status; + } + + /** + * @see FileBackendStore::doPrepareInternal() + * @return Status + */ + protected function doPrepareInternal( $fullCont, $dir, array $params ) { + $status = Status::newGood(); + + try { + $this->proxy->createContainer( $fullCont ); + if ( !empty( $params['noAccess'] ) ) { + // Make container private to end-users... + $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) ); + } else { + // Make container public to end-users... + $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) ); + } + } catch ( ServiceException $e ) { + switch ( $e->getCode() ) { + case 404: + case 409: // container already exists + break; + + default: // some other exception? + $this->handleException( $e, $status, __METHOD__, $params ); + } + } + + return $status; + } + + /** + * @see FileBackendStore::doSecureInternal() + * @return Status + */ + protected function doSecureInternal( $fullCont, $dir, array $params ) { + $status = Status::newGood(); + if ( empty( $params['noAccess'] ) ) { + return $status; // nothing to do + } + + // Restrict container from end-users... + try { + $acl = new ContainerAcl(); + $acl->setPublicAccess( PublicAccessType::NONE ); + $this->proxy->setContainerAcl( $fullCont, $acl ); + } catch ( ServiceException $e ) { + $this->handleException( $e, $status, __METHOD__, $params ); + } + + return $status; + } + + /** + * @see FileBackendStore::doPublishInternal() + * @return Status + */ + protected function doPublishInternal( $fullCont, $dir, array $params ) { + $status = Status::newGood(); + + // Unrestrict container from end-users... + try { + $acl = new ContainerAcl(); + $acl->setPublicAccess( PublicAccessType::BLOBS_ONLY ); + $this->proxy->setContainerAcl( $fullCont, $acl ); + } catch ( ServiceException $e ) { + $this->handleException( $e, $status, __METHOD__, $params ); + } + + return $status; + } + + /** + * @see FileBackendStore::doFileExists() + * @return array|bool|null + */ + protected function doGetFileStat( array $params ) { + list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); + if ( $srcRel === null ) { + return false; // invalid storage path + } + + try { + $this->addMissingMetadata( $srcCont, $srcRel, $params['src'] ); + $properties = $this->proxy->getBlobProperties( $srcCont, $srcRel ); + $timestamp = $properties->getProperties()->getLastModified()->getTimestamp(); + $size = $properties->getProperties()->getContentLength(); + $metadata = $properties->getMetadata(); + $sha1 = $metadata['sha1base36']; + $stat = array( + 'mtime' => wfTimestamp( TS_MW, $timestamp ), + 'size' => $size, + 'sha1' => $sha1 + ); + } catch ( ServiceException $e ) { + switch ( $e->getCode() ) { + case 404: + $stat = false; + break; + + default: // some other exception? + $stat = null; + $this->handleException( $e, null, __METHOD__, $params ); + } + } + + return $stat; + } + + /** + * Fill in any missing blob metadata and save it to Azure + * + * @param $srcCont string Container name + * @param $srcRel string Blob name + * @param $path string Storage path to object + * @return bool Success + * @throws Exception Azure Storage service exception + */ + protected function addMissingMetadata( $srcCont, $srcRel, $path ) { + $metadata = $this->proxy->getBlobMetadata( $srcCont, $srcRel )->getMetadata(); + if ( isset( $metadata['sha1base36'] ) ) { + return true; // nothing to do + } + wfProfileIn( __METHOD__ ); + trigger_error( "$path was not stored with SHA-1 metadata.", E_USER_WARNING ); + $status = Status::newGood(); + $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status ); + if ( $status->isOK() ) { + $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1 ) ); + if ( $tmpFile ) { + $hash = $tmpFile->getSha1Base36(); + if ( $hash !== false ) { + $this->proxy->setBlobMetadata( $srcCont, $srcRel, array( 'sha1base36' => $hash ) ); + wfProfileOut( __METHOD__ ); + return true; // success + } + } + } + trigger_error( "Unable to set SHA-1 metadata for $path", E_USER_WARNING ); + // Set the SHA-1 metadata to 0 (setting to false doesn't seem to work) + $this->proxy->setBlobMetadata( $srcCont, $srcRel, array( 'sha1base36' => 0 ) ); + wfProfileOut( __METHOD__ ); + return false; // failed + } + + /** + * @see FileBackendStore::doDirectoryExists() + * @return bool|null + */ + protected function doDirectoryExists( $fullCont, $dir, array $params ) { + try { + $prefix = ( $dir == '' ) ? null : "{$dir}/"; + + $options = new ListBlobsOptions(); + $options->setMaxResults( 1 ); + $options->setPrefix( $prefix ); + + $blobs = $this->proxy->listBlobs( $fullCont, $options )->getBlobs(); + + return ( count( $blobs ) > 0 ); + } catch ( ServiceException $e ) { + switch ( $e->getCode() ) { + case 404: + return false; + + default: // some other exception? + $this->handleException( $e, null, __METHOD__, + array( 'cont' => $fullCont, 'dir' => $dir ) ); + return null; + } + } + } + + /** + * @see FileBackendStore::getDirectoryListInternal() + * @return AzureFileBackendDirList + */ + public function getDirectoryListInternal( $fullCont, $dir, array $params ) { + return new AzureFileBackendDirList( $this, $fullCont, $dir, $params ); + } + + /** + * @see FileBackendStore::getFileListInternal() + * @return AzureFileBackendFileList + */ + public function getFileListInternal( $fullCont, $dir, array $params ) { + return new AzureFileBackendFileList( $this, $fullCont, $dir, $params ); + } + + public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) { + $dirs = array(); + if ( $after === INF ) { + return $dirs; + } + wfProfileIn( __METHOD__ . '-' . $this->name ); + + try { + $prefix = ( $dir == '' ) ? null : "{$dir}/"; + + $options = new ListBlobsOptions(); + $options->setMaxResults( $limit ); + $options->setMarker( $after ); + $options->setPrefix( $prefix ); + + $objects = array(); + + if ( !empty( $params['topOnly'] ) ) { + // Blobs are listed in alphabetical order in the response body, with + // upper-case letters listed first. + $blobs = $this->proxy->listBlobs( $fullCont, $options )->getBlobs(); + foreach ( $blobs as $blob ) { + $name = $blob->getName(); + if ( $prefix === null ) { + if ( !preg_match( '#\/#', $name ) ) { + continue; + } + $dirray = preg_split( '#\/#', $name ); + $name = $dirray[0] . '/'; + $objects[] = $name; + } + $name = preg_replace( '#[^/]*$#', '', $name ); + if ( preg_match( '#^' . $prefix . '(\/|)$#', $name ) ) continue; + $dirray = preg_split( '#\/#', $name ); + $elements = count( preg_split( '#\/#', $prefix ) ); + $name = ''; + for ( $i = 0; $i < $elements; $i++ ) { + $name = $name . $dirray[$i] . '/'; + } + $objects[] = $name; + } + $dirs = array_unique( $objects ); + } else { + // Get directory from last item of prior page + $lastDir = $this->getParentDir( $after ); // must be first page + $blobs = $this->proxy->listBlobs( $fullCont, $options )->getBlobs(); + + // Generate an array of blob names + foreach ( $blobs as $blob ) { + array_push( $objects, $blob->getName() ); + } + + foreach ( $objects as $object ) { // files + $objectDir = $this->getParentDir( $object ); // directory of object + if ( $objectDir !== false && $objectDir !== $dir ) { + if ( strcmp( $objectDir, $lastDir ) > 0 ) { + $pDir = $objectDir; + do { // add dir and all its parent dirs + $dirs[] = "{$pDir}/"; + $pDir = $this->getParentDir( $pDir ); + } while ( $pDir !== false // sanity + && strcmp( $pDir, $lastDir ) > 0 // not done already + && strlen( $pDir ) > strlen( $dir ) // within $dir + ); + } + $lastDir = $objectDir; + } + } + } + + if ( count( $objects ) < $limit ) { + $after = INF; // avoid a second RTT + } else { + $after = end( $objects ); // update last item + } + } catch ( ServiceException $e ) { + switch ( $e->getCode() ) { + case 404: + break; + + default: // some other exception? + $this->handleException( $e, null, __METHOD__, + array( 'cont' => $fullCont, 'dir' => $dir ) ); + } + } + + wfProfileOut( __METHOD__ . '-' . $this->name ); + return $dirs; + } + + protected function getParentDir( $path ) { + return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false; + } + + /** + * Do not call this function outside of AzureFileBackendFileList + * + * @return array List of relative paths of files under $dir + */ + public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) { + $files = array(); + if ( $after === INF ) { + return $files; + } + wfProfileIn( __METHOD__ . '-' . $this->name ); + + try { + $prefix = ( $dir == '' ) ? null : "{$dir}/"; + + $options = new ListBlobsOptions(); + $options->setMaxResults( $limit ); + $options->setMarker( $after ); + $options->setPrefix( $prefix ); + + $objects = array(); + + if ( !empty( $params['topOnly'] ) ) { + $options->setDelimiter( '/' ); + + $blobs = $this->proxy->listBlobs( $fullCont, $options )->getBlobs(); + + foreach ( $blobs as $blob ) { + array_push( $objects, $blob->getName() ); + } + + foreach ( $objects as $object ) { + if ( substr( $object, -1 ) !== '/' ) { + $files[] = $object; + } + } + } else { + $blobs = $this->proxy->listBlobs( $fullCont, $options )->getBlobs(); + + foreach ( $blobs as $blob ) { + array_push( $objects, $blob->getName() ); + } + + $files = $objects; + } + + if ( count( $objects ) < $limit ) { + $after = INF; + } else { + $after = end( $objects ); + } + } catch ( ServiceException $e ) { + switch ( $e->getCode() ) { + case 404: + break; + + default: // some other exception? + $this->handleException( $e, null, __METHOD__, + array( 'cont' => $fullCont, 'dir' => $dir ) ); + } + } + + wfProfileOut( __METHOD__ . '-' . $this->name ); + return $files; + } + + /** + * @see FileBackendStore::doGetFileSha1base36() + * @return bool + */ + protected function doGetFileSha1base36( array $params ) { + $stat = $this->getFileStat( $params ); + if ( $stat ) { + return $stat['sha1']; + } else { + return false; + } + } + + /** + * @see FileBackendStore::doStreamFile() + * @return Status + */ + protected function doStreamFile( array $params ) { + $status = Status::newGood(); + + list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); + if ( $srcRel === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['src'] ); + } + + try { + $contents = $this->proxy->getBlob( $srcCont, $srcRel )->getContentStream(); + file_put_contents( 'php://output', $contents ); + } catch ( ServiceException $e ) { + switch ( $e->getCode() ) { + case 404: + $status->fatal( 'backend-fail-stream', $params['src'] ); + break; + + default: // some other exception? + $this->handleException( $e, $status, __METHOD__, $params ); + } + } + + return $status; + } + + /** + * @see FileBackendStore::doGetLocalCopyMulti() + * @return null|TempFSFile + */ + protected function doGetLocalCopyMulti( array $params ) { + $tmpFiles = array(); + + $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging + // Blindly create tmp files and stream to them, catching any exception if the file does + // not exist. Doing a stat here is useless causes infinite loops in addMissingMetadata(). + foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) { + foreach ( $pathBatch as $path ) { // each path in this concurrent batch + list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path ); + if ( $srcRel === null ) { + $tmpFiles[$path] = null; + continue; + } + $tmpFile = null; + try { + // Get source file extension + $ext = FileBackend::extensionFromPath( $path ); + // Create a new temporary file... + $tmpFile = TempFSFile::factory( 'localcopy_', $ext ); + if ( $tmpFile ) { + $tmpPath = $tmpFile->getPath(); + $contents = $this->proxy->getBlob( $srcCont, $srcRel )->getContentStream(); + file_put_contents( $tmpPath, $contents ); + } + } catch ( ServiceException $e ) { + $tmpFile = null; + switch ( $e->getCode() ) { + case 404: + break; + + default: // some other exception? + $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep ); + } + } + $tmpFiles[$path] = $tmpFile; + } + } + + return $tmpFiles; + } + + /** + * @see FileBackendStore::directoriesAreVirtual() + * @return bool + */ + protected function directoriesAreVirtual() { + return true; + } + + /** + * Log an unexpected exception for this backend. + * This also sets the Status object to have a fatal error. + * + * @param $e Exception + * @param $status Status|null + * @param $func string + * @param $params Array + * @return void + */ + protected function handleException( Exception $e, $status, $func, array $params ) { + if ( $status instanceof Status ) { + $status->fatal( 'backend-fail-internal', $this->name ); + } + if ( $e->getMessage() ) { + trigger_error( "$func:" . $e->getMessage(), E_USER_WARNING ); + } + wfDebugLog( 'AzureBackend', + get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" . + ( $e->getMessage() ? ": {$e->getMessage()}" : "" ) + ); + } +} + +/* + * AzureFileBackend helper class to page through listsings. + * Do not use this class from places outside AzureFileBackend. + * + * @ingroup FileBackend + */ +abstract class AzureFileBackendList implements Iterator { + /** @var Array */ + protected $bufferIter = array(); + protected $bufferAfter = null; // string; list items *after* this path + protected $pos = 0; // integer + /** @var Array */ + protected $params = array(); + + /** @var AzureFileBackend */ + protected $backend; + protected $container; // string; container name + protected $dir; // string; storage directory + protected $suffixStart; // integer + + const PAGE_SIZE = 9000; // file listing buffer size + + /** + * @param $backend AzureFileBackend + * @param $fullCont string Resolved container name + * @param $dir string Resolved directory relative to container + * @param $params Array + */ + public function __construct( AzureFileBackend $backend, $fullCont, $dir, array $params ) { + $this->backend = $backend; + $this->container = $fullCont; + $this->dir = $dir; + if ( substr( $this->dir, -1 ) === '/' ) { + $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash + } + if ( $this->dir == '' ) { // whole container + $this->suffixStart = 0; + } else { // dir within container + $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/" + } + $this->params = $params; + } + + /** + * @see Iterator::key() + * @return integer + */ + public function key() { + return $this->pos; + } + + /** + * @see Iterator::next() + * @return void + */ + public function next() { + // Advance to the next file in the page + next( $this->bufferIter ); + ++$this->pos; + // Check if there are no files left in this page and + // advance to the next page if this page was not empty. + if ( !$this->valid() && count( $this->bufferIter ) ) { + $this->bufferIter = $this->pageFromList( + $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params + ); // updates $this->bufferAfter + } + } + + /** + * @see Iterator::rewind() + * @return void + */ + public function rewind() { + $this->pos = 0; + $this->bufferAfter = null; + $this->bufferIter = $this->pageFromList( + $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params + ); // updates $this->bufferAfter + } + + /** + * @see Iterator::valid() + * @return bool + */ + public function valid() { + if ( $this->bufferIter === null ) { + return false; // some failure? + } else { + return ( current( $this->bufferIter ) !== false ); // no paths can have this value + } + } + + /** + * Get the given list portion (page) + * + * @param $container string Resolved container name + * @param $dir string Resolved path relative to container + * @param $after string|null + * @param $limit integer + * @param $params Array + * @return Traversable|array|null Returns null on failure + */ + abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params ); +} + +/** + * Iterator for listing directories + */ +class AzureFileBackendDirList extends AzureFileBackendList { + /** + * @see Iterator::current() + * @return string|bool String (relative path) or false + */ + public function current() { + return substr( current( $this->bufferIter ), $this->suffixStart, -1 ); + } + + /** + * @see AzureFileBackendList::pageFromList() + * @return Array|null + */ + public function pageFromList( $container, $dir, &$after, $limit, array $params ) { + return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params ); + } +} + +/** + * Iterator for listing regular files + */ +class AzureFileBackendFileList extends AzureFileBackendList { + /** + * @see Iterator::current() + * @return string|bool String (relative path) or false + */ + public function current() { + return substr( current( $this->bufferIter ), $this->suffixStart ); + } + + /** + * @see AzureFileBackendList::pageFromList() + * @return Array|null + */ + public function pageFromList( $container, $dir, &$after, $limit, array $params ) { + return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params ); + } +} diff --git a/WindowsAzureStorage.php b/WindowsAzureStorage.php index 6c2a81b..c66bdd3 100644 --- a/WindowsAzureStorage.php +++ b/WindowsAzureStorage.php @@ -1,27 +1,11 @@ <?php -/* - (c) Hallo Welt! Medienwerkstatt GmbH, 2011 GPL - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - http://www.gnu.org/copyleft/gpl.html -*/ if ( !defined( 'MEDIAWIKI' ) ) { - echo 'To install WindowsAzureStorage, put the following line in LocalSettings.php: include_once( "$IP/extensions/WindowsAzureStorage/WindowsAzureStorage.php" );'."\n"; + echo "WindowsAzureStorage extension\n"; exit( 1 ); } + +$dir = dirname( __FILE__ ) . '/'; $wgExtensionCredits['other'][] = array( 'path' => __FILE__, @@ -32,7 +16,6 @@ 'descriptionmsg' => 'windowsazurestorage-desc', ); -$dir = dirname(__FILE__) . '/'; $wgExtensionMessagesFiles['WindowsAzureStorage'] = $dir . 'WindowsAzureStorage.i18n.php'; -$wgAutoloadClasses['WindowsAzureFileBackend'] = $dir . 'includes/filerepo/backend/WindowsAzureFileBackend.php'; \ No newline at end of file +$wgAutoloadClasses['WindowsAzureFileBackend'] = $dir . 'WindowsAzureFileBackend.php'; \ No newline at end of file diff --git a/includes/filerepo/backend/WindowsAzureFileBackend.php b/includes/filerepo/backend/WindowsAzureFileBackend.php deleted file mode 100644 index 182f51c..0000000 --- a/includes/filerepo/backend/WindowsAzureFileBackend.php +++ /dev/null @@ -1,359 +0,0 @@ -<?php -/** - * @file - * @ingroup FileBackend - * @author Markus Glaser - * @author Robert Vogel - * @author Hallo Welt! - Medienwerkstatt GmbH - */ - -/** - * Copied and modified from Swift FileBackend: - * - * Class for a Windows Azure Blob Storage based file backend. - * Status messages should avoid mentioning the Azure account name - * Likewise, error suppression should be used to avoid path disclosure. - * - * This requires the PHPAzure library to be present, - * which is available at http://phpazure.codeplex.com/. - * All of the library classes must be registed in $wgAutoloadClasses. - * You may use the WindowsAzureSDK MediaWiki extension to fulfill this - * requirement. - * - * @ingroup FileBackend - */ -class WindowsAzureFileBackend extends FileBackendStore { - - /** @var Microsoft_WindowsAzure_Storage_Blob */ - protected $storageClient = null; - - /** @var Array Map of container names to Azure container names */ - protected $containerPaths = array(); - - /** - * @see FileBackend::__construct() - * Additional $config params include: - * azureHost : Windows Azure server URL - * azureAccount : Windows Azure user used by MediaWiki - * azureKey : Authentication key for the above user (used to get sessions) - * //azureContainer : Identifier of the container. (Optional. If not provided wikiId will be used as container name) - * containerPaths : Map of container names to Azure container names - */ - public function __construct( array $config ) { - parent::__construct( $config ); - $this->storageClient = new Microsoft_WindowsAzure_Storage_Blob( - $config['azureHost'], - $config['azureAccount'], - $config['azureKey'] - ); - - $this->containerPaths = (array)$config['containerPaths']; - } - - /** - * @see FileBackend::resolveContainerPath() - */ - protected function resolveContainerPath( $container, $relStoragePath ) { - //Azure blob naming conventions; http://msdn.microsoft.com/en-us/library/dd135715.aspx - - if ( strlen( urlencode( $relStoragePath ) ) > 1024 ) { - return null; - } - - return $relStoragePath; - } - - /** - * @see FileBackend::doStoreInternal() - */ - function doStoreInternal( array $params ) { - $status = Status::newGood(); - list( $dstCont, $dstRel ) = $this->resolveStoragePath( $params['dst'] ); - if ( $dstRel === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); - return $status; - } - - if ( empty( $params['overwrite'] ) ) { //Blob should not be overridden - // Check if the destination object already exists - if ( $this->storageClient->blobExists( $dstCont, $dstRel ) ) { - $status->fatal( 'backend-fail-alreadyexists', $params['dst'] ); - return $status; - } - } - - try { - $result = $this->storageClient->putBlob( $dstCont, $dstRel, $params['src']); - } - catch ( Exception $e ) { - $status->fatal( 'backend-fail-store', $dstRel, $dstCont ); - } - return $status; - } - - /** - * @see FileBackend::doCopyInternal() - */ - function doCopyInternal( array $params ) { - $status = Status::newGood(); - list( $srcContainer, $srcDir ) = $this->resolveStoragePath( $params['src'] ); - list( $dstContainer, $dstDir ) = $this->resolveStoragePath( $params['dst'] ); - if ( $srcDir === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['src'] ); - return $status; - } - if ( $dstDir === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); - return $status; - } - - if ( empty( $params['overwrite'] ) ) { //Blob should not be overridden - if ( $this->storageClient->blobExists( $dstContainer, $dstDir ) ) { - $status->fatal( 'backend-fail-alreadyexists', $params['dst'] ); - return $status; - } - } - - try { - $result = $this->storageClient->copyBlob( $srcContainer, $srcDir, $dstContainer, $dstDir); - } - catch ( Exception $e ) { - $status->fatal( 'backend-fail-copy', $e->getMessage() ); - } - return $status; - } - - /** - * @see FileBackend::doDeleteInternal() - */ - function doDeleteInternal( array $params ) { - $status = Status::newGood(); - - list( $srcCont, $srcRel ) = $this->resolveStoragePath( $params['src'] ); - if ( $srcRel === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['src'] ); - return $status; - } - - // Check the source container - try { - $container = $this->storageClient->getContainer( $srcCont ); - } - catch ( Exception $e ) { - $status->fatal( 'backend-fail-delete', $srcRel ); - return $status; - } - - // Actually delete the object - try { - $this->storageClient->deleteBlob( $srcCont, $srcRel ); - } - catch ( Exception $e ) { - $status->fatal( 'backend-fail-internal' ); - } - - return $status; - } - - /** - * @see FileBackend::doCreateInternal() - */ - function doCreateInternal( array $params ) { - $status = Status::newGood(); - - list( $dstCont, $dstRel ) = $this->resolveStoragePath( $params['dst'] ); - if ( $dstRel === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); - return $status; - } - - if ( empty( $params['overwrite'] ) ) { //Blob should not be overridden - // Check if the destination object already exists - if ( $this->storageClient->blobExists( $dstCont, $dstRel ) ) { - $status->fatal( 'backend-fail-alreadyexists', $params['dst'] ); - return $status; - } - } - - // Actually create the object - try { - $this->storageClient->putBlobData( $dstCont, $dstRel, $params['content'] ); - } - catch ( Exception $e ) { - $status->fatal( 'backend-fail-internal' ); - } - return $status; - } - - /** - * @see FileBackend::doPrepare() - */ - - function doPrepareInternal( $container, $dir, array $params ) { - $status = Status::newGood(); - - list( $c, $dir ) = $this->resolveStoragePath( $params['dir'] ); - if ( $dir === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); - return $status; // invalid storage path - } - try { - $this->storageClient->createContainerIfNotExists( $c ); - $this->storageClient->setContainerAcl( $c, Microsoft_WindowsAzure_Storage_Blob::ACL_PUBLIC ); - } - catch (Exception $e ) { - $status->fatal( 'directorycreateerror', $params['dir'] ); - return $status; - } - return $status; - } - - - /** - * @see FileBackend::resolveContainerName() - */ - protected function resolveContainerName( $container ) { - //Azure container naming conventions; http://msdn.microsoft.com/en-us/library/dd135715.aspx - $container = strtolower($container); - $container = preg_replace( '#[^a-z0-9\-]#', '', $container ); - $container = preg_replace( '#^-#', '', $container ); - $container = preg_replace( '#-$#', '', $container ); - $container = preg_replace( '#-{2,}#', '-', $container ); - - return $container; - } - - /** - * @see FileBackend::secure() - */ - function doSecureInternal( $container, $dir, array $params ) { - $status = Status::newGood(); - - try { - if ( $this->storageClient->containerExists( $container ) ) { - if ( $params['noAccess'] == true ) { - $this->storageClient->setContainerAcl( $container, Microsoft_WindowsAzure_Storage_Blob::ACL_PRIVATE ); - } - } - } - catch (Exception $e ) { - $status->fatal( 'directorycreateerror', $container ); - return $status; - } - return $status; - } - - /** - * @see FileBackend::getFileList() - */ - function getFileListInternal( $container, $dir, array $params ) { - $files = array(); - - try { - if ( trim($dir) == '' ) { - $blobs = $this->storageClient->listBlobs($container); - } - else { - if ( strrpos($dir, '/') != strlen($dir)-1 ) { - $dir = $dir.'/'; - } - $blobs = $this->storageClient->listBlobs($container, $dir); - } - - foreach( $blobs as $blob ) { - // Only return the actual file name without the / - $tempName = $blob->name; - if ( trim($dir) != '' ) { - if ( strstr( $tempName, $dir ) !== false ) { - $tempName = substr($tempName, strpos( $tempName, $dir ) + strlen( $dir ) ); - $files[] = $tempName; - } - } else { - $files[] = $tempName; - } - - } - } - catch( Exception $e ) { - return null; - } - - // if there are no files matching the prefix, return empty array - return $files; - } - - /** - * @see FileBackend::getLocalCopy() - */ - function getLocalCopy( array $params ) { - list( $srcCont, $srcRel ) = $this->resolveStoragePath( $params['src'] ); - if ( $srcRel === null ) { - return null; - } - - // Get source file extension - $ext = FileBackend::extensionFromPath( $srcRel ); - // Create a new temporary file... - $tmpFile = TempFSFile::factory( wfBaseName( $srcRel ) . '_', $ext ); - if ( !$tmpFile ) { - return null; - } - $tmpPath = $tmpFile->getPath(); - - try { - $this->storageClient->getBlob( $srcCont, $srcRel, $tmpPath ); - } - catch ( Exception $e ) { - return null; - } - - return $tmpFile; - } - - /** - * @see FileBackend::getFileStat() - */ - protected function doGetFileStat( array $params ) { - list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); - if ( $srcRel === null ) { - return false; // invalid storage path - } - - $timestamp= false; - $size = false; - - try { - $blob = $this->storageClient->getBlobInstance( $srcCont, $srcRel ); - $timestamp = wfTimestamp( TS_MW, $blob->LastModified ); - $size = $blob->Size; - return array( - 'mtime' => $timestamp, - 'size' => $size - ); - } catch ( Exception $e ) { - $stat = null; - } - return false; - } - - /** - * Check if a file can be created at a given storage path. - * FS backends should check if the parent directory exists and the file is writable. - * Backends using key/value stores should check if the container exists. - * - * @param $storagePath string - * @return bool - */ - - public function isPathUsableInternal( $storagePath ) { - list( $c, $dir ) = $this->resolveStoragePath( $storagePath ); - if ( $dir === null ) { - return false; - } - if ( !$this->storageClient->containerExists( $c ) ) { - return false; - } - - return true; - } -} -- To view, visit https://gerrit.wikimedia.org/r/56771 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I919fb8e29777e766b478a69167bd6e0c0b1f361f Gerrit-PatchSet: 2 Gerrit-Project: mediawiki/extensions/WindowsAzureStorage Gerrit-Branch: master Gerrit-Owner: Thaiphan <[email protected]> Gerrit-Reviewer: Aaron Schulz <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
