Parent5446 has uploaded a new change for review.
https://gerrit.wikimedia.org/r/52089
Change subject: Added S3 Backend (do not merge).
......................................................................
Added S3 Backend (do not merge).
Change-Id: Id2f1492fa7dc8d1467fcf0856080e4d12fd964cc
---
M AWS.php
A s3/AmazonS3FileBackend.php
A s3/AmazonS3Repo.php
3 files changed, 487 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/AWS
refs/changes/89/52089/1
diff --git a/AWS.php b/AWS.php
index 6ab567e..9a92986 100644
--- a/AWS.php
+++ b/AWS.php
@@ -43,6 +43,20 @@
*/
$wgAWSRegion = false;
+$wgAutoloadClasses['AmazonS3Repo'] = __DIR__ . '/s3/AmazonS3Repo.php';
+$wgAutoloadClasses['AmazonS3FileBackend'] = __DIR__ .
'/s3/AmazonS3FileBackend.php';
+
$wgExtensionMessagesFiles['AWS'] = __DIR__ . '/AWS.i18n.php';
+$wgLockManagers[] = array(
+ 'name' => 'memcLockManager',
+ 'class' => 'MemcLockManager'
+);
+
+$wgFileBackends[] = array(
+ 'name' => 'amazons3-backend',
+ 'class' => 'AmazonS3FileBackend',
+ 'lockManager' => 'memcLockManager',
+);
+
require_once( __DIR__ . '/vendor/autoload.php' );
diff --git a/s3/AmazonS3FileBackend.php b/s3/AmazonS3FileBackend.php
new file mode 100644
index 0000000..333cab4
--- /dev/null
+++ b/s3/AmazonS3FileBackend.php
@@ -0,0 +1,458 @@
+<?php
+
+use Aws\S3\S3Client;
+
+class AmazonS3FileBackend extends FileBackendStore {
+ function __construct( array $config ) {
+ global $wgAWSCredentials, $wgAWSRegion;
+
+ parent::__construct( $config );
+
+ $this->encryption = $wgAWSServerSideEncryption ? 'AES256' :
null;
+ $this->client = S3Client::factory( array(
+ 'key' => $wgAWSCredentials['key'],
+ 'secret' => $wgAWSCredentials['secret'],
+ 'region' => $wgAWSRegion,
+ 'scheme' => WebRequest::detectProtocol(),
+ 'ssl.certificate_authority' => true
+ ) );
+ }
+
+ function directoriesAreVirtual() {
+ return true;
+ }
+
+ function isPathUsableInternal( $storagePath ) {
+ return count( $storagePath ) <= 1024;
+ }
+
+ function resolveContainerPath( $container, $relStoragePath ) {
+ $validLabel = '[a-z0-9]+([a-z0-9\-]+[a-z0-9])?';
+ if(
+ $this->client->isValidBucketName( $container ) &&
+ $this->isPathUsableInternal( $relStoragePath )
+ ) {
+ return $relStoragePath;
+ } else {
+ return null;
+ }
+ }
+
+
+
+ function doCreateInternal( array $params ) {
+ $status = Status::newGood();
+
+ list( $container, $key ) = $this->resolveStoragePathReal(
$params['dst'] );
+ if( $container === null || $key == null ) {
+ $status->fatal( 'backend-fail-invalidpath',
$params['dst'] );
+ return $status;
+ }
+
+ $params['headers'] = isset( $params['headers'] ) ?
$params['headers'] : array();
+ $params['headers'] += array_fill_keys( array(
+ 'Cache-Control' => null,
+ 'Content-Encoding' => null,
+ 'Content-Language' => null,
+ 'Content-Type' => null,
+ 'Expires' => null
+ ), null );
+
+ $res = $this->client->putObject( array(
+ 'ACL' => $this->isSecure( $container ) ? 'private' :
'public-read',
+ 'Body' => $params['content'],
+ 'Bucket' => $container,
+ 'CacheControl' => $params['headers']['Cache-Control'],
+ 'ContentDisposition' => $this->truncDisp( $disposition
),
+ 'ContentEncoding' =>
$params['headers']['ContentEncoding'],
+ 'ContentLanguage' =>
$params['headers']['ContentLanguage'],
+ 'ContentType' => $params['headers']['ContentType'],
+ 'Expires' => $params['headers']['Expires'],
+ 'Key' => $this->client->encodeKey( $key ),
+ 'ServerSideEncryption' => $this->encryption,
+ 'ContentMD5' => md5( $params['content'] ),
+ ) );
+
+ if( !$res['RequestId'] ) {
+ $status->fatal( 'backend-fail-create', $params['dst'] );
+ }
+
+ return $status;
+ }
+
+ function doStoreInternal( array $params ) {
+ $params['content'] = fopen( $params['src'], 'r' );
+ return $this->doCreateInternal( $params );
+ }
+
+ function doCopyInternal( array $params ) {
+ $status = Status::newGood();
+
+ list( $srcContainer, $srcKey ) = $this->resolveStoragePathReal(
$params['src'] );
+ list( $dstContainer, $dstKey ) = $this->resolveStoragePathReal(
$params['dst'] );
+ if( $srcContainer === null || $srcKey == null ) {
+ $status->fatal( 'backend-fail-invalidpath',
$params['src'] );
+ }
+ if( $dstContainer === null || $dstKey == null ) {
+ $status->fatal( 'backend-fail-invalidpath',
$params['dst'] );
+ }
+
+ if( !$status->isOK() ) {
+ return $status;
+ }
+
+ $params['headers'] += array_fill_keys( array(
+ 'Cache-Control',
+ 'Content-Encoding',
+ 'Content-Language',
+ 'Content-Type',
+ 'Expires'
+ ), null );
+
+ if( $srcContainer == $dstContainer ) {
+ $res = $this->client->copyObject( array_filter( array(
+ 'ACL' => $this->isSecure( $container ) ?
'private' : 'public-read',
+ 'Bucket' => $srcContainer,
+ 'CacheControl' =>
$params['headers']['Cache-Control'],
+ 'ContentDisposition' => $this->truncDisp(
$disposition ),
+ 'ContentEncoding' =>
$params['headers']['ContentEncoding'],
+ 'ContentLanguage' =>
$params['headers']['ContentLanguage'],
+ 'ContentType' =>
$params['headers']['ContentType'],
+ 'CopySource' => urlencode( $srcKey ),
+ 'CopySourceIfMatch' =>
$params['headers']['ETag'],
+ 'CopySourceIfModifiedSince' =>
$params['headers']['If-Modified-Since'],
+ 'Expires' => $params['headers']['Expires'],
+ 'Key' => $this->client->encodeKey( $key ),
+ 'ServerSideEncryption' => $this->encryption
+ ) ) );
+
+ if( !$res['RequestId'] ) {
+ $status->fatal( 'backend-fail-create',
$params['dst'] );
+ }
+ } else {
+ // Not in the same bucket, so a manual copy/paste has
to be done.
+ $params['content'] = $this->getFileContents( array(
'src' => $params['src'] ) );
+ $status->merge( $this->createInternal( $params ) );
+ }
+
+ return $status;
+ }
+
+ function doDeleteInternal( array $params ) {
+ $status = Status::newGood();
+
+ list( $container, $key ) = $this->resolveStoragePathReal(
$params['src'] );
+ if( $container === null || $key == null ) {
+ $status->fatal( 'backend-fail-invalidpath',
$params['src'] );
+ return $status;
+ }
+
+ $this->client->deleteObject( array(
+ 'Bucket' => $container,
+ 'Key' => $this->client->encodeKey( $key )
+ ) );
+
+ if( !$res['RequestId'] ) {
+ $status->fatal( 'backend-fail-create', $params['dst'] );
+ }
+
+ return $status;
+ }
+
+
+
+ function doDirectoryExists( $container, $dir, array $params ) {
+ // See if at least one file is in the directory.
+ $it = new AmazonS3FileIterator( $this->client, $container,
$dir, array( 'topOnly' ), 1 );
+ return $it->valid();
+ }
+
+ function doGetFileStat( array $params ) {
+ list( $container, $key ) = $this->resolveStoragePathReal(
$params['src'] );
+ if( $container === null || $key == null ) {
+ return null;
+ } elseif( !$this->client->doesObjectExist( $container, $key ) )
{
+ return false;
+ }
+
+ $res = $this->client->headObject( array(
+ 'Bucket' => $container,
+ 'Key' => $this->client->encodeKey( $key )
+ ) );
+
+ return array(
+ 'mtime' => wfTimestamp( TS_MW, $res['LastModified'] ),
+ 'size' => $res['ContentLength'],
+ 'etag' => $res['Etag']
+ );
+ }
+
+ function getFileHttpUrl( array $params ) {
+ list( $container, $key ) = $this->resolveStoragePathReal(
$params['src'] );
+ if( $container === null ) {
+ return null;
+ } elseif ( $key ) {
+ $key = "/$key";
+ }
+
+ $scheme = WebRequest::detectProtocol();
+ return "$scheme://$container.s3.amazonaws.com$key";
+ }
+
+ function getDirectoryListInternal( $container, $dir, array $params ) {
+ return new AmazonS3DirectoryIterator( $this->client,
$container, $dir, $params );
+ }
+
+ function getFileListInternal( $container, $dir, array $params ) {
+ return new AmazonS3FileIterator( $this->client, $container,
$dir, $params );
+ }
+
+
+
+ function doGetLocalCopyMulti( array $params ) {
+ $fsFiles = array();
+ if ( !isset( $params['srcs'] ) ) {
+ $params['srcs'] = array( $params['src'] );
+ }
+ foreach( $params['srcs'] as $src ) {
+ list( $container, $key ) =
$this->resolveStoragePathReal( $src );
+ if( $container === null || $key === null ) {
+ $fsFiles[$src] = null;
+ } else {
+ $ext = self::extensionFromPath( $src );
+ $tmpFile = TempFSFile::factory( 'localcopy_',
$ext );
+ if( !$tmpFile ) {
+ $fsFiles[$src] = null;
+ } else {
+ $srcPath = $this->getFileHttpUrl(
array( 'src' => $src ) );
+ $dstPath = $tmpFile->getPath();
+
+ wfSuppressWarnings();
+ $ok = copy( $srcPath, $dstPath );
+ wfRestoreWarnings();
+
+ if( !$ok ) {
+ $fsFiles[$src] = null;
+ } else {
+ $fsFiles[$src] = $tmpFile;
+ }
+ }
+ }
+ }
+ return $fsFiles;
+ }
+
+ function doGetLocalReferenceMulti( array $params ) {
+ $fsFiles = array();
+ if ( !isset( $params['srcs'] ) ) {
+ $params['srcs'] = array( $params['src'] );
+ }
+ foreach( $params['srcs'] as $src ) {
+ list( $container, $key ) =
$this->resolveStoragePathReal( $src );
+ if( $container === null || $key === null ) {
+ $fsFiles[$src] = null;
+ } else {
+ $fsFiles[$src] = new FSFile(
$this->getFileHttpUrl( array( 'src' => $src ) ) );
+ }
+ }
+ return $fsFiles;
+ }
+
+
+
+ function doPrepareInternal( $container, $dir, array $params ) {
+ $status = Status::newGood();
+
+ if( $this->client->doesBucketExist( $container ) ) {
+ return $status;
+ }
+
+ $res = $this->client->createBucket( array(
+ 'ACL' => isset( $params['noListing'] ) ? 'private' :
'public-read',
+ 'Bucket' => $container
+ ) );
+
+ if( !$res['Location'] ) {
+ $status->fatal( 'directorycreateerror', $container );
+ }
+
+ $this->client->waitUntilBucketExists( array( 'Bucket' =>
$container ) );
+
+ $status->merge( $this->doSecureInternal( $container, $dir,
$params ) );
+
+ return $status;
+ }
+
+ function doPublishInternal( $container, $dir, array $params ) {
+ $status = Status::newGood();
+
+ if( !empty( $params['listing'] ) ) {
+ $res = $this->client->putBucketAcl( array(
+ 'ACL' => 'public-read',
+ 'Bucket' => $container
+ ) );
+
+ if( !$res['RequestId'] ) {
+ $status->fatal( 'backend-fail-delete',
$container );
+ }
+ }
+
+ if( !empty( $params['access'] ) ) {
+ foreach( new AmazonS3FileIterator( $this->client,
$container, $dir, $params ) as $key ) {
+ $res = $this->client->putObjectAcl( array(
+ 'ACL' => 'public-read',
+ 'Bucket' => $container,
+ 'Key' => $key
+ ) );
+
+ if( !$res['RequestID'] ) {
+ $status->fatal( 'backend-fail-delete',
$key );
+ }
+ }
+ }
+
+ return $status;
+ }
+
+ function doSecureInternal( $container, $dir, array $params ) {
+ $status = Status::newGood();
+
+ if( !empty( $params['noListing'] ) ) {
+ $res = $this->client->putBucketAcl( array(
+ 'ACL' => 'private',
+ 'Bucket' => $container
+ ) );
+
+ if( !$res['RequestId'] ) {
+ $status->fatal( 'backend-fail-create',
$container );
+ }
+ }
+
+ if( !empty( $params['noAccess'] ) ) {
+ foreach( new AmazonS3FileIterator( $this->client,
$container, $dir, $params ) as $key ) {
+ $res = $this->client->putObjectAcl( array(
+ 'ACL' => 'private',
+ 'Bucket' => $container,
+ 'Key' => $key
+ ) );
+
+ if( !$res['RequestID'] ) {
+ $status->fatal( 'backend-fail-create',
$key );
+ }
+ }
+ }
+
+ return $status;
+ }
+
+ private function isSecure( $container ) {
+ static $pubUrl =
"http://acs.amazonaws.com/groups/global/AllUsers";;
+ $acl = $this->client->getBucketAcl( array( 'Bucket' =>
$container ) );
+ foreach( $acl['Grants'] as $grant ) {
+ if( $grant['Grantee']['URI'] == $pubUrl ) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private function truncDisp( $disposition ) {
+ $res = '';
+ foreach( explode( ';', $disposition ) as $part ) {
+ $part = trim( $part );
+ $new = $res ? $part : "{$res};{$part}";
+ if( strlen( $new ) <= 255 ) {
+ $res = $new;
+ } else {
+ break;
+ }
+ }
+ return $res;
+ }
+}
+
+class AmazonS3FileIterator implements Iterator {
+ private $client, $container, $dir, $topOnly, $limit;
+ private $index, $results, $market, $finished;
+
+ public function __construct( S3Client $client, $container, $dir, array
$params, $limit = 500 ) {
+ $this->client = $client;
+ $this->container = $container;
+ $this->dir = $dir;
+ $this->limit = $limit;
+ $this->topOnly = $params['topOnly'];
+
+ $this->rewind();
+ }
+
+ public function key() {
+ $this->init();
+ return $this->index;
+ }
+
+ public function current() {
+ $this->init();
+ return $this->results['Contents'][$this->index]['Key'];
+ }
+
+ public function next() {
+ if( $this->topOnly ) {
+ do {
+ ++$this->index;
+ } while( strpos( $this->current(), '/', strlen(
$this->dir ) ) !== false );
+ } else {
+ ++$this->index;
+ }
+ }
+
+ public function rewind() {
+ $this->results = null;
+ $this->marker = null;
+ $this->index = 0;
+ $this->finished = false;
+ }
+
+ public function valid() {
+ $this->init();
+ return !$this->finished || $this->index < count(
$this->results['Contents'] );
+ }
+
+ private function init() {
+ if(
+ (
+ $this->results === null ||
+ $this->index >= count(
$this->results['Contents'] )
+ ) &&
+ !$this->finished
+ ) {
+ $this->results = $this->client->listObjects( array(
+ 'Bucket' => $this->container,
+ 'Delimiter' => '/',
+ 'Marker' => $this->marker,
+ 'MaxKeys' => $this->limit,
+ 'Prefix' => $this->dir
+ ) );
+
+ $this->index = 0;
+ $this->marker = $this->results['Marker'];
+ $this->finished = !$this->results['IsTruncated'];
+ return true;
+ } else {
+ return false;
+ }
+ }
+}
+
+class AmazonS3DirectoryIterator extends AmazonS3FileIterator {
+ private $directories = array();
+
+ function current() {
+ return dirname( parent::current() );
+ }
+
+ function next() {
+ do {
+ parent::next();
+ } while( array_key_exists( $this->current(), $this->directories
) );
+ }
+}
diff --git a/s3/AmazonS3Repo.php b/s3/AmazonS3Repo.php
new file mode 100644
index 0000000..074211b
--- /dev/null
+++ b/s3/AmazonS3Repo.php
@@ -0,0 +1,15 @@
+<?php
+
+class AmazonS3Repo extends LocalRepo {
+ function __construct( array $info = null ) {
+ settype( $info, 'array' );
+ $info['backend'] = 'amazons3-backend';
+
+ parent::__construct( $info );
+ }
+
+ function getZoneUrl( $zone, $ext = null ) {
+ $params = array( 'src' => $this->getZonePath( $zone ) );
+ return $this->backend->getFileHttpUrl( $params );
+ }
+}
--
To view, visit https://gerrit.wikimedia.org/r/52089
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Id2f1492fa7dc8d1467fcf0856080e4d12fd964cc
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/AWS
Gerrit-Branch: master
Gerrit-Owner: Parent5446 <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
