Updated Branches: refs/heads/4.2 2976d9c79 -> b63270f1d
CLOUDSTACK-4816: Make S3 upload multipart or singlepart configurable. Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/975422a9 Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/975422a9 Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/975422a9 Branch: refs/heads/4.2 Commit: 975422a9500f17c5b595e8c4c6075b3b49463337 Parents: 2976d9c Author: Min Chen <[email protected]> Authored: Fri Oct 4 16:04:36 2013 -0700 Committer: Min Chen <[email protected]> Committed: Fri Oct 4 17:38:17 2013 -0700 ---------------------------------------------------------------------- api/src/com/cloud/agent/api/to/S3TO.java | 14 ++++- .../storage/template/S3TemplateDownloader.java | 25 ++++----- .../driver/S3ImageStoreDriverImpl.java | 5 +- server/src/com/cloud/configuration/Config.java | 1 + .../resource/NfsSecondaryStorageResource.java | 7 ++- utils/src/com/cloud/utils/S3Utils.java | 56 ++++++++++++++++++++ 6 files changed, 92 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cloudstack/blob/975422a9/api/src/com/cloud/agent/api/to/S3TO.java ---------------------------------------------------------------------- diff --git a/api/src/com/cloud/agent/api/to/S3TO.java b/api/src/com/cloud/agent/api/to/S3TO.java index b1b692a..ab08a69 100644 --- a/api/src/com/cloud/agent/api/to/S3TO.java +++ b/api/src/com/cloud/agent/api/to/S3TO.java @@ -39,6 +39,7 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO { private Integer socketTimeout; private Date created; private boolean enableRRS; + private boolean multipartEnabled; public S3TO() { @@ -50,7 +51,7 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO { final String secretKey, final String endPoint, final String bucketName, final Boolean httpsFlag, final Integer connectionTimeout, final Integer maxErrorRetry, - final Integer socketTimeout, final Date created, final boolean enableRRS) { + final Integer socketTimeout, final Date created, final boolean enableRRS, final boolean multipart) { super(); @@ -66,6 +67,7 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO { this.socketTimeout = socketTimeout; this.created = created; this.enableRRS = enableRRS; + this.multipartEnabled = multipart; } @@ -268,7 +270,6 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO { } - public boolean getEnableRRS() { return enableRRS; } @@ -277,5 +278,14 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO { this.enableRRS = enableRRS; } + public boolean isMultipartEnabled() { + return multipartEnabled; + } + + public void setMultipartEnabled(boolean multipartEnabled) { + this.multipartEnabled = multipartEnabled; + } + + } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/975422a9/core/src/com/cloud/storage/template/S3TemplateDownloader.java ---------------------------------------------------------------------- diff --git a/core/src/com/cloud/storage/template/S3TemplateDownloader.java b/core/src/com/cloud/storage/template/S3TemplateDownloader.java index 340e0db..c394368 100644 --- a/core/src/com/cloud/storage/template/S3TemplateDownloader.java +++ b/core/src/com/cloud/storage/template/S3TemplateDownloader.java @@ -43,15 +43,11 @@ import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import com.amazonaws.AmazonClientException; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.ProgressEvent; import com.amazonaws.services.s3.model.ProgressListener; import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.StorageClass; -import com.amazonaws.services.s3.transfer.TransferManager; -import com.amazonaws.services.s3.transfer.Upload; import com.cloud.agent.api.storage.Proxy; import com.cloud.agent.api.to.S3TO; import com.cloud.utils.Pair; @@ -226,9 +222,6 @@ public class S3TemplateDownloader implements TemplateDownloader { // compute s3 key s3Key = join(asList(installPath, fileName), S3Utils.SEPARATOR); - // multi-part upload using S3 api to handle > 5G input stream - TransferManager tm = new TransferManager(S3Utils.acquireClient(s3)); - // download using S3 API ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(remoteSize); @@ -261,11 +254,19 @@ public class S3TemplateDownloader implements TemplateDownloader { } }); - // TransferManager processes all transfers asynchronously, - // so this call will return immediately. - Upload upload = tm.upload(putObjectRequest); - - upload.waitForCompletion(); + + if ( s3.isMultipartEnabled()){ + // use TransferManager to do multipart upload + S3Utils.mputObject(s3, putObjectRequest); + } else{ + // single part upload, with 5GB limit in Amazon + S3Utils.putObject(s3, putObjectRequest); + while (status != TemplateDownloader.Status.DOWNLOAD_FINISHED && + status != TemplateDownloader.Status.UNRECOVERABLE_ERROR && + status != TemplateDownloader.Status.ABORTED) { + // wait for completion + } + } // finished or aborted Date finish = new Date(); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/975422a9/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java ---------------------------------------------------------------------- diff --git a/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java b/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java index ac681cc..117beee 100644 --- a/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java +++ b/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java @@ -64,7 +64,10 @@ public class S3ImageStoreDriverImpl extends BaseImageStoreDriverImpl { details.get(ApiConstants.S3_SOCKET_TIMEOUT) == null ? null : Integer.valueOf(details .get(ApiConstants.S3_SOCKET_TIMEOUT)), imgStore.getCreated(), _configDao.getValue(Config.S3EnableRRS.toString()) == null ? false : Boolean.parseBoolean(_configDao - .getValue(Config.S3EnableRRS.toString()))); + .getValue(Config.S3EnableRRS.toString())), + _configDao.getValue(Config.S3EnableMultiPartUpload.toString()) == null ? true : Boolean.parseBoolean(_configDao + .getValue(Config.S3EnableMultiPartUpload.toString())) + ); } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/975422a9/server/src/com/cloud/configuration/Config.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java index d4f9607..613a882 100755 --- a/server/src/com/cloud/configuration/Config.java +++ b/server/src/com/cloud/configuration/Config.java @@ -425,6 +425,7 @@ public enum Config { // object store S3EnableRRS("Advanced", ManagementServer.class, Boolean.class, "s3.rrs.enabled", "false", "enable s3 reduced redundancy storage", null), + S3EnableMultiPartUpload("Advanced", ManagementServer.class, Boolean.class, "s3.multipart.enabled", "true", "enable s3 multipart upload", null), // VMSnapshots VMSnapshotMax("Advanced", VMSnapshotManager.class, Integer.class, "vmsnapshot.max", "10", "Maximum vm snapshots for a vm", null), http://git-wip-us.apache.org/repos/asf/cloudstack/blob/975422a9/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java ---------------------------------------------------------------------- diff --git a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 3a563cb..a5a9f2b 100755 --- a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.storage.resource; import static com.cloud.utils.S3Utils.putFile; +import static com.cloud.utils.S3Utils.mputFile; import static com.cloud.utils.StringUtils.join; import static com.cloud.utils.db.GlobalLock.executeWithNoWaitLock; import static java.lang.String.format; @@ -821,7 +822,11 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } ImageFormat format = this.getTemplateFormat(srcFile.getName()); String key = destData.getPath() + S3Utils.SEPARATOR + srcFile.getName(); - putFile(s3, srcFile, bucket, key); + if (s3.isMultipartEnabled()){ + mputFile(s3, srcFile, bucket, key); + } else{ + putFile(s3, srcFile, bucket, key); + } DataTO retObj = null; if (destData.getObjectType() == DataObjectType.TEMPLATE) { http://git-wip-us.apache.org/repos/asf/cloudstack/blob/975422a9/utils/src/com/cloud/utils/S3Utils.java ---------------------------------------------------------------------- diff --git a/utils/src/com/cloud/utils/S3Utils.java b/utils/src/com/cloud/utils/S3Utils.java index 5ee5783..ce4d4b7 100644 --- a/utils/src/com/cloud/utils/S3Utils.java +++ b/utils/src/com/cloud/utils/S3Utils.java @@ -48,6 +48,7 @@ import org.apache.commons.lang.ArrayUtils; import org.apache.log4j.Logger; import com.amazonaws.AmazonClientException; +import com.amazonaws.AmazonServiceException; import com.amazonaws.ClientConfiguration; import com.amazonaws.HttpMethod; import com.amazonaws.auth.AWSCredentials; @@ -61,6 +62,9 @@ import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.amazonaws.services.s3.transfer.TransferManager; +import com.amazonaws.services.s3.transfer.Upload; + import com.cloud.utils.exception.CloudRuntimeException; public final class S3Utils { @@ -171,6 +175,58 @@ public final class S3Utils { } + // multi-part upload file + public static void mputFile(final ClientOptions clientOptions, + final File sourceFile, final String bucketName, final String key) throws InterruptedException { + + assert clientOptions != null; + assert sourceFile != null; + assert !isBlank(bucketName); + assert !isBlank(key); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(format("Multipart sending file %1$s as S3 object %2$s in " + + "bucket %3$s", sourceFile.getName(), key, bucketName)); + } + TransferManager tm = new TransferManager(S3Utils.acquireClient(clientOptions)); + Upload upload = tm.upload(bucketName, key, sourceFile); + upload.waitForCompletion(); + } + + // multi-part upload object + public static void mputObject(final ClientOptions clientOptions, + final InputStream sourceStream, final String bucketName, final String key) throws InterruptedException { + + assert clientOptions != null; + assert sourceStream != null; + assert !isBlank(bucketName); + assert !isBlank(key); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(format("Multipart sending stream as S3 object %1$s in " + + "bucket %2$s", key, bucketName)); + } + TransferManager tm = new TransferManager(S3Utils.acquireClient(clientOptions)); + Upload upload = tm.upload(bucketName, key, sourceStream, null); + upload.waitForCompletion(); + } + + // multi-part upload object + public static void mputObject(final ClientOptions clientOptions, + final PutObjectRequest req) throws InterruptedException { + + assert clientOptions != null; + assert req != null; + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Multipart sending object to S3 using PutObjectRequest"); + } + TransferManager tm = new TransferManager(S3Utils.acquireClient(clientOptions)); + Upload upload = tm.upload(req); + upload.waitForCompletion(); + + } + public static void setObjectAcl(final ClientOptions clientOptions, final String bucketName, final String key, final CannedAccessControlList acl) {
