HADOOP-15267. S3A multipart upload fails when SSE-C encryption is enabled. Contributed by Anis Elleuch.
(cherry picked from commit 1dedc68f9d8d8544d715e67ee77cd3f017c21699) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e0307e53 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e0307e53 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e0307e53 Branch: refs/heads/trunk Commit: e0307e53e2110cb6b418861a7471e97a013c16e2 Parents: d69b31f Author: Steve Loughran <ste...@apache.org> Authored: Wed Mar 7 18:51:10 2018 +0000 Committer: Steve Loughran <ste...@apache.org> Committed: Wed Mar 7 18:51:10 2018 +0000 ---------------------------------------------------------------------- .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 18 ++++++ .../apache/hadoop/fs/s3a/MockS3AFileSystem.java | 6 ++ .../scale/ITestS3AHugeFilesSSECDiskBlocks.java | 58 ++++++++++++++++++++ 3 files changed, 82 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0307e53/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 4424eba..eb65cfa 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -1572,6 +1572,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities { long len = request.getPartSize(); incrementPutStartStatistics(len); try { + setOptionalUploadPartRequestParameters(request); UploadPartResult uploadPartResult = s3.uploadPart(request); incrementPutCompletedStatistics(true, len); return uploadPartResult; @@ -2564,6 +2565,23 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities { } /** + * Sets server side encryption parameters to the part upload + * request when encryption is enabled. + * @param request upload part request + */ + protected void setOptionalUploadPartRequestParameters( + UploadPartRequest request) { + switch (serverSideEncryptionAlgorithm) { + case SSE_C: + if (isNotBlank(getServerSideEncryptionKey(bucket, getConf()))) { + request.setSSECustomerKey(generateSSECustomerKey()); + } + break; + default: + } + } + + /** * Initiate a multipart upload from the preconfigured request. * Retry policy: none + untranslated. * @param request request to initiate http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0307e53/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java index 4952580..1062a12 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java @@ -24,6 +24,7 @@ import java.net.URI; import com.amazonaws.AmazonClientException; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; +import com.amazonaws.services.s3.model.UploadPartRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -278,6 +279,11 @@ public class MockS3AFileSystem extends S3AFileSystem { } @Override + protected void setOptionalUploadPartRequestParameters( + UploadPartRequest request) { + } + + @Override @SuppressWarnings("deprecation") public long getDefaultBlockSize() { return mock.getDefaultBlockSize(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0307e53/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesSSECDiskBlocks.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesSSECDiskBlocks.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesSSECDiskBlocks.java new file mode 100644 index 0000000..2e5185b --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesSSECDiskBlocks.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.hadoop.fs.s3a.scale; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.s3a.Constants; +import org.apache.hadoop.fs.s3a.S3AEncryptionMethods; +import org.apache.hadoop.fs.s3a.S3ATestUtils; + +import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfEncryptionTestsDisabled; + +/** + * Concrete class that extends {@link ITestS3AHugeFilesDiskBlocks} + * and tests huge files operations with SSE-C encryption enabled. + * Skipped if the SSE tests are disabled. + */ +public class ITestS3AHugeFilesSSECDiskBlocks + extends ITestS3AHugeFilesDiskBlocks { + + private static final String KEY_1 + = "4niV/jPK5VFRHY+KNb6wtqYd4xXyMgdJ9XQJpcQUVbs="; + + @Override + public void setup() throws Exception { + super.setup(); + skipIfEncryptionTestsDisabled(getConfiguration()); + } + + @Override + protected Configuration createScaleConfiguration() { + Configuration conf = super.createScaleConfiguration(); + S3ATestUtils.disableFilesystemCaching(conf); + conf.set(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM, + getSSEAlgorithm().getMethod()); + conf.set(Constants.SERVER_SIDE_ENCRYPTION_KEY, KEY_1); + return conf; + } + + private S3AEncryptionMethods getSSEAlgorithm() { + return S3AEncryptionMethods.SSE_C; + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
