Author: atm
Date: Sat May 3 00:26:23 2014
New Revision: 1592134
URL: http://svn.apache.org/r1592134
Log:
HADOOP-10568. Add s3 server-side encryption. Contributed by David S. Wang.
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1592134&r1=1592133&r2=1592134&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Sat May 3 00:26:23 2014
@@ -111,6 +111,8 @@ Release 2.5.0 - UNRELEASED
HADOOP-10562. Namenode exits on exception without printing stack trace
in AbstractDelegationTokenSecretManager. (Arpit Agarwal)
+ HADOOP-10568. Add s3 server-side encryption. (David S. Wang via atm)
+
Release 2.4.1 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java?rev=1592134&r1=1592133&r2=1592134&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/Jets3tNativeFileSystemStore.java
Sat May 3 00:26:23 2014
@@ -63,6 +63,8 @@ class Jets3tNativeFileSystemStore implem
private boolean multipartEnabled;
private long multipartCopyBlockSize;
static final long MAX_PART_SIZE = (long)5 * 1024 * 1024 * 1024;
+
+ private String serverSideEncryptionAlgorithm;
public static final Log LOG =
LogFactory.getLog(Jets3tNativeFileSystemStore.class);
@@ -87,6 +89,7 @@ class Jets3tNativeFileSystemStore implem
multipartCopyBlockSize = Math.min(
conf.getLong("fs.s3n.multipart.copy.block.size", MAX_PART_SIZE),
MAX_PART_SIZE);
+ serverSideEncryptionAlgorithm =
conf.get("fs.s3n.server-side-encryption-algorithm");
bucket = new S3Bucket(uri.getHost());
}
@@ -107,6 +110,7 @@ class Jets3tNativeFileSystemStore implem
object.setDataInputStream(in);
object.setContentType("binary/octet-stream");
object.setContentLength(file.length());
+ object.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm);
if (md5Hash != null) {
object.setMd5Hash(md5Hash);
}
@@ -130,6 +134,7 @@ class Jets3tNativeFileSystemStore implem
object.setDataInputFile(file);
object.setContentType("binary/octet-stream");
object.setContentLength(file.length());
+ object.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm);
if (md5Hash != null) {
object.setMd5Hash(md5Hash);
}
@@ -156,6 +161,7 @@ class Jets3tNativeFileSystemStore implem
object.setDataInputStream(new ByteArrayInputStream(new byte[0]));
object.setContentType("binary/octet-stream");
object.setContentLength(0);
+ object.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm);
s3Service.putObject(bucket, object);
} catch (S3ServiceException e) {
handleS3ServiceException(e);
@@ -317,8 +323,11 @@ class Jets3tNativeFileSystemStore implem
return;
}
}
+
+ S3Object dstObject = new S3Object(dstKey);
+
dstObject.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm);
s3Service.copyObject(bucket.getName(), srcKey, bucket.getName(),
- new S3Object(dstKey), false);
+ dstObject, false);
} catch (ServiceException e) {
handleServiceException(srcKey, e);
}
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml?rev=1592134&r1=1592133&r2=1592134&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
Sat May 3 00:26:23 2014
@@ -568,6 +568,14 @@
</property>
<property>
+ <name>fs.s3n.server-side-encryption-algorithm</name>
+ <value></value>
+ <description>Specify a server-side encryption algorithm for S3.
+ The default is NULL, and the only other currently allowable value is AES256.
+ </description>
+</property>
+
+<property>
<name>io.seqfile.compress.blocksize</name>
<value>1000000</value>
<description>The minimum block size for compression in block compressed
