Author: yliu
Date: Wed Jul 16 00:36:30 2014
New Revision: 1610887
URL: http://svn.apache.org/r1610887
Log:
HADOOP-10735. Fall back AesCtrCryptoCodec implementation from OpenSSL to JCE if
non native support. (yliu)
Modified:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/CHANGES-fs-encryption.txt
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CipherSuite.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
Modified:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/CHANGES-fs-encryption.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/CHANGES-fs-encryption.txt?rev=1610887&r1=1610886&r2=1610887&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/CHANGES-fs-encryption.txt
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/CHANGES-fs-encryption.txt
Wed Jul 16 00:36:30 2014
@@ -37,6 +37,9 @@ fs-encryption (Unreleased)
HADOOP-10803. Update OpensslCipher#getInstance to accept CipherSuite#name
format. (Yi Liu)
+ HADOOP-10735. Fall back AesCtrCryptoCodec implementation from OpenSSL to
+ JCE if non native support. (Yi Liu)
+
OPTIMIZATIONS
BUG FIXES
Modified:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CipherSuite.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CipherSuite.java?rev=1610887&r1=1610886&r2=1610887&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CipherSuite.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CipherSuite.java
Wed Jul 16 00:36:30 2014
@@ -72,4 +72,14 @@ public enum CipherSuite {
builder.append("}");
return builder.toString();
}
+
+ public static void checkName(String name) {
+ CipherSuite[] suites = CipherSuite.values();
+ for (CipherSuite suite : suites) {
+ if (suite.getName().equals(name)) {
+ return;
+ }
+ }
+ throw new IllegalArgumentException("Invalid cipher suite name: " + name);
+ }
}
Modified:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java?rev=1610887&r1=1610886&r2=1610887&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java
Wed Jul 16 00:36:30 2014
@@ -18,13 +18,23 @@
package org.apache.hadoop.crypto;
import java.security.GeneralSecurityException;
+import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ReflectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.Lists;
+
import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASS_KEY;
+import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASS_DEFAULT;
+import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY;
+import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_DEFAULT;
/**
* Crypto codec class, encapsulates encryptor/decryptor pair.
@@ -32,12 +42,57 @@ import static org.apache.hadoop.fs.Commo
@InterfaceAudience.Private
@InterfaceStability.Evolving
public abstract class CryptoCodec implements Configurable {
+ public static Logger LOG = LoggerFactory.getLogger(CryptoCodec.class);
public static CryptoCodec getInstance(Configuration conf) {
- final Class<? extends CryptoCodec> klass = conf.getClass(
- HADOOP_SECURITY_CRYPTO_CODEC_CLASS_KEY, JceAesCtrCryptoCodec.class,
- CryptoCodec.class);
- return ReflectionUtils.newInstance(klass, conf);
+ List<Class<? extends CryptoCodec>> klasses = getCodecClasses(conf);
+ String name = conf.get(HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY,
+ HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_DEFAULT);
+ CipherSuite.checkName(name);
+ CryptoCodec codec = null;
+ for (Class<? extends CryptoCodec> klass : klasses) {
+ try {
+ CryptoCodec c = ReflectionUtils.newInstance(klass, conf);
+ if (c.getCipherSuite().getName().equalsIgnoreCase(name)) {
+ if (codec == null) {
+ LOG.debug("Using crypto codec {}.", klass.getName());
+ codec = c;
+ }
+ } else {
+ LOG.warn("Crypto codec {} doesn't meet the cipher suite {}.",
+ klass.getName(), name);
+ }
+ } catch (Exception e) {
+ LOG.warn("Crypto codec {} is not available.", klass.getName());
+ }
+ }
+
+ if (codec != null) {
+ return codec;
+ }
+
+ throw new RuntimeException("No available crypto codec which meets " +
+ "the cipher suite " + name + ".");
+ }
+
+ private static List<Class<? extends CryptoCodec>> getCodecClasses(
+ Configuration conf) {
+ List<Class<? extends CryptoCodec>> result = Lists.newArrayList();
+ String codecString = conf.get(HADOOP_SECURITY_CRYPTO_CODEC_CLASS_KEY,
+ HADOOP_SECURITY_CRYPTO_CODEC_CLASS_DEFAULT);
+ for (String c : Splitter.on(',').trimResults().omitEmptyStrings().
+ split(codecString)) {
+ try {
+ Class<?> cls = conf.getClassByName(c);
+ result.add(cls.asSubclass(CryptoCodec.class));
+ } catch (ClassCastException e) {
+ LOG.warn("Class " + c + " is not a CryptoCodec.");
+ } catch (ClassNotFoundException e) {
+ LOG.warn("Crypto codec " + c + " not found.");
+ }
+ }
+
+ return result;
}
/**
Modified:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java?rev=1610887&r1=1610886&r2=1610887&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslAesCtrCryptoCodec.java
Wed Jul 16 00:36:30 2014
@@ -47,6 +47,9 @@ public class OpensslAesCtrCryptoCodec ex
private Random random;
public OpensslAesCtrCryptoCodec() {
+ if (!OpensslCipher.isNativeCodeLoaded()) {
+ throw new RuntimeException("Failed to load OpenSSL Cipher.");
+ }
}
@Override
Modified:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java?rev=1610887&r1=1610886&r2=1610887&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
Wed Jul 16 00:36:30 2014
@@ -285,6 +285,14 @@ public class CommonConfigurationKeysPubl
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
public static final String HADOOP_SECURITY_CRYPTO_CODEC_CLASS_KEY =
"hadoop.security.crypto.codec.class";
+ public static final String HADOOP_SECURITY_CRYPTO_CODEC_CLASS_DEFAULT =
+ "org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec," +
+ "org.apache.hadoop.crypto.JceAesCtrCryptoCodec";
+ /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ public static final String HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY =
+ "hadoop.security.crypto.cipher.suite";
+ public static final String HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_DEFAULT =
+ "AES/CTR/NoPadding";
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
public static final String HADOOP_SECURITY_CRYPTO_JCE_PROVIDER_KEY =
"hadoop.security.crypto.jce.provider";
@@ -302,8 +310,10 @@ public class CommonConfigurationKeysPubl
/** Defalt value for HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_KEY */
public static final String
HADOOP_SECURITY_JAVA_SECURE_RANDOM_ALGORITHM_DEFAULT =
"SHA1PRNG";
+ /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
public static final String HADOOP_SECURITY_SECURE_RANDOM_IMPL_KEY =
"hadoop.security.secure.random.impl";
+ /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
public static final String
HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_KEY =
"hadoop.security.random.device.file.path";
public static final String
HADOOP_SECURITY_SECURE_RANDOM_DEVICE_FILE_PATH_DEFAULT =
Modified:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml?rev=1610887&r1=1610886&r2=1610887&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
Wed Jul 16 00:36:30 2014
@@ -1453,10 +1453,20 @@ for ldap providers in the same way as ab
<property>
<name>hadoop.security.crypto.codec.class</name>
- <value></value>
+ <value>org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec,
+ org.apache.hadoop.crypto.JceAesCtrCryptoCodec</value>
+ <description>
+ Comma list of CryptoCodec implementations which are used for encryption
+ and decryption. The first implementation will be used if avaiable, others
+ are fallbacks.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.crypto.cipher.suite</name>
+ <value>AES/CTR/NoPadding</value>
<description>
- The default implementation of CryptoCodec which is used for encryption
- and decryption.
+ Cipher suite for crypto codec.
</description>
</property>
Modified:
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java?rev=1610887&r1=1610886&r2=1610887&view=diff
==============================================================================
---
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
(original)
+++
hadoop/common/branches/fs-encryption/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
Wed Jul 16 00:36:30 2014
@@ -29,7 +29,8 @@ public class TestCryptoStreamsWithOpenss
public static void init() throws Exception {
Configuration conf = new Configuration();
conf.set(HADOOP_SECURITY_CRYPTO_CODEC_CLASS_KEY,
- OpensslAesCtrCryptoCodec.class.getName());
+ OpensslAesCtrCryptoCodec.class.getName() + "," +
+ JceAesCtrCryptoCodec.class.getName());
codec = CryptoCodec.getInstance(conf);
}
}
