Repository: hadoop Updated Branches: refs/heads/trunk c1f832323 -> df8c84cba
HADOOP-11071. KMSClientProvider should drain the local generated EEK cache on key rollover. (tucu) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/df8c84cb Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/df8c84cb Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/df8c84cb Branch: refs/heads/trunk Commit: df8c84cba8512058f5097c6faeedf4b65cab3806 Parents: c1f8323 Author: Alejandro Abdelnur <t...@apache.org> Authored: Mon Sep 8 10:12:16 2014 -0700 Committer: Alejandro Abdelnur <t...@apache.org> Committed: Mon Sep 8 11:31:30 2014 -0700 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../crypto/key/KeyProviderCryptoExtension.java | 11 ++++++++++ .../crypto/key/kms/KMSClientProvider.java | 9 +++++++- .../hadoop/crypto/key/kms/ValueQueue.java | 13 ++++++++++++ .../hadoop/crypto/key/TestValueQueue.java | 14 +++++++++++++ ...rKeyGeneratorKeyProviderCryptoExtension.java | 22 ++++++++++++++++++++ .../hadoop/crypto/key/kms/server/TestKMS.java | 17 +++++++++++++++ 7 files changed, 88 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/df8c84cb/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index fe011fd..0417b0a 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -771,6 +771,9 @@ Release 2.6.0 - UNRELEASED HADOOP-11073. Credential Provider related Unit Tests Failure on Windows. (Xiaoyu Yao via cnauroth) + HADOOP-11071. KMSClientProvider should drain the local generated EEK cache + on key rollover. (tucu) + Release 2.5.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/df8c84cb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java index e2fb5cb..fed7e9e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java @@ -179,6 +179,13 @@ public class KeyProviderCryptoExtension extends throws IOException; /** + * Drains the Queue for the provided key. + * + * @param keyName the key to drain the Queue for + */ + public void drain(String keyName); + + /** * Generates a key material and encrypts it using the given key version name * and initialization vector. The generated key material is of the same * length as the <code>KeyVersion</code> material of the latest key version @@ -313,6 +320,10 @@ public class KeyProviderCryptoExtension extends // NO-OP since the default version does not cache any keys } + @Override + public void drain(String keyName) { + // NO-OP since the default version does not cache any keys + } } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/df8c84cb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java index acbe096..899b6c4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java @@ -590,7 +590,9 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension, conn.setRequestProperty(CONTENT_TYPE, APPLICATION_JSON_MIME); Map response = call(conn, jsonMaterial, HttpURLConnection.HTTP_OK, Map.class); - return parseJSONKeyVersion(response); + KeyVersion keyVersion = parseJSONKeyVersion(response); + encKeyVersionQueue.drain(name); + return keyVersion; } @@ -713,6 +715,11 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension, } @Override + public void drain(String keyName) { + encKeyVersionQueue.drain(keyName); + } + + @Override public Token<?>[] addDelegationTokens(String renewer, Credentials credentials) throws IOException { Token<?>[] tokens; http://git-wip-us.apache.org/repos/asf/hadoop/blob/df8c84cb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java index a415e2e..ee10483 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java @@ -228,6 +228,19 @@ public class ValueQueue <E> { } /** + * Drains the Queue for the provided key. + * + * @param keyName the key to drain the Queue for + */ + public void drain(String keyName ) { + try { + keyQueues.get(keyName).clear(); + } catch (ExecutionException ex) { + //NOP + } + } + + /** * This removes the "num" values currently at the head of the Queue for the * provided key. Will immediately fire the Queue filler function if key * does not exist http://git-wip-us.apache.org/repos/asf/hadoop/blob/df8c84cb/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestValueQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestValueQueue.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestValueQueue.java index 7946588..8e3a093 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestValueQueue.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestValueQueue.java @@ -187,4 +187,18 @@ public class TestValueQueue { Assert.assertEquals(10, filler.getTop().num); vq.shutdown(); } + + @Test + public void testDrain() throws Exception { + MockFiller filler = new MockFiller(); + ValueQueue<String> vq = + new ValueQueue<String>(10, 0.1f, 300, 1, + SyncGenerationPolicy.ALL, filler); + Assert.assertEquals("test", vq.getNext("k1")); + Assert.assertEquals(1, filler.getTop().num); + vq.drain("k1"); + Assert.assertNull(filler.getTop()); + vq.shutdown(); + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/df8c84cb/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java index a952cfe..a33f4f1 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java @@ -20,6 +20,7 @@ package org.apache.hadoop.crypto.key.kms.server; import java.io.IOException; import java.security.GeneralSecurityException; +import java.security.NoSuchAlgorithmException; import java.util.LinkedList; import java.util.List; import java.util.Queue; @@ -27,6 +28,7 @@ import java.util.concurrent.ExecutionException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.crypto.key.kms.ValueQueue; import org.apache.hadoop.crypto.key.kms.ValueQueue.SyncGenerationPolicy; @@ -113,6 +115,11 @@ public class EagerKeyGeneratorKeyProviderCryptoExtension } @Override + public void drain(String keyName) { + encKeyVersionQueue.drain(keyName); + } + + @Override public EncryptedKeyVersion generateEncryptedKey(String encryptionKeyName) throws IOException, GeneralSecurityException { try { @@ -146,4 +153,19 @@ public class EagerKeyGeneratorKeyProviderCryptoExtension new CryptoExtension(conf, keyProviderCryptoExtension)); } + @Override + public KeyVersion rollNewVersion(String name) + throws NoSuchAlgorithmException, IOException { + KeyVersion keyVersion = super.rollNewVersion(name); + getExtension().drain(name); + return keyVersion; + } + + @Override + public KeyVersion rollNewVersion(String name, byte[] material) + throws IOException { + KeyVersion keyVersion = super.rollNewVersion(name, material); + getExtension().drain(name); + return keyVersion; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/df8c84cb/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java index b921c84..74eab5c 100644 --- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java +++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java @@ -531,6 +531,7 @@ public class TestKMS { Assert.assertEquals("d", meta.getDescription()); Assert.assertEquals(attributes, meta.getAttributes()); + // test delegation token retrieval KeyProviderDelegationTokenExtension kpdte = KeyProviderDelegationTokenExtension. createKeyProviderDelegationTokenExtension(kp); @@ -542,6 +543,22 @@ public class TestKMS { Assert.assertEquals(new Text("kms-dt"), credentials.getToken( SecurityUtil.buildTokenService(kmsAddr)).getKind()); + + + // test rollover draining + KeyProviderCryptoExtension kpce = KeyProviderCryptoExtension. + createKeyProviderCryptoExtension(kp); + options = new KeyProvider.Options(conf); + options.setCipher("AES/CTR/NoPadding"); + options.setBitLength(128); + kpce.createKey("k6", options); + + EncryptedKeyVersion ekv1 = kpce.generateEncryptedKey("k6"); + kpce.rollNewVersion("k6"); + EncryptedKeyVersion ekv2 = kpce.generateEncryptedKey("k6"); + Assert.assertNotEquals(ekv1.getEncryptionKeyVersionName(), + ekv2.getEncryptionKeyVersionName()); + return null; } });
