hadoop git commit: HDFS-10879. TestEncryptionZonesWithKMS#testReadWrite fails intermittently. Contributed by Xiao Chen.
Repository: hadoop Updated Branches: refs/heads/branch-2.7 f27cf17f7 -> d053d1c3f HDFS-10879. TestEncryptionZonesWithKMS#testReadWrite fails intermittently. Contributed by Xiao Chen. (cherry picked from commit d31bef575478fe78c0600fb602e117e4b81c9887) (cherry picked from commit 11ed4f5d40effcdb26461a393379c6bddaa29bed) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d053d1c3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d053d1c3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d053d1c3 Branch: refs/heads/branch-2.7 Commit: d053d1c3ffb76122f55d98894631676edaf18660 Parents: f27cf17 Author: Xiao ChenAuthored: Tue Sep 20 16:52:05 2016 -0700 Committer: Zhe Zhang Committed: Tue Oct 18 10:16:30 2016 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../apache/hadoop/hdfs/TestEncryptionZones.java | 22 +++- 2 files changed, 24 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d053d1c3/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index a13a566..1776a49 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -157,6 +157,9 @@ Release 2.7.4 - UNRELEASED HDFS-10512. VolumeScanner may terminate due to NPE in DataNode.reportBadBlocks. Contributed by Wei-Chiu Chuang and Yiqun Lin. +HDFS-10879. TestEncryptionZonesWithKMS#testReadWrite fails intermittently. +(xiaochen) + Release 2.7.3 - 2016-08-25 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/d053d1c3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java index a30f396..39f76bd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java @@ -46,6 +46,7 @@ import org.apache.hadoop.crypto.key.JavaKeyStoreProvider; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.crypto.key.KeyProviderFactory; +import org.apache.hadoop.crypto.key.kms.server.EagerKeyGeneratorKeyProviderCryptoExtension; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataOutputStream; @@ -635,14 +636,33 @@ public class TestEncryptionZones { // Roll the key of the encryption zone assertNumZones(1); String keyName = dfsAdmin.listEncryptionZones().next().getKeyName(); +FileEncryptionInfo feInfo1 = getFileEncryptionInfo(encFile1); cluster.getNamesystem().getProvider().rollNewVersion(keyName); +/** + * due to the cache on the server side, client may get old keys. + * @see EagerKeyGeneratorKeyProviderCryptoExtension#rollNewVersion(String) + */ +boolean rollSucceeded = false; +for (int i = 0; i <= EagerKeyGeneratorKeyProviderCryptoExtension +.KMS_KEY_CACHE_SIZE_DEFAULT + CommonConfigurationKeysPublic. +KMS_CLIENT_ENC_KEY_CACHE_SIZE_DEFAULT; ++i) { + KeyProviderCryptoExtension.EncryptedKeyVersion ekv2 = + cluster.getNamesystem().getProvider().generateEncryptedKey(TEST_KEY); + if (!(feInfo1.getEzKeyVersionName() + .equals(ekv2.getEncryptionKeyVersionName( { +rollSucceeded = true; +break; + } +} +Assert.assertTrue("rollover did not generate a new key even after" ++ " queue is drained", rollSucceeded); + // Read them back in and compare byte-by-byte verifyFilesEqual(fs, baseFile, encFile1, len); // Write a new enc file and validate final Path encFile2 = new Path(zone, "myfile2"); DFSTestUtil.createFile(fs, encFile2, len, (short) 1, 0xFEED); // FEInfos should be different -FileEncryptionInfo feInfo1 = getFileEncryptionInfo(encFile1); FileEncryptionInfo feInfo2 = getFileEncryptionInfo(encFile2); assertFalse("EDEKs should be different", Arrays .equals(feInfo1.getEncryptedDataEncryptionKey(), - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[08/13] hadoop git commit: HDFS-10879. TestEncryptionZonesWithKMS#testReadWrite fails intermittently. Contributed by Xiao Chen.
HDFS-10879. TestEncryptionZonesWithKMS#testReadWrite fails intermittently. Contributed by Xiao Chen. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0e918dff Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0e918dff Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0e918dff Branch: refs/heads/HADOOP-12756 Commit: 0e918dff594e9ba5434fdee7fc1f6394b62b32cd Parents: e80386d Author: Xiao ChenAuthored: Tue Sep 20 16:52:05 2016 -0700 Committer: Xiao Chen Committed: Tue Sep 20 16:56:52 2016 -0700 -- .../apache/hadoop/hdfs/TestEncryptionZones.java | 23 +++- 1 file changed, 22 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e918dff/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java index b634dd2..9168ca6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java @@ -45,7 +45,9 @@ import org.apache.hadoop.crypto.CipherSuite; import org.apache.hadoop.crypto.CryptoProtocolVersion; import org.apache.hadoop.crypto.key.JavaKeyStoreProvider; import org.apache.hadoop.crypto.key.KeyProvider; +import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.crypto.key.KeyProviderFactory; +import org.apache.hadoop.crypto.key.kms.server.EagerKeyGeneratorKeyProviderCryptoExtension; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataOutputStream; @@ -734,14 +736,33 @@ public class TestEncryptionZones { // Roll the key of the encryption zone assertNumZones(1); String keyName = dfsAdmin.listEncryptionZones().next().getKeyName(); +FileEncryptionInfo feInfo1 = getFileEncryptionInfo(encFile1); cluster.getNamesystem().getProvider().rollNewVersion(keyName); +/** + * due to the cache on the server side, client may get old keys. + * @see EagerKeyGeneratorKeyProviderCryptoExtension#rollNewVersion(String) + */ +boolean rollSucceeded = false; +for (int i = 0; i <= EagerKeyGeneratorKeyProviderCryptoExtension +.KMS_KEY_CACHE_SIZE_DEFAULT + CommonConfigurationKeysPublic. +KMS_CLIENT_ENC_KEY_CACHE_SIZE_DEFAULT; ++i) { + KeyProviderCryptoExtension.EncryptedKeyVersion ekv2 = + cluster.getNamesystem().getProvider().generateEncryptedKey(TEST_KEY); + if (!(feInfo1.getEzKeyVersionName() + .equals(ekv2.getEncryptionKeyVersionName( { +rollSucceeded = true; +break; + } +} +Assert.assertTrue("rollover did not generate a new key even after" ++ " queue is drained", rollSucceeded); + // Read them back in and compare byte-by-byte verifyFilesEqual(fs, baseFile, encFile1, len); // Write a new enc file and validate final Path encFile2 = new Path(zone, "myfile2"); DFSTestUtil.createFile(fs, encFile2, len, (short) 1, 0xFEED); // FEInfos should be different -FileEncryptionInfo feInfo1 = getFileEncryptionInfo(encFile1); FileEncryptionInfo feInfo2 = getFileEncryptionInfo(encFile2); assertFalse("EDEKs should be different", Arrays .equals(feInfo1.getEncryptedDataEncryptionKey(), - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: HDFS-10879. TestEncryptionZonesWithKMS#testReadWrite fails intermittently. Contributed by Xiao Chen.
Repository: hadoop Updated Branches: refs/heads/trunk e80386d69 -> 0e918dff5 HDFS-10879. TestEncryptionZonesWithKMS#testReadWrite fails intermittently. Contributed by Xiao Chen. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0e918dff Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0e918dff Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0e918dff Branch: refs/heads/trunk Commit: 0e918dff594e9ba5434fdee7fc1f6394b62b32cd Parents: e80386d Author: Xiao ChenAuthored: Tue Sep 20 16:52:05 2016 -0700 Committer: Xiao Chen Committed: Tue Sep 20 16:56:52 2016 -0700 -- .../apache/hadoop/hdfs/TestEncryptionZones.java | 23 +++- 1 file changed, 22 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e918dff/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java index b634dd2..9168ca6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java @@ -45,7 +45,9 @@ import org.apache.hadoop.crypto.CipherSuite; import org.apache.hadoop.crypto.CryptoProtocolVersion; import org.apache.hadoop.crypto.key.JavaKeyStoreProvider; import org.apache.hadoop.crypto.key.KeyProvider; +import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.crypto.key.KeyProviderFactory; +import org.apache.hadoop.crypto.key.kms.server.EagerKeyGeneratorKeyProviderCryptoExtension; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataOutputStream; @@ -734,14 +736,33 @@ public class TestEncryptionZones { // Roll the key of the encryption zone assertNumZones(1); String keyName = dfsAdmin.listEncryptionZones().next().getKeyName(); +FileEncryptionInfo feInfo1 = getFileEncryptionInfo(encFile1); cluster.getNamesystem().getProvider().rollNewVersion(keyName); +/** + * due to the cache on the server side, client may get old keys. + * @see EagerKeyGeneratorKeyProviderCryptoExtension#rollNewVersion(String) + */ +boolean rollSucceeded = false; +for (int i = 0; i <= EagerKeyGeneratorKeyProviderCryptoExtension +.KMS_KEY_CACHE_SIZE_DEFAULT + CommonConfigurationKeysPublic. +KMS_CLIENT_ENC_KEY_CACHE_SIZE_DEFAULT; ++i) { + KeyProviderCryptoExtension.EncryptedKeyVersion ekv2 = + cluster.getNamesystem().getProvider().generateEncryptedKey(TEST_KEY); + if (!(feInfo1.getEzKeyVersionName() + .equals(ekv2.getEncryptionKeyVersionName( { +rollSucceeded = true; +break; + } +} +Assert.assertTrue("rollover did not generate a new key even after" ++ " queue is drained", rollSucceeded); + // Read them back in and compare byte-by-byte verifyFilesEqual(fs, baseFile, encFile1, len); // Write a new enc file and validate final Path encFile2 = new Path(zone, "myfile2"); DFSTestUtil.createFile(fs, encFile2, len, (short) 1, 0xFEED); // FEInfos should be different -FileEncryptionInfo feInfo1 = getFileEncryptionInfo(encFile1); FileEncryptionInfo feInfo2 = getFileEncryptionInfo(encFile2); assertFalse("EDEKs should be different", Arrays .equals(feInfo1.getEncryptedDataEncryptionKey(), - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: HDFS-10879. TestEncryptionZonesWithKMS#testReadWrite fails intermittently. Contributed by Xiao Chen.
Repository: hadoop Updated Branches: refs/heads/branch-2.8 c7bd564b7 -> 11ed4f5d4 HDFS-10879. TestEncryptionZonesWithKMS#testReadWrite fails intermittently. Contributed by Xiao Chen. (cherry picked from commit d31bef575478fe78c0600fb602e117e4b81c9887) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/11ed4f5d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/11ed4f5d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/11ed4f5d Branch: refs/heads/branch-2.8 Commit: 11ed4f5d40effcdb26461a393379c6bddaa29bed Parents: c7bd564 Author: Xiao ChenAuthored: Tue Sep 20 16:52:05 2016 -0700 Committer: Xiao Chen Committed: Tue Sep 20 16:57:44 2016 -0700 -- .../apache/hadoop/hdfs/TestEncryptionZones.java | 23 +++- 1 file changed, 22 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/11ed4f5d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java index 73f2109..1774a85 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java @@ -45,7 +45,9 @@ import org.apache.hadoop.crypto.CipherSuite; import org.apache.hadoop.crypto.CryptoProtocolVersion; import org.apache.hadoop.crypto.key.JavaKeyStoreProvider; import org.apache.hadoop.crypto.key.KeyProvider; +import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.crypto.key.KeyProviderFactory; +import org.apache.hadoop.crypto.key.kms.server.EagerKeyGeneratorKeyProviderCryptoExtension; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataOutputStream; @@ -734,14 +736,33 @@ public class TestEncryptionZones { // Roll the key of the encryption zone assertNumZones(1); String keyName = dfsAdmin.listEncryptionZones().next().getKeyName(); +FileEncryptionInfo feInfo1 = getFileEncryptionInfo(encFile1); cluster.getNamesystem().getProvider().rollNewVersion(keyName); +/** + * due to the cache on the server side, client may get old keys. + * @see EagerKeyGeneratorKeyProviderCryptoExtension#rollNewVersion(String) + */ +boolean rollSucceeded = false; +for (int i = 0; i <= EagerKeyGeneratorKeyProviderCryptoExtension +.KMS_KEY_CACHE_SIZE_DEFAULT + CommonConfigurationKeysPublic. +KMS_CLIENT_ENC_KEY_CACHE_SIZE_DEFAULT; ++i) { + KeyProviderCryptoExtension.EncryptedKeyVersion ekv2 = + cluster.getNamesystem().getProvider().generateEncryptedKey(TEST_KEY); + if (!(feInfo1.getEzKeyVersionName() + .equals(ekv2.getEncryptionKeyVersionName( { +rollSucceeded = true; +break; + } +} +Assert.assertTrue("rollover did not generate a new key even after" ++ " queue is drained", rollSucceeded); + // Read them back in and compare byte-by-byte verifyFilesEqual(fs, baseFile, encFile1, len); // Write a new enc file and validate final Path encFile2 = new Path(zone, "myfile2"); DFSTestUtil.createFile(fs, encFile2, len, (short) 1, 0xFEED); // FEInfos should be different -FileEncryptionInfo feInfo1 = getFileEncryptionInfo(encFile1); FileEncryptionInfo feInfo2 = getFileEncryptionInfo(encFile2); assertFalse("EDEKs should be different", Arrays .equals(feInfo1.getEncryptedDataEncryptionKey(), - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: HDFS-10879. TestEncryptionZonesWithKMS#testReadWrite fails intermittently. Contributed by Xiao Chen.
Repository: hadoop Updated Branches: refs/heads/branch-2 f98afb3be -> 81df2be01 HDFS-10879. TestEncryptionZonesWithKMS#testReadWrite fails intermittently. Contributed by Xiao Chen. (cherry picked from commit d31bef575478fe78c0600fb602e117e4b81c9887) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/81df2be0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/81df2be0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/81df2be0 Branch: refs/heads/branch-2 Commit: 81df2be0174f6e1a5d37e1a17a98c730ad2ac7dc Parents: f98afb3 Author: Xiao ChenAuthored: Tue Sep 20 16:52:05 2016 -0700 Committer: Xiao Chen Committed: Tue Sep 20 16:57:34 2016 -0700 -- .../apache/hadoop/hdfs/TestEncryptionZones.java | 23 +++- 1 file changed, 22 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/81df2be0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java index 73f2109..1774a85 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java @@ -45,7 +45,9 @@ import org.apache.hadoop.crypto.CipherSuite; import org.apache.hadoop.crypto.CryptoProtocolVersion; import org.apache.hadoop.crypto.key.JavaKeyStoreProvider; import org.apache.hadoop.crypto.key.KeyProvider; +import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.crypto.key.KeyProviderFactory; +import org.apache.hadoop.crypto.key.kms.server.EagerKeyGeneratorKeyProviderCryptoExtension; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataOutputStream; @@ -734,14 +736,33 @@ public class TestEncryptionZones { // Roll the key of the encryption zone assertNumZones(1); String keyName = dfsAdmin.listEncryptionZones().next().getKeyName(); +FileEncryptionInfo feInfo1 = getFileEncryptionInfo(encFile1); cluster.getNamesystem().getProvider().rollNewVersion(keyName); +/** + * due to the cache on the server side, client may get old keys. + * @see EagerKeyGeneratorKeyProviderCryptoExtension#rollNewVersion(String) + */ +boolean rollSucceeded = false; +for (int i = 0; i <= EagerKeyGeneratorKeyProviderCryptoExtension +.KMS_KEY_CACHE_SIZE_DEFAULT + CommonConfigurationKeysPublic. +KMS_CLIENT_ENC_KEY_CACHE_SIZE_DEFAULT; ++i) { + KeyProviderCryptoExtension.EncryptedKeyVersion ekv2 = + cluster.getNamesystem().getProvider().generateEncryptedKey(TEST_KEY); + if (!(feInfo1.getEzKeyVersionName() + .equals(ekv2.getEncryptionKeyVersionName( { +rollSucceeded = true; +break; + } +} +Assert.assertTrue("rollover did not generate a new key even after" ++ " queue is drained", rollSucceeded); + // Read them back in and compare byte-by-byte verifyFilesEqual(fs, baseFile, encFile1, len); // Write a new enc file and validate final Path encFile2 = new Path(zone, "myfile2"); DFSTestUtil.createFile(fs, encFile2, len, (short) 1, 0xFEED); // FEInfos should be different -FileEncryptionInfo feInfo1 = getFileEncryptionInfo(encFile1); FileEncryptionInfo feInfo2 = getFileEncryptionInfo(encFile2); assertFalse("EDEKs should be different", Arrays .equals(feInfo1.getEncryptedDataEncryptionKey(), - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org