Repository: hadoop Updated Branches: refs/heads/trunk 2b6bcfdaf -> 05e04f34f
HDFS-8451. DFSClient probe for encryption testing interprets empty URI property for enabled. Contributed by Steve Loughran. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/05e04f34 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/05e04f34 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/05e04f34 Branch: refs/heads/trunk Commit: 05e04f34f27149537fdb89f46af26bee14531ca4 Parents: 2b6bcfd Author: Xiaoyu Yao <x...@apache.org> Authored: Thu May 21 11:58:00 2015 -0700 Committer: Xiaoyu Yao <x...@apache.org> Committed: Thu May 21 11:58:00 2015 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/hdfs/DFSClient.java | 9 ++++++-- .../java/org/apache/hadoop/hdfs/DFSUtil.java | 22 ++++++++++++++++---- .../apache/hadoop/hdfs/KeyProviderCache.java | 4 ++-- .../org/apache/hadoop/hdfs/TestDFSUtil.java | 18 ++++++++++++++++ .../apache/hadoop/hdfs/TestEncryptionZones.java | 2 +- 6 files changed, 49 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/05e04f34/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 9cfad7d..e830421 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -886,6 +886,9 @@ Release 2.7.1 - UNRELEASED HDFS-8404. Pending block replication can get stuck using older genstamp (Nathan Roberts via kihwal) + HDFS-8451. DFSClient probe for encryption testing interprets empty URI + property for "enabled". (Steve Loughran via xyao) + Release 2.7.0 - 2015-04-20 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/05e04f34/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index a2b9760..60e5577 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -3205,10 +3205,15 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, } } + /** + * Probe for encryption enabled on this filesystem. + * See {@link DFSUtil#isHDFSEncryptionEnabled(Configuration)} + * @return true if encryption is enabled + */ public boolean isHDFSEncryptionEnabled() { - return conf.get( - DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, null) != null; + return DFSUtil.isHDFSEncryptionEnabled(this.conf); } + /** * Returns the SaslDataTransferClient configured for this DFSClient. * http://git-wip-us.apache.org/repos/asf/hadoop/blob/05e04f34/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index 5f501c1..cae56c0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -145,8 +145,8 @@ public class DFSUtil { a.isDecommissioned() ? 1 : -1; } }; - - + + /** * Comparator for sorting DataNodeInfo[] based on decommissioned/stale states. * Decommissioned/stale nodes are moved to the end of the array on sorting @@ -1460,9 +1460,9 @@ public class DFSUtil { public static KeyProvider createKeyProvider( final Configuration conf) throws IOException { final String providerUriStr = - conf.get(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, null); + conf.getTrimmed(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, ""); // No provider set in conf - if (providerUriStr == null) { + if (providerUriStr.isEmpty()) { return null; } final URI providerUri; @@ -1513,4 +1513,18 @@ public class DFSUtil { public static int getSmallBufferSize(Configuration conf) { return Math.min(getIoFileBufferSize(conf) / 2, 512); } + + /** + * Probe for HDFS Encryption being enabled; this uses the value of + * the option {@link DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI}, + * returning true if that property contains a non-empty, non-whitespace + * string. + * @param conf configuration to probe + * @return true if encryption is considered enabled. + */ + public static boolean isHDFSEncryptionEnabled(Configuration conf) { + return !conf.getTrimmed( + DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, "").isEmpty(); + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/05e04f34/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java index c7da7af..a2b6c7e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java @@ -83,9 +83,9 @@ public class KeyProviderCache { private URI createKeyProviderURI(Configuration conf) { final String providerUriStr = - conf.get(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, null); + conf.getTrimmed(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, ""); // No provider set in conf - if (providerUriStr == null) { + if (providerUriStr.isEmpty()) { LOG.error("Could not find uri with key [" + DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI + "] to create a keyProvider !!"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/05e04f34/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java index 4128a09..a821c30 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java @@ -897,4 +897,22 @@ public class TestDFSUtil { } catch (IOException ignored) { } } + + @Test + public void testEncryptionProbe() throws Throwable { + Configuration conf = new Configuration(false); + conf.unset(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI); + assertFalse("encryption enabled on no provider key", + DFSUtil.isHDFSEncryptionEnabled(conf)); + conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, ""); + assertFalse("encryption enabled on empty provider key", + DFSUtil.isHDFSEncryptionEnabled(conf)); + conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, "\n\t\n"); + assertFalse("encryption enabled on whitespace provider key", + DFSUtil.isHDFSEncryptionEnabled(conf)); + conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, "http://hadoop.apache.org";); + assertTrue("encryption disabled on valid provider key", + DFSUtil.isHDFSEncryptionEnabled(conf)); + + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/05e04f34/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java index b211ffb..e0bd6f4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java @@ -699,7 +699,7 @@ public class TestEncryptionZones { // Flushing the KP on the NN, since it caches, and init a test one cluster.getNamesystem().getProvider().flush(); KeyProvider provider = KeyProviderFactory - .get(new URI(conf.get(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI)), + .get(new URI(conf.getTrimmed(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI)), conf); List<String> keys = provider.getKeys(); assertEquals("Expected NN to have created one key per zone", 1,
