HADOOP-12555. WASB to read credentials from a credential provider. Contributed by Larry McCay.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27b77751 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27b77751 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27b77751 Branch: refs/heads/HDFS-1312 Commit: 27b77751c1163ab4a1ce081a426e5190d1b8aff4 Parents: 66289a3 Author: cnauroth <[email protected]> Authored: Mon Feb 22 17:30:13 2016 -0800 Committer: cnauroth <[email protected]> Committed: Mon Feb 22 17:30:13 2016 -0800 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++ .../src/site/markdown/CredentialProviderAPI.md | 1 + .../hadoop/fs/azure/SimpleKeyProvider.java | 16 +++++- .../hadoop-azure/src/site/markdown/index.md | 52 ++++++++++++++++++-- .../fs/azure/TestWasbUriAndConfiguration.java | 45 ++++++++++++++++- 5 files changed, 112 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/27b77751/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 01bfacb8..361859a 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -1143,6 +1143,9 @@ Release 2.8.0 - UNRELEASED HADOOP-12668. Support excluding weak Ciphers in HttpServer2 through ssl-server.conf. (Vijay Singh via zhz) + HADOOP-12555. WASB to read credentials from a credential provider. + (Larry McCay via cnauroth) + OPTIMIZATIONS HADOOP-11785. Reduce the number of listStatus operation in distcp http://git-wip-us.apache.org/repos/asf/hadoop/blob/27b77751/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md b/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md index eccc205..5ca95eb 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/CredentialProviderAPI.md @@ -110,6 +110,7 @@ In summary, first, provision the credentials into a provider then configure the |HDFS |DFSUtil leverages Configuration.getPassword method to use the credential provider API and/or fallback to the clear text value stored in ssl-server.xml.|TODO| |YARN |WebAppUtils uptakes the use of the credential provider API through the new method on Configuration called getPassword. This provides an alternative to storing the passwords in clear text within the ssl-server.xml file while maintaining backward compatibility.|TODO| |AWS <br/> S3/S3A |Uses Configuration.getPassword to get the S3 credentials. They may be resolved through the credential provider API or from the config for backward compatibility.|[AWS S3/S3A Usage](../../hadoop-aws/tools/hadoop-aws/index.html)| +|Azure <br/> WASB |Uses Configuration.getPassword to get the WASB credentials. They may be resolved through the credential provider API or from the config for backward compatibility.|[Azure WASB Usage](../../hadoop-azure/index.html)| |Apache <br/> Accumulo|The trace.password property is used by the Tracer to authenticate with Accumulo and persist the traces in the trace table. The credential provider API is used to acquire the trace.password from a provider or from configuration for backward compatibility.|TODO| |Apache <br/> Slider |A capability has been added to Slider to prompt the user for needed passwords and store them using CredentialProvider so they can be retrieved by an app later.|TODO| |Apache <br/> Hive |Protection of the metastore password, SSL related passwords and JDO string password has been added through the use of the Credential Provider API|TODO| http://git-wip-us.apache.org/repos/asf/hadoop/blob/27b77751/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SimpleKeyProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SimpleKeyProvider.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SimpleKeyProvider.java index 3cd3eda..28e307e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SimpleKeyProvider.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SimpleKeyProvider.java @@ -18,6 +18,10 @@ package org.apache.hadoop.fs.azure; +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; @@ -27,6 +31,7 @@ import org.apache.hadoop.conf.Configuration; */ @InterfaceAudience.Private public class SimpleKeyProvider implements KeyProvider { + private static final Log LOG = LogFactory.getLog(SimpleKeyProvider.class); protected static final String KEY_ACCOUNT_KEY_PREFIX = "fs.azure.account.key."; @@ -34,7 +39,16 @@ public class SimpleKeyProvider implements KeyProvider { @Override public String getStorageAccountKey(String accountName, Configuration conf) throws KeyProviderException { - return conf.get(getStorageAccountKeyName(accountName)); + String key = null; + try { + char[] keyChars = conf.getPassword(getStorageAccountKeyName(accountName)); + if (keyChars != null) { + key = new String(keyChars); + } + } catch(IOException ioe) { + LOG.warn("Unable to get key from credential providers.", ioe); + } + return key; } protected String getStorageAccountKeyName(String accountName) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/27b77751/hadoop-tools/hadoop-azure/src/site/markdown/index.md ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/index.md b/hadoop-tools/hadoop-azure/src/site/markdown/index.md index 4402467..f813c88 100644 --- a/hadoop-tools/hadoop-azure/src/site/markdown/index.md +++ b/hadoop-tools/hadoop-azure/src/site/markdown/index.md @@ -88,9 +88,55 @@ For example: <value>YOUR ACCESS KEY</value> </property> -In many Hadoop clusters, the core-site.xml file is world-readable. If it's -undesirable for the access key to be visible in core-site.xml, then it's also -possible to configure it in encrypted form. An additional configuration property +In many Hadoop clusters, the core-site.xml file is world-readable. It is possible to +protect the access key within a credential provider as well. This provides an encrypted +file format along with protection with file permissions. + +#### Protecting the Azure Credentials for WASB with Credential Providers + +To protect these credentials from prying eyes, it is recommended that you use +the credential provider framework to securely store them and access them +through configuration. The following describes its use for Azure credentials +in WASB FileSystem. + +For additional reading on the credential provider API see: +[Credential Provider API](../hadoop-project-dist/hadoop-common/CredentialProviderAPI.html). + +##### End to End Steps for Distcp and WASB with Credential Providers + +###### provision + +``` +% hadoop credential create fs.azure.account.key.youraccount.blob.core.windows.net -value 123 + -provider localjceks://file/home/lmccay/wasb.jceks +``` + +###### configure core-site.xml or command line system property + +``` +<property> + <name>hadoop.security.credential.provider.path</name> + <value>localjceks://file/home/lmccay/wasb.jceks</value> + <description>Path to interrogate for protected credentials.</description> +</property> +``` + +###### distcp + +``` +% hadoop distcp + [-D hadoop.security.credential.provider.path=localjceks://file/home/lmccay/wasb.jceks] + hdfs://hostname:9001/user/lmccay/007020615 wasb://[email protected]/testDir/ +``` + +NOTE: You may optionally add the provider path property to the distcp command line instead of +added job specific configuration to a generic core-site.xml. The square brackets above illustrate +this capability. + +#### Protecting the Azure Credentials for WASB within an Encrypted File + +In addition to using the credential provider framework to protect your credentials, it's +also possible to configure it in encrypted form. An additional configuration property specifies an external program to be invoked by Hadoop processes to decrypt the key. The encrypted key value is passed to this external program as a command line argument: http://git-wip-us.apache.org/repos/asf/hadoop/blob/27b77751/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java index 25eee47..06a5b62 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java @@ -34,6 +34,11 @@ import java.io.OutputStream; import java.net.URI; import java.util.Date; import java.util.EnumSet; +import java.io.File; + +import org.apache.hadoop.security.ProviderUtils; +import org.apache.hadoop.security.alias.CredentialProvider; +import org.apache.hadoop.security.alias.CredentialProviderFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.AbstractFileSystem; @@ -43,7 +48,9 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.CreateOptions; import org.junit.After; import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import com.microsoft.azure.storage.blob.CloudBlobContainer; import com.microsoft.azure.storage.blob.CloudBlockBlob; @@ -57,6 +64,9 @@ public class TestWasbUriAndConfiguration { protected String accountKey; protected static Configuration conf = null; + @Rule + public final TemporaryFolder tempDir = new TemporaryFolder(); + private AzureBlobStorageTestAccount testAccount; @After @@ -307,6 +317,40 @@ public class TestWasbUriAndConfiguration { } @Test + public void testCredsFromCredentialProvider() throws Exception { + String account = "testacct"; + String key = "testkey"; + // set up conf to have a cred provider + final Configuration conf = new Configuration(); + final File file = tempDir.newFile("test.jks"); + final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider( + file.toURI()); + conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, + jks.toString()); + + provisionAccountKey(conf, account, key); + + // also add to configuration as clear text that should be overridden + conf.set(SimpleKeyProvider.KEY_ACCOUNT_KEY_PREFIX + account, + key + "cleartext"); + + String result = AzureNativeFileSystemStore.getAccountKeyFromConfiguration( + account, conf); + // result should contain the credential provider key not the config key + assertEquals("AccountKey incorrect.", key, result); + } + + void provisionAccountKey( + final Configuration conf, String account, String key) throws Exception { + // add our creds to the provider + final CredentialProvider provider = + CredentialProviderFactory.getProviders(conf).get(0); + provider.createCredentialEntry( + SimpleKeyProvider.KEY_ACCOUNT_KEY_PREFIX + account, key.toCharArray()); + provider.flush(); + } + + @Test public void testValidKeyProvider() throws Exception { Configuration conf = new Configuration(); String account = "testacct"; @@ -366,7 +410,6 @@ public class TestWasbUriAndConfiguration { String authority = testAccount.getFileSystem().getUri().getAuthority(); URI defaultUri = new URI(defaultScheme, authority, null, null, null); conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString()); - // Add references to file system implementations for wasb and wasbs. conf.addResource("azure-test.xml"); URI wantedUri = new URI(wantedScheme + ":///random/path");
