HADOOP-12807 S3AFileSystem should read AWS credentials from environment variables. Contributed by Tobin Baker.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a3f78d8f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a3f78d8f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a3f78d8f Branch: refs/heads/HDFS-7240 Commit: a3f78d8fa83f07f9183f3546203a191fcf50008c Parents: 4a1cedc Author: Steve Loughran <ste...@apache.org> Authored: Mon Jun 6 23:40:49 2016 +0200 Committer: Steve Loughran <ste...@apache.org> Committed: Mon Jun 6 23:42:36 2016 +0200 ---------------------------------------------------------------------- .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 2 ++ .../src/site/markdown/tools/hadoop-aws/index.md | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3f78d8f/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index c028544..0281a3a 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -31,6 +31,7 @@ import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; +import com.amazonaws.auth.EnvironmentVariableCredentialsProvider; import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonServiceException; import com.amazonaws.ClientConfiguration; @@ -464,6 +465,7 @@ public class S3AFileSystem extends FileSystem { new BasicAWSCredentialsProvider( creds.getAccessKey(), creds.getAccessSecret()), new InstanceProfileCredentialsProvider(), + new EnvironmentVariableCredentialsProvider(), new AnonymousAWSCredentialsProvider() ); http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3f78d8f/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 7a5e455..7d63a86 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -202,6 +202,25 @@ credentials in S3AFileSystem. For additional reading on the credential provider API see: [Credential Provider API](../../../hadoop-project-dist/hadoop-common/CredentialProviderAPI.html). +#### Authenticating via environment variables + +S3A supports configuration via [the standard AWS environment variables](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-environment). + +The core environment variables are for the access key and associated secret: + +``` +export AWS_ACCESS_KEY_ID=my.aws.key +export AWS_SECRET_ACCESS_KEY=my.secret.key +``` + +These environment variables can be used to set the authentication credentials +instead of properties in the Hadoop configuration. *Important:* these +environment variables are not propagated from client to server when +YARN applications are launched. That is: having the AWS environment variables +set when an application is launched will not permit the launched application +to access S3 resources. The environment variables must (somehow) be set +on the hosts/processes where the work is executed. + ##### End to End Steps for Distcp and S3 with Credential Providers ###### provision --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org