HADOOP-12537 S3A to support Amazon STS temporary credentials. Contributed by Sean Mackrory.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/31ffaf76 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/31ffaf76 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/31ffaf76 Branch: refs/heads/trunk Commit: 31ffaf76f2b6e1fd2a141daa4daaebdfecefe727 Parents: 9378d94 Author: Steve Loughran <ste...@apache.org> Authored: Thu Jun 9 20:58:30 2016 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Thu Jun 9 21:00:47 2016 +0100 ---------------------------------------------------------------------- .../src/main/resources/core-default.xml | 5 + hadoop-project/pom.xml | 8 +- hadoop-tools/hadoop-aws/pom.xml | 5 + .../fs/s3a/BasicAWSCredentialsProvider.java | 3 +- .../org/apache/hadoop/fs/s3a/Constants.java | 3 + .../s3a/CredentialInitializationException.java | 46 ++++++ .../fs/s3a/TemporaryAWSCredentialsProvider.java | 70 +++++++++ .../src/site/markdown/tools/hadoop-aws/index.md | 71 ++++++++- .../fs/s3a/TestS3ATemporaryCredentials.java | 150 +++++++++++++++++++ 9 files changed, 357 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ffaf76/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 8bb27ea..39b7132 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -806,6 +806,11 @@ </property> <property> + <name>fs.s3a.session.token</name> + <description>The session token used with temporary credentials. Used only with provider org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider.</description> +</property> + +<property> <name>fs.s3a.connection.maximum</name> <value>15</value> <description>Controls the maximum number of simultaneous connections to S3.</description> http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ffaf76/hadoop-project/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 2b6b162..4c618a1 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -116,6 +116,7 @@ <make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version> <native-maven-plugin.version>1.0-alpha-8</native-maven-plugin.version> <surefire.fork.timeout>900</surefire.fork.timeout> + <aws-java-sdk.version>1.10.6</aws-java-sdk.version> </properties> <dependencyManagement> @@ -690,7 +691,12 @@ <dependency> <groupId>com.amazonaws</groupId> <artifactId>aws-java-sdk-s3</artifactId> - <version>1.10.6</version> + <version>${aws-java-sdk.version}</version> + </dependency> + <dependency> + <groupId>com.amazonaws</groupId> + <artifactId>aws-java-sdk-sts</artifactId> + <version>${aws-java-sdk.version}</version> </dependency> <dependency> <groupId>org.apache.mina</groupId> http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ffaf76/hadoop-tools/hadoop-aws/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index c95f1e6..7c25e60 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -231,6 +231,11 @@ <scope>compile</scope> </dependency> <dependency> + <groupId>com.amazonaws</groupId> + <artifactId>aws-java-sdk-sts</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <scope>test</scope> http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ffaf76/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java index 3a5ee8c..61be43f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java @@ -18,7 +18,6 @@ package org.apache.hadoop.fs.s3a; -import com.amazonaws.AmazonClientException; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.auth.AWSCredentials; @@ -49,7 +48,7 @@ public class BasicAWSCredentialsProvider implements AWSCredentialsProvider { if (!StringUtils.isEmpty(accessKey) && !StringUtils.isEmpty(secretKey)) { return new BasicAWSCredentials(accessKey, secretKey); } - throw new AmazonClientException( + throw new CredentialInitializationException( "Access key or secret key is null"); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ffaf76/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index eb859ac..4abb550 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -41,6 +41,9 @@ public final class Constants { public static final String AWS_CREDENTIALS_PROVIDER = "fs.s3a.aws.credentials.provider"; + // session token for when using TemporaryAWSCredentialsProvider + public static final String SESSION_TOKEN = "fs.s3a.session.token"; + // number of simultaneous connections to s3 public static final String MAXIMUM_CONNECTIONS = "fs.s3a.connection.maximum"; public static final int DEFAULT_MAXIMUM_CONNECTIONS = 15; http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ffaf76/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CredentialInitializationException.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CredentialInitializationException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CredentialInitializationException.java new file mode 100644 index 0000000..46655bc --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CredentialInitializationException.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a; + +import com.amazonaws.AmazonClientException; + +/** + * Exception which Hadoop's AWSCredentialsProvider implementations should + * throw when there is a problem with the credential setup. This + * is a subclass of {@link AmazonClientException} which sets + * {@link #isRetryable()} to false, so as to fail fast. + */ +public class CredentialInitializationException extends AmazonClientException { + public CredentialInitializationException(String message, Throwable t) { + super(message, t); + } + + public CredentialInitializationException(String message) { + super(message); + } + + /** + * This exception is not going to go away if you try calling it again. + * @return false, always. + */ + @Override + public boolean isRetryable() { + return false; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ffaf76/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java new file mode 100644 index 0000000..190f7bc --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a; + +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.BasicSessionCredentials; +import com.amazonaws.auth.AWSCredentials; +import org.apache.commons.lang.StringUtils; + +import java.net.URI; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; + +import static org.apache.hadoop.fs.s3a.Constants.*; + +/** + * Support session credentials for authenticating with AWS. + */ +@InterfaceAudience.Private +@InterfaceStability.Stable +public class TemporaryAWSCredentialsProvider implements AWSCredentialsProvider { + + public static final String NAME + = "org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider"; + private final String accessKey; + private final String secretKey; + private final String sessionToken; + + public TemporaryAWSCredentialsProvider(URI uri, Configuration conf) { + this.accessKey = conf.get(ACCESS_KEY, null); + this.secretKey = conf.get(SECRET_KEY, null); + this.sessionToken = conf.get(SESSION_TOKEN, null); + } + + public AWSCredentials getCredentials() { + if (!StringUtils.isEmpty(accessKey) && !StringUtils.isEmpty(secretKey) + && !StringUtils.isEmpty(sessionToken)) { + return new BasicSessionCredentials(accessKey, secretKey, sessionToken); + } + throw new CredentialInitializationException( + "Access key, secret key or session token is unset"); + } + + @Override + public void refresh() {} + + @Override + public String toString() { + return getClass().getSimpleName(); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ffaf76/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 4086bc0..606275c 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -201,6 +201,46 @@ If you do any of these: change your credentials immediately! </description> </property> + <property> + <name>fs.s3a.session.token</name> + <description>Session token, when using org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider as the providers.</description> + </property> + +#### Authentication methods + +The standard way to authenticate is with an access key and secret key using the +properties above. You can also avoid configuring credentials if the EC2 +instances in your cluster are configured with IAM instance profiles that grant +the appropriate S3 access. + +A temporary set of credentials can also be obtained from Amazon STS; these +consist of an access key, a secret key, and a session token. To use these +temporary credentials you must include the `aws-java-sdk-sts` JAR in your +classpath (consult the POM for the current version) and set the +`TemporaryAWSCredentialsProvider` class as the provider. The session key +must be set in the property `fs.s3a.session.token` âand the access and secret +key properties to those of this temporary session. + + <property> + <name>fs.s3a.aws.credentials.provider</name> + <value>org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider</value> + </property> + + <property> + <name>fs.s3a.access.key</name> + <value>SESSION-ACCESS-KEY</value> + </property> + + <property> + <name>fs.s3a.secret.key</name> + <value>SESSION-SECRET-KEY</value> + </property> + + <property> + <name>fs.s3a.session.token</name> + <value>SECRET-SESSION-TOKEN</value> + </property> + #### Protecting the AWS Credentials in S3A To protect the access/secret keys from prying eyes, it is recommended that you @@ -605,6 +645,13 @@ Example: <description>AWS secret key. Omit for IAM role-based authentication.</description> <value>DONOTEVERSHARETHISSECRETKEY!</value> </property> + + <property> + <name>test.sts.endpoint</name> + <description>Specific endpoint to use for STS requests.</description> + <value>sts.amazonaws.com</value> + </property> + </configuration> ### File `contract-test-options.xml` @@ -714,8 +761,30 @@ that the file `contract-test-options.xml` does not contain any secret credentials itself. As the auth keys XML file is kept out of the source code tree, it is not going to get accidentally committed. -### Running Performance Tests against non-AWS storage infrastructures +### Running Tests against non-AWS storage infrastructures + +### S3A session tests + +The test `TestS3ATemporaryCredentials` requests a set of temporary +credentials from the STS service, then uses them to authenticate with S3. + +If an S3 implementation does not support STS, then the functional test +cases must be disabled: + + <property> + <name>test.fs.s3a.sts.enabled</name> + <value>false</value> + </property> + +These tests reqest a temporary set of credentials from the STS service endpoint. +An alternate endpoint may be defined in `test.fs.s3a.sts.endpoint`. + + <property> + <name>test.fs.s3a.sts.endpoint</name> + <value>https://sts.example.org/</value> + </property> +The default is ""; meaning "use the amazon default value". #### CSV Data source http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ffaf76/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ATemporaryCredentials.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ATemporaryCredentials.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ATemporaryCredentials.java new file mode 100644 index 0000000..be5c599 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ATemporaryCredentials.java @@ -0,0 +1,150 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a; + +import java.io.IOException; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSCredentialsProviderChain; +import com.amazonaws.auth.InstanceProfileCredentialsProvider; +import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient; +import com.amazonaws.services.securitytoken.model.GetSessionTokenRequest; +import com.amazonaws.services.securitytoken.model.GetSessionTokenResult; +import com.amazonaws.services.securitytoken.model.Credentials; + +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.contract.AbstractFSContractTestBase; +import org.apache.hadoop.fs.contract.s3a.S3AContract; +import org.apache.hadoop.conf.Configuration; + +import org.junit.Test; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.hadoop.fs.contract.ContractTestUtils.*; +import static org.apache.hadoop.fs.s3a.Constants.*; + +/** + * Tests use of temporary credentials (for example, AWS STS & S3). + * This test extends a class that "does things to the root directory", and + * should only be used against transient filesystems where you don't care about + * the data. + */ +public class TestS3ATemporaryCredentials extends AbstractFSContractTestBase { + public static final String TEST_STS_ENABLED = "test.fs.s3a.sts.enabled"; + public static final String TEST_STS_ENDPOINT = "test.fs.s3a.sts.endpoint"; + + private static final Logger LOG = + LoggerFactory.getLogger(TestS3ATemporaryCredentials.class); + + private S3AFileSystem fs; + + + private static final String PROVIDER_CLASS = + "org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider"; + + private static final long TEST_FILE_SIZE = 1024; + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new S3AContract(conf); + } + + /** + * Test use of STS for requesting temporary credentials. + * + * The property test.sts.endpoint can be set to point this at different + * STS endpoints. This test will use the AWS credentials (if provided) for + * S3A tests to request temporary credentials, then attempt to use those + * credentials instead. + * + * @throws IOException + */ + @Test + public void testSTS() throws IOException { + Configuration conf = getContract().getConf(); + if (!conf.getBoolean(TEST_STS_ENABLED, true)) { + skip("STS functional tests disabled"); + } + + String parentAccessKey = conf.getTrimmed(ACCESS_KEY, null); + String parentSecretKey = conf.getTrimmed(SECRET_KEY, null); + String stsEndpoint = conf.getTrimmed(TEST_STS_ENDPOINT, ""); + AWSCredentialsProviderChain parentCredentials; + parentCredentials = new AWSCredentialsProviderChain( + new BasicAWSCredentialsProvider(parentAccessKey, parentSecretKey), + new InstanceProfileCredentialsProvider() + ); + + AWSSecurityTokenServiceClient stsClient; + stsClient = new AWSSecurityTokenServiceClient(parentCredentials); + if (!stsEndpoint.isEmpty()) { + LOG.debug("STS Endpoint ={}", stsEndpoint); + stsClient.setEndpoint(stsEndpoint); + } + GetSessionTokenRequest sessionTokenRequest = new GetSessionTokenRequest(); + sessionTokenRequest.setDurationSeconds(900); + GetSessionTokenResult sessionTokenResult; + sessionTokenResult = stsClient.getSessionToken(sessionTokenRequest); + Credentials sessionCreds = sessionTokenResult.getCredentials(); + + String childAccessKey = sessionCreds.getAccessKeyId(); + conf.set(ACCESS_KEY, childAccessKey); + String childSecretKey = sessionCreds.getSecretAccessKey(); + conf.set(SECRET_KEY, childSecretKey); + String sessionToken = sessionCreds.getSessionToken(); + conf.set(SESSION_TOKEN, sessionToken); + + conf.set(AWS_CREDENTIALS_PROVIDER, PROVIDER_CLASS); + + try(S3AFileSystem fs = S3ATestUtils.createTestFileSystem(conf)) { + createAndVerifyFile(fs, path("testSTS"), TEST_FILE_SIZE); + } + + // now create an invalid set of credentials by changing the session + // token + conf.set(SESSION_TOKEN, "invalid-" + sessionToken); + try (S3AFileSystem fs = S3ATestUtils.createTestFileSystem(conf)) { + createAndVerifyFile(fs, path("testSTSInvalidToken"), TEST_FILE_SIZE); + fail("Expected an access exception, but file access to " + + fs.getUri() + " was allowed: " + fs); + } catch (AWSS3IOException ex) { + LOG.info("Expected Exception: {}", ex.toString()); + LOG.debug("Expected Exception: {}", ex, ex); + } + } + + @Test + public void testTemporaryCredentialValidation() throws Throwable { + Configuration conf = new Configuration(); + conf.set(ACCESS_KEY, "accesskey"); + conf.set(SECRET_KEY, "secretkey"); + conf.set(SESSION_TOKEN, ""); + TemporaryAWSCredentialsProvider provider + = new TemporaryAWSCredentialsProvider(getFileSystem().getUri(), conf); + try { + AWSCredentials credentials = provider.getCredentials(); + fail("Expected a CredentialInitializationException," + + " got " + credentials); + } catch (CredentialInitializationException expected) { + // expected + } + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
