HADOOP-13237: s3a initialization against public bucket fails if caller lacks any credentials. Contributed by Chris Nauroth
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7e09601a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7e09601a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7e09601a Branch: refs/heads/branch-2.8 Commit: 7e09601a90303874f37c647d860a217bffe85311 Parents: 3b2a25b Author: Steve Loughran <ste...@apache.org> Authored: Thu Jun 9 16:36:27 2016 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Thu Jun 9 16:36:51 2016 +0100 ---------------------------------------------------------------------- .../src/main/resources/core-default.xml | 13 ++++- .../fs/s3a/AnonymousAWSCredentialsProvider.java | 11 ++++ .../fs/s3a/BasicAWSCredentialsProvider.java | 8 +++ .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 22 +++++--- .../src/site/markdown/tools/hadoop-aws/index.md | 14 ++++- .../fs/s3a/TestS3AAWSCredentialsProvider.java | 55 ++++++++++++++++++++ 6 files changed, 113 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e09601a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 3294bb5..7963f33 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -746,7 +746,18 @@ <property> <name>fs.s3a.aws.credentials.provider</name> - <description>Class name of a credentials provider that implements com.amazonaws.auth.AWSCredentialsProvider. Omit if using access/secret keys or another authentication mechanism.</description> + <description> + Class name of a credentials provider that implements + com.amazonaws.auth.AWSCredentialsProvider. Omit if using access/secret keys + or another authentication mechanism. The specified class must provide an + accessible constructor accepting java.net.URI and + org.apache.hadoop.conf.Configuration, or an accessible default constructor. + Specifying org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider allows + anonymous access to a publicly accessible S3 bucket without any credentials. + Please note that allowing anonymous access to an S3 bucket compromises + security and therefore is unsuitable for most use cases. It can be useful + for accessing public data sets without requiring AWS credentials. + </description> </property> <property> http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e09601a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java index e62ec77..2c863fc 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java @@ -24,6 +24,17 @@ import com.amazonaws.auth.AWSCredentials; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +/** + * AnonymousAWSCredentialsProvider supports anonymous access to AWS services + * through the AWS SDK. AWS requests will not be signed. This is not suitable + * for most cases, because allowing anonymous access to an S3 bucket compromises + * security. This can be useful for accessing public data sets without + * requiring AWS credentials. + * + * Please note that users may reference this class name from configuration + * property fs.s3a.aws.credentials.provider. Therefore, changing the class name + * would be a backward-incompatible change. + */ @InterfaceAudience.Private @InterfaceStability.Stable public class AnonymousAWSCredentialsProvider implements AWSCredentialsProvider { http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e09601a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java index 2f721e4..3a5ee8c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java @@ -26,6 +26,14 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +/** + * BasicAWSCredentialsProvider supports static configuration of access key ID + * and secret access key for use with the AWS SDK. + * + * Please note that users may reference this class name from configuration + * property fs.s3a.aws.credentials.provider. Therefore, changing the class name + * would be a backward-incompatible change. + */ @InterfaceAudience.Private @InterfaceStability.Stable public class BasicAWSCredentialsProvider implements AWSCredentialsProvider { http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e09601a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index d477679..c88e0f0 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -527,20 +527,28 @@ public class S3AFileSystem extends FileSystem { new BasicAWSCredentialsProvider( creds.getAccessKey(), creds.getAccessSecret()), new InstanceProfileCredentialsProvider(), - new EnvironmentVariableCredentialsProvider(), - new AnonymousAWSCredentialsProvider() - ); + new EnvironmentVariableCredentialsProvider()); } else { try { LOG.debug("Credential provider class is {}", className); - credentials = (AWSCredentialsProvider) Class.forName(className) - .getDeclaredConstructor(URI.class, Configuration.class) - .newInstance(this.uri, conf); + Class<?> credClass = Class.forName(className); + try { + credentials = + (AWSCredentialsProvider)credClass.getDeclaredConstructor( + URI.class, Configuration.class).newInstance(this.uri, conf); + } catch (NoSuchMethodException | SecurityException e) { + credentials = + (AWSCredentialsProvider)credClass.getDeclaredConstructor() + .newInstance(); + } } catch (ClassNotFoundException e) { throw new IOException(className + " not found.", e); } catch (NoSuchMethodException | SecurityException e) { - throw new IOException(className + " constructor exception.", e); + throw new IOException(String.format("%s constructor exception. A " + + "class specified in %s must provide an accessible constructor " + + "accepting URI and Configuration, or an accessible default " + + "constructor.", className, AWS_CREDENTIALS_PROVIDER), e); } catch (ReflectiveOperationException | IllegalArgumentException e) { throw new IOException(className + " instantiation exception.", e); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e09601a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 8ca10a7..c8c3865 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -184,8 +184,18 @@ If you do any of these: change your credentials immediately! <property> <name>fs.s3a.aws.credentials.provider</name> - <description>Class name of a credentials provider that implements com.amazonaws.auth.AWSCredentialsProvider. - Omit if using access/secret keys or another authentication mechanism.</description> + <description> + Class name of a credentials provider that implements + com.amazonaws.auth.AWSCredentialsProvider. Omit if using access/secret keys + or another authentication mechanism. The specified class must provide an + accessible constructor accepting java.net.URI and + org.apache.hadoop.conf.Configuration, or an accessible default constructor. + Specifying org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider allows + anonymous access to a publicly accessible S3 bucket without any credentials. + Please note that allowing anonymous access to an S3 bucket compromises + security and therefore is unsuitable for most use cases. It can be useful + for accessing public data sets without requiring AWS credentials. + </description> </property> #### Protecting the AWS Credentials in S3A http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e09601a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java index 1a11a45..a25ca9c 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs.s3a; import static org.apache.hadoop.fs.s3a.Constants.*; +import static org.apache.hadoop.fs.s3a.S3ATestConstants.*; import static org.junit.Assert.*; import java.io.IOException; @@ -26,8 +27,13 @@ import java.net.URI; import java.nio.file.AccessDeniedException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.Timeout; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; @@ -45,6 +51,12 @@ public class TestS3AAWSCredentialsProvider { private static final Logger LOG = LoggerFactory.getLogger(TestS3AAWSCredentialsProvider.class); + @Rule + public Timeout testTimeout = new Timeout(1 * 60 * 1000); + + @Rule + public ExpectedException exception = ExpectedException.none(); + @Test public void testBadConfiguration() throws IOException { Configuration conf = new Configuration(); @@ -113,4 +125,47 @@ public class TestS3AAWSCredentialsProvider { conf.set(AWS_CREDENTIALS_PROVIDER, GoodCredentialsProvider.class.getName()); S3ATestUtils.createTestFileSystem(conf); } + + @Test + public void testAnonymousProvider() throws Exception { + Configuration conf = new Configuration(); + conf.set(AWS_CREDENTIALS_PROVIDER, + AnonymousAWSCredentialsProvider.class.getName()); + Path testFile = new Path( + conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE)); + FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf); + assertNotNull(fs); + assertTrue(fs instanceof S3AFileSystem); + FileStatus stat = fs.getFileStatus(testFile); + assertNotNull(stat); + assertEquals(testFile, stat.getPath()); + } + + static class ConstructorErrorProvider implements AWSCredentialsProvider { + + @SuppressWarnings("unused") + public ConstructorErrorProvider(String str) { + } + + @Override + public AWSCredentials getCredentials() { + return null; + } + + @Override + public void refresh() { + } + } + + @Test + public void testProviderConstructorError() throws Exception { + Configuration conf = new Configuration(); + conf.set(AWS_CREDENTIALS_PROVIDER, + ConstructorErrorProvider.class.getName()); + Path testFile = new Path( + conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE)); + exception.expect(IOException.class); + exception.expectMessage("constructor exception"); + FileSystem fs = FileSystem.newInstance(testFile.toUri(), conf); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
