[
https://issues.apache.org/jira/browse/HADOOP-19384?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17912040#comment-17912040
]
ASF GitHub Bot commented on HADOOP-19384:
-----------------------------------------
steveloughran commented on code in PR #7284:
URL: https://github.com/apache/hadoop/pull/7284#discussion_r1910631412
##########
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProfileAWSCredentialsProvider.java:
##########
@@ -0,0 +1,46 @@
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.commons.lang3.SystemUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
+import software.amazon.awssdk.profiles.ProfileFile;
+
+import java.net.URI;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+
[email protected]
[email protected]
Review Comment:
evolving
##########
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProfileAWSCredentialsProvider.java:
##########
@@ -0,0 +1,46 @@
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.commons.lang3.SystemUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
+import software.amazon.awssdk.profiles.ProfileFile;
+
+import java.net.URI;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+
[email protected]
[email protected]
+public class ProfileAWSCredentialsProvider implements AwsCredentialsProvider {
+ public static final String NAME
+ = "org.apache.hadoop.fs.s3a.ProfileAWSCredentialsProvider";
+
+ private ProfileCredentialsProvider pcp;
+
+ private static Path getCredentialsPath() {
+ String credentialsFile =
SystemUtils.getEnvironmentVariable("AWS_SHARED_CREDENTIALS_FILE", null);
+ Path path = (credentialsFile == null) ?
+
FileSystems.getDefault().getPath(SystemUtils.getUserHome().getPath(),".aws","credentials")
+ : FileSystems.getDefault().getPath(credentialsFile);
+ return path;
+ }
+
+ public ProfileAWSCredentialsProvider(URI uri, Configuration conf) {
+ ProfileCredentialsProvider.Builder builder =
ProfileCredentialsProvider.builder();
+
builder.profileName("default").profileFile(ProfileFile.builder().content(getCredentialsPath()).type(ProfileFile.Type.CREDENTIALS).build());
+ pcp = builder.build();
+ }
+
+ public ProfileAWSCredentialsProvider(Configuration conf) {
+ ProfileCredentialsProvider.Builder builder =
ProfileCredentialsProvider.builder();
+
builder.profileName("default").profileFile(ProfileFile.builder().content(getCredentialsPath()).build());
Review Comment:
profile name should be configurable
##########
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/CredentialProviderListFactory.java:
##########
@@ -84,7 +85,8 @@ public final class CredentialProviderListFactory {
EnvironmentVariableCredentialsProvider.class,
IAMInstanceCredentialsProvider.class,
SimpleAWSCredentialsProvider.class,
- TemporaryAWSCredentialsProvider.class));
+ TemporaryAWSCredentialsProvider.class,
+ ProfileAWSCredentialsProvider.class));
Review Comment:
not a standard one...will need to be explicitly configured
##########
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProfileAWSCredentialsProvider.java:
##########
@@ -0,0 +1,46 @@
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.commons.lang3.SystemUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
+import software.amazon.awssdk.profiles.ProfileFile;
+
+import java.net.URI;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+
[email protected]
[email protected]
+public class ProfileAWSCredentialsProvider implements AwsCredentialsProvider {
Review Comment:
`extends AbstractAWSCredentialProvider`
and move into package auth
##########
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml:
##########
@@ -1430,7 +1430,8 @@
org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider,
org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,
software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider,
- org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider
+ org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider,
+ org.apache.hadoop.fs.s3a.ProfileAWSCredentialsProvider
Review Comment:
make this optional but document it
##########
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProfileAWSCredentialsProvider.java:
##########
@@ -0,0 +1,46 @@
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.commons.lang3.SystemUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
+import software.amazon.awssdk.profiles.ProfileFile;
+
+import java.net.URI;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+
[email protected]
[email protected]
+public class ProfileAWSCredentialsProvider implements AwsCredentialsProvider {
+ public static final String NAME
+ = "org.apache.hadoop.fs.s3a.ProfileAWSCredentialsProvider";
+
+ private ProfileCredentialsProvider pcp;
Review Comment:
make final
##########
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProfileAWSCredentialsProvider.java:
##########
@@ -0,0 +1,46 @@
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.commons.lang3.SystemUtils;
Review Comment:
nit: use same import ordering as other classes
##########
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ProfileAWSCredentialsProvider.java:
##########
@@ -0,0 +1,46 @@
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.commons.lang3.SystemUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
+import software.amazon.awssdk.profiles.ProfileFile;
+
+import java.net.URI;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+
[email protected]
[email protected]
+public class ProfileAWSCredentialsProvider implements AwsCredentialsProvider {
+ public static final String NAME
+ = "org.apache.hadoop.fs.s3a.ProfileAWSCredentialsProvider";
+
+ private ProfileCredentialsProvider pcp;
+
+ private static Path getCredentialsPath() {
+ String credentialsFile =
SystemUtils.getEnvironmentVariable("AWS_SHARED_CREDENTIALS_FILE", null);
+ Path path = (credentialsFile == null) ?
+
FileSystems.getDefault().getPath(SystemUtils.getUserHome().getPath(),".aws","credentials")
Review Comment:
this path should be configurable
> Add support for ProfileCredentialsProvider
> ------------------------------------------
>
> Key: HADOOP-19384
> URL: https://issues.apache.org/jira/browse/HADOOP-19384
> Project: Hadoop Common
> Issue Type: Improvement
> Components: fs/s3
> Affects Versions: 3.4.1
> Reporter: Venkatasubrahmanian Narayanan
> Assignee: Venkatasubrahmanian Narayanan
> Priority: Minor
> Labels: pull-request-available
>
> Hadoop currently doesn't support AWS' ProfileCredentialsProvider. A thin
> wrapper is sufficient to get it to work, since it just needs us to fetch the
> credentials file.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
