[
https://issues.apache.org/jira/browse/HADOOP-18154?focusedWorklogId=741520&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-741520
]
ASF GitHub Bot logged work on HADOOP-18154:
-------------------------------------------
Author: ASF GitHub Bot
Created on: 15/Mar/22 10:18
Start Date: 15/Mar/22 10:18
Worklog Time Spent: 10m
Work Description: steveloughran commented on a change in pull request
#4070:
URL: https://github.com/apache/hadoop/pull/4070#discussion_r826785122
##########
File path:
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java
##########
@@ -0,0 +1,79 @@
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.commons.lang3.StringUtils;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.WebIdentityTokenCredentialsProvider;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.ProviderUtils;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+
+/**
+ * WebIdentityTokenCredentialsProvider supports static configuration
+ * of OIDC token path, role ARN and role session name.
+ *
+ */
+//@InterfaceAudience.Public
+//@InterfaceStability.Stable
+public class OIDCTokenCredentialsProvider implements AWSCredentialsProvider {
+ public static final String NAME
+ = "org.apache.hadoop.fs.s3a.OIDCTokenCredentialsProvider";
+
+ //these are the parameters to document and to pass along with the class
+ //usually from import static org.apache.hadoop.fs.s3a.Constants.*;
+ public static final String JWT_PATH = "fs.s3a.jwt.path";
+ public static final String ROLE_ARN = "fs.s3a.role.arn";
Review comment:
and reference existing constants from the same class
##########
File path:
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java
##########
@@ -0,0 +1,79 @@
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.commons.lang3.StringUtils;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.WebIdentityTokenCredentialsProvider;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.ProviderUtils;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+
+/**
+ * WebIdentityTokenCredentialsProvider supports static configuration
+ * of OIDC token path, role ARN and role session name.
+ *
+ */
+//@InterfaceAudience.Public
+//@InterfaceStability.Stable
+public class OIDCTokenCredentialsProvider implements AWSCredentialsProvider {
+ public static final String NAME
+ = "org.apache.hadoop.fs.s3a.OIDCTokenCredentialsProvider";
+
+ //these are the parameters to document and to pass along with the class
+ //usually from import static org.apache.hadoop.fs.s3a.Constants.*;
+ public static final String JWT_PATH = "fs.s3a.jwt.path";
Review comment:
move new constants into org.apache.hadoop.fs.s3a.Constants
##########
File path:
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java
##########
@@ -0,0 +1,79 @@
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.commons.lang3.StringUtils;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.WebIdentityTokenCredentialsProvider;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.ProviderUtils;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+
+/**
+ * WebIdentityTokenCredentialsProvider supports static configuration
+ * of OIDC token path, role ARN and role session name.
+ *
+ */
+//@InterfaceAudience.Public
+//@InterfaceStability.Stable
+public class OIDCTokenCredentialsProvider implements AWSCredentialsProvider {
+ public static final String NAME
+ = "org.apache.hadoop.fs.s3a.OIDCTokenCredentialsProvider";
+
+ //these are the parameters to document and to pass along with the class
+ //usually from import static org.apache.hadoop.fs.s3a.Constants.*;
+ public static final String JWT_PATH = "fs.s3a.jwt.path";
+ public static final String ROLE_ARN = "fs.s3a.role.arn";
+ public static final String SESSION_NAME = "fs.s3a.session.name";
+
+ /** Reuse the S3AFileSystem log. */
+ private static final Logger LOG = S3AFileSystem.LOG;
+
+ private String jwtPath;
+ private String roleARN;
+ private String sessionName;
+ private IOException lookupIOE;
+
+ public OIDCTokenCredentialsProvider(Configuration conf) {
+ try {
+ Configuration c =
ProviderUtils.excludeIncompatibleCredentialProviders(
+ conf, S3AFileSystem.class);
+ this.jwtPath = S3AUtils.lookupPassword(c, JWT_PATH, null);
+ this.roleARN = S3AUtils.lookupPassword(c, ROLE_ARN, null);
+ this.sessionName = S3AUtils.lookupPassword(c, SESSION_NAME, null);
+ } catch (IOException e) {
+ lookupIOE = e;
+ }
+ }
+
+ public AWSCredentials getCredentials() {
+ if (lookupIOE != null) {
+ // propagate any initialization problem
+ throw new CredentialInitializationException(lookupIOE.toString(),
+ lookupIOE);
+ }
+
+ LOG.debug("jwtPath {} roleARN {}", jwtPath, roleARN);
+
+ if (!StringUtils.isEmpty(jwtPath) && !StringUtils.isEmpty(roleARN)) {
+ final AWSCredentialsProvider credentialsProvider =
+ WebIdentityTokenCredentialsProvider.builder()
+ .webIdentityTokenFile(jwtPath)
+ .roleArn(roleARN)
+ .roleSessionName(sessionName)
+ .build();
+ return credentialsProvider.getCredentials();
+ }
+ else throw new CredentialInitializationException(
+ "OIDC token path or role ARN is null");
+ }
+
+ public void refresh() {}
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName();
Review comment:
be nice to include any non-secret values here, e.g. role name, just to
help with logging
##########
File path:
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java
##########
@@ -0,0 +1,79 @@
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.commons.lang3.StringUtils;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.WebIdentityTokenCredentialsProvider;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.ProviderUtils;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+
+/**
+ * WebIdentityTokenCredentialsProvider supports static configuration
+ * of OIDC token path, role ARN and role session name.
+ *
+ */
+//@InterfaceAudience.Public
+//@InterfaceStability.Stable
+public class OIDCTokenCredentialsProvider implements AWSCredentialsProvider {
+ public static final String NAME
+ = "org.apache.hadoop.fs.s3a.OIDCTokenCredentialsProvider";
+
+ //these are the parameters to document and to pass along with the class
+ //usually from import static org.apache.hadoop.fs.s3a.Constants.*;
+ public static final String JWT_PATH = "fs.s3a.jwt.path";
+ public static final String ROLE_ARN = "fs.s3a.role.arn";
+ public static final String SESSION_NAME = "fs.s3a.session.name";
+
+ /** Reuse the S3AFileSystem log. */
+ private static final Logger LOG = S3AFileSystem.LOG;
+
+ private String jwtPath;
+ private String roleARN;
+ private String sessionName;
+ private IOException lookupIOE;
+
+ public OIDCTokenCredentialsProvider(Configuration conf) {
Review comment:
should credential providers be allowed to raise IOEs? we should be able
to fix that
##########
File path:
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java
##########
@@ -0,0 +1,79 @@
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.commons.lang3.StringUtils;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.WebIdentityTokenCredentialsProvider;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.ProviderUtils;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+
+/**
+ * WebIdentityTokenCredentialsProvider supports static configuration
+ * of OIDC token path, role ARN and role session name.
+ *
+ */
+//@InterfaceAudience.Public
+//@InterfaceStability.Stable
+public class OIDCTokenCredentialsProvider implements AWSCredentialsProvider {
+ public static final String NAME
+ = "org.apache.hadoop.fs.s3a.OIDCTokenCredentialsProvider";
+
+ //these are the parameters to document and to pass along with the class
+ //usually from import static org.apache.hadoop.fs.s3a.Constants.*;
+ public static final String JWT_PATH = "fs.s3a.jwt.path";
+ public static final String ROLE_ARN = "fs.s3a.role.arn";
+ public static final String SESSION_NAME = "fs.s3a.session.name";
+
+ /** Reuse the S3AFileSystem log. */
+ private static final Logger LOG = S3AFileSystem.LOG;
+
+ private String jwtPath;
+ private String roleARN;
+ private String sessionName;
+ private IOException lookupIOE;
+
+ public OIDCTokenCredentialsProvider(Configuration conf) {
+ try {
+ Configuration c =
ProviderUtils.excludeIncompatibleCredentialProviders(
+ conf, S3AFileSystem.class);
+ this.jwtPath = S3AUtils.lookupPassword(c, JWT_PATH, null);
+ this.roleARN = S3AUtils.lookupPassword(c, ROLE_ARN, null);
+ this.sessionName = S3AUtils.lookupPassword(c, SESSION_NAME, null);
+ } catch (IOException e) {
+ lookupIOE = e;
+ }
+ }
+
+ public AWSCredentials getCredentials() {
+ if (lookupIOE != null) {
+ // propagate any initialization problem
+ throw new CredentialInitializationException(lookupIOE.toString(),
+ lookupIOE);
+ }
+
+ LOG.debug("jwtPath {} roleARN {}", jwtPath, roleARN);
+
+ if (!StringUtils.isEmpty(jwtPath) && !StringUtils.isEmpty(roleARN)) {
+ final AWSCredentialsProvider credentialsProvider =
+ WebIdentityTokenCredentialsProvider.builder()
+ .webIdentityTokenFile(jwtPath)
+ .roleArn(roleARN)
+ .roleSessionName(sessionName)
+ .build();
+ return credentialsProvider.getCredentials();
+ }
Review comment:
nit: same line as the }
##########
File path:
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java
##########
@@ -0,0 +1,79 @@
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.commons.lang3.StringUtils;
Review comment:
nit, we have some layout rules for imports. here's my full intellij
settings for this if it helps
https://gist.github.com/steveloughran/817dd90e0f1775ce2b6f24684dfb078c
##########
File path:
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/OIDCTokenCredentialsProvider.java
##########
@@ -0,0 +1,79 @@
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.commons.lang3.StringUtils;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.WebIdentityTokenCredentialsProvider;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.ProviderUtils;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+
+/**
+ * WebIdentityTokenCredentialsProvider supports static configuration
+ * of OIDC token path, role ARN and role session name.
+ *
+ */
+//@InterfaceAudience.Public
+//@InterfaceStability.Stable
+public class OIDCTokenCredentialsProvider implements AWSCredentialsProvider {
+ public static final String NAME
+ = "org.apache.hadoop.fs.s3a.OIDCTokenCredentialsProvider";
+
+ //these are the parameters to document and to pass along with the class
+ //usually from import static org.apache.hadoop.fs.s3a.Constants.*;
+ public static final String JWT_PATH = "fs.s3a.jwt.path";
+ public static final String ROLE_ARN = "fs.s3a.role.arn";
+ public static final String SESSION_NAME = "fs.s3a.session.name";
+
+ /** Reuse the S3AFileSystem log. */
+ private static final Logger LOG = S3AFileSystem.LOG;
+
+ private String jwtPath;
+ private String roleARN;
+ private String sessionName;
+ private IOException lookupIOE;
+
+ public OIDCTokenCredentialsProvider(Configuration conf) {
+ try {
+ Configuration c =
ProviderUtils.excludeIncompatibleCredentialProviders(
+ conf, S3AFileSystem.class);
+ this.jwtPath = S3AUtils.lookupPassword(c, JWT_PATH, null);
+ this.roleARN = S3AUtils.lookupPassword(c, ROLE_ARN, null);
+ this.sessionName = S3AUtils.lookupPassword(c, SESSION_NAME, null);
+ } catch (IOException e) {
+ lookupIOE = e;
+ }
+ }
+
+ public AWSCredentials getCredentials() {
+ if (lookupIOE != null) {
+ // propagate any initialization problem
+ throw new CredentialInitializationException(lookupIOE.toString(),
+ lookupIOE);
+ }
+
+ LOG.debug("jwtPath {} roleARN {}", jwtPath, roleARN);
+
+ if (!StringUtils.isEmpty(jwtPath) && !StringUtils.isEmpty(roleARN)) {
+ final AWSCredentialsProvider credentialsProvider =
+ WebIdentityTokenCredentialsProvider.builder()
+ .webIdentityTokenFile(jwtPath)
Review comment:
this will handle local files only, so won't work for jobs across a
cluster unless the token is already there.
either cluster fs paths will be needed (download locally and then reference)
or we require it on the host of the user launching a job and then include the
token data in a delegation token which goes with it. that's a lot more powerful
-but a lot more work. best to leave that for a followup patch
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 741520)
Remaining Estimate: 0h
Time Spent: 10m
> S3A Authentication to support WebIdentity
> -----------------------------------------
>
> Key: HADOOP-18154
> URL: https://issues.apache.org/jira/browse/HADOOP-18154
> Project: Hadoop Common
> Issue Type: Improvement
> Components: fs/s3
> Affects Versions: 2.10.1
> Reporter: Ju Clarysse
> Assignee: Ju Clarysse
> Priority: Major
> Time Spent: 10m
> Remaining Estimate: 0h
>
> We are using the latest version of
> [delta-sharing|https://github.com/delta-io/delta-sharing] which takes
> advantage of
> [hadoop-aws|https://hadoop.apache.org/docs/current/hadoop-aws/tools/hadoop-aws/index.html]
> (S3A) connector in [Hadoop release version
> 2.10.1|https://github.com/apache/hadoop/tree/rel/release-2.10.1] to mount an
> AWS S3 File System. In our particular setup, all services are operated in
> Amazon Elastic Kubernetes Service (EKS) and need to comply to the AWS
> security concept [IAM roles for service
> accounts|https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html]
> (IRSA).
> As [Delta sharing S3 connection|https://github.com/delta-io/delta-sharing#s3]
> doesn't offer any corresponding support, we patched hadoop-aws-2.10.1 to
> address this need via a new credentials provider class
> org.apache.hadoop.fs.s3a.OIDCTokenCredentialsProvider. We also upgraded
> dependency aws-java-sdk-bundle to its latest version 1.12.167 as [AWS
> WebIdentityTokenCredentialsProvider
> class|https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/WebIdentityTokenCredentialsProvider.html%E2%80%A6]
> was not yet available in original version 1.11.271.
> We believe that other delta-sharing users could benefit from this short-term
> contribution. Then sooner or later, delta-sharing owners will have to upgrade
> their project to a more recent version of hadoop-aws that is probably more
> widely used. The effort to promote this change is probably low.
> Additional note: AWS WebIdentityTokenCredentialsProvider class is directly
> supported by Spark applications submitted with configuration properties
> `spark.hadoop.fs.s3a.aws.credentials.provider`and
> `spark.kubernetes.authenticate.submission.oauthToken`
> ([doc|https://spark.apache.org/docs/latest/running-on-kubernetes.html#spark-properties]).
> So bringing this support to Hadoop will primarily be interesting for
> non-Spark users.
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
