virajjasani commented on PR #4705:
URL: https://github.com/apache/hadoop/pull/4705#issuecomment-1205754299
Tested against `us-west-2` endpoint:
```
mvn -Dparallel-tests -DtestsThreadCount=8 clean verify
[INFO] Tests run: 2, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.213
s - in org.apache.hadoop.mapreduce.filecache.TestS3AResourceScope
[INFO] Tests run: 4, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 24.88
s - in org.apache.hadoop.fs.s3a.audit.impl.TestActiveAuditManagerThreadLeakage
[INFO] Tests run: 3, Failures: 0, Errors: 0, Skipped: 0, Time elapsed:
25.786 s - in org.apache.hadoop.fs.s3a.TestS3AInputStreamRetry
[INFO] Tests run: 63, Failures: 0, Errors: 0, Skipped: 0, Time elapsed:
50.546 s - in org.apache.hadoop.fs.s3a.commit.staging.TestStagingCommitter
[INFO] Tests run: 4, Failures: 0, Errors: 0, Skipped: 0, Time elapsed:
54.939 s - in
org.apache.hadoop.fs.s3a.commit.staging.TestDirectoryCommitterScale
[INFO]
[INFO] Results:
[INFO]
[WARNING] Tests run: 406, Failures: 0, Errors: 0, Skipped: 4
[INFO]
[INFO]
[ERROR] Errors:
[ERROR] ITestS3ATemporaryCredentials.testSTS:133 » AccessDenied : request
session cred...
[ERROR]
ITestDelegatedMRJob.setup:182->AbstractS3ATestBase.setup:111->AbstractFSContractTestBase.setup:189
» ServiceState
[ERROR]
ITestDelegatedMRJob.setup:182->AbstractS3ATestBase.setup:111->AbstractFSContractTestBase.setup:189
» ServiceState
[ERROR]
ITestSessionDelegationInFileystem.testDelegationBindingMismatch2:511->AbstractDelegationIT.newS3AInstance:103
» ServiceState
[INFO]
[ERROR] Tests run: 1139, Failures: 3, Errors: 4, Skipped: 190
[INFO]
[INFO]
[INFO] Running org.apache.hadoop.fs.s3a.ITestS3AEncryptionSSEC
[INFO] Tests run: 24, Failures: 0, Errors: 0, Skipped: 0, Time elapsed:
124.403 s - in org.apache.hadoop.fs.s3a.ITestS3AEncryptionSSEC
[INFO]
[INFO] Results:
[INFO]
[WARNING] Tests run: 124, Failures: 0, Errors: 0, Skipped: 84
[INFO]
[INFO]
Errors:
[ERROR] Tests run: 11, Failures: 0, Errors: 1, Skipped: 0, Time elapsed:
30.672 s <<< FAILURE! - in
org.apache.hadoop.fs.s3a.auth.delegation.ITestSessionDelegationInFileystem
[ERROR]
testDelegationBindingMismatch2(org.apache.hadoop.fs.s3a.auth.delegation.ITestSessionDelegationInFileystem)
Time elapsed: 1.139 s <<< ERROR!
org.apache.hadoop.service.ServiceStateException:
org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenIOException: no
credentials in configuration or environment variables: No AWS credentials
at
org.apache.hadoop.service.ServiceStateException.convert(ServiceStateException.java:105)
at
org.apache.hadoop.service.AbstractService.start(AbstractService.java:204)
at
org.apache.hadoop.fs.s3a.S3AFileSystem.bindAWSClient(S3AFileSystem.java:858)
at
org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:523)
at
org.apache.hadoop.fs.s3a.auth.delegation.AbstractDelegationIT.newS3AInstance(AbstractDelegationIT.java:103)
at
org.apache.hadoop.fs.s3a.auth.delegation.ITestSessionDelegationInFileystem.testDelegationBindingMismatch2(ITestSessionDelegationInFileystem.java:511)
[ERROR] Tests run: 6, Failures: 0, Errors: 2, Skipped: 2, Time elapsed:
29.185 s <<< FAILURE! - in
org.apache.hadoop.fs.s3a.auth.delegation.ITestDelegatedMRJob
[ERROR]
testCommonCrawlLookup[1](org.apache.hadoop.fs.s3a.auth.delegation.ITestDelegatedMRJob)
Time elapsed: 0.187 s <<< ERROR!
org.apache.hadoop.service.ServiceStateException:
org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenIOException: no
credentials in configuration or environment variables: No AWS credentials
at
org.apache.hadoop.service.ServiceStateException.convert(ServiceStateException.java:105)
at
org.apache.hadoop.service.AbstractService.start(AbstractService.java:204)
at
org.apache.hadoop.fs.s3a.S3AFileSystem.bindAWSClient(S3AFileSystem.java:858)
at
org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:523)
at
org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3563)
at org.apache.hadoop.fs.FileSystem.access$300(FileSystem.java:172)
at
org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3668)
at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3615)
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:556)
at
org.apache.hadoop.fs.contract.AbstractBondedFSContract.init(AbstractBondedFSContract.java:72)
at
org.apache.hadoop.fs.contract.AbstractFSContractTestBase.setup(AbstractFSContractTestBase.java:189)
at
org.apache.hadoop.fs.s3a.AbstractS3ATestBase.setup(AbstractS3ATestBase.java:111)
at
org.apache.hadoop.fs.s3a.auth.delegation.ITestDelegatedMRJob.setup(ITestDelegatedMRJob.java:182)
[ERROR]
testJobSubmissionCollectsTokens[1](org.apache.hadoop.fs.s3a.auth.delegation.ITestDelegatedMRJob)
Time elapsed: 0.097 s <<< ERROR!
org.apache.hadoop.service.ServiceStateException:
org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenIOException: no
credentials in configuration or environment variables: No AWS credentials
at
org.apache.hadoop.service.ServiceStateException.convert(ServiceStateException.java:105)
at
org.apache.hadoop.service.AbstractService.start(AbstractService.java:204)
at
org.apache.hadoop.fs.s3a.S3AFileSystem.bindAWSClient(S3AFileSystem.java:858)
at
org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:523)
at
org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3563)
at org.apache.hadoop.fs.FileSystem.access$300(FileSystem.java:172)
at
org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3668)
at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3615)
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:556)
at
org.apache.hadoop.fs.contract.AbstractBondedFSContract.init(AbstractBondedFSContract.java:72)
at
org.apache.hadoop.fs.contract.AbstractFSContractTestBase.setup(AbstractFSContractTestBase.java:189)
at
org.apache.hadoop.fs.s3a.AbstractS3ATestBase.setup(AbstractS3ATestBase.java:111)
at
org.apache.hadoop.fs.s3a.auth.delegation.ITestDelegatedMRJob.setup(ITestDelegatedMRJob.java:182)
[ERROR] Tests run: 14, Failures: 0, Errors: 1, Skipped: 4, Time elapsed:
153.197 s <<< FAILURE! - in
org.apache.hadoop.fs.s3a.ITestS3ATemporaryCredentials
[ERROR] testSTS(org.apache.hadoop.fs.s3a.ITestS3ATemporaryCredentials) Time
elapsed: 2.979 s <<< ERROR!
java.nio.file.AccessDeniedException: : request session credentials:
com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException:
Cannot call GetSessionToken with session credentials (Service:
AWSSecurityTokenService; Status Code: 403; Error Code: AccessDenied; Request
ID: 4ea755eb-da74-435c-a48c-443a76f6c194; Proxy: null):AccessDenied
at
org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:247)
at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:124)
at org.apache.hadoop.fs.s3a.Invoker.lambda$retry$4(Invoker.java:376)
at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:468)
at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:372)
at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:347)
at
org.apache.hadoop.fs.s3a.auth.STSClientFactory$STSClient.requestSessionCredentials(STSClientFactory.java:202)
at
org.apache.hadoop.fs.s3a.ITestS3ATemporaryCredentials.testSTS(ITestS3ATemporaryCredentials.java:133)
[ERROR] Tests run: 12, Failures: 3, Errors: 0, Skipped: 0, Time elapsed:
125.266 s <<< FAILURE! - in
org.apache.hadoop.fs.s3a.commit.integration.ITestS3ACommitterMRJob
[ERROR]
test_200_execute[directory](org.apache.hadoop.fs.s3a.commit.integration.ITestS3ACommitterMRJob)
Time elapsed: 20.188 s <<< FAILURE!
java.lang.AssertionError:
Job job_1659642728132_0001 failed in state FAILED with cause Job setup
failed : java.nio.file.AccessDeniedException:
s3a://{s3a_bucket}/fork-0003/test/ITestS3ACommitterMRJob-execute-directory:
org.apache.hadoop.fs.s3a.auth.NoAuthWithAWSException: No AWS Credentials
provided by TemporaryAWSCredentialsProvider SimpleAWSCredentialsProvider
EnvironmentVariableCredentialsProvider IAMInstanceCredentialsProvider :
com.amazonaws.SdkClientException: Unable to load AWS credentials from
environment variables (AWS_ACCESS_KEY_ID (or AWS_ACCESS_KEY) and AWS_SECRET_KEY
(or AWS_SECRET_ACCESS_KEY))
at
org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:209)
at
org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:172)
at
org.apache.hadoop.fs.s3a.S3AFileSystem.s3GetFileStatus(S3AFileSystem.java:3570)
at
org.apache.hadoop.fs.s3a.S3AFileSystem.innerGetFileStatus(S3AFileSystem.java:3473)
at
org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$getFileStatus$24(S3AFileSystem.java:3450)
at
org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.invokeTrackingDuration(IOStatisticsBinding.java:547)
at
org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.lambda$trackDurationOfOperation$5(IOStatisticsBinding.java:528)
at
org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDuration(IOStatisticsBinding.java:449)
at
org.apache.hadoop.fs.s3a.S3AFileSystem.trackDurationAndSpan(S3AFileSystem.java:2341)
at
org.apache.hadoop.fs.s3a.S3AFileSystem.trackDurationAndSpan(S3AFileSystem.java:2360)
at
org.apache.hadoop.fs.s3a.S3AFileSystem.getFileStatus(S3AFileSystem.java:3448)
at
org.apache.hadoop.fs.s3a.commit.staging.DirectoryStagingCommitter.setupJob(DirectoryStagingCommitter.java:74)
at
org.apache.hadoop.mapreduce.v2.app.commit.CommitterEventHandler$EventProcessor.handleJobSetup(CommitterEventHandler.java:255)
at
org.apache.hadoop.mapreduce.v2.app.commit.CommitterEventHandler$EventProcessor.run(CommitterEventHandler.java:235)
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
