[
https://issues.apache.org/jira/browse/HADOOP-18863?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17758094#comment-17758094
]
Steve Loughran commented on HADOOP-18863:
-----------------------------------------
run with args -Dtest=moo -Dprefetch -Dscale
-Dit.test=ITestS3AHugeFilesNoMultipart
{code}
[ERROR]
test_030_postCreationAssertions(org.apache.hadoop.fs.s3a.scale.ITestS3AHugeFilesNoMultipart)
Time elapsed: 139.8 s <<< ERROR!
org.apache.hadoop.fs.s3a.AWSClientIOException:
copyFile(tests3ascale/disk/hugefile, tests3ascale/disk/hugefileRenamed) on
tests3ascale/disk/hugefile:
software.amazon.awssdk.core.exception.SdkClientException: Failed to initiate
multipart upload: Failed to initiate multipart upload
at
org.apache.hadoop.fs.s3a.S3AUtils.translateException(S3AUtils.java:194)
at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:124)
at org.apache.hadoop.fs.s3a.Invoker.lambda$retry$4(Invoker.java:376)
at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:468)
at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:372)
at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:347)
at
org.apache.hadoop.fs.s3a.S3AFileSystem.copyFile(S3AFileSystem.java:4439)
at
org.apache.hadoop.fs.s3a.S3AFileSystem.access$2300(S3AFileSystem.java:283)
at
org.apache.hadoop.fs.s3a.S3AFileSystem$OperationCallbacksImpl.copyFile(S3AFileSystem.java:2432)
at
org.apache.hadoop.fs.s3a.impl.RenameOperation.copySource(RenameOperation.java:561)
at
org.apache.hadoop.fs.s3a.impl.RenameOperation.renameFileToDest(RenameOperation.java:312)
at
org.apache.hadoop.fs.s3a.impl.RenameOperation.execute(RenameOperation.java:266)
at
org.apache.hadoop.fs.s3a.S3AFileSystem.innerRename(S3AFileSystem.java:2351)
at
org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$rename$8(S3AFileSystem.java:2202)
at
org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.invokeTrackingDuration(IOStatisticsBinding.java:547)
at
org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.lambda$trackDurationOfOperation$5(IOStatisticsBinding.java:528)
at
org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDuration(IOStatisticsBinding.java:449)
at
org.apache.hadoop.fs.s3a.S3AFileSystem.trackDurationAndSpan(S3AFileSystem.java:2623)
at
org.apache.hadoop.fs.s3a.S3AFileSystem.rename(S3AFileSystem.java:2200)
at
org.apache.hadoop.fs.s3a.scale.ITestS3AHugeFilesNoMultipart.lambda$test_030_postCreationAssertions$0(ITestS3AHugeFilesNoMultipart.java:108)
at
org.apache.hadoop.test.LambdaTestUtils.intercept(LambdaTestUtils.java:498)
at
org.apache.hadoop.test.LambdaTestUtils.intercept(LambdaTestUtils.java:384)
at
org.apache.hadoop.fs.s3a.scale.ITestS3AHugeFilesNoMultipart.test_030_postCreationAssertions(ITestS3AHugeFilesNoMultipart.java:107)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:568)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
at
org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at
org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:833)
Caused by: software.amazon.awssdk.core.exception.SdkClientException: Failed to
initiate multipart upload
at
software.amazon.awssdk.core.exception.SdkClientException$BuilderImpl.build(SdkClientException.java:111)
at
software.amazon.awssdk.core.exception.SdkClientException.create(SdkClientException.java:47)
at
software.amazon.awssdk.services.s3.internal.multipart.GenericMultipartHelper.handleException(GenericMultipartHelper.java:65)
at
software.amazon.awssdk.services.s3.internal.multipart.CopyObjectHelper.lambda$copyInParts$6(CopyObjectHelper.java:115)
at
java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)
at
java.base/java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887)
at
java.base/java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325)
at
software.amazon.awssdk.services.s3.internal.multipart.CopyObjectHelper.copyInParts(CopyObjectHelper.java:113)
at
software.amazon.awssdk.services.s3.internal.multipart.CopyObjectHelper.doCopyObject(CopyObjectHelper.java:98)
at
software.amazon.awssdk.services.s3.internal.multipart.CopyObjectHelper.lambda$copyObject$1(CopyObjectHelper.java:79)
at
java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)
at
java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)
at
java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at
java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147)
at
software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncApiCallMetricCollectionStage.lambda$execute$0(AsyncApiCallMetricCollectionStage.java:58)
at
java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)
at
java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)
at
java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at
java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147)
at
software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncApiCallTimeoutTrackingStage.lambda$execute$2(AsyncApiCallTimeoutTrackingStage.java:69)
at
java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)
at
java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)
at
java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at
java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147)
at
software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncRetryableStage$RetryingExecutor.lambda$attemptExecute$1(AsyncRetryableStage.java:177)
at
java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)
at
java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)
at
java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at
java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147)
at
software.amazon.awssdk.core.internal.http.pipeline.stages.MakeAsyncHttpRequestStage.lambda$null$0(MakeAsyncHttpRequestStage.java:105)
at
java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)
at
java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)
at
java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at
java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147)
at
software.amazon.awssdk.core.internal.http.pipeline.stages.MakeAsyncHttpRequestStage.completeResponseFuture(MakeAsyncHttpRequestStage.java:238)
at
software.amazon.awssdk.core.internal.http.pipeline.stages.MakeAsyncHttpRequestStage.lambda$executeHttpRequest$3(MakeAsyncHttpRequestStage.java:163)
at
java.base/java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:934)
at
java.base/java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:911)
at
java.base/java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:482)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.hadoop.fs.s3a.audit.AuditOperationRejectedException:
Multipart IO request CreateMultipartUploadRequest(Bucket=stevel-london,
ContentType=application/octet-stream, Key=tests3ascale/disk/hugefileRenamed,
Metadata={}, ServerSideEncryption=aws:kms, SSEKMSKeyId=*** Sensitive Data
Redacted ***) rejected
https://audit.example.org/hadoop/1/op_rename/51bd92f0-d3d2-4df5-ad13-6f6be3101fa6-00000017/?op=op_rename&p1=s3a://stevel-london/tests3ascale/disk/hugefile&pr=stevel&p2=s3a://stevel-london/tests3ascale/disk/hugefileRenamed&ps=59bae7c8-3098-4209-98f2-c9c9e7604bbd&id=51bd92f0-d3d2-4df5-ad13-6f6be3101fa6-00000017&t0=61&fs=51bd92f0-d3d2-4df5-ad13-6f6be3101fa6&t1=105&ts=1692800788644
at
org.apache.hadoop.fs.s3a.audit.impl.LoggingAuditor$LoggingAuditSpan.modifyHttpRequest(LoggingAuditor.java:405)
at
org.apache.hadoop.fs.s3a.audit.impl.ActiveAuditManagerS3A$WrappingAuditSpan.modifyHttpRequest(ActiveAuditManagerS3A.java:871)
at
org.apache.hadoop.fs.s3a.audit.impl.ActiveAuditManagerS3A.modifyHttpRequest(ActiveAuditManagerS3A.java:612)
at
software.amazon.awssdk.core.interceptor.ExecutionInterceptorChain.modifyHttpRequestAndHttpContent(ExecutionInterceptorChain.java:90)
at
software.amazon.awssdk.core.internal.handler.BaseClientHandler.runModifyHttpRequestAndHttpContentInterceptors(BaseClientHandler.java:157)
at
software.amazon.awssdk.core.internal.handler.BaseClientHandler.finalizeSdkHttpFullRequest(BaseClientHandler.java:83)
at
software.amazon.awssdk.core.internal.handler.BaseAsyncClientHandler.doExecute(BaseAsyncClientHandler.java:201)
at
software.amazon.awssdk.core.internal.handler.BaseAsyncClientHandler.lambda$execute$1(BaseAsyncClientHandler.java:82)
at
software.amazon.awssdk.core.internal.handler.BaseAsyncClientHandler.measureApiCallSuccess(BaseAsyncClientHandler.java:291)
at
software.amazon.awssdk.core.internal.handler.BaseAsyncClientHandler.execute(BaseAsyncClientHandler.java:75)
at
software.amazon.awssdk.awscore.client.handler.AwsAsyncClientHandler.execute(AwsAsyncClientHandler.java:52)
at
software.amazon.awssdk.services.s3.DefaultS3AsyncClient.createMultipartUpload(DefaultS3AsyncClient.java:1550)
at
software.amazon.awssdk.services.s3.DelegatingS3AsyncClient.lambda$createMultipartUpload$4(DelegatingS3AsyncClient.java:1273)
at
software.amazon.awssdk.services.s3.internal.multipart.MultipartS3AsyncClient$1.invokeOperation(MultipartS3AsyncClient.java:91)
at
software.amazon.awssdk.services.s3.DelegatingS3AsyncClient.createMultipartUpload(DelegatingS3AsyncClient.java:1273)
at
software.amazon.awssdk.services.s3.internal.multipart.CopyObjectHelper.copyInParts(CopyObjectHelper.java:108)
... 34 more
[INFO]
[INFO] Results:
[INFO]
[ERROR] Errors:
[ERROR]
ITestS3AHugeFilesNoMultipart.test_030_postCreationAssertions:107->lambda$test_030_postCreationAssertions$0:108
ยป AWSClientIO
[INFO]
{code}
> AWS SDK V2 - AuditFailureExceptions aren't being translated properly
> --------------------------------------------------------------------
>
> Key: HADOOP-18863
> URL: https://issues.apache.org/jira/browse/HADOOP-18863
> Project: Hadoop Common
> Issue Type: Sub-task
> Components: fs/s3
> Affects Versions: 3.4.0
> Reporter: Steve Loughran
> Priority: Major
>
> {{ITestS3AHugeFilesNoMultipart}} is failing because the
> {{AuditFailureException}} variant raised in the sdk handler is being wrapped
> as it makes its way back to the s3a code -but S3AUtiis.translateException()
> isn't looking at the inner cause.
> looks like aws v2 sdk class {{.GenericMultipartHelper.handleException}} is
> wrapping an SdkException with a SdkClientException even though it is not
> needed.
> we probably have to start looking at the inner cause of any exception during
> translation to see if that is also a AuditFailureException.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
