[
https://issues.apache.org/jira/browse/HADOOP-19514?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17938626#comment-17938626
]
Steve Loughran commented on HADOOP-19514:
-----------------------------------------
First read
{code}
2025-03-26 14:12:49,868 [setup] DEBUG s3a.Invoker
(DurationInfo.java:<init>(80)) - Starting: PUT 0-byte object
2025-03-26 14:12:49,876 [setup] DEBUG s3a.S3AFileSystem
(S3AFileSystem.java:putObjectDirect(3245)) - PUT 0 bytes to test/
2025-03-26 14:12:49,876 [setup] DEBUG impl.S3AStoreImpl
(S3AStoreImpl.java:incrementPutStartStatistics(508)) - PUT start 0 bytes
2025-03-26 14:12:49,885 [setup] DEBUG impl.LoggingAuditor
(LoggingAuditor.java:modifyHttpRequest(404)) - [13]
787e4f03-40bb-4030-a460-c7d488b4cda1-00000005 Executing op_mkdirs with
{object_put_request 'test/' size=0, mutating=true};
https://audit.example.org/hadoop/1/op_mkdirs/787e4f03-40bb-4030-a460-c7d488b4cda1-00000005/?op=op_mkdirs&p1=test&pr=stevel&ps=52fbb2d1-1894-4cc5-bc6f-c4da90b934ec&id=787e4f03-40bb-4030-a460-c7d488b4cda1-00000005&t0=13&fs=787e4f03-40bb-4030-a460-c7d488b4cda1&t1=13&ts=1742998368140
Breakpoint reached at
org.apache.hadoop.fs.s3a.impl.UploadContentProviders$BaseContentProvider.newStream(UploadContentProviders.java:284)
Breakpoint reached
at
org.apache.hadoop.fs.s3a.impl.UploadContentProviders$BaseContentProvider.newStream(UploadContentProviders.java:284)
at
software.amazon.awssdk.core.internal.handler.BaseClientHandler.lambda$getBody$3(BaseClientHandler.java:139)
at
software.amazon.awssdk.core.internal.handler.BaseClientHandler$$Lambda$968.448457250.newStream(Unknown
Source:-1)
at
software.amazon.awssdk.core.internal.http.StreamManagingStage$ClosingStreamProvider.newStream(StreamManagingStage.java:75)
at
software.amazon.awssdk.http.auth.aws.internal.signer.AwsChunkedV4PayloadSigner.beforeSigning(AwsChunkedV4PayloadSigner.java:126)
at
software.amazon.awssdk.http.auth.aws.internal.signer.DefaultAwsV4HttpSigner.doSign(DefaultAwsV4HttpSigner.java:267)
at
software.amazon.awssdk.http.auth.aws.internal.signer.DefaultAwsV4HttpSigner.sign(DefaultAwsV4HttpSigner.java:66)
at
software.amazon.awssdk.services.s3.internal.s3express.DefaultS3ExpressHttpSigner.sign(DefaultS3ExpressHttpSigner.java:56)
at
software.amazon.awssdk.core.internal.http.pipeline.stages.SigningStage.doSraSign(SigningStage.java:113)
at
software.amazon.awssdk.core.internal.http.pipeline.stages.SigningStage.lambda$sraSignRequest$1(SigningStage.java:93)
at
software.amazon.awssdk.core.internal.http.pipeline.stages.SigningStage$$Lambda$573.1402072620.get(Unknown
Source:-1)
at
software.amazon.awssdk.core.internal.util.MetricUtils.measureDuration(MetricUtils.java:63)
at
software.amazon.awssdk.core.internal.http.pipeline.stages.SigningStage.sraSignRequest(SigningStage.java:92)
at
software.amazon.awssdk.core.internal.http.pipeline.stages.SigningStage.execute(SigningStage.java:79)
at
software.amazon.awssdk.core.internal.http.pipeline.stages.SigningStage.execute(SigningStage.java:50)
at
software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
at
software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
at
software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
at
software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
at
software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
at
software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptTimeoutTrackingStage.execute(ApiCallAttemptTimeoutTrackingStage.java:74)
at
software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptTimeoutTrackingStage.execute(ApiCallAttemptTimeoutTrackingStage.java:43)
at
software.amazon.awssdk.core.internal.http.pipeline.stages.TimeoutExceptionHandlingStage.execute(TimeoutExceptionHandlingStage.java:79)
at
software.amazon.awssdk.core.internal.http.pipeline.stages.TimeoutExceptionHandlingStage.execute(TimeoutExceptionHandlingStage.java:41)
at
software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptMetricCollectionStage.execute(ApiCallAttemptMetricCollectionStage.java:55)
at
software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptMetricCollectionStage.execute(ApiCallAttemptMetricCollectionStage.java:39)
at
software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage2.executeRequest(RetryableStage2.java:93)
at
software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage2.execute(RetryableStage2.java:56)
at
software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage2.execute(RetryableStage2.java:36)
at
software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
at
software.amazon.awssdk.core.internal.http.StreamManagingStage.execute(StreamManagingStage.java:53)
at
software.amazon.awssdk.core.internal.http.StreamManagingStage.execute(StreamManagingStage.java:35)
at
software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.executeWithTimer(ApiCallTimeoutTrackingStage.java:82)
at
software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.execute(ApiCallTimeoutTrackingStage.java:62)
at
software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.execute(ApiCallTimeoutTrackingStage.java:43)
at
software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallMetricCollectionStage.execute(ApiCallMetricCollectionStage.java:50)
at
software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallMetricCollectionStage.execute(ApiCallMetricCollectionStage.java:32)
at
software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
at
software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
at
software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:37)
at
software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:26)
at
software.amazon.awssdk.core.internal.http.AmazonSyncHttpClient$RequestExecutionBuilderImpl.execute(AmazonSyncHttpClient.java:210)
at
software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.invoke(BaseSyncClientHandler.java:103)
at
software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.doExecute(BaseSyncClientHandler.java:173)
at
software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.lambda$execute$1(BaseSyncClientHandler.java:80)
at
software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler$$Lambda$381.77983043.get(Unknown
Source:-1)
at
software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.measureApiCallSuccess(BaseSyncClientHandler.java:182)
at
software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.execute(BaseSyncClientHandler.java:74)
at
software.amazon.awssdk.core.client.handler.SdkSyncClientHandler.execute(SdkSyncClientHandler.java:45)
at
software.amazon.awssdk.awscore.client.handler.AwsSyncClientHandler.execute(AwsSyncClientHandler.java:53)
at
software.amazon.awssdk.services.s3.DefaultS3Client.putObject(DefaultS3Client.java:11159)
at
software.amazon.awssdk.services.s3.DelegatingS3Client.lambda$putObject$86(DelegatingS3Client.java:9090)
at
software.amazon.awssdk.services.s3.DelegatingS3Client$$Lambda$963.108910578.apply(Unknown
Source:-1)
at
software.amazon.awssdk.services.s3.internal.crossregion.S3CrossRegionSyncClient.invokeOperation(S3CrossRegionSyncClient.java:67)
at
software.amazon.awssdk.services.s3.DelegatingS3Client.putObject(DelegatingS3Client.java:9090)
at
org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$putObjectDirect$14(S3AFileSystem.java:3253)
at
org.apache.hadoop.fs.s3a.S3AFileSystem$$Lambda$962.1954981459.get(Unknown
Source:-1)
at
org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfSupplier(IOStatisticsBinding.java:651)
at
org.apache.hadoop.fs.s3a.S3AFileSystem.putObjectDirect(S3AFileSystem.java:3251)
at
org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$createEmptyObject$30(S3AFileSystem.java:4638)
at
org.apache.hadoop.fs.s3a.S3AFileSystem$$Lambda$880.2071922674.apply(Unknown
Source:-1)
at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:122)
at org.apache.hadoop.fs.s3a.Invoker.lambda$retry$4(Invoker.java:376)
at
org.apache.hadoop.fs.s3a.Invoker$$Lambda$881.1245808814.apply(Unknown Source:-1)
at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:468)
at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:372)
at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:347)
at
org.apache.hadoop.fs.s3a.S3AFileSystem.createEmptyObject(S3AFileSystem.java:4637)
at
org.apache.hadoop.fs.s3a.S3AFileSystem.createFakeDirectory(S3AFileSystem.java:4620)
at
org.apache.hadoop.fs.s3a.S3AFileSystem.access$2600(S3AFileSystem.java:291)
at
org.apache.hadoop.fs.s3a.S3AFileSystem$MkdirOperationCallbacksImpl.createFakeDirectory(S3AFileSystem.java:3761)
at
org.apache.hadoop.fs.s3a.impl.MkdirOperation.execute(MkdirOperation.java:162)
at
org.apache.hadoop.fs.s3a.impl.MkdirOperation.execute(MkdirOperation.java:59)
at
org.apache.hadoop.fs.s3a.impl.ExecutingStoreOperation.apply(ExecutingStoreOperation.java:76)
at
org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.invokeTrackingDuration(IOStatisticsBinding.java:547)
at
org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.lambda$trackDurationOfOperation$5(IOStatisticsBinding.java:528)
at
org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding$$Lambda$93.901145732.apply(Unknown
Source:-1)
at
org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDuration(IOStatisticsBinding.java:449)
at
org.apache.hadoop.fs.s3a.S3AFileSystem.trackDurationAndSpan(S3AFileSystem.java:2802)
at
org.apache.hadoop.fs.s3a.S3AFileSystem.trackDurationAndSpan(S3AFileSystem.java:2821)
at
org.apache.hadoop.fs.s3a.S3AFileSystem.mkdirs(S3AFileSystem.java:3726)
at org.apache.hadoop.fs.FileSystem.mkdirs(FileSystem.java:2496)
at
org.apache.hadoop.fs.contract.AbstractFSContractTestBase.mkdirs(AbstractFSContractTestBase.java:363)
at
org.apache.hadoop.fs.contract.AbstractFSContractTestBase.setup(AbstractFSContractTestBase.java:205)
at
sun.reflect.NativeMethodAccessorImpl.invoke0(NativeMethodAccessorImpl.java:-1)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at
org.junit.internal.runners.statements.RunBefores.invokeMethod(RunBefores.java:33)
at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24)
at
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
at
org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at
org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run$$$capture(FutureTask.java:266)
at java.util.concurrent.FutureTask.run(FutureTask.java:-1)
at java.lang.Thread.run(Thread.java:750)
{code}
> SecretManager logs at INFO in bin/hadoop calls
> ----------------------------------------------
>
> Key: HADOOP-19514
> URL: https://issues.apache.org/jira/browse/HADOOP-19514
> Project: Hadoop Common
> Issue Type: Bug
> Components: bin
> Affects Versions: 3.5.0
> Reporter: Steve Loughran
> Assignee: Chris Nauroth
> Priority: Blocker
> Labels: pull-request-available
>
> When I invoke a CLI command, I now get told information about HMAC keys
> which are
> * utterly meaningless to me
> * completely unrelated to what I am doing
> {code}
> bin/hadoop s3guard bucket-info $BUCKET
> 2025-03-25 12:05:44,838 [main] INFO token.SecretManager
> (SecretManager.java:<clinit>(126)) - Selected hash algorithm: HmacSHA1
> 2025-03-25 12:05:44,838 [main] INFO token.SecretManager
> (SecretManager.java:<clinit>(131)) - Selected hash key length:64
> {code}
> Looks like the changes in YARN-11738 have created this
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-issues-h...@hadoop.apache.org
