This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 9b217163be2abfd17c6ddaf049d5b338ccb50104
Author: Steve Loughran <ste...@cloudera.com>
AuthorDate: Wed Apr 2 19:42:19 2025 +0100
Revert "HADOOP-19455. S3A: SDK client to log metrics at TRACE"
This reverts commit 5a4108c0213e63dcd3d2cc57a2082746353020fc.
---
.../hadoop/fs/s3a/DefaultS3ClientFactory.java | 11 +-----
.../tools/hadoop-aws/troubleshooting_s3a.md | 42 ----------------------
.../hadoop-aws/src/test/resources/log4j.properties | 4 ---
3 files changed, 1 insertion(+), 56 deletions(-)
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
index 7b5aa5ff934..7b686130ce3 100644
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
+++
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java
@@ -38,7 +38,6 @@
import software.amazon.awssdk.http.auth.spi.scheme.AuthScheme;
import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient;
import software.amazon.awssdk.identity.spi.AwsCredentialsIdentity;
-import software.amazon.awssdk.metrics.LoggingMetricPublisher;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.s3accessgrants.plugin.S3AccessGrantsPlugin;
import software.amazon.awssdk.services.s3.S3AsyncClient;
@@ -212,20 +211,12 @@ private <BuilderT extends S3BaseClientBuilder<BuilderT,
ClientT>, ClientT> Build
final ClientOverrideConfiguration.Builder override =
createClientOverrideConfiguration(parameters, conf);
- S3BaseClientBuilder<BuilderT, ClientT> s3BaseClientBuilder = builder
+ S3BaseClientBuilder s3BaseClientBuilder = builder
.overrideConfiguration(override.build())
.credentialsProvider(parameters.getCredentialSet())
.disableS3ExpressSessionAuth(!parameters.isExpressCreateSession())
.serviceConfiguration(serviceConfiguration);
- if (LOG.isTraceEnabled()) {
- // if this log is set to "trace" then we turn on logging of SDK metrics.
- // The metrics itself will log at info; it is just that reflection work
- // would be needed to change that setting safely for shaded and unshaded
aws artifacts.
- s3BaseClientBuilder.overrideConfiguration(o ->
- o.addMetricPublisher(LoggingMetricPublisher.create()));
- }
-
if (conf.getBoolean(HTTP_SIGNER_ENABLED, HTTP_SIGNER_ENABLED_DEFAULT)) {
// use an http signer through an AuthScheme
final AuthScheme<AwsCredentialsIdentity> signer =
diff --git
a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md
b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md
index 0638c2e4681..e8ec9f98127 100644
---
a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md
+++
b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md
@@ -1381,48 +1381,6 @@ execchain.MainClientExec
(MainClientExec.java:execute(284)) - Connection can be
```
-
-To log the output of the AWS SDK metrics, set the log
-`org.apache.hadoop.fs.s3a.DefaultS3ClientFactory` to `TRACE`.
-This will then turn on logging of the internal SDK metrics.4
-
-These will actually be logged at INFO in the log
-```
-software.amazon.awssdk.metrics.LoggingMetricPublisher
-```
-
-```text
-INFO metrics.LoggingMetricPublisher (LoggerAdapter.java:info(165)) - Metrics
published:
-MetricCollection(name=ApiCall, metrics=[
-MetricRecord(metric=MarshallingDuration, value=PT0.000092041S),
-MetricRecord(metric=RetryCount, value=0),
-MetricRecord(metric=ApiCallSuccessful, value=true),
-MetricRecord(metric=OperationName, value=DeleteObject),
-MetricRecord(metric=EndpointResolveDuration, value=PT0.000132792S),
-MetricRecord(metric=ApiCallDuration, value=PT0.064890875S),
-MetricRecord(metric=CredentialsFetchDuration, value=PT0.000017458S),
-MetricRecord(metric=ServiceEndpoint,
value=https://buckets3.eu-west-2.amazonaws.com),
-MetricRecord(metric=ServiceId, value=S3)], children=[
-MetricCollection(name=ApiCallAttempt, metrics=[
- MetricRecord(metric=TimeToFirstByte, value=PT0.06260225S),
- MetricRecord(metric=SigningDuration, value=PT0.000293083S),
- MetricRecord(metric=ReadThroughput, value=0.0),
- MetricRecord(metric=ServiceCallDuration, value=PT0.06260225S),
- MetricRecord(metric=HttpStatusCode, value=204),
- MetricRecord(metric=BackoffDelayDuration, value=PT0S),
- MetricRecord(metric=TimeToLastByte, value=PT0.064313667S),
- MetricRecord(metric=AwsRequestId, value=RKZD44SE5DW91K1G)], children=[
- MetricCollection(name=HttpClient, metrics=[
- MetricRecord(metric=AvailableConcurrency, value=1),
- MetricRecord(metric=LeasedConcurrency, value=0),
- MetricRecord(metric=ConcurrencyAcquireDuration, value=PT0S),
- MetricRecord(metric=PendingConcurrencyAcquires, value=0),
- MetricRecord(metric=MaxConcurrency, value=512),
- MetricRecord(metric=HttpClientName, value=Apache)], children=[])
- ])
- ])
-```
-
### <a name="audit-logging"></a> Enable S3 Server-side Logging
The [Auditing](auditing) feature of the S3A connector can be used to generate
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties
b/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties
index f61668643a1..7b8dd3c11fc 100644
--- a/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties
+++ b/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties
@@ -102,7 +102,3 @@
log4j.logger.org.apache.hadoop.fs.s3a.S3AStorageStatistics=INFO
# services it launches itself.
# log4.logger.org.apache.hadoop.service=DEBUG
-# log this at trace to trigger enabling printing of the low-level
-# performance metrics in the AWS SDK itself.
-# log4j.logger.org.apache.hadoop.fs.s3a.DefaultS3ClientFactory=TRACE
-
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
