(iceberg) branch main updated: Add Iceberg version to UserAgent in S3 requests (#9963)

[amoghj]

This is an automated email from the ASF dual-hosted git repository.

amoghj pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/main by this push:
     new 83bacf501e Add Iceberg version to UserAgent in S3 requests (#9963)
83bacf501e is described below

commit 83bacf501e116c0923eb6e77bff4500cd3d8042f
Author: Csenger Geza <csengerg...@gmail.com>
AuthorDate: Mon Mar 25 19:01:42 2024 +0000

    Add Iceberg version to UserAgent in S3 requests (#9963)
    
    This allows developers to monitor which version of Iceberg they have
    deployed to a cluster (for example, S3 Access Logs contain the user agent
    field).
    
    Co-authored-by: Geza Csenger <gccse...@amazon.com>
---
 .../java/org/apache/iceberg/aws/AwsClientFactories.java    |  1 +
 .../iceberg/aws/s3/DefaultS3FileIOAwsClientFactory.java    |  1 +
 .../java/org/apache/iceberg/aws/s3/S3FileIOProperties.java | 14 ++++++++++++++
 .../org/apache/iceberg/aws/s3/TestS3FileIOProperties.java  | 10 ++++++++++
 4 files changed, 26 insertions(+)

diff --git a/aws/src/main/java/org/apache/iceberg/aws/AwsClientFactories.java 
b/aws/src/main/java/org/apache/iceberg/aws/AwsClientFactories.java
index 580e7303ff..81c7bd6b4b 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/AwsClientFactories.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/AwsClientFactories.java
@@ -113,6 +113,7 @@ public class AwsClientFactories {
               b -> 
s3FileIOProperties.applyCredentialConfigurations(awsClientProperties, b))
           .applyMutation(s3FileIOProperties::applySignerConfiguration)
           .applyMutation(s3FileIOProperties::applyS3AccessGrantsConfigurations)
+          .applyMutation(s3FileIOProperties::applyUserAgentConfigurations)
           .build();
     }
 
diff --git 
a/aws/src/main/java/org/apache/iceberg/aws/s3/DefaultS3FileIOAwsClientFactory.java
 
b/aws/src/main/java/org/apache/iceberg/aws/s3/DefaultS3FileIOAwsClientFactory.java
index a65910612f..18b40000a9 100644
--- 
a/aws/src/main/java/org/apache/iceberg/aws/s3/DefaultS3FileIOAwsClientFactory.java
+++ 
b/aws/src/main/java/org/apache/iceberg/aws/s3/DefaultS3FileIOAwsClientFactory.java
@@ -54,6 +54,7 @@ class DefaultS3FileIOAwsClientFactory implements 
S3FileIOAwsClientFactory {
                     awsClientProperties, s3ClientBuilder))
         .applyMutation(s3FileIOProperties::applySignerConfiguration)
         .applyMutation(s3FileIOProperties::applyS3AccessGrantsConfigurations)
+        .applyMutation(s3FileIOProperties::applyUserAgentConfigurations)
         .build();
   }
 }
diff --git 
a/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java 
b/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java
index 9aad784be8..857f35e710 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java
@@ -24,6 +24,7 @@ import java.util.Collections;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
+import org.apache.iceberg.EnvironmentContext;
 import org.apache.iceberg.aws.AwsClientProperties;
 import org.apache.iceberg.aws.glue.GlueCatalog;
 import org.apache.iceberg.aws.s3.signer.S3V4RestSignerClient;
@@ -375,6 +376,14 @@ public class S3FileIOProperties implements Serializable {
 
   public static final boolean PRELOAD_CLIENT_ENABLED_DEFAULT = false;
 
+  /**
+   * User Agent Prefix set by the S3 client.
+   *
+   * <p>This allows developers to monitor which version of Iceberg they have 
deployed to a cluster
+   * (for example, through the S3 Access Logs, which contain the user agent 
field).
+   */
+  private static final String S3_FILE_IO_USER_AGENT = "s3fileio/" + 
EnvironmentContext.get();
+
   private String sseType;
   private String sseKey;
   private String sseMd5;
@@ -819,6 +828,11 @@ public class S3FileIOProperties implements Serializable {
     }
   }
 
+  public <T extends S3ClientBuilder> void applyUserAgentConfigurations(T 
builder) {
+    builder.overrideConfiguration(
+        c -> c.putAdvancedOption(SdkAdvancedClientOption.USER_AGENT_PREFIX, 
S3_FILE_IO_USER_AGENT));
+  }
+
   /**
    * Dynamically load the http client builder to avoid runtime deps 
requirements of any optional SDK
    * Plugins
diff --git 
a/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java 
b/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java
index 2ed8a9471d..658b5b7819 100644
--- a/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java
+++ b/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java
@@ -478,4 +478,14 @@ public class TestS3FileIOProperties {
     s3FileIOProperties.applyEndpointConfigurations(mockS3ClientBuilder);
     
Mockito.verify(mockS3ClientBuilder).endpointOverride(Mockito.any(URI.class));
   }
+
+  @Test
+  public void testApplyUserAgentConfigurations() {
+    Map<String, String> properties = Maps.newHashMap();
+    S3FileIOProperties s3FileIOProperties = new S3FileIOProperties(properties);
+    S3ClientBuilder mockS3ClientBuilder = Mockito.mock(S3ClientBuilder.class);
+    s3FileIOProperties.applyUserAgentConfigurations(mockS3ClientBuilder);
+
+    
Mockito.verify(mockS3ClientBuilder).overrideConfiguration(Mockito.any(Consumer.class));
+  }
 }