This is an automated email from the ASF dual-hosted git repository.
amoghj pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new b9a6645a53 AWS: Include http-auth-aws-crt module into
iceberg-aws-bundle (#10972)
b9a6645a53 is described below
commit b9a6645a53a1a7a5143a58b23adaae1ea0af99d4
Author: Akira Ajisaka <[email protected]>
AuthorDate: Sun Aug 25 11:47:35 2024 +0900
AWS: Include http-auth-aws-crt module into iceberg-aws-bundle (#10972)
---
aws-bundle/LICENSE | 140 +++++++++++++++------
aws-bundle/NOTICE | 67 +++++-----
aws-bundle/build.gradle | 1 +
.../org/apache/iceberg/aws/AwsIntegTestUtil.java | 11 ++
.../iceberg/aws/s3/TestS3FileIOIntegration.java | 90 +++++++++----
build.gradle | 1 +
kafka-connect/build.gradle | 1 +
7 files changed, 221 insertions(+), 90 deletions(-)
diff --git a/aws-bundle/LICENSE b/aws-bundle/LICENSE
index a4ca14bf8f..0a9656c2c7 100644
--- a/aws-bundle/LICENSE
+++ b/aws-bundle/LICENSE
@@ -207,7 +207,7 @@ This binary artifact contains code from the following
projects:
--------------------------------------------------------------------------------
-Group: commons-codec Name: commons-codec Version: 1.15
+Group: commons-codec Name: commons-codec Version: 1.17.1
Project URL: https://commons.apache.org/proper/commons-codec/
License: Apache License, Version 2.0 -
https://www.apache.org/licenses/LICENSE-2.0.txt
@@ -219,61 +219,61 @@ License: The Apache Software License, Version 2.0 -
http://www.apache.org/licens
--------------------------------------------------------------------------------
-Group: io.netty Name: netty-buffer Version: 4.1.86.Final
+Group: io.netty Name: netty-buffer Version: 4.1.112.Final
Project URL: https://netty.io/
License: Apache License, Version 2.0 -
https://www.apache.org/licenses/LICENSE-2.0
--------------------------------------------------------------------------------
-Group: io.netty Name: netty-codec Version: 4.1.86.Final
+Group: io.netty Name: netty-codec Version: 4.1.112.Final
Project URL: https://netty.io/
License: Apache License, Version 2.0 -
https://www.apache.org/licenses/LICENSE-2.0
--------------------------------------------------------------------------------
-Group: io.netty Name: netty-codec-http Version: 4.1.86.Final
+Group: io.netty Name: netty-codec-http Version: 4.1.112.Final
Project URL: https://netty.io/
License: Apache License, Version 2.0 -
https://www.apache.org/licenses/LICENSE-2.0
--------------------------------------------------------------------------------
-Group: io.netty Name: netty-codec-http2 Version: 4.1.86.Final
+Group: io.netty Name: netty-codec-http2 Version: 4.1.112.Final
Project URL: https://netty.io/
License: Apache License, Version 2.0 -
https://www.apache.org/licenses/LICENSE-2.0
--------------------------------------------------------------------------------
-Group: io.netty Name: netty-common Version: 4.1.86.Final
+Group: io.netty Name: netty-common Version: 4.1.112.Final
Project URL: https://netty.io/
License: Apache License, Version 2.0 -
https://www.apache.org/licenses/LICENSE-2.0
--------------------------------------------------------------------------------
-Group: io.netty Name: netty-handler Version: 4.1.86.Final
+Group: io.netty Name: netty-handler Version: 4.1.112.Final
Project URL: https://netty.io/
License: Apache License, Version 2.0 -
https://www.apache.org/licenses/LICENSE-2.0
--------------------------------------------------------------------------------
-Group: io.netty Name: netty-resolver Version: 4.1.86.Final
+Group: io.netty Name: netty-resolver Version: 4.1.112.Final
Project URL: https://netty.io/
License: Apache License, Version 2.0 -
https://www.apache.org/licenses/LICENSE-2.0
--------------------------------------------------------------------------------
-Group: io.netty Name: netty-transport Version: 4.1.86.Final
+Group: io.netty Name: netty-transport Version: 4.1.112.Final
Project URL: https://netty.io/
License: Apache License, Version 2.0 -
https://www.apache.org/licenses/LICENSE-2.0
--------------------------------------------------------------------------------
-Group: io.netty Name: netty-transport-classes-epoll Version: 4.1.86.Final
+Group: io.netty Name: netty-transport-classes-epoll Version: 4.1.112.Final
Project URL: https://netty.io/
License: Apache License, Version 2.0 -
https://www.apache.org/licenses/LICENSE-2.0
--------------------------------------------------------------------------------
-Group: io.netty Name: netty-transport-native-unix-common Version:
4.1.86.Final
+Group: io.netty Name: netty-transport-native-unix-common Version:
4.1.112.Final
Project URL: https://netty.io/
License: Apache License, Version 2.0 -
https://www.apache.org/licenses/LICENSE-2.0
@@ -285,13 +285,13 @@ License: Apache License, Version 2.0 -
http://www.apache.org/licenses/LICENSE-2.
--------------------------------------------------------------------------------
-Group: org.apache.httpcomponents Name: httpcore Version: 4.4.13
+Group: org.apache.httpcomponents Name: httpcore Version: 4.4.16
Project URL: http://hc.apache.org/httpcomponents-core-ga
License: Apache License, Version 2.0 -
http://www.apache.org/licenses/LICENSE-2.0.txt
--------------------------------------------------------------------------------
-Group: org.reactivestreams Name: reactive-streams Version: 1.0.3
+Group: org.reactivestreams Name: reactive-streams Version: 1.0.4
Project URL: http://reactive-streams.org
License: CC0 - http://creativecommons.org/publicdomain/zero/1.0/
@@ -303,160 +303,220 @@ License: MIT License -
http://www.opensource.org/licenses/mit-license.php
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: annotations Version: 2.20.131
+Group: software.amazon.awssdk Name: annotations Version: 2.27.7
License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: apache-client Version: 2.20.131
+Group: software.amazon.awssdk Name: apache-client Version: 2.27.7
License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: arns Version: 2.20.131
+Group: software.amazon.awssdk Name: arns Version: 2.27.7
Project URL: https://aws.amazon.com/sdkforjava
License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: auth Version: 2.20.131
+Group: software.amazon.awssdk Name: auth Version: 2.27.7
Project URL: https://aws.amazon.com/sdkforjava
License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: aws-core Version: 2.20.131
+Group: software.amazon.awssdk Name: aws-core Version: 2.27.7
Project URL: https://aws.amazon.com/sdkforjava
License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: aws-json-protocol Version: 2.20.131
+Group: software.amazon.awssdk Name: aws-json-protocol Version: 2.27.7
Project URL: https://aws.amazon.com/sdkforjava
License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: aws-query-protocol Version: 2.20.131
+Group: software.amazon.awssdk Name: aws-query-protocol Version: 2.27.7
Project URL: https://aws.amazon.com/sdkforjava
License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: aws-xml-protocol Version: 2.20.131
+Group: software.amazon.awssdk Name: aws-xml-protocol Version: 2.27.7
Project URL: https://aws.amazon.com/sdkforjava
License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: dynamodb Version: 2.20.131
+Group: software.amazon.awssdk Name: checksums Version: 2.27.7
Project URL: https://aws.amazon.com/sdkforjava
License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: endpoints-spi Version: 2.20.131
+Group: software.amazon.awssdk Name: checksums-spi Version: 2.27.7
+Project URL: https://aws.amazon.com/sdkforjava
License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: glue Version: 2.20.131
+Group: software.amazon.awssdk Name: crt-core Version: 2.27.7
Project URL: https://aws.amazon.com/sdkforjava
License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: http-client-spi Version: 2.20.131
+Group: software.amazon.awssdk Name: dynamodb Version: 2.27.7
+Project URL: https://aws.amazon.com/sdkforjava
License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: iam Version: 2.20.131
+Group: software.amazon.awssdk Name: endpoints-spi Version: 2.27.7
+License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
+
+--------------------------------------------------------------------------------
+
+Group: software.amazon.awssdk Name: glue Version: 2.27.7
Project URL: https://aws.amazon.com/sdkforjava
License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: json-utils Version: 2.20.131
+Group: software.amazon.awssdk Name: http-auth Version: 2.27.7
Project URL: https://aws.amazon.com/sdkforjava
License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: kms Version: 2.20.131
+Group: software.amazon.awssdk Name: http-auth-aws Version: 2.27.7
Project URL: https://aws.amazon.com/sdkforjava
License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: lakeformation Version: 2.20.131
+Group: software.amazon.awssdk Name: http-auth-aws-crt Version: 2.27.7
Project URL: https://aws.amazon.com/sdkforjava
License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: metrics-spi Version: 2.20.131
+Group: software.amazon.awssdk Name: http-auth-aws-eventstream Version: 2.27.7
+Project URL: https://aws.amazon.com/sdkforjava
License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: netty-nio-client Version: 2.20.131
+Group: software.amazon.awssdk Name: http-auth-spi Version: 2.27.7
+Project URL: https://aws.amazon.com/sdkforjava
+License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
+
+--------------------------------------------------------------------------------
+
+Group: software.amazon.awssdk Name: http-client-spi Version: 2.27.7
License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: profiles Version: 2.20.131
+Group: software.amazon.awssdk Name: iam Version: 2.27.7
Project URL: https://aws.amazon.com/sdkforjava
License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: protocol-core Version: 2.20.131
+Group: software.amazon.awssdk Name: identity-spi Version: 2.27.7
Project URL: https://aws.amazon.com/sdkforjava
License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: regions Version: 2.20.131
+Group: software.amazon.awssdk Name: json-utils Version: 2.27.7
+Project URL: https://aws.amazon.com/sdkforjava
License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: s3 Version: 2.20.131
+Group: software.amazon.awssdk Name: kms Version: 2.27.7
Project URL: https://aws.amazon.com/sdkforjava
License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: sdk-core Version: 2.20.131
+Group: software.amazon.awssdk Name: lakeformation Version: 2.27.7
Project URL: https://aws.amazon.com/sdkforjava
License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: sso Version: 2.20.131
+Group: software.amazon.awssdk Name: metrics-spi Version: 2.27.7
+License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
+
+--------------------------------------------------------------------------------
+
+Group: software.amazon.awssdk Name: netty-nio-client Version: 2.27.7
+License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
+
+--------------------------------------------------------------------------------
+
+Group: software.amazon.awssdk Name: profiles Version: 2.27.7
Project URL: https://aws.amazon.com/sdkforjava
License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: sts Version: 2.20.131
+Group: software.amazon.awssdk Name: protocol-core Version: 2.27.7
Project URL: https://aws.amazon.com/sdkforjava
License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: third-party-jackson-core Version:
2.20.131
+Group: software.amazon.awssdk Name: regions Version: 2.27.7
+License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
+
+--------------------------------------------------------------------------------
+
+Group: software.amazon.awssdk Name: s3 Version: 2.27.7
Project URL: https://aws.amazon.com/sdkforjava
License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: utils Version: 2.20.131
+Group: software.amazon.awssdk Name: sdk-core Version: 2.27.7
+Project URL: https://aws.amazon.com/sdkforjava
+License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
+
+--------------------------------------------------------------------------------
+
+Group: software.amazon.awssdk Name: sso Version: 2.27.7
+Project URL: https://aws.amazon.com/sdkforjava
+License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
+
+--------------------------------------------------------------------------------
+
+Group: software.amazon.awssdk Name: sts Version: 2.27.7
+Project URL: https://aws.amazon.com/sdkforjava
+License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
+
+--------------------------------------------------------------------------------
+
+Group: software.amazon.awssdk Name: third-party-jackson-core Version: 2.27.7
+Project URL: https://aws.amazon.com/sdkforjava
+License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
+
+--------------------------------------------------------------------------------
+
+Group: software.amazon.awssdk Name: utils Version: 2.27.7
License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
--------------------------------------------------------------------------------
+Group: software.amazon.awssdk.crt Name: aws-crt Version: 0.30.6
+Project URL: https://github.com/awslabs/aws-crt-java
+License: The Apache Software License, Version 2.0 -
http://www.apache.org/licenses/LICENSE-2.0.txt
+
+--------------------------------------------------------------------------------
+
Group: software.amazon.eventstream Name: eventstream Version: 1.0.1
Project URL: https://github.com/awslabs/aws-eventstream-java
License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
diff --git a/aws-bundle/NOTICE b/aws-bundle/NOTICE
index ed353940ba..070d109c8e 100644
--- a/aws-bundle/NOTICE
+++ b/aws-bundle/NOTICE
@@ -7,7 +7,7 @@ The Apache Software Foundation (http://www.apache.org/).
--------------------------------------------------------------------------------
-NOTICE for Group: commons-codec Name: commons-codec Version: 1.15
+NOTICE for Group: commons-codec Name: commons-codec Version: 1.17.1
src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java
contains test data from http://aspell.net/test/orig/batch0.tab.
@@ -23,32 +23,43 @@ Copyright (c) 2008 Alexander Beider & Stephen P. Morse.
--------------------------------------------------------------------------------
-NOTICE for Group: software.amazon.awssdk Name: annotations Version: 2.20.131
-NOTICE for Group: software.amazon.awssdk Name: apache-client Version:
2.20.131
-NOTICE for Group: software.amazon.awssdk Name: arns Version: 2.20.131
-NOTICE for Group: software.amazon.awssdk Name: auth Version: 2.20.131
-NOTICE for Group: software.amazon.awssdk Name: aws-core Version: 2.20.131
-NOTICE for Group: software.amazon.awssdk Name: aws-json-protocol Version:
2.20.131
-NOTICE for Group: software.amazon.awssdk Name: aws-query-protocol Version:
2.20.131
-NOTICE for Group: software.amazon.awssdk Name: aws-xml-protocol Version:
2.20.131
-NOTICE for Group: software.amazon.awssdk Name: dynamodb Version: 2.20.131
-NOTICE for Group: software.amazon.awssdk Name: endpoints-spi Version:
2.20.131
-NOTICE for Group: software.amazon.awssdk Name: glue Version: 2.20.131
-NOTICE for Group: software.amazon.awssdk Name: http-client-spi Version:
2.20.131
-NOTICE for Group: software.amazon.awssdk Name: iam Version: 2.20.131
-NOTICE for Group: software.amazon.awssdk Name: json-utils Version: 2.20.131
-NOTICE for Group: software.amazon.awssdk Name: kms Version: 2.20.131
-NOTICE for Group: software.amazon.awssdk Name: lakeformation Version:
2.20.131
-NOTICE for Group: software.amazon.awssdk Name: metrics-spi Version: 2.20.131
-NOTICE for Group: software.amazon.awssdk Name: netty-nio-client Version:
2.20.131
-NOTICE for Group: software.amazon.awssdk Name: profiles Version: 2.20.131
-NOTICE for Group: software.amazon.awssdk Name: protocol-core Version:
2.20.131
-NOTICE for Group: software.amazon.awssdk Name: regions Version: 2.20.131
-NOTICE for Group: software.amazon.awssdk Name: s3 Version: 2.20.131
-NOTICE for Group: software.amazon.awssdk Name: sdk-core Version: 2.20.131
-NOTICE for Group: software.amazon.awssdk Name: sso Version: 2.20.131
-NOTICE for Group: software.amazon.awssdk Name: sts Version: 2.20.131
-NOTICE for Group: software.amazon.awssdk Name: utils Version: 2.20.131
+NOTICE for Group: software.amazon.awssdk Name: annotations Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: apache-client Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: arns Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: auth Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: aws-core Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: aws-json-protocol Version:
2.27.7
+NOTICE for Group: software.amazon.awssdk Name: aws-query-protocol Version:
2.27.7
+NOTICE for Group: software.amazon.awssdk Name: aws-xml-protocol Version:
2.27.7
+NOTICE for Group: software.amazon.awssdk Name: checksums Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: checksums-spi Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: crt-core Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: dynamodb Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: endpoints-spi Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: glue Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: http-auth Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: http-auth-aws Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: http-auth-aws-crt Version:
2.27.7
+NOTICE for Group: software.amazon.awssdk Name: http-auth-aws-eventstream
Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: http-auth-spi Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: http-client-spi Version:
2.27.7
+NOTICE for Group: software.amazon.awssdk Name: iam Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: identity-spi Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: json-utils Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: kms Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: lakeformation Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: metrics-spi Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: netty-nio-client Version:
2.27.7
+NOTICE for Group: software.amazon.awssdk Name: profiles Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: protocol-core Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: regions Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: retries Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: retries-spi Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: s3 Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: sdk-core Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: sso Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: sts Version: 2.27.7
+NOTICE for Group: software.amazon.awssdk Name: utils Version: 2.27.7
AWS SDK for Java 2.0
Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
@@ -71,7 +82,7 @@ The licenses for these third party components are included in
LICENSE.txt
--------------------------------------------------------------------------------
-NOTICE for Group: software.amazon.awssdk Name: third-party-jackson-core
Version: 2.20.18
+NOTICE for Group: software.amazon.awssdk Name: third-party-jackson-core
Version: 2.27.7
# Jackson JSON processor
diff --git a/aws-bundle/build.gradle b/aws-bundle/build.gradle
index eea591a7b7..16952d78d8 100644
--- a/aws-bundle/build.gradle
+++ b/aws-bundle/build.gradle
@@ -27,6 +27,7 @@ project(":iceberg-aws-bundle") {
implementation platform(libs.awssdk.bom)
implementation "software.amazon.awssdk:apache-client"
implementation "software.amazon.awssdk:auth"
+ implementation "software.amazon.awssdk:http-auth-aws-crt"
implementation "software.amazon.awssdk:iam"
implementation "software.amazon.awssdk:sso"
implementation "software.amazon.awssdk:s3"
diff --git
a/aws/src/integration/java/org/apache/iceberg/aws/AwsIntegTestUtil.java
b/aws/src/integration/java/org/apache/iceberg/aws/AwsIntegTestUtil.java
index 7e0ca6ed10..e9cf474add 100644
--- a/aws/src/integration/java/org/apache/iceberg/aws/AwsIntegTestUtil.java
+++ b/aws/src/integration/java/org/apache/iceberg/aws/AwsIntegTestUtil.java
@@ -95,6 +95,17 @@ public class AwsIntegTestUtil {
return System.getenv("AWS_TEST_ACCOUNT_ID");
}
+ /**
+ * Set the environment variable AWS_TEST_MULTI_REGION_ACCESS_POINT_ALIAS for
a default account to
+ * use for testing. Developers need to create a S3 multi region access point
before running
+ * integration tests because creating it takes a few minutes
+ *
+ * @return The alias of S3 multi region access point route to the default S3
bucket
+ */
+ public static String testMultiRegionAccessPointAlias() {
+ return System.getenv("AWS_TEST_MULTI_REGION_ACCESS_POINT_ALIAS");
+ }
+
public static void cleanS3Bucket(S3Client s3, String bucketName, String
prefix) {
ListObjectVersionsIterable response =
s3.listObjectVersionsPaginator(
diff --git
a/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3FileIOIntegration.java
b/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3FileIOIntegration.java
index cacf048918..fc2b9f97f1 100644
---
a/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3FileIOIntegration.java
+++
b/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3FileIOIntegration.java
@@ -41,6 +41,7 @@ import org.apache.iceberg.io.InputFile;
import org.apache.iceberg.io.OutputFile;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.assertj.core.api.Assumptions;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
@@ -75,11 +76,13 @@ public class TestS3FileIOIntegration {
private static S3Client s3;
private static S3ControlClient s3Control;
private static S3ControlClient crossRegionS3Control;
+ private static S3ControlClient multiRegionS3Control;
private static KmsClient kms;
private static String bucketName;
private static String crossRegionBucketName;
private static String accessPointName;
private static String crossRegionAccessPointName;
+ private static String multiRegionAccessPointAlias;
private static String prefix;
private static byte[] contentBytes;
private static String content;
@@ -109,6 +112,7 @@ public class TestS3FileIOIntegration {
AwsIntegTestUtil.createAccessPoint(s3Control, accessPointName, bucketName);
AwsIntegTestUtil.createAccessPoint(
crossRegionS3Control, crossRegionAccessPointName,
crossRegionBucketName);
+ multiRegionAccessPointAlias =
AwsIntegTestUtil.testMultiRegionAccessPointAlias();
s3.putBucketVersioning(
PutBucketVersioningRequest.builder()
.bucket(bucketName)
@@ -202,15 +206,32 @@ public class TestS3FileIOIntegration {
validateRead(s3FileIO);
}
+ @Test
+ public void testNewInputStreamWithMultiRegionAccessPoint() throws Exception {
+ Assumptions.assumeThat(multiRegionAccessPointAlias).isNotEmpty();
+
clientFactory.initialize(ImmutableMap.of(S3FileIOProperties.USE_ARN_REGION_ENABLED,
"true"));
+ S3Client s3Client = clientFactory.s3();
+ s3Client.putObject(
+ PutObjectRequest.builder().bucket(bucketName).key(objectKey).build(),
+ RequestBody.fromBytes(contentBytes));
+ S3FileIO s3FileIO = new S3FileIO(clientFactory::s3);
+ s3FileIO.initialize(
+ ImmutableMap.of(
+ S3FileIOProperties.ACCESS_POINTS_PREFIX + bucketName,
+ testMultiRegionAccessPointARN(
+ AwsIntegTestUtil.testRegion(), multiRegionAccessPointAlias)));
+ validateRead(s3FileIO);
+ }
+
@Test
public void testNewOutputStream() throws Exception {
S3FileIO s3FileIO = new S3FileIO(clientFactory::s3);
write(s3FileIO);
- InputStream stream =
-
s3.getObject(GetObjectRequest.builder().bucket(bucketName).key(objectKey).build());
- String result = IoUtils.toUtf8String(stream);
- stream.close();
- assertThat(result).isEqualTo(content);
+ try (InputStream stream =
+
s3.getObject(GetObjectRequest.builder().bucket(bucketName).key(objectKey).build()))
{
+ String result = IoUtils.toUtf8String(stream);
+ assertThat(result).isEqualTo(content);
+ }
}
@Test
@@ -221,11 +242,11 @@ public class TestS3FileIOIntegration {
S3FileIOProperties.ACCESS_POINTS_PREFIX + bucketName,
testAccessPointARN(AwsIntegTestUtil.testRegion(),
accessPointName)));
write(s3FileIO);
- InputStream stream =
-
s3.getObject(GetObjectRequest.builder().bucket(bucketName).key(objectKey).build());
- String result = IoUtils.toUtf8String(stream);
- stream.close();
- assertThat(result).isEqualTo(content);
+ try (InputStream stream =
+
s3.getObject(GetObjectRequest.builder().bucket(bucketName).key(objectKey).build()))
{
+ String result = IoUtils.toUtf8String(stream);
+ assertThat(result).isEqualTo(content);
+ }
}
@Test
@@ -238,17 +259,35 @@ public class TestS3FileIOIntegration {
S3FileIOProperties.ACCESS_POINTS_PREFIX + bucketName,
testAccessPointARN(AwsIntegTestUtil.testCrossRegion(),
crossRegionAccessPointName)));
write(s3FileIO);
- InputStream stream =
+ try (InputStream stream =
s3Client.getObject(
GetObjectRequest.builder()
.bucket(
testAccessPointARN(
AwsIntegTestUtil.testCrossRegion(),
crossRegionAccessPointName))
.key(objectKey)
- .build());
- String result = IoUtils.toUtf8String(stream);
- stream.close();
- assertThat(result).isEqualTo(content);
+ .build())) {
+ String result = IoUtils.toUtf8String(stream);
+ assertThat(result).isEqualTo(content);
+ }
+ }
+
+ @Test
+ public void testNewOutputStreamWithMultiRegionAccessPoint() throws Exception
{
+ Assumptions.assumeThat(multiRegionAccessPointAlias).isNotEmpty();
+
clientFactory.initialize(ImmutableMap.of(S3FileIOProperties.USE_ARN_REGION_ENABLED,
"true"));
+ S3FileIO s3FileIO = new S3FileIO(clientFactory::s3);
+ s3FileIO.initialize(
+ ImmutableMap.of(
+ S3FileIOProperties.ACCESS_POINTS_PREFIX + bucketName,
+ testMultiRegionAccessPointARN(
+ AwsIntegTestUtil.testRegion(), multiRegionAccessPointAlias)));
+ write(s3FileIO);
+ try (InputStream stream =
+
s3.getObject(GetObjectRequest.builder().bucket(bucketName).key(objectKey).build()))
{
+ String result = IoUtils.toUtf8String(stream);
+ assertThat(result).isEqualTo(content);
+ }
}
@Test
@@ -508,18 +547,18 @@ public class TestS3FileIOIntegration {
private void write(S3FileIO s3FileIO, String uri) throws Exception {
OutputFile outputFile = s3FileIO.newOutputFile(uri);
- OutputStream outputStream = outputFile.create();
- IoUtils.copy(new ByteArrayInputStream(contentBytes), outputStream);
- outputStream.close();
+ try (OutputStream outputStream = outputFile.create()) {
+ IoUtils.copy(new ByteArrayInputStream(contentBytes), outputStream);
+ }
}
private void validateRead(S3FileIO s3FileIO) throws Exception {
InputFile file = s3FileIO.newInputFile(objectUri);
assertThat(file.getLength()).isEqualTo(contentBytes.length);
- InputStream stream = file.newStream();
- String result = IoUtils.toUtf8String(stream);
- stream.close();
- assertThat(result).isEqualTo(content);
+ try (InputStream stream = file.newStream()) {
+ String result = IoUtils.toUtf8String(stream);
+ assertThat(result).isEqualTo(content);
+ }
}
private String testAccessPointARN(String region, String accessPoint) {
@@ -532,6 +571,13 @@ public class TestS3FileIOIntegration {
accessPoint);
}
+ private String testMultiRegionAccessPointARN(String region, String alias) {
+ // format: arn:aws:s3::account-id:accesspoint/MultiRegionAccessPoint_alias
+ return String.format(
+ "arn:%s:s3::%s:accesspoint/%s",
+ PartitionMetadata.of(Region.of(region)).id(),
AwsIntegTestUtil.testAccountId(), alias);
+ }
+
private void createRandomObjects(String objectPrefix, int count) {
S3URI s3URI = new S3URI(objectPrefix);
random
diff --git a/build.gradle b/build.gradle
index 7a11943cf8..3b89dc4e05 100644
--- a/build.gradle
+++ b/build.gradle
@@ -471,6 +471,7 @@ project(':iceberg-aws') {
compileOnly("software.amazon.awssdk:url-connection-client")
compileOnly("software.amazon.awssdk:apache-client")
compileOnly("software.amazon.awssdk:auth")
+ compileOnly("software.amazon.awssdk:http-auth-aws-crt")
compileOnly("software.amazon.awssdk:s3")
compileOnly("software.amazon.awssdk:kms")
compileOnly("software.amazon.awssdk:glue")
diff --git a/kafka-connect/build.gradle b/kafka-connect/build.gradle
index 60aaeee8c6..c687bb4aea 100644
--- a/kafka-connect/build.gradle
+++ b/kafka-connect/build.gradle
@@ -115,6 +115,7 @@
project(':iceberg-kafka-connect:iceberg-kafka-connect-runtime') {
implementation platform(libs.awssdk.bom)
implementation 'software.amazon.awssdk:apache-client'
implementation 'software.amazon.awssdk:auth'
+ implementation "software.amazon.awssdk:http-auth-aws-crt"
implementation 'software.amazon.awssdk:iam'
implementation 'software.amazon.awssdk:sso'
implementation 'software.amazon.awssdk:s3'
