This is an automated email from the ASF dual-hosted git repository.
corgy pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 41c251cc8a [Improve][Core] Unify the aws-sdk-v2 version to 2.31.30
(#9698)
41c251cc8a is described below
commit 41c251cc8ab1eec1cfc8cb470b810a3f56b1a4a5
Author: litiliu <[email protected]>
AuthorDate: Fri Aug 15 18:14:50 2025 +0800
[Improve][Core] Unify the aws-sdk-v2 version to 2.31.30 (#9698)
---
pom.xml | 1 +
.../connector-amazondynamodb/pom.xml | 6 +-
.../connector-amazonsqs/pom.xml | 6 +-
seatunnel-connectors-v2/connector-iceberg/pom.xml | 1 -
.../connector-amazondynamodb-e2e/pom.xml | 5 +-
.../connector-amazonsqs-e2e/pom.xml | 5 +-
.../e2e/connector/amazonsqs/AmazonsqsIT.java | 36 +++++++----
seatunnel-transforms-v2/pom.xml | 5 +-
tools/dependencies/known-dependencies.txt | 72 +++++++++++-----------
9 files changed, 68 insertions(+), 69 deletions(-)
diff --git a/pom.xml b/pom.xml
index 10d796c9c6..497cfd0972 100644
--- a/pom.xml
+++ b/pom.xml
@@ -164,6 +164,7 @@
<enableSourceJarCreation>true</enableSourceJarCreation>
<hadoop-aws.version>3.1.4</hadoop-aws.version>
+
<software.amazon.awssdk.version>2.31.30</software.amazon.awssdk.version>
<arrow.version>15.0.1</arrow.version>
</properties>
diff --git a/seatunnel-connectors-v2/connector-amazondynamodb/pom.xml
b/seatunnel-connectors-v2/connector-amazondynamodb/pom.xml
index 559402cc6b..90d97c3889 100644
--- a/seatunnel-connectors-v2/connector-amazondynamodb/pom.xml
+++ b/seatunnel-connectors-v2/connector-amazondynamodb/pom.xml
@@ -29,16 +29,12 @@
<artifactId>connector-amazondynamodb</artifactId>
<name>SeaTunnel : Connectors V2 : Amazon Dynamo DB</name>
- <properties>
- <amazon.awssdk>2.18.1</amazon.awssdk>
- </properties>
-
<dependencyManagement>
<dependencies>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>bom</artifactId>
- <version>${amazon.awssdk}</version>
+ <version>${software.amazon.awssdk.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
diff --git a/seatunnel-connectors-v2/connector-amazonsqs/pom.xml
b/seatunnel-connectors-v2/connector-amazonsqs/pom.xml
index bacce442ee..ff7c55457d 100644
--- a/seatunnel-connectors-v2/connector-amazonsqs/pom.xml
+++ b/seatunnel-connectors-v2/connector-amazonsqs/pom.xml
@@ -29,16 +29,12 @@
<artifactId>connector-amazonsqs</artifactId>
<name>SeaTunnel : Connectors V2 : Amazon SQS</name>
- <properties>
- <amazon.awssdk>2.18.1</amazon.awssdk>
- </properties>
-
<dependencyManagement>
<dependencies>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>bom</artifactId>
- <version>${amazon.awssdk}</version>
+ <version>${software.amazon.awssdk.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
diff --git a/seatunnel-connectors-v2/connector-iceberg/pom.xml
b/seatunnel-connectors-v2/connector-iceberg/pom.xml
index 23e461491d..529dafd1eb 100644
--- a/seatunnel-connectors-v2/connector-iceberg/pom.xml
+++ b/seatunnel-connectors-v2/connector-iceberg/pom.xml
@@ -35,7 +35,6 @@
<avro.version>1.11.3</avro.version>
<hive.version>2.3.9</hive.version>
<connector.name>connector.iceberg</connector.name>
-
<software.amazon.awssdk.version>2.31.30</software.amazon.awssdk.version>
</properties>
<dependencyManagement>
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/pom.xml
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/pom.xml
index 07c28bdbfe..ff0bcbe41c 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/pom.xml
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazondynamodb-e2e/pom.xml
@@ -24,15 +24,12 @@
<artifactId>connector-amazondynamodb-e2e</artifactId>
<name>SeaTunnel : E2E : Connector V2 : Amazon Dynamo DB</name>
- <properties>
- <amazon.awssdk>2.18.1</amazon.awssdk>
- </properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>bom</artifactId>
- <version>${amazon.awssdk}</version>
+ <version>${software.amazon.awssdk.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazonsqs-e2e/pom.xml
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazonsqs-e2e/pom.xml
index 02fb1aaa8d..5c7396ab35 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazonsqs-e2e/pom.xml
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazonsqs-e2e/pom.xml
@@ -24,15 +24,12 @@
<artifactId>connector-amazonsqs-e2e</artifactId>
<name>SeaTunnel : E2E : Connector V2 : Amazon SQS</name>
- <properties>
- <amazon.awssdk>2.18.1</amazon.awssdk>
- </properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>bom</artifactId>
- <version>${amazon.awssdk}</version>
+ <version>${software.amazon.awssdk.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazonsqs-e2e/src/test/java/org/apache/seatunnel/e2e/connector/amazonsqs/AmazonsqsIT.java
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazonsqs-e2e/src/test/java/org/apache/seatunnel/e2e/connector/amazonsqs/AmazonsqsIT.java
index 8d6b05b7f7..4724f01ea3 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazonsqs-e2e/src/test/java/org/apache/seatunnel/e2e/connector/amazonsqs/AmazonsqsIT.java
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazonsqs-e2e/src/test/java/org/apache/seatunnel/e2e/connector/amazonsqs/AmazonsqsIT.java
@@ -44,11 +44,15 @@ import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
+import static org.awaitility.Awaitility.await;
import static org.awaitility.Awaitility.given;
@Slf4j
public class AmazonsqsIT extends TestSuiteBase implements TestResource {
- private static final String LOCALSTACK_DOCKER_IMAGE =
"localstack/localstack:0.11.2";
+
+ private static final String LOCALSTACK_DOCKER_IMAGE_VERSION = "3.7";
+ private static final String LOCALSTACK_DOCKER_IMAGE =
+ "localstack/localstack:" + LOCALSTACK_DOCKER_IMAGE_VERSION;
private static final String AMAZONSQS_JOB_CONFIG =
"/amazonsqsIT_source_to_sink.conf";
private static final String AMAZONSQS_CONTAINER_HOST = "sqs-host";
private static final int AMAZONSQS_CONTAINER_PORT = 4566;
@@ -66,7 +70,7 @@ public class AmazonsqsIT extends TestSuiteBase implements
TestResource {
public void startUp() throws Exception {
// start a localstack docker container
localstack =
- new LocalStackContainer()
+ new LocalStackContainer(LOCALSTACK_DOCKER_IMAGE_VERSION)
.withServices(LocalStackContainer.Service.SQS)
.withEnv("AWS_DEFAULT_REGION", "us-east-1")
.withEnv("AWS_ACCESS_KEY_ID", "1234")
@@ -97,7 +101,8 @@ public class AmazonsqsIT extends TestSuiteBase implements
TestResource {
// create a sqs client
sqsClient =
SqsClient.builder()
- .endpointOverride(localstack.getEndpoint())
+ .endpointOverride(
+
localstack.getEndpointOverride(LocalStackContainer.Service.SQS))
.region(Region.US_EAST_1)
.credentialsProvider(
StaticCredentialsProvider.create(
@@ -107,12 +112,23 @@ public class AmazonsqsIT extends TestSuiteBase implements
TestResource {
// create source and sink queue
sqsClient.createQueue(r -> r.queueName(SOURCE_QUEUE));
sqsClient.createQueue(r -> r.queueName(SINK_QUEUE));
+ // wait for create complete
+ await().atMost(10, TimeUnit.SECONDS)
+ .pollInterval(1, TimeUnit.SECONDS)
+ .ignoreExceptionsInstanceOf(Exception.class)
+ .untilAsserted(
+ () -> {
+ getQueueUrl(SOURCE_QUEUE);
+ getQueueUrl(SINK_QUEUE);
+ });
// insert message to source queue
- sqsClient.sendMessage(
- r ->
- r.queueUrl(sqsClient.listQueues().queueUrls().get(0))
- .messageBody(TEST_MESSAGE));
+ String sourceQueueUrl = getQueueUrl(SOURCE_QUEUE);
+ sqsClient.sendMessage(r ->
r.queueUrl(sourceQueueUrl).messageBody(TEST_MESSAGE));
+ }
+
+ private String getQueueUrl(String queueName) {
+ return sqsClient.getQueueUrl(r -> r.queueName(queueName)).queueUrl();
}
@AfterAll
@@ -134,10 +150,8 @@ public class AmazonsqsIT extends TestSuiteBase implements
TestResource {
// check if there is message in sink queue, and compare the sink
record with the source
// record
// the message is invisible after reception, so don't call it twice.
- List<Message> messages =
- sqsClient
- .receiveMessage(r ->
r.queueUrl(sqsClient.listQueues().queueUrls().get(1)))
- .messages();
+ String sinkQueueUrl = getQueueUrl(SINK_QUEUE);
+ List<Message> messages = sqsClient.receiveMessage(r ->
r.queueUrl(sinkQueueUrl)).messages();
Assertions.assertEquals(1, messages.size());
Assertions.assertEquals(TEST_MESSAGE, messages.get(0).body());
}
diff --git a/seatunnel-transforms-v2/pom.xml b/seatunnel-transforms-v2/pom.xml
index 9bc4c6aa06..1707d350f7 100644
--- a/seatunnel-transforms-v2/pom.xml
+++ b/seatunnel-transforms-v2/pom.xml
@@ -34,7 +34,6 @@
<httpcore.version>4.4.16</httpcore.version>
<mockwebserver.version>3.6.0</mockwebserver.version>
<zhipu.version>release-V4-2.3.0</zhipu.version>
- <aws.sdk.version>2.27.21</aws.sdk.version>
</properties>
<dependencyManagement>
@@ -52,12 +51,12 @@
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>bedrockruntime</artifactId>
- <version>${aws.sdk.version}</version>
+ <version>${software.amazon.awssdk.version}</version>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>apache-client</artifactId>
- <version>${aws.sdk.version}</version>
+ <version>${software.amazon.awssdk.version}</version>
</dependency>
<dependency>
<groupId>org.apache.seatunnel</groupId>
diff --git a/tools/dependencies/known-dependencies.txt
b/tools/dependencies/known-dependencies.txt
index 353008ae0a..319f9e99d8 100755
--- a/tools/dependencies/known-dependencies.txt
+++ b/tools/dependencies/known-dependencies.txt
@@ -87,43 +87,43 @@ flatbuffers-java-23.5.26.jar
netty-buffer-4.1.104.Final.jar
netty-common-4.1.104.Final.jar
seatunnel-arrow-2.3.12-SNAPSHOT-optional.jar
-sdk-core-2.27.21.jar
-third-party-jackson-core-2.27.21.jar
-utils-2.27.21.jar
+sdk-core-2.31.30.jar
+third-party-jackson-core-2.31.30.jar
+utils-2.31.30.jar
reactive-streams-1.0.4.jar
-regions-2.27.21.jar
-retries-2.27.21.jar
-retries-spi-2.27.21.jar
-annotations-2.27.21.jar
-apache-client-2.27.21.jar
-auth-2.27.21.jar
-aws-core-2.27.21.jar
-aws-json-protocol-2.27.21.jar
-bedrockruntime-2.27.21.jar
-checksums-2.27.21.jar
-checksums-spi-2.27.21.jar
-endpoints-spi-2.27.21.jar
-http-auth-2.27.21.jar
-http-auth-aws-2.27.21.jar
-http-auth-aws-eventstream-2.27.21.jar
-http-auth-spi-2.27.21.jar
-http-client-spi-2.27.21.jar
-identity-spi-2.27.21.jar
-json-utils-2.27.21.jar
-metrics-spi-2.27.21.jar
-netty-nio-client-2.27.21.jar
-profiles-2.27.21.jar
-protocol-core-2.27.21.jar
-netty-transport-4.1.112.Final.jar
-netty-transport-classes-epoll-4.1.112.Final.jar
-netty-transport-native-unix-common-4.1.112.Final.jar
-netty-buffer-4.1.112.Final.jar
-netty-codec-4.1.112.Final.jar
-netty-codec-http-4.1.112.Final.jar
-netty-codec-http2-4.1.112.Final.jar
-netty-common-4.1.112.Final.jar
-netty-handler-4.1.112.Final.jar
-netty-resolver-4.1.112.Final.jar
+regions-2.31.30.jar
+retries-2.31.30.jar
+retries-spi-2.31.30.jar
+auth-2.31.30.jar
+annotations-2.31.30.jar
+apache-client-2.31.30.jar
+aws-core-2.31.30.jar
+aws-json-protocol-2.31.30.jar
+bedrockruntime-2.31.30.jar
+checksums-2.31.30.jar
+checksums-spi-2.31.30.jar
+endpoints-spi-2.31.30.jar
+http-auth-2.31.30.jar
+http-auth-aws-2.31.30.jar
+http-auth-aws-eventstream-2.31.30.jar
+http-auth-spi-2.31.30.jar
+http-client-spi-2.31.30.jar
+identity-spi-2.31.30.jar
+json-utils-2.31.30.jar
+metrics-spi-2.31.30.jar
+netty-nio-client-2.31.30.jar
+profiles-2.31.30.jar
+protocol-core-2.31.30.jar
+netty-transport-4.1.118.Final.jar
+netty-transport-classes-epoll-4.1.118.Final.jar
+netty-transport-native-unix-common-4.1.118.Final.jar
+netty-buffer-4.1.118.Final.jar
+netty-codec-4.1.118.Final.jar
+netty-codec-http-4.1.118.Final.jar
+netty-codec-http2-4.1.118.Final.jar
+netty-common-4.1.118.Final.jar
+netty-handler-4.1.118.Final.jar
+netty-resolver-4.1.118.Final.jar
eventstream-1.0.1.jar
java-diff-utils-4.12.jar
jline-3.22.0.jar
\ No newline at end of file
