This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-11201-cf67b549a7a6c35fa0beb12d83c62892427ea919 in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit eea324dfaf9d7cacdd3415c3b63f2d5c25d7e922 Author: zhangdonghao <[email protected]> AuthorDate: Thu Sep 10 10:29:42 2026 +0000 [Fix][E2E] Unify testcontainers version to 1.21.4 across all e2e connectors (#11201) Co-authored-by: Zhang Donghao <[email protected]> Co-authored-by: Claude <[email protected]> --- pom.xml | 2 +- .../connector-activemq-e2e/pom.xml | 2 +- .../connector-amazonsqs-e2e/pom.xml | 2 +- .../connector-bigquery-e2e/pom.xml | 2 +- .../connector-databend-e2e/pom.xml | 1 - .../connector-elasticsearch-e2e/pom.xml | 2 +- .../connector-file-local-e2e/pom.xml | 2 +- .../connector-hudi-e2e/pom.xml | 2 +- .../connector-iceberg-s3-e2e/pom.xml | 1 - .../connector-jdbc-e2e-part-2/pom.xml | 7 +- .../connector-milvus-e2e/pom.xml | 7 +- .../connector-paimon-e2e/pom.xml | 2 +- .../connector-qdrant-e2e/pom.xml | 2 +- .../seatunnel/engine/e2e/k8s/KubernetesIT.java | 131 +++++++++++++++++++-- 14 files changed, 134 insertions(+), 31 deletions(-) diff --git a/pom.xml b/pom.xml index 2606f89943..baf66a1ff9 100644 --- a/pom.xml +++ b/pom.xml @@ -155,7 +155,7 @@ <maven-dependency-plugin.version>3.1.1</maven-dependency-plugin.version> <p3c-pmd.version>1.3.0</p3c-pmd.version> <maven-scm-provider-jgit.version>2.0.0</maven-scm-provider-jgit.version> - <testcontainer.version>1.17.6</testcontainer.version> + <testcontainer.version>1.21.4</testcontainer.version> <spotless.version>2.29.0</spotless.version> <jsqlparser.version>4.9</jsqlparser.version> <json-path.version>2.7.0</json-path.version> diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-activemq-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-activemq-e2e/pom.xml index d94d24b29d..26b6f078c7 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-activemq-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-activemq-e2e/pom.xml @@ -29,7 +29,7 @@ <dependency> <groupId>org.testcontainers</groupId> <artifactId>activemq</artifactId> - <version>1.20.1</version> + <version>${testcontainer.version}</version> <scope>test</scope> </dependency> diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazonsqs-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazonsqs-e2e/pom.xml index 5c7396ab35..be29f9a523 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazonsqs-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-amazonsqs-e2e/pom.xml @@ -55,7 +55,7 @@ <dependency> <groupId>org.testcontainers</groupId> <artifactId>localstack</artifactId> - <version>1.19.0</version> + <version>${testcontainer.version}</version> <scope>test</scope> </dependency> <dependency> diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-bigquery-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-bigquery-e2e/pom.xml index 7d50f0bc84..e42431d7a9 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-bigquery-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-bigquery-e2e/pom.xml @@ -61,7 +61,7 @@ <dependency> <groupId>org.testcontainers</groupId> <artifactId>gcloud</artifactId> - <version>1.19.0</version> + <version>${testcontainer.version}</version> <scope>test</scope> </dependency> <dependency> diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-databend-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-databend-e2e/pom.xml index 416b48d54d..8880d35ce5 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-databend-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-databend-e2e/pom.xml @@ -27,7 +27,6 @@ <properties> <databend.jdbc.version>0.3.7</databend.jdbc.version> - <testcontainer.version>1.20.2</testcontainer.version> </properties> <dependencies> diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-elasticsearch-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-elasticsearch-e2e/pom.xml index 03c0ff2644..f4335559a3 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-elasticsearch-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-elasticsearch-e2e/pom.xml @@ -53,7 +53,7 @@ <dependency> <groupId>org.testcontainers</groupId> <artifactId>elasticsearch</artifactId> - <version>1.17.3</version> + <version>${testcontainer.version}</version> <scope>test</scope> </dependency> diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/pom.xml index 7b216d167c..010f0d8ebe 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/pom.xml @@ -63,7 +63,7 @@ <dependency> <groupId>org.testcontainers</groupId> <artifactId>mysql</artifactId> - <version>1.19.1</version> + <version>${testcontainer.version}</version> <scope>test</scope> </dependency> <!-- MySQL JDBC driver --> diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/pom.xml index e0dc3ae24f..c23ec53c80 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/pom.xml @@ -26,7 +26,6 @@ <name>SeaTunnel : E2E : Connector V2 : Hudi</name> <properties> - <testcontainer.version>1.19.1</testcontainer.version> <minio.version>8.5.6</minio.version> </properties> @@ -98,6 +97,7 @@ <groupId>org.testcontainers</groupId> <artifactId>mysql</artifactId> <version>${testcontainer.version}</version> + <scope>test</scope> </dependency> <dependency> diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-s3-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-s3-e2e/pom.xml index e08d05f319..81048f2947 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-s3-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-s3-e2e/pom.xml @@ -26,7 +26,6 @@ <name>SeaTunnel : E2E : Connector V2 : Iceberg : S3</name> <properties> - <testcontainer.version>1.19.1</testcontainer.version> <minio.version>8.5.6</minio.version> <hadoop3.version>3.1.4</hadoop3.version> </properties> diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/pom.xml index 45fc846efd..79c2f684de 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/pom.xml @@ -26,8 +26,6 @@ <name>SeaTunnel : E2E : Connector V2 : Jdbc : Part 2</name> <properties> <mysql.legacy.e2e.version>8.0.16</mysql.legacy.e2e.version> - <testcontainer.milvus.version>1.19.8</testcontainer.milvus.version> - <testcontainer.oceanbase.version>1.20.1</testcontainer.oceanbase.version> </properties> <dependencies> <dependency> @@ -52,12 +50,13 @@ <dependency> <groupId>org.testcontainers</groupId> <artifactId>milvus</artifactId> - <version>${testcontainer.milvus.version}</version> + <version>${testcontainer.version}</version> + <scope>test</scope> </dependency> <dependency> <groupId>org.testcontainers</groupId> <artifactId>oceanbase</artifactId> - <version>${testcontainer.oceanbase.version}</version> + <version>${testcontainer.version}</version> <scope>test</scope> </dependency> <!-- drivers --> diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-milvus-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-milvus-e2e/pom.xml index 2175811c6c..bbf5d9ea41 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-milvus-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-milvus-e2e/pom.xml @@ -25,10 +25,6 @@ <artifactId>connector-milvus-e2e</artifactId> <name>SeaTunnel : E2E : Connector V2 : Milvus</name> - <properties> - <testcontainer.milvus.version>1.19.8</testcontainer.milvus.version> - </properties> - <dependencies> <dependency> <groupId>org.apache.seatunnel</groupId> @@ -47,7 +43,8 @@ <dependency> <groupId>org.testcontainers</groupId> <artifactId>milvus</artifactId> - <version>${testcontainer.milvus.version}</version> + <version>${testcontainer.version}</version> + <scope>test</scope> </dependency> <dependency> diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/pom.xml index 1c7b5a867e..bc1661947f 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-paimon-e2e/pom.xml @@ -27,7 +27,6 @@ <properties> <libfb303.version>0.9.0</libfb303.version> - <testcontainer.version>1.19.1</testcontainer.version> <minio.version>8.5.6</minio.version> </properties> @@ -103,6 +102,7 @@ <groupId>org.testcontainers</groupId> <artifactId>mysql</artifactId> <version>${testcontainer.version}</version> + <scope>test</scope> </dependency> <dependency> diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-qdrant-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-qdrant-e2e/pom.xml index 4dd8093c0d..384d4d4f70 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-qdrant-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-qdrant-e2e/pom.xml @@ -46,7 +46,7 @@ <dependency> <groupId>org.testcontainers</groupId> <artifactId>qdrant</artifactId> - <version>1.20.1</version> + <version>${testcontainer.version}</version> <scope>test</scope> </dependency> diff --git a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-k8s-e2e/src/test/java/org/apache/seatunnel/engine/e2e/k8s/KubernetesIT.java b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-k8s-e2e/src/test/java/org/apache/seatunnel/engine/e2e/k8s/KubernetesIT.java index 7beea86086..0bd883b2cc 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-k8s-e2e/src/test/java/org/apache/seatunnel/engine/e2e/k8s/KubernetesIT.java +++ b/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-k8s-e2e/src/test/java/org/apache/seatunnel/engine/e2e/k8s/KubernetesIT.java @@ -28,15 +28,19 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.testcontainers.DockerClientFactory; import org.testcontainers.shaded.org.awaitility.Awaitility; +import org.testcontainers.shaded.org.awaitility.core.ConditionTimeoutException; import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.command.BuildImageCmd; +import com.github.dockerjava.api.model.Image; import com.github.dockerjava.api.model.Info; import io.kubernetes.client.openapi.ApiClient; import io.kubernetes.client.openapi.ApiException; import io.kubernetes.client.openapi.Configuration; import io.kubernetes.client.openapi.apis.AppsV1Api; import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.models.V1Pod; +import io.kubernetes.client.openapi.models.V1PodList; import io.kubernetes.client.openapi.models.V1Service; import io.kubernetes.client.openapi.models.V1StatefulSet; import io.kubernetes.client.util.Config; @@ -50,6 +54,7 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.util.Collections; +import java.util.List; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; @@ -94,7 +99,14 @@ public class KubernetesIT { Info info = dockerClient.infoCmd().exec(); log.info("Docker's environmental information"); log.info(info.toString()); - if (dockerClient.listImagesCmd().withImageNameFilter(tag).exec().isEmpty()) { + List<Image> matchedImages = dockerClient.listImagesCmd().withReferenceFilter(tag).exec(); + log.info( + "Check image existence with withReferenceFilter({}): matched {} images, isEmpty={}", + tag, + matchedImages.size(), + matchedImages.isEmpty()); + if (matchedImages.isEmpty()) { + log.info("Image '{}' not found in Docker daemon, starting manual docker build...", tag); copyFileToCurrentResources(hazelCastConfigFile, targetPath); File file = new File( @@ -104,6 +116,12 @@ public class KubernetesIT { buildImageCmd.withTags(Collections.singleton(tag)); String imageId = buildImageCmd.start().awaitImageId(); Assertions.assertNotNull(imageId); + log.info("Image '{}' built successfully, imageId={}", tag, imageId); + } else { + log.info( + "Image '{}' already exists in Docker daemon (matched {} images), skipping manual build", + tag, + matchedImages.size()); } Configuration.setDefaultApiClient(client); V1Service yamlSvc = @@ -118,20 +136,31 @@ public class KubernetesIT { new File( PROJECT_ROOT_PATH + "/seatunnel-e2e/seatunnel-engine-e2e/seatunnel-engine-k8s-e2e/src/test/resources/seatunnel-statefulset.yaml")); + // Drop resources left over from a previous (possibly aborted) run so the creates below + // do not fail with a 409 Conflict. + cleanupResources(appsV1Api, coreV1Api); try { coreV1Api.createNamespacedService(namespace, yamlSvc, null, null, null, null); appsV1Api.createNamespacedStatefulSet( namespace, yamlStatefulSet, null, null, null, null); - Awaitility.await() - .atMost(360, TimeUnit.SECONDS) - .untilAsserted( - () -> { - V1StatefulSet v1StatefulSet = - appsV1Api.readNamespacedStatefulSet( - stsName, namespace, null); - Assertions.assertEquals( - 2, v1StatefulSet.getStatus().getReadyReplicas()); - }); + try { + Awaitility.await() + .atMost(360, TimeUnit.SECONDS) + .untilAsserted( + () -> { + V1StatefulSet v1StatefulSet = + appsV1Api.readNamespacedStatefulSet( + stsName, namespace, null); + Assertions.assertEquals( + 2, v1StatefulSet.getStatus().getReadyReplicas()); + }); + } catch (ConditionTimeoutException e) { + // Dump pod state so a timeout is diagnosable instead of a bare + // "expected: <2> but was: <null>" (the StatefulSet status is not populated + // when the pods never become Ready). + logPodStates(coreV1Api); + throw e; + } // submit job String command = "/opt/seatunnel/bin/seatunnel.sh --config /opt/seatunnel/config/v2.batch.config.template"; @@ -166,6 +195,86 @@ public class KubernetesIT { } } + private void cleanupResources(AppsV1Api appsV1Api, CoreV1Api coreV1Api) throws ApiException { + try { + appsV1Api.deleteNamespacedStatefulSet( + stsName, namespace, null, null, null, null, null, null); + } catch (ApiException e) { + if (e.getCode() != 404) { + throw e; + } + } + try { + coreV1Api.deleteNamespacedService( + svcName, namespace, null, null, null, null, null, null); + } catch (ApiException e) { + if (e.getCode() != 404) { + throw e; + } + } + // The name stays reserved while the object terminates, so wait for the resources to be + // actually gone before re-creating them. + Awaitility.await() + .atMost(60, TimeUnit.SECONDS) + .pollInterval(2, TimeUnit.SECONDS) + .until( + () -> { + try { + appsV1Api.readNamespacedStatefulSet(stsName, namespace, null); + return false; + } catch (ApiException e) { + return e.getCode() == 404; + } + }); + Awaitility.await() + .atMost(60, TimeUnit.SECONDS) + .pollInterval(2, TimeUnit.SECONDS) + .until( + () -> { + try { + coreV1Api.readNamespacedService(svcName, namespace, null); + return false; + } catch (ApiException e) { + return e.getCode() == 404; + } + }); + } + + private void logPodStates(CoreV1Api coreV1Api) { + try { + V1PodList podList = + coreV1Api.listNamespacedPod( + namespace, + null, + null, + null, + null, + "app=seatunnel", + null, + null, + null, + null, + null, + null); + log.error("Seatunnel pods are not ready. Pod states:"); + if (podList.getItems() != null) { + for (V1Pod pod : podList.getItems()) { + log.error( + " pod={} phase={} conditions={} containerStatuses={}", + pod.getMetadata().getName(), + pod.getStatus() == null ? null : pod.getStatus().getPhase(), + pod.getStatus() == null ? null : pod.getStatus().getConditions(), + pod.getStatus() == null + ? null + : pod.getStatus().getContainerStatuses()); + } + } + } catch (Exception e) { + // Best effort only - never let the diagnostic mask the real timeout failure. + log.error("Failed to fetch pod states for diagnostics", e); + } + } + private void copyFileToCurrentResources(String hazelCastConfigFile, String targetPath) throws IOException { File jarsPath = new File(targetPath + "/jars");
