This is an automated email from the ASF dual-hosted git repository.
gaojun2048 pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new b65f40c3c9 [Improve][CDC]Remove driver for cdc connector (#4952)
b65f40c3c9 is described below
commit b65f40c3c9eb7180d8467a5d9cdfa50143beb422
Author: Guangdong Liu <[email protected]>
AuthorDate: Wed Jun 28 11:01:33 2023 +0800
[Improve][CDC]Remove driver for cdc connector (#4952)
---
.../connector-cdc/connector-cdc-mysql/pom.xml | 6 ++++++
.../connector-cdc/connector-cdc-sqlserver/pom.xml | 6 ++++++
seatunnel-connectors-v2/connector-jdbc/pom.xml | 2 +-
.../connector-cdc-mysql-e2e/pom.xml | 18 +++++++++++++++++
.../connectors/seatunnel/cdc/mysql/MysqlCDCIT.java | 19 ++++++++++++++++++
.../src/test/resources/log4j2-test.properties | 4 ++--
.../connector-cdc-sqlserver-e2e/pom.xml | 17 ++++++++++++++++
.../connector/cdc/sqlserver/SqlServerCDCIT.java | 19 ++++++++++++++++++
.../connector-kafka-e2e/pom.xml | 20 +++++++++++++++----
.../connector-pulsar-e2e/pom.xml | 23 +++++++++++++++-------
.../src/test/resources/log4j2.properties | 2 +-
.../src/test/resources/log4j2-test.properties | 4 ++--
.../src/test/resources/log4j2.properties | 4 ++--
13 files changed, 125 insertions(+), 19 deletions(-)
diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/pom.xml
b/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/pom.xml
index 58b5ca3f40..4fe4176da9 100644
--- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/pom.xml
+++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/pom.xml
@@ -64,6 +64,12 @@
<dependency>
<groupId>io.debezium</groupId>
<artifactId>debezium-connector-mysql</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.seatunnel</groupId>
diff --git
a/seatunnel-connectors-v2/connector-cdc/connector-cdc-sqlserver/pom.xml
b/seatunnel-connectors-v2/connector-cdc/connector-cdc-sqlserver/pom.xml
index 3d54cd4b1b..11abf8b890 100644
--- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-sqlserver/pom.xml
+++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-sqlserver/pom.xml
@@ -57,6 +57,12 @@
<dependency>
<groupId>io.debezium</groupId>
<artifactId>debezium-connector-sqlserver</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>com.microsoft.sqlserver</groupId>
+ <artifactId>mssql-jdbc</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
diff --git a/seatunnel-connectors-v2/connector-jdbc/pom.xml
b/seatunnel-connectors-v2/connector-jdbc/pom.xml
index 7b451df120..e76237e7e0 100644
--- a/seatunnel-connectors-v2/connector-jdbc/pom.xml
+++ b/seatunnel-connectors-v2/connector-jdbc/pom.xml
@@ -30,7 +30,7 @@
<name>SeaTunnel : Connectors V2 : Jdbc</name>
<properties>
- <mysql.version>8.0.16</mysql.version>
+ <mysql.version>8.0.27</mysql.version>
<postgresql.version>42.4.3</postgresql.version>
<dm-jdbc.version>8.1.2.141</dm-jdbc.version>
<sqlserver.version>9.2.1.jre8</sqlserver.version>
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/pom.xml
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/pom.xml
index e6588aeecc..539fce1890 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/pom.xml
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/pom.xml
@@ -25,6 +25,18 @@
<artifactId>connector-cdc-mysql-e2e</artifactId>
<name>SeaTunnel : E2E : Connector V2 : CDC MySql</name>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>connector-jdbc</artifactId>
+ <version>${project.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
<dependencies>
<!-- SeaTunnel connectors -->
<dependency>
@@ -47,5 +59,11 @@
<artifactId>mysql</artifactId>
<version>${testcontainer.version}</version>
</dependency>
+
+ <dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/MysqlCDCIT.java
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/MysqlCDCIT.java
index adb3a59869..1d0d90853f 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/MysqlCDCIT.java
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/MysqlCDCIT.java
@@ -22,14 +22,17 @@ import
org.apache.seatunnel.connectors.seatunnel.cdc.mysql.testutils.MySqlVersio
import
org.apache.seatunnel.connectors.seatunnel.cdc.mysql.testutils.UniqueDatabase;
import org.apache.seatunnel.e2e.common.TestResource;
import org.apache.seatunnel.e2e.common.TestSuiteBase;
+import org.apache.seatunnel.e2e.common.container.ContainerExtendedFactory;
import org.apache.seatunnel.e2e.common.container.EngineType;
import org.apache.seatunnel.e2e.common.container.TestContainer;
import org.apache.seatunnel.e2e.common.junit.DisabledOnContainer;
+import org.apache.seatunnel.e2e.common.junit.TestContainerExtension;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.TestTemplate;
+import org.testcontainers.containers.Container;
import org.testcontainers.containers.output.Slf4jLogConsumer;
import org.testcontainers.lifecycle.Startables;
import org.testcontainers.utility.DockerLoggerFactory;
@@ -102,6 +105,22 @@ public class MysqlCDCIT extends TestSuiteBase implements
TestResource {
return mySqlContainer;
}
+ private String driverUrl() {
+ return
"https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.32/mysql-connector-j-8.0.32.jar";;
+ }
+
+ @TestContainerExtension
+ protected final ContainerExtendedFactory extendedFactory =
+ container -> {
+ Container.ExecResult extraCommands =
+ container.execInContainer(
+ "bash",
+ "-c",
+ "mkdir -p /tmp/seatunnel/plugins/MySQL-CDC/lib
&& cd /tmp/seatunnel/plugins/MySQL-CDC/lib && wget "
+ + driverUrl());
+ Assertions.assertEquals(0, extraCommands.getExitCode(),
extraCommands.getStderr());
+ };
+
@BeforeAll
@Override
public void startUp() throws ClassNotFoundException, InterruptedException {
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/log4j2-test.properties
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/log4j2-test.properties
index d1ca535f2b..d01e745df7 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/log4j2-test.properties
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/log4j2-test.properties
@@ -18,8 +18,8 @@
################################################################################
# Set root logger level to OFF to not flood build logs
-# set manually to INFO for debugging purposes
-rootLogger.level=INFO
+# The amount of logs is too large to be suitable for troubleshooting, manually
set to WARN.
+rootLogger.level=WARN
rootLogger.appenderRef.test.ref = TestLogger
appender.testlogger.name = TestLogger
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/pom.xml
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/pom.xml
index 7d7376f1ad..0c5a0fa8e0 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/pom.xml
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/pom.xml
@@ -25,6 +25,18 @@
<artifactId>connector-cdc-sqlserver-e2e</artifactId>
<name>SeaTunnel : E2E : Connector V2 : CDC SqlServer</name>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>connector-jdbc</artifactId>
+ <version>${project.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
<dependencies>
<!-- SeaTunnel connectors -->
<dependency>
@@ -59,5 +71,10 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>com.microsoft.sqlserver</groupId>
+ <artifactId>mssql-jdbc</artifactId>
+ </dependency>
</dependencies>
</project>
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/java/org/apache/seatunnel/e2e/connector/cdc/sqlserver/SqlServerCDCIT.java
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/java/org/apache/seatunnel/e2e/connector/cdc/sqlserver/SqlServerCDCIT.java
index d1f0fd130d..8bca3e3b03 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/java/org/apache/seatunnel/e2e/connector/cdc/sqlserver/SqlServerCDCIT.java
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/java/org/apache/seatunnel/e2e/connector/cdc/sqlserver/SqlServerCDCIT.java
@@ -19,9 +19,11 @@ package org.apache.seatunnel.e2e.connector.cdc.sqlserver;
import org.apache.seatunnel.e2e.common.TestResource;
import org.apache.seatunnel.e2e.common.TestSuiteBase;
+import org.apache.seatunnel.e2e.common.container.ContainerExtendedFactory;
import org.apache.seatunnel.e2e.common.container.EngineType;
import org.apache.seatunnel.e2e.common.container.TestContainer;
import org.apache.seatunnel.e2e.common.junit.DisabledOnContainer;
+import org.apache.seatunnel.e2e.common.junit.TestContainerExtension;
import org.awaitility.Awaitility;
import org.awaitility.core.ConditionTimeoutException;
@@ -29,6 +31,7 @@ import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.TestTemplate;
+import org.testcontainers.containers.Container;
import org.testcontainers.containers.MSSQLServerContainer;
import org.testcontainers.containers.output.Slf4jLogConsumer;
import org.testcontainers.lifecycle.Startables;
@@ -92,6 +95,22 @@ public class SqlServerCDCIT extends TestSuiteBase implements
TestResource {
new Slf4jLogConsumer(
DockerLoggerFactory.getLogger("sqlserver-docker-image")));
+ private String driverUrl() {
+ return
"https://repo1.maven.org/maven2/com/microsoft/sqlserver/mssql-jdbc/9.4.1.jre8/mssql-jdbc-9.4.1.jre8.jar";;
+ }
+
+ @TestContainerExtension
+ protected final ContainerExtendedFactory extendedFactory =
+ container -> {
+ Container.ExecResult extraCommands =
+ container.execInContainer(
+ "bash",
+ "-c",
+ "mkdir -p
/tmp/seatunnel/plugins/SqlServer-CDC/lib && cd
/tmp/seatunnel/plugins/SqlServer-CDC/lib && wget "
+ + driverUrl());
+ Assertions.assertEquals(0, extraCommands.getExitCode(),
extraCommands.getStderr());
+ };
+
@Override
@BeforeAll
public void startUp() throws Exception {
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/pom.xml
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/pom.xml
index ead4c654c2..81cbb78569 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/pom.xml
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/pom.xml
@@ -25,9 +25,17 @@
<artifactId>connector-kafka-e2e</artifactId>
<name>SeaTunnel : E2E : Connector V2 : Kafka</name>
- <properties>
- <postgresql.version>42.4.3</postgresql.version>
- </properties>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>connector-jdbc</artifactId>
+ <version>${project.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
<dependencies>
<dependency>
@@ -82,7 +90,6 @@
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
- <version>${postgresql.version}</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -97,6 +104,11 @@
<version>${testcontainer.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-pulsar-e2e/pom.xml
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-pulsar-e2e/pom.xml
index f4db6fc0f4..f817fdc155 100644
--- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-pulsar-e2e/pom.xml
+++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-pulsar-e2e/pom.xml
@@ -25,12 +25,17 @@
<artifactId>connector-pulsar-e2e</artifactId>
<name>SeaTunnel : E2E : Connector V2 : Pulsar</name>
- <properties>
- <maven.compiler.source>8</maven.compiler.source>
- <maven.compiler.target>8</maven.compiler.target>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <postgresql.version>42.4.3</postgresql.version>
- </properties>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.seatunnel</groupId>
+ <artifactId>connector-jdbc</artifactId>
+ <version>${project.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
<dependencies>
<dependency>
@@ -93,7 +98,11 @@
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
- <version>${postgresql.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
diff --git
a/seatunnel-e2e/seatunnel-e2e-common/src/test/resources/log4j2.properties
b/seatunnel-e2e/seatunnel-e2e-common/src/test/resources/log4j2.properties
index 2dc1b8ca50..26190157fd 100644
--- a/seatunnel-e2e/seatunnel-e2e-common/src/test/resources/log4j2.properties
+++ b/seatunnel-e2e/seatunnel-e2e-common/src/test/resources/log4j2.properties
@@ -16,7 +16,7 @@
# limitations under the License.
################################################################################
-rootLogger.level = INFO
+rootLogger.level = WARN
rootLogger.appenderRef.consoleStdout.ref = consoleStdoutAppender
rootLogger.appenderRef.consoleStderr.ref = consoleStderrAppender
diff --git
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/log4j2-test.properties
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/log4j2-test.properties
index 0999f94876..f0090af024 100644
---
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/log4j2-test.properties
+++
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/log4j2-test.properties
@@ -16,13 +16,13 @@
# limitations under the License.
################################################################################
-rootLogger.level = INFO
+rootLogger.level = WARN
rootLogger.appenderRef.consoleStdout.ref = consoleStdoutAppender
rootLogger.appenderRef.consoleStderr.ref = consoleStderrAppender
logger.zeta.name=org.apache.seatunnel.engine
-logger.zeta.level=DEBUG
+logger.zeta.level=WARN
appender.consoleStdout.name = consoleStdoutAppender
appender.consoleStdout.type = CONSOLE
diff --git
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/log4j2.properties
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/log4j2.properties
index 7b9977bd6e..fc0c2063b7 100644
---
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/log4j2.properties
+++
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/log4j2.properties
@@ -16,10 +16,10 @@
# limitations under the License.
################################################################################
-rootLogger.level = INFO
+rootLogger.level = WARN
logger.zeta.name=org.apache.seatunnel.engine
-logger.zeta.level=INFO
+logger.zeta.level=WARN
logger.debezium.name=io.debezium.connector
logger.debezium.level=WARN
