This is an automated email from the ASF dual-hosted git repository.
leonard pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-cdc.git
The following commit(s) were added to refs/heads/master by this push:
new af02ce1bc [build][e2e] Separate Pipeline and Source E2e tests and
cover flink 1.20 version
af02ce1bc is described below
commit af02ce1bc948c8b70807ef11f839544e3a22e836
Author: yuxiqian <[email protected]>
AuthorDate: Thu Aug 8 17:45:07 2024 +0800
[build][e2e] Separate Pipeline and Source E2e tests and cover flink 1.20
version
This closes #3514.
---
.github/workflows/flink_cdc.yml | 21 ++++++++++++++-------
.../flink-cdc-pipeline-e2e-tests/pom.xml | 3 ++-
.../tests/utils/PipelineTestEnvironment.java | 2 +-
.../flink-cdc-source-e2e-tests/pom.xml | 20 ++++++++++++++++----
.../tests/utils/FlinkContainerTestEnvironment.java | 10 ++--------
5 files changed, 35 insertions(+), 21 deletions(-)
diff --git a/.github/workflows/flink_cdc.yml b/.github/workflows/flink_cdc.yml
index ef67eb55d..ac0dfdc82 100644
--- a/.github/workflows/flink_cdc.yml
+++ b/.github/workflows/flink_cdc.yml
@@ -94,8 +94,10 @@ env:
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-vitess-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-vitess-cdc"
- MODULES_E2E: "\
- flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests,\
+ MODULES_PIPELINE_E2E: "\
+ flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests"
+
+ MODULES_SOURCE_E2E: "\
flink-cdc-e2e-tests/flink-cdc-source-e2e-tests"
jobs:
@@ -134,7 +136,8 @@ jobs:
"oceanbase",
"db2",
"vitess",
- "e2e"
+ "pipeline_e2e",
+ "source_e2e"
]
timeout-minutes: 120
env:
@@ -222,13 +225,17 @@ jobs:
("vitess")
modules=${{ env.MODULES_VITESS }}
;;
- ("e2e")
- compile_modules="${{ env.MODULES_CORE }},${{
env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL }},${{
env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB
}},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{
env.MODULES_OCEANBASE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{
env.MODULES_E2E }}"
- modules=${{ env.MODULES_E2E }}
+ ("pipeline_e2e")
+ compile_modules="${{ env.MODULES_CORE }},${{
env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL }},${{
env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB
}},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{
env.MODULES_OCEANBASE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{
env.MODULES_PIPELINE_E2E }}"
+ modules=${{ env.MODULES_PIPELINE_E2E }}
+ ;;
+ ("source_e2e")
+ compile_modules="${{ env.MODULES_CORE }},${{
env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL }},${{
env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB
}},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{
env.MODULES_OCEANBASE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{
env.MODULES_SOURCE_E2E }}"
+ modules=${{ env.MODULES_SOURCE_E2E }}
;;
esac
- if [ ${{ matrix.module }} != "e2e" ]; then
+ if [ ${{ matrix.module }} != "pipeline_e2e" ] && [ ${{ matrix.module
}} != "source_e2e" ]; then
compile_modules=$modules
fi
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/pom.xml
b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/pom.xml
index 2326240b6..557ecafa4 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/pom.xml
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/pom.xml
@@ -30,7 +30,8 @@ limitations under the License.
<properties>
<flink-1.17>1.17.2</flink-1.17>
<flink-1.18>1.18.1</flink-1.18>
- <flink-1.19>1.19.0</flink-1.19>
+ <flink-1.19>1.19.1</flink-1.19>
+ <flink-1.20>1.20.0</flink-1.20>
<mysql.driver.version>8.0.27</mysql.driver.version>
<starrocks.connector.version>1.2.9_flink-${flink.major.version}</starrocks.connector.version>
</properties>
diff --git
a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/utils/PipelineTestEnvironment.java
b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/utils/PipelineTestEnvironment.java
index 1143012a6..d1c0bb7e7 100644
---
a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/utils/PipelineTestEnvironment.java
+++
b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/utils/PipelineTestEnvironment.java
@@ -88,7 +88,7 @@ public abstract class PipelineTestEnvironment extends
TestLogger {
@Parameterized.Parameters(name = "flinkVersion: {0}")
public static List<String> getFlinkVersion() {
- return Arrays.asList("1.17.2", "1.18.1", "1.19.0");
+ return Arrays.asList("1.17.2", "1.18.1", "1.19.1", "1.20.0");
}
@Before
diff --git a/flink-cdc-e2e-tests/flink-cdc-source-e2e-tests/pom.xml
b/flink-cdc-e2e-tests/flink-cdc-source-e2e-tests/pom.xml
index a7f35cf04..c1e50101a 100644
--- a/flink-cdc-e2e-tests/flink-cdc-source-e2e-tests/pom.xml
+++ b/flink-cdc-e2e-tests/flink-cdc-source-e2e-tests/pom.xml
@@ -31,10 +31,12 @@ limitations under the License.
<flink-1.16>1.16.3</flink-1.16>
<flink-1.17>1.17.2</flink-1.17>
<flink-1.18>1.18.1</flink-1.18>
- <flink-1.19>1.19.0</flink-1.19>
- <jdbc.version-1.17>3.1.1-1.17</jdbc.version-1.17>
- <jdbc.version-1.18>3.1.2-1.18</jdbc.version-1.18>
- <jdbc.version-1.19>3.1.2-1.18</jdbc.version-1.19>
+ <flink-1.19>1.19.1</flink-1.19>
+ <flink-1.20>1.20.0</flink-1.20>
+ <jdbc.version-1.17>3.1.2-1.17</jdbc.version-1.17>
+ <jdbc.version-1.18>3.2.0-1.18</jdbc.version-1.18>
+ <jdbc.version-1.19>3.2.0-1.19</jdbc.version-1.19>
+ <jdbc.version-1.20>3.2.0-1.19</jdbc.version-1.20>
<mysql.driver.version>8.0.27</mysql.driver.version>
<postgresql.driver.version>42.7.3</postgresql.driver.version>
</properties>
@@ -277,6 +279,16 @@ limitations under the License.
</outputDirectory>
</artifactItem>
+ <artifactItem>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-jdbc</artifactId>
+ <version>${jdbc.version-1.20}</version>
+
<destFileName>jdbc-connector_${flink-1.20}.jar</destFileName>
+ <type>jar</type>
+
<outputDirectory>${project.build.directory}/dependencies
+ </outputDirectory>
+ </artifactItem>
+
<artifactItem>
<groupId>org.apache.flink</groupId>
<artifactId>flink-sql-connector-mysql-cdc</artifactId>
diff --git
a/flink-cdc-e2e-tests/flink-cdc-source-e2e-tests/src/test/java/org/apache/flink/cdc/connectors/tests/utils/FlinkContainerTestEnvironment.java
b/flink-cdc-e2e-tests/flink-cdc-source-e2e-tests/src/test/java/org/apache/flink/cdc/connectors/tests/utils/FlinkContainerTestEnvironment.java
index 6175aec2c..91e7e5c6f 100644
---
a/flink-cdc-e2e-tests/flink-cdc-source-e2e-tests/src/test/java/org/apache/flink/cdc/connectors/tests/utils/FlinkContainerTestEnvironment.java
+++
b/flink-cdc-e2e-tests/flink-cdc-source-e2e-tests/src/test/java/org/apache/flink/cdc/connectors/tests/utils/FlinkContainerTestEnvironment.java
@@ -120,12 +120,9 @@ public abstract class FlinkContainerTestEnvironment
extends TestLogger {
@Parameterized.Parameters(name = "flinkVersion: {0}")
public static List<String> getFlinkVersion() {
- return Arrays.asList("1.16.3", "1.17.2", "1.18.1", "1.19.0");
+ return Arrays.asList("1.16.3", "1.17.2", "1.18.1", "1.19.1", "1.20.0");
}
- private static final List<String> FLINK_VERSION_WITH_SCALA_212 =
- Arrays.asList("1.16.3", "1.17.2", "1.18.1", "1.19.0");
-
@Before
public void before() {
mysqlInventoryDatabase.createAndInitialize();
@@ -309,10 +306,7 @@ public abstract class FlinkContainerTestEnvironment
extends TestLogger {
}
private String getFlinkDockerImageTag() {
- if (FLINK_VERSION_WITH_SCALA_212.contains(flinkVersion)) {
- return String.format("flink:%s-scala_2.12", flinkVersion);
- }
- return String.format("flink:%s-scala_2.11", flinkVersion);
+ return String.format("flink:%s-scala_2.12", flinkVersion);
}
protected String getJdbcConnectorResourceName() {
