This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 505f217bc5 [INLONG-10284][Manager][Sort] Upgrade flink default version
to 1.15 (#10310)
505f217bc5 is described below
commit 505f217bc545d58ddc1e08b8c2c7d477b567f9d8
Author: wallezhang <[email protected]>
AuthorDate: Wed Jun 12 10:07:57 2024 +0800
[INLONG-10284][Manager][Sort] Upgrade flink default version to 1.15 (#10310)
Co-authored-by: Charles Zhang <[email protected]>
---
docker/docker-compose/docker-compose.yml | 4 ++--
inlong-manager/manager-docker/Dockerfile | 8 +++-----
inlong-manager/manager-docker/pom.xml | 20 +++++---------------
inlong-manager/manager-plugins/base/pom.xml | 6 +++---
.../src/main/resources/flink-sort-plugin.properties | 2 +-
inlong-sort/pom.xml | 6 +++---
inlong-sort/sort-core/pom.xml | 6 +++---
inlong-sort/sort-dist/pom.xml | 6 +++---
inlong-sort/sort-flink/base/pom.xml | 6 +++---
.../format-row/format-json-v1.13/pom.xml | 2 +-
.../format-row/format-json-v1.15/pom.xml | 2 +-
.../format-row/format-json-v1.18/pom.xml | 2 +-
.../CanalJsonEnhancedDeserializationSchema.java | 9 +++++++++
.../canal/CanalJsonEnhancedSerializationSchema.java | 5 +++++
.../json/canal/CanalJsonEnhancedSerDeSchemaTest.java | 1 +
inlong-sort/sort-formats/format-row/pom.xml | 10 +++++++++-
inlong-sort/sort-formats/pom.xml | 6 +++---
pom.xml | 6 +++---
18 files changed, 59 insertions(+), 48 deletions(-)
diff --git a/docker/docker-compose/docker-compose.yml
b/docker/docker-compose/docker-compose.yml
index 6140e13c02..1cff746abe 100644
--- a/docker/docker-compose/docker-compose.yml
+++ b/docker/docker-compose/docker-compose.yml
@@ -123,7 +123,7 @@ services:
# flink jobmanager
jobmanager:
- image: apache/flink:1.13-scala_2.11
+ image: apache/flink:1.15-scala_2.12
container_name: jobmanager
environment:
- |
@@ -135,7 +135,7 @@ services:
# flink taskmanager
taskmanager:
- image: apache/flink:1.13-scala_2.11
+ image: apache/flink:1.15-scala_2.12
container_name: taskmanager
environment:
- |
diff --git a/inlong-manager/manager-docker/Dockerfile
b/inlong-manager/manager-docker/Dockerfile
index 2345581f7c..05fda7b120 100644
--- a/inlong-manager/manager-docker/Dockerfile
+++ b/inlong-manager/manager-docker/Dockerfile
@@ -48,11 +48,9 @@ RUN wget -P lib/ ${default_jdbc_connector_url}
# add sort resource
ADD target/sort-dist-${VERSION}.jar /opt/inlong-sort/
RUN wget -P /opt/inlong-sort/connectors/ ${default_jdbc_connector_url}
-ADD target/sort-connector-pulsar-${VERSION}.jar /opt/inlong-sort/connectors/
-ADD target/sort-connector-kafka-${VERSION}.jar /opt/inlong-sort/connectors/
-ADD target/sort-connector-jdbc-${VERSION}.jar /opt/inlong-sort/connectors/
-ADD target/sort-connector-hive-${VERSION}.jar /opt/inlong-sort/connectors/
-ADD target/sort-connector-mysql-cdc-${VERSION}.jar /opt/inlong-sort/connectors/
+ADD target/sort-connector-pulsar-v1.15-${VERSION}.jar
/opt/inlong-sort/connectors/
+ADD target/sort-connector-kafka-v1.15-${VERSION}.jar
/opt/inlong-sort/connectors/
+ADD target/sort-connector-mysql-cdc-v1.15-${VERSION}.jar
/opt/inlong-sort/connectors/
# audit clickhouse
ENV AUDIT_CK_URL=127.0.0.1:8123
ENV AUDIT_CK_USERNAME=default
diff --git a/inlong-manager/manager-docker/pom.xml
b/inlong-manager/manager-docker/pom.xml
index ef703c2d08..201cb05256 100644
--- a/inlong-manager/manager-docker/pom.xml
+++ b/inlong-manager/manager-docker/pom.xml
@@ -54,27 +54,17 @@
<!-- copy sort-connectors to the manager-docker/target path -->
<dependency>
<groupId>org.apache.inlong</groupId>
- <artifactId>sort-connector-pulsar</artifactId>
+ <artifactId>sort-connector-pulsar-v1.15</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
- <artifactId>sort-connector-kafka</artifactId>
+ <artifactId>sort-connector-kafka-v1.15</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>org.apache.inlong</groupId>
- <artifactId>sort-connector-jdbc</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.inlong</groupId>
- <artifactId>sort-connector-hive</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.inlong</groupId>
- <artifactId>sort-connector-mysql-cdc</artifactId>
+ <artifactId>sort-connector-mysql-cdc-v1.15</artifactId>
<version>${project.parent.version}</version>
</dependency>
</dependencies>
@@ -130,8 +120,8 @@
<configuration>
<outputDirectory>target/</outputDirectory>
<includeArtifactIds>manager-web,
-
sort-dist,sort-connector-pulsar,sort-connector-kafka,
-
sort-connector-jdbc,sort-connector-hive,sort-connector-mysql-cdc</includeArtifactIds>
+
sort-dist,sort-connector-pulsar-v1.15,sort-connector-kafka-v1.15,
+
sort-connector-mysql-cdc-v1.15</includeArtifactIds>
<excludeTransitive>true</excludeTransitive>
</configuration>
</execution>
diff --git a/inlong-manager/manager-plugins/base/pom.xml
b/inlong-manager/manager-plugins/base/pom.xml
index 9c738b7e69..d76eb8983b 100644
--- a/inlong-manager/manager-plugins/base/pom.xml
+++ b/inlong-manager/manager-plugins/base/pom.xml
@@ -109,9 +109,6 @@
<profiles>
<profile>
<id>v1.13</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
<dependencies>
<dependency>
<groupId>org.apache.inlong</groupId>
@@ -122,6 +119,9 @@
</profile>
<profile>
<id>v1.15</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
<dependencies>
<dependency>
<groupId>org.apache.inlong</groupId>
diff --git
a/inlong-manager/manager-plugins/base/src/main/resources/flink-sort-plugin.properties
b/inlong-manager/manager-plugins/base/src/main/resources/flink-sort-plugin.properties
index 4840f19ae1..ff34f913f8 100644
---
a/inlong-manager/manager-plugins/base/src/main/resources/flink-sort-plugin.properties
+++
b/inlong-manager/manager-plugins/base/src/main/resources/flink-sort-plugin.properties
@@ -22,7 +22,7 @@
# flink config
########################
# Flink version, support [1.13|1.15]
-flink.version=1.13
+flink.version=1.15
# the REST server address for Flink
flink.rest.address=127.0.0.1
# the REST server Port for Flink
diff --git a/inlong-sort/pom.xml b/inlong-sort/pom.xml
index ced807a53b..6360d4263e 100644
--- a/inlong-sort/pom.xml
+++ b/inlong-sort/pom.xml
@@ -163,9 +163,6 @@
<profiles>
<profile>
<id>v1.13</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
<!-- End-To-End after refactoring, you need to mention this module
under modules. -->
<properties>
<sort.flink.version>v1.13</sort.flink.version>
@@ -183,6 +180,9 @@
</profile>
<profile>
<id>v1.15</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
<properties>
<sort.flink.version>v1.15</sort.flink.version>
<flink.version>1.15.4</flink.version>
diff --git a/inlong-sort/sort-core/pom.xml b/inlong-sort/sort-core/pom.xml
index fa7396ec70..c84738449d 100644
--- a/inlong-sort/sort-core/pom.xml
+++ b/inlong-sort/sort-core/pom.xml
@@ -99,9 +99,6 @@
<profiles>
<profile>
<id>v1.13</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
<dependencies>
<!--flink core-->
<dependency>
@@ -244,6 +241,9 @@
</profile>
<profile>
<id>v1.15</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
<dependencies>
<!--flink core-->
<dependency>
diff --git a/inlong-sort/sort-dist/pom.xml b/inlong-sort/sort-dist/pom.xml
index 9314c7f8d1..4a2106e139 100644
--- a/inlong-sort/sort-dist/pom.xml
+++ b/inlong-sort/sort-dist/pom.xml
@@ -128,9 +128,6 @@
<profiles>
<profile>
<id>v1.13</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
<dependencies>
<dependency>
<groupId>org.apache.inlong</groupId>
@@ -166,6 +163,9 @@
</profile>
<profile>
<id>v1.15</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
<dependencies>
<dependency>
<groupId>org.apache.inlong</groupId>
diff --git a/inlong-sort/sort-flink/base/pom.xml
b/inlong-sort/sort-flink/base/pom.xml
index 5f3eef5bf8..590d35e5f3 100644
--- a/inlong-sort/sort-flink/base/pom.xml
+++ b/inlong-sort/sort-flink/base/pom.xml
@@ -61,9 +61,6 @@
<profiles>
<profile>
<id>v1.13</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
<dependencies>
<dependency>
<groupId>org.apache.flink</groupId>
@@ -90,6 +87,9 @@
</profile>
<profile>
<id>v1.15</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
<dependencies>
<dependency>
<groupId>org.apache.flink</groupId>
diff --git a/inlong-sort/sort-formats/format-row/format-json-v1.13/pom.xml
b/inlong-sort/sort-formats/format-row/format-json-v1.13/pom.xml
index e9824e64cb..8903069a38 100644
--- a/inlong-sort/sort-formats/format-row/format-json-v1.13/pom.xml
+++ b/inlong-sort/sort-formats/format-row/format-json-v1.13/pom.xml
@@ -68,7 +68,7 @@
<dependency>
<groupId>org.apache.inlong</groupId>
-
<artifactId>sort-flink-dependencies-${sort.flink.version}</artifactId>
+ <artifactId>sort-flink-dependencies-v1.13</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
diff --git a/inlong-sort/sort-formats/format-row/format-json-v1.15/pom.xml
b/inlong-sort/sort-formats/format-row/format-json-v1.15/pom.xml
index 4d49fe96fb..fd0e33bd5f 100644
--- a/inlong-sort/sort-formats/format-row/format-json-v1.15/pom.xml
+++ b/inlong-sort/sort-formats/format-row/format-json-v1.15/pom.xml
@@ -69,7 +69,7 @@
<dependency>
<groupId>org.apache.inlong</groupId>
-
<artifactId>sort-flink-dependencies-${sort.flink.version}</artifactId>
+ <artifactId>sort-flink-dependencies-v1.15</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
diff --git a/inlong-sort/sort-formats/format-row/format-json-v1.18/pom.xml
b/inlong-sort/sort-formats/format-row/format-json-v1.18/pom.xml
index 4e19988579..a528d56490 100644
--- a/inlong-sort/sort-formats/format-row/format-json-v1.18/pom.xml
+++ b/inlong-sort/sort-formats/format-row/format-json-v1.18/pom.xml
@@ -73,7 +73,7 @@
<dependency>
<groupId>org.apache.inlong</groupId>
-
<artifactId>sort-flink-dependencies-${sort.flink.version}</artifactId>
+ <artifactId>sort-flink-dependencies-v1.18</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
diff --git
a/inlong-sort/sort-formats/format-row/format-json-v1.18/src/main/java/org/apache/inlong/sort/formats/json/canal/CanalJsonEnhancedDeserializationSchema.java
b/inlong-sort/sort-formats/format-row/format-json-v1.18/src/main/java/org/apache/inlong/sort/formats/json/canal/CanalJsonEnhancedDeserializationSchema.java
index a4c326ef79..642f5bf749 100644
---
a/inlong-sort/sort-formats/format-row/format-json-v1.18/src/main/java/org/apache/inlong/sort/formats/json/canal/CanalJsonEnhancedDeserializationSchema.java
+++
b/inlong-sort/sort-formats/format-row/format-json-v1.18/src/main/java/org/apache/inlong/sort/formats/json/canal/CanalJsonEnhancedDeserializationSchema.java
@@ -301,6 +301,15 @@ public final class CanalJsonEnhancedDeserializationSchema
implements Deserializa
}
}
+ @Override
+ public void open(InitializationContext context) {
+ try {
+ this.jsonDeserializer.open(context);
+ } catch (Exception e) {
+ throw new RuntimeException("JsonRowDataDeserializationSchema
failed to open.", e);
+ }
+ }
+
private void emitRow(
GenericRowData rootRow, GenericRowData physicalRow,
Collector<RowData> out) {
// shortcut in case no output projection is required
diff --git
a/inlong-sort/sort-formats/format-row/format-json-v1.18/src/main/java/org/apache/inlong/sort/formats/json/canal/CanalJsonEnhancedSerializationSchema.java
b/inlong-sort/sort-formats/format-row/format-json-v1.18/src/main/java/org/apache/inlong/sort/formats/json/canal/CanalJsonEnhancedSerializationSchema.java
index a70c7bebf0..6f97e722c6 100644
---
a/inlong-sort/sort-formats/format-row/format-json-v1.18/src/main/java/org/apache/inlong/sort/formats/json/canal/CanalJsonEnhancedSerializationSchema.java
+++
b/inlong-sort/sort-formats/format-row/format-json-v1.18/src/main/java/org/apache/inlong/sort/formats/json/canal/CanalJsonEnhancedSerializationSchema.java
@@ -145,6 +145,11 @@ public class CanalJsonEnhancedSerializationSchema
implements SerializationSchema
size--;
}
reuse = new GenericRowData(size);
+ try {
+ this.jsonSerializer.open(context);
+ } catch (Exception e) {
+ throw new RuntimeException("JsonRowDataSerializationSchema failed
to open", e);
+ }
}
/**
diff --git
a/inlong-sort/sort-formats/format-row/format-json-v1.18/src/test/java/org/apache/inlong/sort/formats/json/canal/CanalJsonEnhancedSerDeSchemaTest.java
b/inlong-sort/sort-formats/format-row/format-json-v1.18/src/test/java/org/apache/inlong/sort/formats/json/canal/CanalJsonEnhancedSerDeSchemaTest.java
index f730340f4c..31aa25f6f4 100644
---
a/inlong-sort/sort-formats/format-row/format-json-v1.18/src/test/java/org/apache/inlong/sort/formats/json/canal/CanalJsonEnhancedSerDeSchemaTest.java
+++
b/inlong-sort/sort-formats/format-row/format-json-v1.18/src/test/java/org/apache/inlong/sort/formats/json/canal/CanalJsonEnhancedSerDeSchemaTest.java
@@ -117,6 +117,7 @@ public class CanalJsonEnhancedSerDeSchemaTest {
List<String> lines = readLines("canal-json-inlong-data.txt");
DeserializationSchema<RowData> deserializationSchema =
createCanalJsonDeserializationSchema(
PHYSICAL_DATA_TYPE, READABLE_METADATA);
+ deserializationSchema.open(null);
// deserialize
SimpleCollector out = new SimpleCollector();
for (String line : lines) {
diff --git a/inlong-sort/sort-formats/format-row/pom.xml
b/inlong-sort/sort-formats/format-row/pom.xml
index 22207262a0..2576017328 100644
--- a/inlong-sort/sort-formats/format-row/pom.xml
+++ b/inlong-sort/sort-formats/format-row/pom.xml
@@ -52,10 +52,18 @@
<profiles>
<profile>
- <id>v1.13</id>
+ <id>flink-all-version</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
+ <modules>
+ <module>format-json-v1.13</module>
+ <module>format-json-v1.15</module>
+ <module>format-json-v1.18</module>
+ </modules>
+ </profile>
+ <profile>
+ <id>v1.13</id>
<modules>
<module>format-json-v1.13</module>
</modules>
diff --git a/inlong-sort/sort-formats/pom.xml b/inlong-sort/sort-formats/pom.xml
index a1e69fb0c7..d81c0e2005 100644
--- a/inlong-sort/sort-formats/pom.xml
+++ b/inlong-sort/sort-formats/pom.xml
@@ -190,9 +190,6 @@
<profiles>
<profile>
<id>v1.13</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
<modules>
<module>format-common</module>
<module>format-row</module>
@@ -220,6 +217,9 @@
</profile>
<profile>
<id>v1.15</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
<modules>
<module>format-common</module>
<module>format-row</module>
diff --git a/pom.xml b/pom.xml
index c82151cda5..a217bdcd84 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1541,9 +1541,6 @@
<profiles>
<profile>
<id>v1.13</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
<properties>
<flink.version>1.13.5</flink.version>
<sort.flink.version>v1.13</sort.flink.version>
@@ -1551,6 +1548,9 @@
</profile>
<profile>
<id>v1.15</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
<properties>
<flink.version>1.15.4</flink.version>
<sort.flink.version>v1.15</sort.flink.version>
