This is an automated email from the ASF dual-hosted git repository.
yuzelin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 32ff166f9 [flink] Support big decimal for flink cdc sync action (#1876)
32ff166f9 is described below
commit 32ff166f962ee9e79dfff80bc9f431dd38158980
Author: JunZhang <[email protected]>
AuthorDate: Fri Sep 1 10:41:20 2023 +0800
[flink] Support big decimal for flink cdc sync action (#1876)
---
.../flink/action/cdc/mysql/MySqlDebeziumJsonEventParser.java | 2 ++
.../paimon/flink/action/cdc/mysql/MySqlCdcTypeMappingITCase.java | 9 +++++----
.../flink/action/cdc/mysql/MySqlSyncTableActionITCase.java | 8 +++++---
.../src/test/resources/mysql/sync_table_setup.sql | 5 +++--
.../src/test/resources/mysql/type_mapping_test_setup.sql | 5 +++--
5 files changed, 18 insertions(+), 11 deletions(-)
diff --git
a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlDebeziumJsonEventParser.java
b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlDebeziumJsonEventParser.java
index 4bdfc3671..d6a807652 100644
---
a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlDebeziumJsonEventParser.java
+++
b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlDebeziumJsonEventParser.java
@@ -39,6 +39,7 @@ import org.apache.paimon.utils.Preconditions;
import org.apache.paimon.utils.StringUtils;
import
org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.type.TypeReference;
+import
org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.DeserializationFeature;
import
org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode;
import
org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
@@ -159,6 +160,7 @@ public class MySqlDebeziumJsonEventParser implements
EventParser<String> {
@Override
public void setRawEvent(String rawEvent) {
try {
+
objectMapper.configure(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS, true);
root = objectMapper.readValue(rawEvent, JsonNode.class);
payload = root.get("payload");
currentTable = payload.get("source").get("table").asText();
diff --git
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlCdcTypeMappingITCase.java
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlCdcTypeMappingITCase.java
index 9c0bdfafa..f7a4dcf32 100644
---
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlCdcTypeMappingITCase.java
+++
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlCdcTypeMappingITCase.java
@@ -127,7 +127,7 @@ public class MySqlCdcTypeMappingITCase extends
MySqlActionITCaseBase {
.build();
runActionWithDefaultEnv(action);
- int allTypeNums = 77;
+ int allTypeNums = 78;
DataType[] types =
IntStream.range(0, allTypeNums)
.mapToObj(i -> DataTypes.STRING())
@@ -183,6 +183,7 @@ public class MySqlCdcTypeMappingITCase extends
MySqlActionITCaseBase {
"_decimal",
"_decimal_unsigned",
"_decimal_unsigned_zerofill",
+ "_big_decimal",
"_date",
"_datetime",
"_datetime3",
@@ -233,8 +234,8 @@ public class MySqlCdcTypeMappingITCase extends
MySqlActionITCaseBase {
+ "1.000011, 2.000022, 3.000033, "
+ "1.000111, 2.000222, 3.000333, "
+ "12345.11, 12345.22, 12345.33, "
- + "1.2345678987654322E32,
1.2345678987654322E32, 1.2345678987654322E32, "
- + "11111, 22222, 33333, "
+ + "123456789876543212345678987654321.11,
123456789876543212345678987654321.22, 123456789876543212345678987654321.33, "
+ + "11111, 22222, 33333,
2222222222222222300000001111.123456789, "
+ "2023-03-23, "
// display value of datetime is not affected
by timezone
+ "2023-03-23 14:30:05.000, 2023-03-23
14:30:05.123, 2023-03-23 14:30:05.123456, "
@@ -274,7 +275,7 @@ public class MySqlCdcTypeMappingITCase extends
MySqlActionITCaseBase {
+ "NULL, NULL, NULL, "
+ "NULL, NULL, NULL, "
+ "NULL, NULL, NULL, "
- + "NULL, NULL, NULL, "
+ + "NULL, NULL, NULL, NULL, "
+ "NULL, "
+ "NULL, NULL, NULL, "
+ "NULL, NULL, "
diff --git
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableActionITCase.java
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableActionITCase.java
index a22b568f0..88133d222 100644
---
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableActionITCase.java
+++
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableActionITCase.java
@@ -385,6 +385,7 @@ public class MySqlSyncTableActionITCase extends
MySqlActionITCaseBase {
DataTypes.DECIMAL(8, 0), // _decimal
DataTypes.DECIMAL(8, 0), // _decimal_unsigned
DataTypes.DECIMAL(8, 0), //
_decimal_unsigned_zerofill
+ DataTypes.DECIMAL(38, 10), // _big_decimal
DataTypes.DATE(), // _date
DataTypes.TIMESTAMP(0), // _datetime
DataTypes.TIMESTAMP(3), // _datetime3
@@ -464,6 +465,7 @@ public class MySqlSyncTableActionITCase extends
MySqlActionITCaseBase {
"_decimal",
"_decimal_unsigned",
"_decimal_unsigned_zerofill",
+ "_big_decimal",
"_date",
"_datetime",
"_datetime3",
@@ -518,8 +520,8 @@ public class MySqlSyncTableActionITCase extends
MySqlActionITCaseBase {
+ "1.000011, 2.000022, 3.000033, "
+ "1.000111, 2.000222, 3.000333, "
+ "12345.110, 12345.220, 12345.330, "
- + "1.2345678987654322E32,
1.2345678987654322E32, 1.2345678987654322E32, "
- + "11111, 22222, 33333, "
+ + "123456789876543212345678987654321.11,
123456789876543212345678987654321.22, 123456789876543212345678987654321.33, "
+ + "11111, 22222, 33333,
2222222222222222300000001111.1234567890, "
+ "19439, "
// display value of datetime is not affected
by timezone
+ "2023-03-23T14:30:05,
2023-03-23T14:30:05.123, 2023-03-23T14:30:05.123456, "
@@ -564,7 +566,7 @@ public class MySqlSyncTableActionITCase extends
MySqlActionITCaseBase {
+ "NULL, NULL, NULL, "
+ "NULL, NULL, NULL, "
+ "NULL, NULL, NULL, "
- + "NULL, NULL, NULL, "
+ + "NULL, NULL, NULL, NULL, "
+ "NULL, "
+ "NULL, NULL, NULL, "
+ "NULL, NULL, "
diff --git
a/paimon-flink/paimon-flink-common/src/test/resources/mysql/sync_table_setup.sql
b/paimon-flink/paimon-flink-common/src/test/resources/mysql/sync_table_setup.sql
index a0b06595e..121f87560 100644
---
a/paimon-flink/paimon-flink-common/src/test/resources/mysql/sync_table_setup.sql
+++
b/paimon-flink/paimon-flink-common/src/test/resources/mysql/sync_table_setup.sql
@@ -107,6 +107,7 @@ CREATE TABLE all_types_table (
_decimal DECIMAL(8),
_decimal_unsigned DECIMAL(8) UNSIGNED,
_decimal_unsigned_zerofill DECIMAL(8) UNSIGNED ZEROFILL,
+ _big_decimal DECIMAL(38,10),
-- DATE
_date DATE,
-- DATETIME
@@ -180,7 +181,7 @@ INSERT INTO all_types_table VALUES (
-- FIXED
123456789876543212345678987654321.11,
123456789876543212345678987654321.22, 123456789876543212345678987654321.33,
-- DECIMAL
- 11111, 22222, 33333,
+ 11111, 22222, 33333, 2222222222222222300000001111.1234567890,
-- DATE
'2023-03-23',
-- DATETIME
@@ -224,7 +225,7 @@ INSERT INTO all_types_table VALUES (
NULL, NULL, NULL,
NULL, NULL, NULL,
NULL, NULL, NULL,
- NULL, NULL, NULL,
+ NULL, NULL, NULL, NULL,
NULL,
NULL, NULL, NULL,
NULL, NULL,
diff --git
a/paimon-flink/paimon-flink-common/src/test/resources/mysql/type_mapping_test_setup.sql
b/paimon-flink/paimon-flink-common/src/test/resources/mysql/type_mapping_test_setup.sql
index 2bd004f2d..5b74751d3 100644
---
a/paimon-flink/paimon-flink-common/src/test/resources/mysql/type_mapping_test_setup.sql
+++
b/paimon-flink/paimon-flink-common/src/test/resources/mysql/type_mapping_test_setup.sql
@@ -104,6 +104,7 @@ CREATE TABLE all_types_table (
_decimal DECIMAL(8),
_decimal_unsigned DECIMAL(8) UNSIGNED,
_decimal_unsigned_zerofill DECIMAL(8) UNSIGNED ZEROFILL,
+ _big_decimal DECIMAL(38,10),
-- DATE
_date DATE,
-- DATETIME
@@ -176,7 +177,7 @@ INSERT INTO all_types_table VALUES (
-- FIXED
123456789876543212345678987654321.11,
123456789876543212345678987654321.22, 123456789876543212345678987654321.33,
-- DECIMAL
- 11111, 22222, 33333,
+ 11111, 22222, 33333, 2222222222222222300000001111.1234567890,
-- DATE
'2023-03-23',
-- DATETIME
@@ -220,7 +221,7 @@ INSERT INTO all_types_table VALUES (
NULL, NULL, NULL,
NULL, NULL, NULL,
NULL, NULL, NULL,
- NULL, NULL, NULL,
+ NULL, NULL, NULL, NULL,
NULL,
NULL, NULL, NULL,
NULL, NULL,
