This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 1f1dd5ef5 [flink][bug] Datetime parsing should not be affected by
server time zone in MySqlDebeziumJsonEventParser (#962)
1f1dd5ef5 is described below
commit 1f1dd5ef5dbbafda90c226f998c4e58c50345c62
Author: tsreaper <[email protected]>
AuthorDate: Thu Apr 20 20:02:46 2023 +0800
[flink][bug] Datetime parsing should not be affected by server time zone in
MySqlDebeziumJsonEventParser (#962)
---
.../cdc/mysql/MySqlDebeziumJsonEventParser.java | 20 ++++++++++++++++++--
.../action/cdc/mysql/MySqlActionITCaseBase.java | 4 +++-
.../action/cdc/mysql/MySqlSyncTableActionITCase.java | 7 ++++++-
.../src/test/resources/mysql/my.cnf | 3 +++
4 files changed, 30 insertions(+), 4 deletions(-)
diff --git
a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlDebeziumJsonEventParser.java
b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlDebeziumJsonEventParser.java
index 4dddc5c9f..667cca59b 100644
---
a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlDebeziumJsonEventParser.java
+++
b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlDebeziumJsonEventParser.java
@@ -37,6 +37,7 @@ import java.math.BigDecimal;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
+import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.Base64;
import java.util.HashMap;
@@ -234,9 +235,14 @@ public class MySqlDebeziumJsonEventParser implements
EventParser<String> {
newValue =
DateTimeUtils.toLocalDate(Integer.parseInt(oldValue)).toString();
} else if ("io.debezium.time.Timestamp".equals(className)) {
// MySQL datetime (precision 0-3)
+
+ // display value of datetime is not affected by timezone,
see
+ // https://dev.mysql.com/doc/refman/8.0/en/datetime.html
for standard, and
+ // RowDataDebeziumDeserializeSchema#convertToTimestamp in
flink-cdc-connector
+ // for implementation
LocalDateTime localDateTime =
Instant.ofEpochMilli(Long.parseLong(oldValue))
- .atZone(serverTimeZone)
+ .atZone(ZoneOffset.UTC)
.toLocalDateTime();
newValue =
DateTimeUtils.formatLocalDateTime(localDateTime, 3);
} else if
("io.debezium.time.MicroTimestamp".equals(className)) {
@@ -247,13 +253,23 @@ public class MySqlDebeziumJsonEventParser implements
EventParser<String> {
long seconds = microseconds / microsecondsPerSecond;
long nanoAdjustment =
(microseconds % microsecondsPerSecond) *
nanosecondsPerMicros;
+
+ // display value of datetime is not affected by timezone,
see
+ // https://dev.mysql.com/doc/refman/8.0/en/datetime.html
for standard, and
+ // RowDataDebeziumDeserializeSchema#convertToTimestamp in
flink-cdc-connector
+ // for implementation
LocalDateTime localDateTime =
Instant.ofEpochSecond(seconds, nanoAdjustment)
- .atZone(serverTimeZone)
+ .atZone(ZoneOffset.UTC)
.toLocalDateTime();
newValue =
DateTimeUtils.formatLocalDateTime(localDateTime, 6);
} else if
("io.debezium.time.ZonedTimestamp".equals(className)) {
// MySQL timestamp
+
+ // dispaly value of timestamp is affected by timezone, see
+ // https://dev.mysql.com/doc/refman/8.0/en/datetime.html
for standard, and
+ // RowDataDebeziumDeserializeSchema#convertToTimestamp in
flink-cdc-connector
+ // for implementation
newValue =
Instant.parse(oldValue)
.atZone(serverTimeZone)
diff --git
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlActionITCaseBase.java
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlActionITCaseBase.java
index 4b5adc79b..d53203702 100644
---
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlActionITCaseBase.java
+++
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlActionITCaseBase.java
@@ -72,6 +72,7 @@ public class MySqlActionITCaseBase extends ActionITCaseBase {
.withSetupSQL("mysql/setup.sql")
.withUsername(USER)
.withPassword(PASSWORD)
+ .withEnv("TZ", "America/Los_Angeles")
.withLogConsumer(new Slf4jLogConsumer(LOG));
}
@@ -124,7 +125,8 @@ public class MySqlActionITCaseBase extends ActionITCaseBase
{
config.put("port", String.valueOf(MYSQL_CONTAINER.getDatabasePort()));
config.put("username", USER);
config.put("password", PASSWORD);
- config.put("server-time-zone", ZoneId.of("+00:00").toString());
+ // see mysql/my.cnf in test resources
+ config.put("server-time-zone",
ZoneId.of("America/New_York").toString());
return config;
}
}
diff --git
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableActionITCase.java
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableActionITCase.java
index cbc8772d5..fe4c0492b 100644
---
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableActionITCase.java
+++
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableActionITCase.java
@@ -505,9 +505,14 @@ public class MySqlSyncTableActionITCase extends
MySqlActionITCaseBase {
+ "1.2345678987654322E32,
1.2345678987654322E32, 1.2345678987654322E32, "
+ "11111, 22222, 33333, "
+ "19439, "
+ // display value of datetime is not affected
by timezone
+ "2023-03-23T14:30:05,
2023-03-23T14:30:05.123, 2023-03-23T14:30:05.123456, "
+ "2023-03-24T14:30, 2023-03-24T14:30:05.120, "
- + "2023-03-23T15:00:10.123456, "
+ // display value of timestamp is affected by
timezone
+ // we store 2023-03-23T15:00:10.123456 in
UTC-8 system timezone
+ // and query this timestamp in UTC-5 MySQL
server timezone
+ // so the display value should increase by 3
hour
+ + "2023-03-23T18:00:10.123456, "
+ "Paimon, Apache Paimon, Apache Paimon MySQL
Test Data, Apache Paimon MySQL Long Test Data, "
+ "[98, 121, 116, 101, 115, 0, 0, 0, 0, 0], "
+ "[109, 111, 114, 101, 32, 98, 121, 116, 101,
115], "
diff --git a/paimon-flink/paimon-flink-common/src/test/resources/mysql/my.cnf
b/paimon-flink/paimon-flink-common/src/test/resources/mysql/my.cnf
index 9c9a8747b..38beb8630 100644
--- a/paimon-flink/paimon-flink-common/src/test/resources/mysql/my.cnf
+++ b/paimon-flink/paimon-flink-common/src/test/resources/mysql/my.cnf
@@ -63,3 +63,6 @@ binlog_format = row
# MySQL will set current timestamp to any timestamp field with NULL value
# which is bad for our tests
explicit_defaults_for_timestamp = ON
+
+# We need a default time zone different from UTC+0 for test
+default-time-zone = America/New_York
