This is an automated email from the ASF dual-hosted git repository.
terrymanu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new c1e419280f7 Decode MySQL binlog dates that have a zero month or a zero
day (#39676)
c1e419280f7 is described below
commit c1e419280f76a03120f89eb406984b2cb0b31a91
Author: ZIHAN DAI <[email protected]>
AuthorDate: Fri Oct 16 03:09:13 2026 +1100
Decode MySQL binlog dates that have a zero month or a zero day (#39676)
* Decode MySQL binlog dates that have a zero month or a zero day
MySQL stores 2023-00-15 and 2023-05-00 whenever NO_ZERO_IN_DATE is not in
sql_mode, and writes them to the binlog like any other date. The DATE,
DATETIME and DATETIME2 decoders pass the parts straight to LocalDate,
which rejects a zero month or day, so one such row aborts the binlog
stream rather than affecting one column.
Each decoder already returns text for the all-zero date, which needs the
same relaxed sql_mode to exist. Return text for a partially zero one too,
and keep the java.sql types for every date they can hold.
* Format incomplete dates with Locale.ROOT and cover fractional DATETIME2
String.format without a locale uses the default format locale, so under
an Arabic or Thai locale a zero-month date came out in non-ASCII digits.
Format both text forms with Locale.ROOT.
Also cover a DATETIME2 value with a zero month or day and a non-zero
fraction at each storage width, checking the full string and that the
fraction bytes are consumed.
* Make the date helpers package-private and unwrap the DATE return
The three MySQLTimeValueUtils helpers are only called from decoders in
the same package. The DATE return expression fits on one line.
* Move readFully after its first caller
---
.../value/time/MySQLDateBinlogProtocolValue.java | 5 ++-
.../time/MySQLDatetime2BinlogProtocolValue.java | 4 +-
.../time/MySQLDatetimeBinlogProtocolValue.java | 7 +--
.../row/column/value/time/MySQLTimeValueUtils.java | 43 ++++++++++++++++++
.../time/MySQLDateBinlogProtocolValueTest.java | 28 ++++++++++++
.../MySQLDatetime2BinlogProtocolValueTest.java | 52 ++++++++++++++++++++++
.../time/MySQLDatetimeBinlogProtocolValueTest.java | 25 +++++++++++
7 files changed, 159 insertions(+), 5 deletions(-)
diff --git
a/database/protocol/dialect/mysql/src/main/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/time/MySQLDateBinlogProtocolValue.java
b/database/protocol/dialect/mysql/src/main/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/time/MySQLDateBinlogProtocolValue.java
index 503af06aa97..3ce663f4f1e 100644
---
a/database/protocol/dialect/mysql/src/main/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/time/MySQLDateBinlogProtocolValue.java
+++
b/database/protocol/dialect/mysql/src/main/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/time/MySQLDateBinlogProtocolValue.java
@@ -39,6 +39,9 @@ public final class MySQLDateBinlogProtocolValue implements
MySQLBinlogProtocolVa
int year = date / 16 / 32;
int month = date / 32 % 16;
int day = date % 32;
- return 0 == date ? MySQLTimeValueUtils.ZERO_OF_DATE :
Date.valueOf(LocalDate.of(year, month, day));
+ if (0 == date) {
+ return MySQLTimeValueUtils.ZERO_OF_DATE;
+ }
+ return MySQLTimeValueUtils.isCompleteDate(month, day) ?
Date.valueOf(LocalDate.of(year, month, day)) :
MySQLTimeValueUtils.formatIncompleteDate(year, month, day);
}
}
diff --git
a/database/protocol/dialect/mysql/src/main/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/time/MySQLDatetime2BinlogProtocolValue.java
b/database/protocol/dialect/mysql/src/main/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/time/MySQLDatetime2BinlogProtocolValue.java
index 9520f17b91c..93b60d509f0 100644
---
a/database/protocol/dialect/mysql/src/main/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/time/MySQLDatetime2BinlogProtocolValue.java
+++
b/database/protocol/dialect/mysql/src/main/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/time/MySQLDatetime2BinlogProtocolValue.java
@@ -68,7 +68,9 @@ public final class MySQLDatetime2BinlogProtocolValue
implements MySQLBinlogProto
int minute = (int) ((time >> 6) % (1L << 6L));
int second = (int) (time % (1L << 6L));
MySQLFractionalSeconds fractionalSeconds = new
MySQLFractionalSeconds(columnDef.getColumnMeta(), payload);
- return Timestamp.valueOf(LocalDateTime.of(year, month, day, hour,
minute, second, fractionalSeconds.getNanos()));
+ return MySQLTimeValueUtils.isCompleteDate(month, day)
+ ? Timestamp.valueOf(LocalDateTime.of(year, month, day, hour,
minute, second, fractionalSeconds.getNanos()))
+ : MySQLTimeValueUtils.formatIncompleteDatetime(year, month,
day, hour, minute, second, fractionalSeconds.getNanos());
}
private void skipFractionalSeconds(final MySQLBinlogColumnDef columnDef,
final MySQLPacketPayload payload) {
diff --git
a/database/protocol/dialect/mysql/src/main/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/time/MySQLDatetimeBinlogProtocolValue.java
b/database/protocol/dialect/mysql/src/main/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/time/MySQLDatetimeBinlogProtocolValue.java
index d29f96c6b10..917145dac0f 100644
---
a/database/protocol/dialect/mysql/src/main/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/time/MySQLDatetimeBinlogProtocolValue.java
+++
b/database/protocol/dialect/mysql/src/main/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/time/MySQLDatetimeBinlogProtocolValue.java
@@ -24,7 +24,6 @@ import
org.apache.shardingsphere.database.protocol.mysql.payload.MySQLPacketPayl
import java.io.Serializable;
import java.sql.Timestamp;
import java.time.LocalDateTime;
-import java.util.Date;
/**
* MySQL DATETIME binlog protocol value.
@@ -40,7 +39,7 @@ public final class MySQLDatetimeBinlogProtocolValue
implements MySQLBinlogProtoc
return 0L == datetime ? MySQLTimeValueUtils.DATETIME_OF_ZERO :
readDateTime(datetime);
}
- private Date readDateTime(final long datetime) {
+ private Serializable readDateTime(final long datetime) {
int date = (int) (datetime / 1000000L);
int year = date / 10000;
int month = (date % 10000) / 100;
@@ -49,6 +48,8 @@ public final class MySQLDatetimeBinlogProtocolValue
implements MySQLBinlogProtoc
int hour = time / 10000;
int minute = (time % 10000) / 100;
int second = time % 100;
- return Timestamp.valueOf(LocalDateTime.of(year, month, day, hour,
minute, second));
+ return MySQLTimeValueUtils.isCompleteDate(month, day)
+ ? Timestamp.valueOf(LocalDateTime.of(year, month, day, hour,
minute, second))
+ : MySQLTimeValueUtils.formatIncompleteDatetime(year, month,
day, hour, minute, second, 0);
}
}
diff --git
a/database/protocol/dialect/mysql/src/main/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/time/MySQLTimeValueUtils.java
b/database/protocol/dialect/mysql/src/main/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/time/MySQLTimeValueUtils.java
index 3097d285bb8..bb426d447bc 100644
---
a/database/protocol/dialect/mysql/src/main/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/time/MySQLTimeValueUtils.java
+++
b/database/protocol/dialect/mysql/src/main/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/time/MySQLTimeValueUtils.java
@@ -20,6 +20,8 @@ package
org.apache.shardingsphere.database.protocol.mysql.packet.binlog.row.colu
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
+import java.util.Locale;
+
/**
* Time value utility class of MySQL.
*/
@@ -33,4 +35,45 @@ public final class MySQLTimeValueUtils {
public static final String YEAR_OF_ZERO = "0000";
public static final String DATETIME_OF_ZERO = "0000-00-00 00:00:00";
+
+ /**
+ * Judge whether a date has both a month and a day, which is what {@link
java.time.LocalDate} requires.
+ * MySQL stores a zero month or a zero day whenever NO_ZERO_IN_DATE is not
in sql_mode.
+ *
+ * @param month month
+ * @param day day
+ * @return whether the date is complete
+ */
+ static boolean isCompleteDate(final int month, final int day) {
+ return month > 0 && day > 0;
+ }
+
+ /**
+ * Format a date that {@link java.time.LocalDate} cannot hold, in the text
form MySQL prints it in, with ASCII digits whatever the default locale.
+ *
+ * @param year year
+ * @param month month
+ * @param day day
+ * @return date in MySQL text form
+ */
+ static String formatIncompleteDate(final int year, final int month, final
int day) {
+ return String.format(Locale.ROOT, "%04d-%02d-%02d", year, month, day);
+ }
+
+ /**
+ * Format a datetime whose date part {@link java.time.LocalDate} cannot
hold, in the text form MySQL prints it in, with ASCII digits whatever the
default locale.
+ *
+ * @param year year
+ * @param month month
+ * @param day day
+ * @param hour hour
+ * @param minute minute
+ * @param second second
+ * @param nanos nanoseconds, always a whole number of microseconds
+ * @return datetime in MySQL text form
+ */
+ static String formatIncompleteDatetime(final int year, final int month,
final int day, final int hour, final int minute, final int second, final int
nanos) {
+ String result = String.format(Locale.ROOT, "%04d-%02d-%02d
%02d:%02d:%02d", year, month, day, hour, minute, second);
+ return 0 == nanos ? result : result + String.format(Locale.ROOT,
".%06d", nanos / 1000);
+ }
}
diff --git
a/database/protocol/dialect/mysql/src/test/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/time/MySQLDateBinlogProtocolValueTest.java
b/database/protocol/dialect/mysql/src/test/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/time/MySQLDateBinlogProtocolValueTest.java
index 5dba3e69bc8..72938f07365 100644
---
a/database/protocol/dialect/mysql/src/test/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/time/MySQLDateBinlogProtocolValueTest.java
+++
b/database/protocol/dialect/mysql/src/test/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/time/MySQLDateBinlogProtocolValueTest.java
@@ -27,6 +27,7 @@ import org.mockito.junit.jupiter.MockitoExtension;
import java.sql.Date;
import java.time.LocalDate;
+import java.util.Locale;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.is;
@@ -52,6 +53,33 @@ class MySQLDateBinlogProtocolValueTest {
assertThat(new MySQLDateBinlogProtocolValue().read(columnDef,
payload), is(expected));
}
+ @Test
+ void assertReadDateWithZeroMonth() {
+ when(payload.getByteBuf()).thenReturn(byteBuf);
+ when(byteBuf.readUnsignedMediumLE()).thenReturn(2023 * 16 * 32 + 15);
+ assertThat(new MySQLDateBinlogProtocolValue().read(columnDef,
payload), is("2023-00-15"));
+ }
+
+ @Test
+ void assertReadDateWithZeroDay() {
+ when(payload.getByteBuf()).thenReturn(byteBuf);
+ when(byteBuf.readUnsignedMediumLE()).thenReturn(2023 * 16 * 32 + 5 *
32);
+ assertThat(new MySQLDateBinlogProtocolValue().read(columnDef,
payload), is("2023-05-00"));
+ }
+
+ @Test
+ void assertReadDateWithZeroMonthUnderNonLatinDefaultFormatLocale() {
+ when(payload.getByteBuf()).thenReturn(byteBuf);
+ when(byteBuf.readUnsignedMediumLE()).thenReturn(2023 * 16 * 32 + 15);
+ Locale originalLocale = Locale.getDefault(Locale.Category.FORMAT);
+ Locale.setDefault(Locale.Category.FORMAT,
Locale.forLanguageTag("th-TH-u-nu-thai"));
+ try {
+ assertThat(new MySQLDateBinlogProtocolValue().read(columnDef,
payload), is("2023-00-15"));
+ } finally {
+ Locale.setDefault(Locale.Category.FORMAT, originalLocale);
+ }
+ }
+
@Test
void assertReadNullDate() {
when(payload.getByteBuf()).thenReturn(byteBuf);
diff --git
a/database/protocol/dialect/mysql/src/test/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/time/MySQLDatetime2BinlogProtocolValueTest.java
b/database/protocol/dialect/mysql/src/test/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/time/MySQLDatetime2BinlogProtocolValueTest.java
index 8e199760253..e4a35b397be 100644
---
a/database/protocol/dialect/mysql/src/test/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/time/MySQLDatetime2BinlogProtocolValueTest.java
+++
b/database/protocol/dialect/mysql/src/test/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/time/MySQLDatetime2BinlogProtocolValueTest.java
@@ -18,6 +18,7 @@
package
org.apache.shardingsphere.database.protocol.mysql.packet.binlog.row.column.value.time;
import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
import
org.apache.shardingsphere.database.protocol.mysql.constant.MySQLBinaryColumnType;
import
org.apache.shardingsphere.database.protocol.mysql.packet.binlog.row.column.MySQLBinlogColumnDef;
import
org.apache.shardingsphere.database.protocol.mysql.payload.MySQLPacketPayload;
@@ -31,8 +32,10 @@ import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.mockito.stubbing.OngoingStubbing;
+import java.nio.charset.StandardCharsets;
import java.sql.Timestamp;
import java.time.LocalDateTime;
+import java.util.Locale;
import java.util.stream.Stream;
import static org.hamcrest.MatcherAssert.assertThat;
@@ -84,6 +87,55 @@ class MySQLDatetime2BinlogProtocolValueTest {
}
}
+ @Test
+ void assertReadWithZeroMonth() {
+ when(payload.readInt1()).thenReturn(0x99, 0xae, 0xde, 0xa0, 0x00);
+ assertThat(protocolValue.read(columnDef, payload), is("2023-00-15
10:00:00"));
+ }
+
+ @Test
+ void assertReadWithZeroDay() {
+ when(payload.readInt1()).thenReturn(0x99, 0xb0, 0x00, 0xa0, 0x00);
+ assertThat(protocolValue.read(columnDef, payload), is("2023-05-00
10:00:00"));
+ }
+
+ @Test
+ void assertReadWithZeroMonthAndOneByteFraction() {
+ assertThat(readFully(2, 0x99, 0xae, 0xde, 0xa0, 0x00, 0x2d),
is("2023-00-15 10:00:00.450000"));
+ }
+
+ private Object readFully(final int fractionalSecondsPrecision, final
int... bytes) {
+ ByteBuf buffer = Unpooled.buffer(bytes.length);
+ for (int each : bytes) {
+ buffer.writeByte(each);
+ }
+ columnDef.setColumnMeta(fractionalSecondsPrecision);
+ Object result = protocolValue.read(columnDef, new
MySQLPacketPayload(buffer, StandardCharsets.UTF_8));
+ assertThat(buffer.readableBytes(), is(0));
+ return result;
+ }
+
+ @Test
+ void assertReadWithZeroDayAndTwoByteFraction() {
+ assertThat(readFully(4, 0x99, 0xb0, 0x00, 0xa0, 0x00, 0x04, 0xd2),
is("2023-05-00 10:00:00.123400"));
+ }
+
+ @Test
+ void assertReadWithZeroMonthAndThreeByteFraction() {
+ assertThat(readFully(6, 0x99, 0xae, 0xde, 0xa0, 0x00, 0x01, 0xe2,
0x40), is("2023-00-15 10:00:00.123456"));
+ }
+
+ @Test
+ void assertReadWithZeroMonthUnderNonLatinDefaultFormatLocale() {
+ Locale originalLocale = Locale.getDefault(Locale.Category.FORMAT);
+ Locale.setDefault(Locale.Category.FORMAT,
Locale.forLanguageTag("th-TH-u-nu-thai"));
+ try {
+ assertThat(readFully(6, 0x99, 0xae, 0xde, 0xa0, 0x00, 0x01, 0xe2,
0x40), is("2023-00-15 10:00:00.123456"));
+ } finally {
+ Locale.setDefault(Locale.Category.FORMAT, originalLocale);
+ }
+ }
+
@Test
void assertReadWithZeroDatetime() {
assertThat(protocolValue.read(columnDef, payload),
is(MySQLTimeValueUtils.DATETIME_OF_ZERO));
diff --git
a/database/protocol/dialect/mysql/src/test/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/time/MySQLDatetimeBinlogProtocolValueTest.java
b/database/protocol/dialect/mysql/src/test/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/time/MySQLDatetimeBinlogProtocolValueTest.java
index 82011a475ee..b566da75d2e 100644
---
a/database/protocol/dialect/mysql/src/test/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/time/MySQLDatetimeBinlogProtocolValueTest.java
+++
b/database/protocol/dialect/mysql/src/test/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/time/MySQLDatetimeBinlogProtocolValueTest.java
@@ -26,6 +26,7 @@ import org.mockito.junit.jupiter.MockitoExtension;
import java.sql.Timestamp;
import java.time.LocalDateTime;
+import java.util.Locale;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.is;
@@ -40,6 +41,30 @@ class MySQLDatetimeBinlogProtocolValueTest {
@Mock
private MySQLBinlogColumnDef columnDef;
+ @Test
+ void assertReadDatetimeWithZeroMonth() {
+ when(payload.readInt8()).thenReturn(20230015100000L);
+ assertThat(new MySQLDatetimeBinlogProtocolValue().read(columnDef,
payload), is("2023-00-15 10:00:00"));
+ }
+
+ @Test
+ void assertReadDatetimeWithZeroDay() {
+ when(payload.readInt8()).thenReturn(20230500100000L);
+ assertThat(new MySQLDatetimeBinlogProtocolValue().read(columnDef,
payload), is("2023-05-00 10:00:00"));
+ }
+
+ @Test
+ void assertReadDatetimeWithZeroMonthUnderNonLatinDefaultFormatLocale() {
+ when(payload.readInt8()).thenReturn(20230015100000L);
+ Locale originalLocale = Locale.getDefault(Locale.Category.FORMAT);
+ Locale.setDefault(Locale.Category.FORMAT,
Locale.forLanguageTag("th-TH-u-nu-thai"));
+ try {
+ assertThat(new MySQLDatetimeBinlogProtocolValue().read(columnDef,
payload), is("2023-00-15 10:00:00"));
+ } finally {
+ Locale.setDefault(Locale.Category.FORMAT, originalLocale);
+ }
+ }
+
@Test
void assertRead() {
when(payload.readInt8()).thenReturn(99991231235959L);
