This is an automated email from the ASF dual-hosted git repository.
diwu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris-spark-connector.git
The following commit(s) were added to refs/heads/master by this push:
new 7cd355b [bug] Fix read datetime field occur LocalDateTime convert
String error. (#226)
7cd355b is described below
commit 7cd355b55b5688137fc5b7e166e667183b8a7b89
Author: Hong Liu <[email protected]>
AuthorDate: Thu Aug 15 14:55:07 2024 +0800
[bug] Fix read datetime field occur LocalDateTime convert String error.
(#226)
---
.../org/apache/doris/spark/serialization/RowBatch.java | 12 ++++++++----
.../apache/doris/spark/serialization/TestRowBatch.java | 18 +++++++++---------
2 files changed, 17 insertions(+), 13 deletions(-)
diff --git
a/spark-doris-connector/src/main/java/org/apache/doris/spark/serialization/RowBatch.java
b/spark-doris-connector/src/main/java/org/apache/doris/spark/serialization/RowBatch.java
index 742d180..319bd3c 100644
---
a/spark-doris-connector/src/main/java/org/apache/doris/spark/serialization/RowBatch.java
+++
b/spark-doris-connector/src/main/java/org/apache/doris/spark/serialization/RowBatch.java
@@ -81,6 +81,11 @@ public class RowBatch {
private static final Logger logger =
LoggerFactory.getLogger(RowBatch.class);
private static final ZoneId DEFAULT_ZONE_ID = ZoneId.systemDefault();
+ private static final DateTimeFormatter DATE_TIME_FORMATTER = new
DateTimeFormatterBuilder()
+ .appendPattern("yyyy-MM-dd HH:mm:ss")
+ .appendFraction(ChronoField.MICRO_OF_SECOND, 0, 6, true)
+ .toFormatter();
+
private static final String DATETIME_PATTERN = "yyyy-MM-dd HH:mm:ss";
private static final String DATETIMEV2_PATTERN = "yyyy-MM-dd
HH:mm:ss.SSSSSS";
private final DateTimeFormatter dateTimeFormatter =
@@ -374,9 +379,7 @@ public class RowBatch {
continue;
}
String value = new
String(varCharVector.get(rowIndex), StandardCharsets.UTF_8);
- value = completeMilliseconds(value);
- LocalDateTime parse =
LocalDateTime.parse(value, dateTimeV2Formatter);
- addValueToRow(rowIndex, parse);
+ addValueToRow(rowIndex, value);
}
} else if (curFieldVector instanceof TimeStampVector) {
TimeStampVector timeStampVector =
(TimeStampVector) curFieldVector;
@@ -388,7 +391,8 @@ public class RowBatch {
continue;
}
LocalDateTime dateTime = getDateTime(rowIndex,
timeStampVector);
- addValueToRow(rowIndex, dateTime);
+ String formatted =
DATE_TIME_FORMATTER.format(dateTime);
+ addValueToRow(rowIndex, formatted);
}
}
diff --git
a/spark-doris-connector/src/test/java/org/apache/doris/spark/serialization/TestRowBatch.java
b/spark-doris-connector/src/test/java/org/apache/doris/spark/serialization/TestRowBatch.java
index 0320cd8..f12014f 100644
---
a/spark-doris-connector/src/test/java/org/apache/doris/spark/serialization/TestRowBatch.java
+++
b/spark-doris-connector/src/test/java/org/apache/doris/spark/serialization/TestRowBatch.java
@@ -273,7 +273,7 @@ public class TestRowBatch {
(float) 1.1,
(double) 1.1,
Date.valueOf("2008-08-08"),
- LocalDateTime.of(2008, 8, 8, 0, 0, 0),
+ "2008-08-08 00:00:00",
Decimal.apply(1234L, 4, 2),
"char1"
);
@@ -287,7 +287,7 @@ public class TestRowBatch {
(float) 2.2,
(double) 2.2,
Date.valueOf("1900-08-08"),
- LocalDateTime.of(1900, 8, 8, 0, 0, 0),
+ "1900-08-08 00:00:00",
Decimal.apply(8888L, 4, 2),
"char2"
);
@@ -301,7 +301,7 @@ public class TestRowBatch {
(float) 3.3,
(double) 3.3,
Date.valueOf("2100-08-08"),
- LocalDateTime.of(2100, 8, 8, 0, 0, 0),
+ "2100-08-08 00:00:00",
Decimal.apply(10L, 2, 0),
"char3"
);
@@ -832,16 +832,16 @@ public class TestRowBatch {
Assert.assertTrue(rowBatch.hasNext());
List<Object> actualRow0 = rowBatch.next();
- Assert.assertEquals(LocalDateTime.of(2024, 3, 20, 0, 0, 0),
actualRow0.get(0));
- Assert.assertEquals(LocalDateTime.of(2024, 3, 20, 0, 0, 0),
actualRow0.get(1));
+ Assert.assertEquals("2024-03-20 00:00:00", actualRow0.get(0));
+ Assert.assertEquals("2024-03-20 00:00:00", actualRow0.get(1));
List<Object> actualRow1 = rowBatch.next();
- Assert.assertEquals(LocalDateTime.of(2024, 3, 20, 0, 0, 1),
actualRow1.get(0));
- Assert.assertEquals(LocalDateTime.of(2024, 3, 20, 0, 0, 0, 123000000),
actualRow1.get(1));
+ Assert.assertEquals("2024-03-20 00:00:01", actualRow1.get(0));
+ Assert.assertEquals("2024-03-20 00:00:00.123", actualRow1.get(1));
List<Object> actualRow2 = rowBatch.next();
- Assert.assertEquals(LocalDateTime.of(2024, 3, 20, 0, 0, 2),
actualRow2.get(0));
- Assert.assertEquals(LocalDateTime.of(2024, 3, 20, 0, 0, 0, 123456000),
actualRow2.get(1));
+ Assert.assertEquals("2024-03-20 00:00:02", actualRow2.get(0));
+ Assert.assertEquals("2024-03-20 00:00:00.123456", actualRow2.get(1));
Assert.assertFalse(rowBatch.hasNext());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
