This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/tsfile.git
The following commit(s) were added to refs/heads/develop by this push:
new 854f7b73 Tablet.serialize() may throw an exception due to null values
in the Date column (#329)
854f7b73 is described below
commit 854f7b7379d33401afe47ca5f2ea5cc1038b97fe
Author: shuwenwei <[email protected]>
AuthorDate: Fri Dec 27 17:07:36 2024 +0800
Tablet.serialize() may throw an exception due to null values in the Date
column (#329)
* Tablet.serialize() may throw an exception due to null values in the Date
column
* fix DateDataPoint
* modify AlignedChunkGroupWriterImpl
---
.../java/org/apache/tsfile/utils/DateUtils.java | 1 +
.../write/chunk/AlignedChunkGroupWriterImpl.java | 2 +-
.../org/apache/tsfile/write/record/Tablet.java | 6 ++++-
.../write/record/datapoint/DateDataPoint.java | 2 +-
.../apache/tsfile/write/TsFileWriteApiTest.java | 27 ++++++++++++++++++++++
.../org/apache/tsfile/write/record/TabletTest.java | 19 +++++++++++++++
6 files changed, 54 insertions(+), 3 deletions(-)
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/DateUtils.java
b/java/tsfile/src/main/java/org/apache/tsfile/utils/DateUtils.java
index cc7705cc..3dfc709f 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/utils/DateUtils.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/DateUtils.java
@@ -28,6 +28,7 @@ import java.time.format.DateTimeParseException;
public class DateUtils {
private static final DateTimeFormatter DATE_FORMATTER =
DateTimeFormatter.ofPattern("yyyy-MM-dd");
+ public static final int EMPTY_DATE_INT = 10000101;
public static String formatDate(int date) {
return date / 10000
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java
b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java
index 0fbaf94d..0b53b75a 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/write/chunk/AlignedChunkGroupWriterImpl.java
@@ -171,7 +171,7 @@ public class AlignedChunkGroupWriterImpl implements
IChunkGroupWriter {
break;
case INT32:
case DATE:
- valueChunkWriter.write(time, (int) point.getValue(), isNull);
+ valueChunkWriter.write(time, isNull ? 0 : (int) point.getValue(),
isNull);
break;
case INT64:
case TIMESTAMP:
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
index d5803edf..2580a2a4 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java
@@ -705,7 +705,11 @@ public class Tablet {
case DATE:
LocalDate[] dateValues = (LocalDate[]) column;
for (int j = 0; j < rowSize; j++) {
-
ReadWriteIOUtils.write(DateUtils.parseDateExpressionToInt(dateValues[j]),
stream);
+ ReadWriteIOUtils.write(
+ dateValues[j] == null
+ ? DateUtils.EMPTY_DATE_INT
+ : DateUtils.parseDateExpressionToInt(dateValues[j]),
+ stream);
}
break;
case INT64:
diff --git
a/java/tsfile/src/main/java/org/apache/tsfile/write/record/datapoint/DateDataPoint.java
b/java/tsfile/src/main/java/org/apache/tsfile/write/record/datapoint/DateDataPoint.java
index 4892b15c..34599f3e 100644
---
a/java/tsfile/src/main/java/org/apache/tsfile/write/record/datapoint/DateDataPoint.java
+++
b/java/tsfile/src/main/java/org/apache/tsfile/write/record/datapoint/DateDataPoint.java
@@ -52,7 +52,7 @@ public class DateDataPoint extends DataPoint {
@Override
public Object getValue() {
- return DateUtils.parseDateExpressionToInt(value);
+ return value == null ? null : DateUtils.parseDateExpressionToInt(value);
}
@Override
diff --git
a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java
b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java
index bd133f15..0a4d92f1 100644
--- a/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java
+++ b/java/tsfile/src/test/java/org/apache/tsfile/write/TsFileWriteApiTest.java
@@ -43,6 +43,8 @@ import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl;
import org.apache.tsfile.write.chunk.ChunkWriterImpl;
import org.apache.tsfile.write.record.TSRecord;
import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.record.datapoint.DateDataPoint;
+import org.apache.tsfile.write.record.datapoint.StringDataPoint;
import org.apache.tsfile.write.schema.IMeasurementSchema;
import org.apache.tsfile.write.schema.MeasurementSchema;
import org.apache.tsfile.write.writer.TsFileIOWriter;
@@ -549,6 +551,31 @@ public class TsFileWriteApiTest {
}
}
+ @Test
+ public void writeRecordWithNullValue() {
+ setEnv(100, 30);
+ measurementSchemas.add(new MeasurementSchema("s1", TSDataType.TEXT,
TSEncoding.PLAIN));
+ measurementSchemas.add(new MeasurementSchema("s2", TSDataType.STRING,
TSEncoding.PLAIN));
+ measurementSchemas.add(new MeasurementSchema("s3", TSDataType.BLOB,
TSEncoding.PLAIN));
+ measurementSchemas.add(new MeasurementSchema("s4", TSDataType.DATE,
TSEncoding.PLAIN));
+ try (TsFileWriter tsFileWriter = new TsFileWriter(f)) {
+
+ // register aligned timeseries
+ tsFileWriter.registerAlignedTimeseries(new Path(deviceId),
measurementSchemas);
+
+ TSRecord record = new TSRecord(deviceId, 0);
+ record.addTuple(new StringDataPoint("s1", null));
+ record.addTuple(new StringDataPoint("s2", null));
+ record.addTuple(new StringDataPoint("s3", null));
+ record.addTuple(new DateDataPoint("s4", null));
+
+ tsFileWriter.writeRecord(record);
+ } catch (Throwable e) {
+ e.printStackTrace();
+ Assert.fail("Meet errors in test: " + e.getMessage());
+ }
+ }
+
@Test
public void writeDataToTabletsWithNegativeTimestamps() {
setEnv(100, 30);
diff --git
a/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java
b/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java
index 64572c8c..a73e22f9 100644
--- a/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java
+++ b/java/tsfile/src/test/java/org/apache/tsfile/write/record/TabletTest.java
@@ -30,6 +30,7 @@ import org.apache.tsfile.write.schema.MeasurementSchema;
import org.junit.Assert;
import org.junit.Test;
+import java.io.IOException;
import java.nio.ByteBuffer;
import java.time.LocalDate;
import java.util.ArrayList;
@@ -278,4 +279,22 @@ public class TabletTest {
}
Assert.fail();
}
+
+ @Test
+ public void testSerializeDateColumnWithNullValue() throws IOException {
+ final List<IMeasurementSchema> measurementSchemas = new ArrayList<>();
+ measurementSchemas.add(new MeasurementSchema("s1", TSDataType.DATE,
TSEncoding.PLAIN));
+ measurementSchemas.add(new MeasurementSchema("s2", TSDataType.DATE,
TSEncoding.PLAIN));
+ Tablet tablet = new Tablet("root.testsg.d1", measurementSchemas);
+ tablet.addTimestamp(0, 0);
+ tablet.addValue(0, 0, LocalDate.now());
+ tablet.addTimestamp(1, 1);
+ tablet.addValue(1, 1, LocalDate.now());
+ ByteBuffer serialized = tablet.serialize();
+ Tablet deserializeTablet = Tablet.deserialize(serialized);
+ Assert.assertEquals(tablet.getValue(0, 0), deserializeTablet.getValue(0,
0));
+ Assert.assertTrue(deserializeTablet.isNull(0, 1));
+ Assert.assertEquals(tablet.getValue(1, 1), deserializeTablet.getValue(1,
1));
+ Assert.assertTrue(deserializeTablet.isNull(1, 0));
+ }
}
