This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch rc/1.3.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rc/1.3.3 by this push:
new 3d949e17077 Pipe: Fixed the NPE in TsFileInsertionScanDataContainer
when there are null values in parts of the alignedChunkData (#13306) (#13314)
3d949e17077 is described below
commit 3d949e17077c6f613b866b8b4377f088eb55be72
Author: Caideyipi <[email protected]>
AuthorDate: Tue Aug 27 17:15:02 2024 +0800
Pipe: Fixed the NPE in TsFileInsertionScanDataContainer when there are null
values in parts of the alignedChunkData (#13306) (#13314)
---
.../scan/TsFileInsertionScanDataContainer.java | 10 ++++--
.../event/TsFileInsertionDataContainerTest.java | 37 ++++++++++++++++++++++
2 files changed, 45 insertions(+), 2 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/TsFileInsertionScanDataContainer.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/TsFileInsertionScanDataContainer.java
index 61206afe0ea..699e16e000f 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/TsFileInsertionScanDataContainer.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/TsFileInsertionScanDataContainer.java
@@ -42,6 +42,7 @@ import org.apache.tsfile.read.reader.IChunkReader;
import org.apache.tsfile.read.reader.chunk.AlignedChunkReader;
import org.apache.tsfile.read.reader.chunk.ChunkReader;
import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.utils.BitMap;
import org.apache.tsfile.utils.DateUtils;
import org.apache.tsfile.utils.TsPrimitiveType;
import org.apache.tsfile.write.UnSupportedDataTypeException;
@@ -185,7 +186,7 @@ public class TsFileInsertionScanDataContainer extends
TsFileInsertionDataContain
final int rowIndex = tablet.rowSize;
tablet.addTimestamp(rowIndex, data.currentTime());
- putValueToColumns(data, tablet.values, rowIndex);
+ putValueToColumns(data, tablet.values, tablet.bitMaps, rowIndex);
tablet.rowSize++;
}
@@ -229,11 +230,16 @@ public class TsFileInsertionScanDataContainer extends
TsFileInsertionDataContain
} while (!data.hasCurrent());
}
- private void putValueToColumns(final BatchData data, final Object[] columns,
final int rowIndex) {
+ private void putValueToColumns(
+ final BatchData data, final Object[] columns, final BitMap[] bitMaps,
final int rowIndex) {
final TSDataType type = data.getDataType();
if (type == TSDataType.VECTOR) {
for (int i = 0; i < columns.length; ++i) {
final TsPrimitiveType primitiveType = data.getVector()[i];
+ if (Objects.isNull(primitiveType)) {
+ bitMaps[i].mark(rowIndex);
+ continue;
+ }
switch (primitiveType.getDataType()) {
case BOOLEAN:
((boolean[]) columns[i])[rowIndex] = primitiveType.getBoolean();
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/TsFileInsertionDataContainerTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/TsFileInsertionDataContainerTest.java
index 8e6ecbd80b7..da49f31cdad 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/TsFileInsertionDataContainerTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/TsFileInsertionDataContainerTest.java
@@ -19,6 +19,8 @@
package org.apache.iotdb.db.pipe.event;
+import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.pipe.pattern.IoTDBPipePattern;
import org.apache.iotdb.commons.pipe.pattern.PipePattern;
import org.apache.iotdb.commons.pipe.pattern.PrefixPipePattern;
@@ -31,6 +33,8 @@ import
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
import org.apache.iotdb.pipe.api.access.Row;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.exception.write.WriteProcessException;
import org.apache.tsfile.file.metadata.PlainDeviceID;
import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
@@ -39,6 +43,9 @@ import org.apache.tsfile.read.common.Path;
import org.apache.tsfile.read.common.TimeRange;
import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.utils.TsFileGeneratorUtils;
+import org.apache.tsfile.write.TsFileWriter;
+import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.MeasurementSchema;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
@@ -47,8 +54,10 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
+import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
@@ -101,6 +110,9 @@ public class TsFileInsertionDataContainerTest {
// Test empty chunk
testMixedTsFileWithEmptyChunk(isQuery);
+ // Test partial null value
+ testPartialNullValue(isQuery);
+
// Test the combinations of pipe and tsFile settings
final Set<Integer> deviceNumbers = new HashSet<>();
deviceNumbers.add(1);
@@ -508,6 +520,31 @@ public class TsFileInsertionDataContainerTest {
resource = null;
}
+ private void testPartialNullValue(final boolean isQuery)
+ throws IOException, WriteProcessException, IllegalPathException {
+ alignedTsFile = new File("0-0-2-0.tsfile");
+
+ final List<MeasurementSchema> schemaList = new ArrayList<>();
+ schemaList.add(new MeasurementSchema("s1", TSDataType.INT32));
+ schemaList.add(new MeasurementSchema("s2", TSDataType.INT64));
+
+ final Tablet t = new Tablet("root.sg.d", schemaList, 1024);
+ t.rowSize = 2;
+ t.addTimestamp(0, 1000);
+ t.addTimestamp(1, 2000);
+ t.addValue("s1", 0, 2);
+ t.addValue("s2", 0, null);
+ t.addValue("s1", 1, null);
+ t.addValue("s2", 1, 2L);
+
+ try (final TsFileWriter writer = new TsFileWriter(alignedTsFile)) {
+ writer.registerAlignedTimeseries(new PartialPath("root.sg.d"),
schemaList);
+ writer.writeAligned(t);
+ }
+ testTsFilePointNum(
+ alignedTsFile, new PrefixPipePattern("root"), Long.MIN_VALUE,
Long.MAX_VALUE, isQuery, 2);
+ }
+
private void testTsFilePointNum(
final File tsFile,
final PipePattern pattern,
