This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 854f1dc60f8 Fix NPE when encoding continous nulls in Tablets (#16107)
854f1dc60f8 is described below
commit 854f1dc60f8d69d1942c3072c7d1eaf89da637fd
Author: Jiang Tian <[email protected]>
AuthorDate: Wed Aug 6 11:26:35 2025 +0800
Fix NPE when encoding continous nulls in Tablets (#16107)
---
.../iotdb/session/it/IoTDBSessionCompressedIT.java | 177 +++++++++------------
.../apache/iotdb/session/util/SessionUtils.java | 82 +++-------
2 files changed, 98 insertions(+), 161 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionCompressedIT.java
b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionCompressedIT.java
index 058ebb8e7b0..c82bdfcdd07 100644
---
a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionCompressedIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionCompressedIT.java
@@ -27,6 +27,7 @@ import org.apache.iotdb.rpc.IoTDBConnectionException;
import org.apache.iotdb.rpc.StatementExecutionException;
import org.apache.iotdb.session.TableSessionBuilder;
+import org.apache.tsfile.enums.ColumnCategory;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
@@ -43,16 +44,13 @@ import org.junit.Test;
import java.time.LocalDate;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
public class IoTDBSessionCompressedIT {
- private static ITableSession session1;
- private static ITableSession session2;
- private static ITableSession session3;
- private static ITableSession session4;
- private static ITableSession session5;
+ private static List<ITableSession> sessions;
@BeforeClass
public static void setUpClass() throws IoTDBConnectionException {
@@ -62,8 +60,7 @@ public class IoTDBSessionCompressedIT {
EnvFactory.getEnv().getDataNodeWrapperList().stream()
.map(DataNodeWrapper::getIpAndPortString)
.collect(Collectors.toList());
- // List<String> nodeUrls = Collections.singletonList("127.0.0.1:6667");
- session1 =
+ ITableSession session1 =
new TableSessionBuilder()
.nodeUrls(nodeUrls)
.username(CommonDescriptor.getInstance().getConfig().getAdminName())
@@ -83,7 +80,7 @@ public class IoTDBSessionCompressedIT {
.withDateEncoding(TSEncoding.PLAIN)
.withTimeStampEncoding(TSEncoding.PLAIN)
.build();
- session2 =
+ ITableSession session2 =
new TableSessionBuilder()
.nodeUrls(nodeUrls)
.username(CommonDescriptor.getInstance().getConfig().getAdminName())
@@ -103,7 +100,7 @@ public class IoTDBSessionCompressedIT {
.withDateEncoding(TSEncoding.PLAIN)
.withTimeStampEncoding(TSEncoding.SPRINTZ)
.build();
- session3 =
+ ITableSession session3 =
new TableSessionBuilder()
.nodeUrls(nodeUrls)
.username(CommonDescriptor.getInstance().getConfig().getAdminName())
@@ -123,7 +120,7 @@ public class IoTDBSessionCompressedIT {
.withDateEncoding(TSEncoding.RLE)
.withTimeStampEncoding(TSEncoding.RLE)
.build();
- session4 =
+ ITableSession session4 =
new TableSessionBuilder()
.nodeUrls(nodeUrls)
.username(CommonDescriptor.getInstance().getConfig().getAdminName())
@@ -143,7 +140,7 @@ public class IoTDBSessionCompressedIT {
.withDateEncoding(TSEncoding.RLE)
.withTimeStampEncoding(TSEncoding.ZIGZAG)
.build();
- session5 =
+ ITableSession session5 =
new TableSessionBuilder()
.nodeUrls(nodeUrls)
.username(CommonDescriptor.getInstance().getConfig().getAdminName())
@@ -153,26 +150,55 @@ public class IoTDBSessionCompressedIT {
.enableAutoFetch(false)
.enableCompaction(false)
.build();
+ sessions = Arrays.asList(session1, session2, session3, session4, session5);
}
@AfterClass
public static void tearDownClass() throws IoTDBConnectionException {
- if (session1 != null) {
- session1.close();
+ for (ITableSession session : sessions) {
+ session.close();
}
- if (session2 != null) {
- session2.close();
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ @Test
+ public void testAllNullColumn() throws IoTDBConnectionException,
StatementExecutionException {
+ Tablet tablet =
+ new Tablet(
+ "t1",
+ Arrays.asList("tag1", "attr1", "s1"),
+ Arrays.asList(TSDataType.STRING, TSDataType.STRING,
TSDataType.DATE),
+ Arrays.asList(ColumnCategory.TAG, ColumnCategory.ATTRIBUTE,
ColumnCategory.FIELD),
+ 100);
+ for (int i = 0; i < 10; i++) {
+ tablet.addTimestamp(i, i);
+ tablet.addValue("tag1", i, "d1");
+ tablet.addValue("attr1", i, "blue");
}
- if (session3 != null) {
- session3.close();
+ for (ITableSession session : sessions) {
+ session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS test");
+ session.executeNonQueryStatement("USE test");
+ session.insert(tablet);
}
- if (session4 != null) {
- session4.close();
+ }
+
+ @Test
+ public void testAllNull() throws IoTDBConnectionException,
StatementExecutionException {
+ Tablet tablet =
+ new Tablet(
+ "t1",
+ Arrays.asList("tag1", "attr1", "s1"),
+ Arrays.asList(TSDataType.STRING, TSDataType.STRING,
TSDataType.DATE),
+ Arrays.asList(ColumnCategory.TAG, ColumnCategory.ATTRIBUTE,
ColumnCategory.FIELD),
+ 100);
+ for (int i = 0; i < 10; i++) {
+ tablet.addTimestamp(i, i);
}
- if (session5 != null) {
- session5.close();
+ for (ITableSession session : sessions) {
+ session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS test");
+ session.executeNonQueryStatement("USE test");
+ session.insert(tablet);
}
- EnvFactory.getEnv().cleanClusterEnvironment();
}
@Test
@@ -273,91 +299,36 @@ public class IoTDBSessionCompressedIT {
String tableName = "table_13";
Tablet tablet = new Tablet(tableName, schemas, timestamp, values,
partBitMap, 4);
- session1.executeNonQueryStatement("create database IF NOT EXISTS
dbTest_0");
- session1.executeNonQueryStatement("use dbTest_0");
- session2.executeNonQueryStatement("use dbTest_0");
- session3.executeNonQueryStatement("use dbTest_0");
- session4.executeNonQueryStatement("use dbTest_0");
- session5.executeNonQueryStatement("use dbTest_0");
+ sessions.get(0).executeNonQueryStatement("create database IF NOT EXISTS
dbTest_0");
+ for (ITableSession session : sessions) {
+ session.executeNonQueryStatement("use dbTest_0");
+ }
// 1. insert
- session1.insert(tablet);
- session2.insert(tablet);
- session3.insert(tablet);
- session4.insert(tablet);
- session5.insert(tablet);
+ for (ITableSession session : sessions) {
+ session.insert(tablet);
+ }
// 2. assert
- SessionDataSet sessionDataSet1 =
- session1.executeQueryStatement("select * from dbTest_0." + tableName);
- SessionDataSet sessionDataSet2 =
- session2.executeQueryStatement("select * from dbTest_0." + tableName);
- SessionDataSet sessionDataSet3 =
- session3.executeQueryStatement("select * from dbTest_0." + tableName);
- SessionDataSet sessionDataSet4 =
- session4.executeQueryStatement("select * from dbTest_0." + tableName);
- SessionDataSet sessionDataSet5 =
- session5.executeQueryStatement("select * from dbTest_0." + tableName);
-
- if (sessionDataSet1.hasNext()) {
- RowRecord next = sessionDataSet1.next();
- Assert.assertEquals(3L, next.getFields().get(0).getLongV());
- Assert.assertEquals(1, next.getFields().get(1).getIntV());
- Assert.assertEquals(1L, next.getFields().get(2).getLongV());
- Assert.assertEquals(1.1f, next.getFields().get(3).getFloatV(), 0.01);
- Assert.assertEquals(0.707, next.getFields().get(4).getDoubleV(), 0.01);
- Assert.assertEquals(new Binary(new byte[] {(byte) 32}),
next.getFields().get(5).getBinaryV());
- Assert.assertEquals(true, next.getFields().get(6).getBoolV());
- Assert.assertEquals(new Binary(new byte[] {(byte) 32}),
next.getFields().get(7).getBinaryV());
- Assert.assertEquals(new Binary(new byte[] {(byte) 32}),
next.getFields().get(8).getBinaryV());
- }
- if (sessionDataSet2.hasNext()) {
- RowRecord next = sessionDataSet2.next();
- Assert.assertEquals(3L, next.getFields().get(0).getLongV());
- Assert.assertEquals(1, next.getFields().get(1).getIntV());
- Assert.assertEquals(1L, next.getFields().get(2).getLongV());
- Assert.assertEquals(1.1f, next.getFields().get(3).getFloatV(), 0.01);
- Assert.assertEquals(0.707, next.getFields().get(4).getDoubleV(), 0.01);
- Assert.assertEquals(new Binary(new byte[] {(byte) 32}),
next.getFields().get(5).getBinaryV());
- Assert.assertEquals(true, next.getFields().get(6).getBoolV());
- Assert.assertEquals(new Binary(new byte[] {(byte) 32}),
next.getFields().get(7).getBinaryV());
- Assert.assertEquals(new Binary(new byte[] {(byte) 32}),
next.getFields().get(8).getBinaryV());
- }
- if (sessionDataSet3.hasNext()) {
- RowRecord next = sessionDataSet3.next();
- Assert.assertEquals(3L, next.getFields().get(0).getLongV());
- Assert.assertEquals(1, next.getFields().get(1).getIntV());
- Assert.assertEquals(1L, next.getFields().get(2).getLongV());
- Assert.assertEquals(1.1f, next.getFields().get(3).getFloatV(), 0.01);
- Assert.assertEquals(0.707, next.getFields().get(4).getDoubleV(), 0.01);
- Assert.assertEquals(new Binary(new byte[] {(byte) 32}),
next.getFields().get(5).getBinaryV());
- Assert.assertEquals(true, next.getFields().get(6).getBoolV());
- Assert.assertEquals(new Binary(new byte[] {(byte) 32}),
next.getFields().get(7).getBinaryV());
- Assert.assertEquals(new Binary(new byte[] {(byte) 32}),
next.getFields().get(8).getBinaryV());
- }
- if (sessionDataSet4.hasNext()) {
- RowRecord next = sessionDataSet4.next();
- Assert.assertEquals(3L, next.getFields().get(0).getLongV());
- Assert.assertEquals(1, next.getFields().get(1).getIntV());
- Assert.assertEquals(1L, next.getFields().get(2).getLongV());
- Assert.assertEquals(1.1f, next.getFields().get(3).getFloatV(), 0.01);
- Assert.assertEquals(0.707, next.getFields().get(4).getDoubleV(), 0.01);
- Assert.assertEquals(new Binary(new byte[] {(byte) 32}),
next.getFields().get(5).getBinaryV());
- Assert.assertEquals(true, next.getFields().get(6).getBoolV());
- Assert.assertEquals(new Binary(new byte[] {(byte) 32}),
next.getFields().get(7).getBinaryV());
- Assert.assertEquals(new Binary(new byte[] {(byte) 32}),
next.getFields().get(8).getBinaryV());
- }
- if (sessionDataSet5.hasNext()) {
- RowRecord next = sessionDataSet5.next();
- Assert.assertEquals(3L, next.getFields().get(0).getLongV());
- Assert.assertEquals(1, next.getFields().get(1).getIntV());
- Assert.assertEquals(1L, next.getFields().get(2).getLongV());
- Assert.assertEquals(1.1f, next.getFields().get(3).getFloatV(), 0.01);
- Assert.assertEquals(0.707, next.getFields().get(4).getDoubleV(), 0.01);
- Assert.assertEquals(new Binary(new byte[] {(byte) 32}),
next.getFields().get(5).getBinaryV());
- Assert.assertEquals(true, next.getFields().get(6).getBoolV());
- Assert.assertEquals(new Binary(new byte[] {(byte) 32}),
next.getFields().get(7).getBinaryV());
- Assert.assertEquals(new Binary(new byte[] {(byte) 32}),
next.getFields().get(8).getBinaryV());
+ for (ITableSession session : sessions) {
+ try (SessionDataSet sessionDataSet =
+ session.executeQueryStatement("select * from dbTest_0." +
tableName)) {
+ if (sessionDataSet.hasNext()) {
+ RowRecord next = sessionDataSet.next();
+ Assert.assertEquals(3L, next.getFields().get(0).getLongV());
+ Assert.assertEquals(1, next.getFields().get(1).getIntV());
+ Assert.assertEquals(1L, next.getFields().get(2).getLongV());
+ Assert.assertEquals(1.1f, next.getFields().get(3).getFloatV(), 0.01);
+ Assert.assertEquals(0.707, next.getFields().get(4).getDoubleV(),
0.01);
+ Assert.assertEquals(
+ new Binary(new byte[] {(byte) 32}),
next.getFields().get(5).getBinaryV());
+ Assert.assertTrue(next.getFields().get(6).getBoolV());
+ Assert.assertEquals(
+ new Binary(new byte[] {(byte) 32}),
next.getFields().get(7).getBinaryV());
+ Assert.assertEquals(
+ new Binary(new byte[] {(byte) 32}),
next.getFields().get(8).getBinaryV());
+ }
+ }
}
}
}
diff --git
a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java
b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java
index 6359a60f890..dd30496090c 100644
---
a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java
+++
b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java
@@ -363,6 +363,7 @@ public class SessionUtils {
}
}
+ @SuppressWarnings({"java:S3776", "java:S6541"})
public static void encodeValue(
TSDataType dataType,
Tablet tablet,
@@ -373,111 +374,76 @@ public class SessionUtils {
switch (dataType) {
case INT32:
int[] intValues = (int[]) tablet.getValues()[i];
+ int lastNonNullIntValue = 0;
for (int index = 0; index < tablet.getRowSize(); index++) {
if (!tablet.isNull(index, i)) {
- encoder.encode(intValues[index], outputStream);
- } else {
- // use the previous value as the placeholder of nulls to increase
encoding performance
- if (index == 0) {
- encoder.encode(intValues[index], outputStream);
- } else {
- encoder.encode(intValues[index - 1], outputStream);
- }
+ lastNonNullIntValue = intValues[index];
}
+ encoder.encode(lastNonNullIntValue, outputStream);
}
break;
case INT64:
case TIMESTAMP:
long[] longValues = (long[]) tablet.getValues()[i];
+ long lastNonNullLongValue = 0;
for (int index = 0; index < tablet.getRowSize(); index++) {
if (!tablet.isNull(index, i)) {
- encoder.encode(longValues[index], outputStream);
- } else {
- // use the previous value as the placeholder of nulls to increase
encoding performance
- if (index == 0) {
- encoder.encode(longValues[index], outputStream);
- } else {
- encoder.encode(longValues[index - 1], outputStream);
- }
+ lastNonNullLongValue = longValues[index];
}
+ encoder.encode(lastNonNullLongValue, outputStream);
}
break;
case FLOAT:
float[] floatValues = (float[]) tablet.getValues()[i];
+ float lastNonNullFloatValue = 0.0f;
for (int index = 0; index < tablet.getRowSize(); index++) {
if (!tablet.isNull(index, i)) {
- encoder.encode(floatValues[index], outputStream);
- } else {
- // use the previous value as the placeholder of nulls to increase
encoding performance
- if (index == 0) {
- encoder.encode(floatValues[index], outputStream);
- } else {
- encoder.encode(floatValues[index - 1], outputStream);
- }
+ lastNonNullFloatValue = floatValues[index];
}
+ encoder.encode(lastNonNullFloatValue, outputStream);
}
break;
case DOUBLE:
double[] doubleValues = (double[]) tablet.getValues()[i];
+ double lastNonNullDoubleValue = 0.0;
for (int index = 0; index < tablet.getRowSize(); index++) {
if (!tablet.isNull(index, i)) {
- encoder.encode(doubleValues[index], outputStream);
- } else {
- // use the previous value as the placeholder of nulls to increase
encoding performance
- if (index == 0) {
- encoder.encode(doubleValues[index], outputStream);
- } else {
- encoder.encode(doubleValues[index - 1], outputStream);
- }
+ lastNonNullDoubleValue = doubleValues[index];
}
+ encoder.encode(lastNonNullDoubleValue, outputStream);
}
break;
case BOOLEAN:
boolean[] boolValues = (boolean[]) tablet.getValues()[i];
+ boolean lastNonNullBooleanValue = false;
for (int index = 0; index < tablet.getRowSize(); index++) {
if (!tablet.isNull(index, i)) {
- encoder.encode(boolValues[index], outputStream);
- } else {
- // use the previous value as the placeholder of nulls to increase
encoding performance
- if (index == 0) {
- encoder.encode(boolValues[index], outputStream);
- } else {
- encoder.encode(boolValues[index - 1], outputStream);
- }
+ lastNonNullBooleanValue = boolValues[index];
}
+ encoder.encode(lastNonNullBooleanValue, outputStream);
}
break;
case TEXT:
case STRING:
case BLOB:
Binary[] binaryValues = (Binary[]) tablet.getValues()[i];
+ Binary lastNonNullBinaryValue = Binary.EMPTY_VALUE;
for (int index = 0; index < tablet.getRowSize(); index++) {
- if (!tablet.isNull(index, i)) {
- encoder.encode(binaryValues[index], outputStream);
- } else {
- // use the previous value as the placeholder of nulls to increase
encoding performance
- if (index == 0) {
- encoder.encode(Binary.EMPTY_VALUE, outputStream);
- } else {
- encoder.encode(binaryValues[index - 1], outputStream);
- }
+ if (!tablet.isNull(index, i) && binaryValues[index] != null) {
+ lastNonNullBinaryValue = binaryValues[index];
}
+ encoder.encode(lastNonNullBinaryValue, outputStream);
}
break;
case DATE:
LocalDate[] dateValues = (LocalDate[]) tablet.getValues()[i];
+ int lastNonNullDateValue = EMPTY_DATE_INT;
for (int index = 0; index < tablet.getRowSize(); index++) {
if (!tablet.isNull(index, i)) {
-
encoder.encode(DateUtils.parseDateExpressionToInt(dateValues[index]),
outputStream);
- } else {
- // use the previous value as the placeholder of nulls to increase
encoding performance
- if (index == 0) {
- encoder.encode(EMPTY_DATE_INT, outputStream);
- } else {
- encoder.encode(
- DateUtils.parseDateExpressionToInt(dateValues[index - 1]),
outputStream);
- }
+ lastNonNullDateValue =
DateUtils.parseDateExpressionToInt(dateValues[index]);
}
+ // use the previous value as the placeholder of nulls to increase
encoding performance
+ encoder.encode(lastNonNullDateValue, outputStream);
}
break;
default:
