This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 7a8079d1524 Table: Ignore null attribute values in insert (#17790)
7a8079d1524 is described below
commit 7a8079d1524cbd280e3b2a689f203340e6838794
Author: Caideyipi <[email protected]>
AuthorDate: Tue Jun 2 15:06:53 2026 +0800
Table: Ignore null attribute values in insert (#17790)
---
.../relational/it/db/it/IoTDBInsertTableIT.java | 35 +++++++++++
.../fetcher/TableDeviceSchemaValidator.java | 44 +++++++++++++-
.../plan/relational/sql/ast/InsertTablet.java | 17 ++++--
.../fetcher/TableDeviceSchemaValidatorTest.java | 29 ++++++++-
.../plan/relational/sql/ast/InsertTabletTest.java | 69 ++++++++++++++++++++++
5 files changed, 185 insertions(+), 9 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java
b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java
index 3de74f71bf9..802d388c08d 100644
---
a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java
@@ -1018,6 +1018,41 @@ public class IoTDBInsertTableIT {
}
}
+ @Test
+ public void testInsertNullAttributeDoesNotOverwriteDeviceAttribute() throws
SQLException {
+ try (Connection connection =
EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+ Statement statement = connection.createStatement()) {
+ statement.execute("use \"test\"");
+ statement.execute(
+ "create table attr_null_insert(tag1 string tag, attr1 string
attribute, attr2 string attribute, s1 int32 field)");
+
+ statement.execute(
+ "insert into attr_null_insert(time, tag1, attr1, attr2, s1)
values(1, 'd1', 'old1', 'old2', 1)");
+ statement.execute(
+ "insert into attr_null_insert(time, tag1, attr1, attr2, s1)
values(2, 'd1', null, null, 2)");
+ assertDeviceAttributes(statement, "old1", "old2");
+
+ statement.execute(
+ "insert into attr_null_insert(time, tag1, attr1, attr2, s1)
values(3, 'd1', null, 'new2', 3)");
+ assertDeviceAttributes(statement, "old1", "new2");
+
+ statement.execute("update attr_null_insert set attr1 = null where tag1 =
'd1'");
+ assertDeviceAttributes(statement, null, "new2");
+ }
+ }
+
+ private void assertDeviceAttributes(
+ final Statement statement, final String expectedAttr1, final String
expectedAttr2)
+ throws SQLException {
+ try (final ResultSet resultSet = statement.executeQuery("show devices from
attr_null_insert")) {
+ assertTrue(resultSet.next());
+ assertEquals("d1", resultSet.getString("tag1"));
+ assertEquals(expectedAttr1, resultSet.getString("attr1"));
+ assertEquals(expectedAttr2, resultSet.getString("attr2"));
+ assertFalse(resultSet.next());
+ }
+ }
+
@Test
public void testInsertWithTTL() {
try (Connection connection =
EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java
index 5c95fbb62d0..e783b4589c5 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java
@@ -36,6 +36,7 @@ import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.utils.Constants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -74,7 +75,9 @@ public class TableDeviceSchemaValidator {
// High-cost operations, shall only be called once
final List<Object[]> deviceIdList = schemaValidation.getDeviceIdList();
final List<String> attributeKeyList =
schemaValidation.getAttributeColumnNameList();
- final List<Object[]> attributeValueList =
schemaValidation.getAttributeValueList();
+ // In normal inserts, null attributes mean no-op. Raw null is reserved for
explicit clears.
+ final List<Object[]> attributeValueList =
+
normalizeNullAttributeValuesForInsert(schemaValidation.getAttributeValueList());
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(
@@ -194,6 +197,9 @@ public class TableDeviceSchemaValidator {
for (int j = 0, size = attributeKeyList.size(); j < size; j++) {
final Object inputValue =
deviceAttributeValueList.length > j ? deviceAttributeValueList[j] :
null;
+ if (inputValue == null || inputValue == Constants.NONE) {
+ continue;
+ }
if (!Objects.equals(attributeMap.get(attributeKeyList.get(j)),
inputValue)) {
return true;
}
@@ -201,6 +207,42 @@ public class TableDeviceSchemaValidator {
return false;
}
+ private List<Object[]> normalizeNullAttributeValuesForInsert(
+ final List<Object[]> attributeValueList) {
+ List<Object[]> result = null;
+ for (int i = 0; i < attributeValueList.size(); i++) {
+ final Object[] deviceAttributeValueList = attributeValueList.get(i);
+ final Object[] normalizedAttributeValueList =
+ normalizeNullAttributeValuesForInsert(deviceAttributeValueList);
+ if (result != null) {
+ result.add(normalizedAttributeValueList);
+ } else if (normalizedAttributeValueList != deviceAttributeValueList) {
+ result = new ArrayList<>(attributeValueList.size());
+ for (int j = 0; j < i; j++) {
+ result.add(attributeValueList.get(j));
+ }
+ result.add(normalizedAttributeValueList);
+ }
+ }
+ return result == null ? attributeValueList : result;
+ }
+
+ private Object[] normalizeNullAttributeValuesForInsert(final Object[]
deviceAttributeValueList) {
+ for (int i = 0; i < deviceAttributeValueList.length; i++) {
+ if (deviceAttributeValueList[i] == null) {
+ final Object[] result =
+ Arrays.copyOf(deviceAttributeValueList,
deviceAttributeValueList.length);
+ for (int j = i; j < result.length; j++) {
+ if (result[j] == null) {
+ result[j] = Constants.NONE;
+ }
+ }
+ return result;
+ }
+ }
+ return deviceAttributeValueList;
+ }
+
private void autoCreateOrUpdateDeviceSchema(
final ITableDeviceSchemaValidation schemaValidation,
final ValidateResult previousValidateResult,
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java
index f77bd3aa143..5d25b01ecf3 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java
@@ -90,10 +90,17 @@ public class InsertTablet extends WrappedInsertStatement {
public List<Object[]> getAttributeValueList() {
prepareDeviceID2LastIdxMap();
final InsertTabletStatement insertTabletStatement =
getInnerTreeStatement();
- List<Object[]> result = new
ArrayList<>(insertTabletStatement.getRowCount());
final List<Integer> attrColumnIndices =
insertTabletStatement.getAttrColumnIndices();
- for (Integer rowIndex : deviceID2LastIdxMap.values()) {
- Object[] attrValues = new Object[attrColumnIndices.size()];
+
+ final Map<IDeviceID, Object[]> deviceID2AttributeValues =
+ new LinkedHashMap<>(deviceID2LastIdxMap.size());
+ for (final IDeviceID deviceID : deviceID2LastIdxMap.keySet()) {
+ deviceID2AttributeValues.put(deviceID, new
Object[attrColumnIndices.size()]);
+ }
+
+ for (int rowIndex = 0; rowIndex < insertTabletStatement.getRowCount();
rowIndex++) {
+ final Object[] attrValues =
+
deviceID2AttributeValues.get(insertTabletStatement.getTableDeviceID(rowIndex));
for (int attrColNum = 0; attrColNum < attrColumnIndices.size();
attrColNum++) {
final int columnIndex = attrColumnIndices.get(attrColNum);
if (!insertTabletStatement.isNull(rowIndex, columnIndex)) {
@@ -101,9 +108,9 @@ public class InsertTablet extends WrappedInsertStatement {
((Object[])
insertTabletStatement.getColumns()[columnIndex])[rowIndex];
}
}
- result.add(attrValues);
}
- return result;
+
+ return new ArrayList<>(deviceID2AttributeValues.values());
}
// The map cannot be maintained during construction because the IDeviceID
may be reset later.
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidatorTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidatorTest.java
index 6242ab8960b..32fed0a8d08 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidatorTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidatorTest.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher;
import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.utils.Constants;
import org.junit.Assert;
import org.junit.Test;
@@ -32,17 +33,39 @@ import java.util.Map;
public class TableDeviceSchemaValidatorTest {
@Test
- public void testNullAttributeNeedsUpdate() {
+ public void testNullAttributeSkipsUpdate() {
final Map<String, Binary> attributeMap = new HashMap<>();
attributeMap.put("attr", new Binary("x", StandardCharsets.UTF_8));
- Assert.assertTrue(
+ Assert.assertFalse(
TableDeviceSchemaValidator.isAttributeUpdateRequired(
Collections.singletonList("attr"), new Object[] {null},
attributeMap));
}
@Test
- public void testMissingTailAttributeValueEqualsNull() {
+ public void testNoneAttributeSkipsUpdate() {
+ final Map<String, Binary> attributeMap = new HashMap<>();
+ attributeMap.put("attr", new Binary("x", StandardCharsets.UTF_8));
+
+ Assert.assertFalse(
+ TableDeviceSchemaValidator.isAttributeUpdateRequired(
+ Collections.singletonList("attr"), new Object[] {Constants.NONE},
attributeMap));
+ }
+
+ @Test
+ public void testNonNullAttributeNeedsUpdate() {
+ final Map<String, Binary> attributeMap = new HashMap<>();
+ attributeMap.put("attr", new Binary("x", StandardCharsets.UTF_8));
+
+ Assert.assertTrue(
+ TableDeviceSchemaValidator.isAttributeUpdateRequired(
+ Collections.singletonList("attr"),
+ new Object[] {new Binary("y", StandardCharsets.UTF_8)},
+ attributeMap));
+ }
+
+ @Test
+ public void testMissingTailAttributeValueSkipsUpdate() {
final Map<String, Binary> attributeMap = new HashMap<>();
attributeMap.put("attr1", new Binary("x", StandardCharsets.UTF_8));
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTabletTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTabletTest.java
index a78fabeb865..f16e680502a 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTabletTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTabletTest.java
@@ -127,4 +127,73 @@ public class InsertTabletTest {
assertArrayEquals(new Object[] {"id3_1"}, deviceIdList.get(2));
assertArrayEquals(new Object[] {}, deviceIdList.get(3));
}
+
+ @Test
+ public void testDuplicateDeviceUsesLastNonNullAttributeValue() {
+ InsertTabletStatement innerStmt = new InsertTabletStatement();
+ innerStmt.setDevicePath(new PartialPath("table1", false));
+ innerStmt.setTimes(new long[] {1, 2, 3, 4});
+ innerStmt.setRowCount(4);
+ innerStmt.setMeasurements(new String[] {"deviceId", "attr1", "attr2",
"measurement"});
+ innerStmt.setColumnCategories(
+ new TsTableColumnCategory[] {
+ TsTableColumnCategory.TAG,
+ TsTableColumnCategory.ATTRIBUTE,
+ TsTableColumnCategory.ATTRIBUTE,
+ TsTableColumnCategory.FIELD
+ });
+ innerStmt.setDataTypes(
+ new TSDataType[] {
+ TSDataType.STRING, TSDataType.STRING, TSDataType.STRING,
TSDataType.STRING
+ });
+ innerStmt.setColumns(
+ new Object[] {
+ new Binary[] {
+ new Binary("d1", StandardCharsets.UTF_8),
+ new Binary("d1", StandardCharsets.UTF_8),
+ new Binary("d1", StandardCharsets.UTF_8),
+ new Binary("d1", StandardCharsets.UTF_8)
+ },
+ new Binary[] {
+ new Binary("attr1_1", StandardCharsets.UTF_8),
+ Binary.EMPTY_VALUE,
+ new Binary("attr1_3", StandardCharsets.UTF_8),
+ Binary.EMPTY_VALUE
+ },
+ new Binary[] {
+ new Binary("attr2_1", StandardCharsets.UTF_8),
+ new Binary("attr2_2", StandardCharsets.UTF_8),
+ Binary.EMPTY_VALUE,
+ Binary.EMPTY_VALUE
+ },
+ new Binary[] {
+ new Binary("m1", StandardCharsets.UTF_8),
+ new Binary("m2", StandardCharsets.UTF_8),
+ new Binary("m3", StandardCharsets.UTF_8),
+ new Binary("m4", StandardCharsets.UTF_8)
+ },
+ });
+ innerStmt.setBitMaps(
+ new BitMap[] {
+ new BitMap(4, new byte[] {0x00}),
+ new BitMap(4, new byte[] {1 << 1 | 1 << 3}),
+ new BitMap(4, new byte[] {1 << 2 | 1 << 3}),
+ new BitMap(4, new byte[] {0x00}),
+ });
+
+ InsertTablet insertTablet = new InsertTablet(innerStmt, null);
+ assertEquals(Arrays.asList("attr1", "attr2"),
insertTablet.getAttributeColumnNameList());
+ List<Object[]> attributeValueList = insertTablet.getAttributeValueList();
+ assertEquals(1, attributeValueList.size());
+ assertArrayEquals(
+ new Object[] {
+ new Binary("attr1_3", StandardCharsets.UTF_8),
+ new Binary("attr2_2", StandardCharsets.UTF_8)
+ },
+ attributeValueList.get(0));
+
+ List<Object[]> deviceIdList = insertTablet.getDeviceIdList();
+ assertEquals(1, deviceIdList.size());
+ assertArrayEquals(new Object[] {"d1"}, deviceIdList.get(0));
+ }
}
