This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 4b2946e799d Fix that partial insert with nulls may result in negative
inserted point count (#17640)
4b2946e799d is described below
commit 4b2946e799d48432a2682af7a586f3bf28848a50
Author: Jiang Tian <[email protected]>
AuthorDate: Wed May 13 10:21:18 2026 +0800
Fix that partial insert with nulls may result in negative inserted point
count (#17640)
* Fix that partial insert with nulls may result in negative inserted point
count
* fix test
---
.../plan/planner/plan/node/write/InsertNode.java | 4 +
.../dataregion/memtable/AbstractMemTable.java | 8 +-
.../write/InsertNodeIsMeasurementFailedTest.java | 174 ++++++++++++
.../AbstractMemTablePartialInsertTest.java | 297 +++++++++++++++++++++
4 files changed, 482 insertions(+), 1 deletion(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java
index 296e74995ff..bbec93afe53 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNode.java
@@ -337,6 +337,10 @@ public abstract class InsertNode extends SearchNode {
return failedMeasurementNumber;
}
+ public boolean isMeasurementFailed(int index) {
+ return measurements[index] == null;
+ }
+
public boolean allMeasurementFailed() {
if (measurements != null) {
return failedMeasurementNumber
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
index e564d23007c..7d5fb840fdd 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java
@@ -242,7 +242,8 @@ public abstract class AbstractMemTable implements IMemTable
{
|| values[i] == null
|| insertRowNode.getColumnCategories() != null
&& insertRowNode.getColumnCategories()[i] !=
TsTableColumnCategory.FIELD) {
- if (values[i] == null) {
+ if (measurements[i] != null && values[i] == null) {
+ // do not include failed measurement to avoid a negative
pointsInserted
nullPointsNumber++;
}
schemaList.add(null);
@@ -321,6 +322,11 @@ public abstract class AbstractMemTable implements
IMemTable {
int nullPointsNumber = 0;
if (values != null) {
for (int i = 0; i < insertTabletNode.getMeasurements().length; i++) {
+ if (insertTabletNode.isMeasurementFailed(i)) {
+ // do not include failed measurement to avoid a negative
pointsInserted
+ continue;
+ }
+
BitMap bitMap = (BitMap) values[i];
if (bitMap != null && !bitMap.isAllUnmarked()) {
for (int j = start; j < end; j++) {
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNodeIsMeasurementFailedTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNodeIsMeasurementFailedTest.java
new file mode 100644
index 00000000000..e5ed818323b
--- /dev/null
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertNodeIsMeasurementFailedTest.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.planner.plan.node.write;
+
+import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.junit.Test;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests for {@link InsertNode#isMeasurementFailed(int)}.
+ *
+ * <p>The method was added to fix a bug where failed (partial-insert)
measurements were incorrectly
+ * counted when computing {@code pointsInserted}, which could produce a
negative value.
+ */
+public class InsertNodeIsMeasurementFailedTest {
+
+ // -----------------------------------------------------------------------
+ // InsertRowNode
+ // -----------------------------------------------------------------------
+
+ @Test
+ public void testInsertRowNode_noFailure_allReturnFalse() throws
IllegalPathException {
+ InsertRowNode node = buildInsertRowNode(new String[] {"s0", "s1", "s2"});
+
+ assertFalse("s0 should not be failed", node.isMeasurementFailed(0));
+ assertFalse("s1 should not be failed", node.isMeasurementFailed(1));
+ assertFalse("s2 should not be failed", node.isMeasurementFailed(2));
+ }
+
+ @Test
+ public void testInsertRowNode_markFirstFailed_firstReturnsTrue() throws
IllegalPathException {
+ InsertRowNode node = buildInsertRowNode(new String[] {"s0", "s1", "s2"});
+ node.markFailedMeasurement(0);
+
+ assertTrue("s0 should be failed after markFailedMeasurement",
node.isMeasurementFailed(0));
+ assertFalse("s1 should not be failed", node.isMeasurementFailed(1));
+ assertFalse("s2 should not be failed", node.isMeasurementFailed(2));
+ }
+
+ @Test
+ public void testInsertRowNode_markAllFailed_allReturnTrue() throws
IllegalPathException {
+ InsertRowNode node = buildInsertRowNode(new String[] {"s0", "s1"});
+ node.markFailedMeasurement(0);
+ node.markFailedMeasurement(1);
+
+ assertTrue(node.isMeasurementFailed(0));
+ assertTrue(node.isMeasurementFailed(1));
+ }
+
+ @Test
+ public void testInsertRowNode_markSameTwice_idempotent() throws
IllegalPathException {
+ // markFailedMeasurement is a no-op when already null; isMeasurementFailed
must stay true
+ InsertRowNode node = buildInsertRowNode(new String[] {"s0", "s1"});
+ node.markFailedMeasurement(0);
+ node.markFailedMeasurement(0); // second call should be a no-op
+
+ assertTrue(node.isMeasurementFailed(0));
+ assertFalse(node.isMeasurementFailed(1));
+ }
+
+ // -----------------------------------------------------------------------
+ // InsertTabletNode
+ // -----------------------------------------------------------------------
+
+ @Test
+ public void testInsertTabletNode_noFailure_allReturnFalse() throws
IllegalPathException {
+ InsertTabletNode node = buildInsertTabletNode(new String[] {"s0", "s1",
"s2"});
+
+ assertFalse(node.isMeasurementFailed(0));
+ assertFalse(node.isMeasurementFailed(1));
+ assertFalse(node.isMeasurementFailed(2));
+ }
+
+ @Test
+ public void testInsertTabletNode_markMiddleFailed_onlyMiddleReturnsTrue()
+ throws IllegalPathException {
+ InsertTabletNode node = buildInsertTabletNode(new String[] {"s0", "s1",
"s2"});
+ node.markFailedMeasurement(1);
+
+ assertFalse(node.isMeasurementFailed(0));
+ assertTrue("s1 should be failed", node.isMeasurementFailed(1));
+ assertFalse(node.isMeasurementFailed(2));
+ }
+
+ @Test
+ public void testInsertTabletNode_markLastFailed_lastReturnsTrue() throws
IllegalPathException {
+ InsertTabletNode node = buildInsertTabletNode(new String[] {"s0", "s1"});
+ node.markFailedMeasurement(1);
+
+ assertFalse(node.isMeasurementFailed(0));
+ assertTrue(node.isMeasurementFailed(1));
+ }
+
+ // -----------------------------------------------------------------------
+ // Helpers
+ // -----------------------------------------------------------------------
+
+ private static InsertRowNode buildInsertRowNode(String[] measurementNames)
+ throws IllegalPathException {
+ int n = measurementNames.length;
+ TSDataType[] dataTypes = new TSDataType[n];
+ Object[] values = new Object[n];
+ MeasurementSchema[] schemas = new MeasurementSchema[n];
+ for (int i = 0; i < n; i++) {
+ dataTypes[i] = TSDataType.INT32;
+ values[i] = i;
+ schemas[i] = new MeasurementSchema(measurementNames[i],
TSDataType.INT32);
+ }
+ InsertRowNode node =
+ new InsertRowNode(
+ new PlanNodeId("test"),
+ new PartialPath("root.sg.d1"),
+ false,
+ measurementNames,
+ dataTypes,
+ schemas,
+ 1L,
+ values,
+ false);
+ return node;
+ }
+
+ private static InsertTabletNode buildInsertTabletNode(String[]
measurementNames)
+ throws IllegalPathException {
+ int n = measurementNames.length;
+ int rowCount = 3;
+ TSDataType[] dataTypes = new TSDataType[n];
+ Object[] columns = new Object[n];
+ MeasurementSchema[] schemas = new MeasurementSchema[n];
+ for (int i = 0; i < n; i++) {
+ dataTypes[i] = TSDataType.INT32;
+ columns[i] = new int[rowCount];
+ schemas[i] = new MeasurementSchema(measurementNames[i],
TSDataType.INT32);
+ }
+ long[] times = {1L, 2L, 3L};
+ InsertTabletNode node =
+ new InsertTabletNode(
+ new PlanNodeId("test"),
+ new PartialPath("root.sg.d1"),
+ false,
+ measurementNames,
+ dataTypes,
+ schemas,
+ times,
+ null,
+ columns,
+ rowCount);
+ return node;
+ }
+}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTablePartialInsertTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTablePartialInsertTest.java
new file mode 100644
index 00000000000..2f4a85d5840
--- /dev/null
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTablePartialInsertTest.java
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.memtable;
+
+import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.exception.WriteProcessException;
+import
org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode;
+import
org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.utils.BitMap;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests that verify the fix for the negative {@code pointsInserted} bug when
a partial insert
+ * contains failed measurements (i.e. {@code measurements[i] == null}).
+ *
+ * <p>Before the fix:
+ *
+ * <ul>
+ * <li>{@code insertAlignedRow}: a failed measurement whose value slot is
also {@code null} was
+ * incorrectly counted as a {@code nullPoint}, making {@code
pointsInserted} go negative.
+ * <li>{@code computeTabletNullPointsNumber}: failed measurements were not
skipped, so their
+ * bitmap marks were counted, again producing a negative {@code
pointsInserted}.
+ * </ul>
+ *
+ * <p>After the fix both paths skip failed measurements, so {@code
pointsInserted >= 0} always.
+ */
+public class AbstractMemTablePartialInsertTest {
+
+ private PrimitiveMemTable memTable;
+ private static boolean prevEnableNullValueInWriteThroughputMetric;
+
+ @Before
+ public void setUp() {
+ memTable = new PrimitiveMemTable("root.sg", "0");
+ prevEnableNullValueInWriteThroughputMetric =
+
IoTDBDescriptor.getInstance().getConfig().isIncludeNullValueInWriteThroughputMetric();
+
IoTDBDescriptor.getInstance().getConfig().setIncludeNullValueInWriteThroughputMetric(false);
+ }
+
+ @After
+ public void tearDown() {
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+
.setIncludeNullValueInWriteThroughputMetric(prevEnableNullValueInWriteThroughputMetric);
+ }
+
+ // =========================================================================
+ // insertAlignedRow – failed measurement must not be counted as nullPoint
+ // =========================================================================
+
+ /** All measurements succeed, no null values → pointsInserted == total
measurements (3). */
+ @Test
+ public void
testInsertAlignedRow_noFailure_noNullValue_pointsInsertedEqualsTotal()
+ throws IllegalPathException {
+ // 3 measurements, all valid, all have values
+ // formula: getMeasurementColumnCnt(3) - failedNum(0) - nullPoints(0) = 3
+ InsertRowNode node =
+ buildAlignedInsertRowNode(
+ new String[] {"s0", "s1", "s2"}, new Object[] {1, 2, 3}, -1 /* no
failure */);
+
+ int points = memTable.insertAlignedRow(node);
+
+ assertEquals(3, points);
+ assertEquals(3, memTable.getTotalPointsNum());
+ }
+
+ /**
+ * One measurement fails (partial insert). The failed slot has
measurements[i]==null and
+ * values[i]==null. Before the fix this null value was counted as a
nullPoint, making
+ * pointsInserted negative. After the fix it is skipped.
+ *
+ * <p>formula: getMeasurementColumnCnt(2) - failedNum(1) - nullPoints(0) = 1
+ */
+ @Test
+ public void
testInsertAlignedRow_oneFailedMeasurement_pointsInsertedNotNegative()
+ throws IllegalPathException {
+ // 2 measurements, first one fails
+ InsertRowNode node =
+ buildAlignedInsertRowNode(
+ new String[] {"s0", "s1"}, new Object[] {1, 2}, 0 /* mark index 0
as failed */);
+
+ int points = memTable.insertAlignedRow(node);
+
+ assertEquals(1, points);
+ assertEquals(1, memTable.getTotalPointsNum());
+ }
+
+ /** All measurements fail → insertAlignedRow returns 0 early (schemaList is
empty). */
+ @Test
+ public void testInsertAlignedRow_allMeasurementsFailed_pointsInsertedIsZero()
+ throws IllegalPathException {
+ InsertRowNode node =
+ buildAlignedInsertRowNode(
+ new String[] {"s0", "s1"},
+ new Object[] {1, 2L},
+ -1 /* mark all failed manually below */);
+ // mark both as failed
+ node.markFailedMeasurement(0);
+ node.markFailedMeasurement(1);
+ node.setFailedMeasurementNumber(2);
+
+ int points = memTable.insertAlignedRow(node);
+
+ assertEquals(0, points);
+ assertEquals(0, memTable.getTotalPointsNum());
+ }
+
+ // =========================================================================
+ // insertTablet – failed measurement must be skipped in null-point counting
+ // =========================================================================
+
+ /** Normal tablet insert with no failures and no null values →
pointsInserted == cols * rows. */
+ @Test
+ public void
testInsertTablet_noFailure_noNullBits_pointsInsertedEqualsColsTimesRows()
+ throws IllegalPathException, WriteProcessException {
+ // formula: (dataTypes.length(2) - failedNum(0)) * rows(3) - nullPoints(0)
= 6
+ InsertTabletNode node =
+ buildInsertTabletNode(
+ new String[] {"s0", "s1"}, 3, null /* no bitmaps */, -1 /* no
failure */);
+
+ int points = memTable.insertTablet(node, 0, 3);
+
+ assertEquals(6, points);
+ assertEquals(6, memTable.getTotalPointsNum());
+ }
+
+ /**
+ * One measurement fails. Before the fix, if the failed column's bitmap had
marks, those marks
+ * were counted as null points, making pointsInserted negative. After the
fix the failed column is
+ * skipped entirely.
+ *
+ * <p>formula: (dataTypes.length(2) - failedNum(1)) * rows(3) -
nullPoints(0, col-0 skipped) = 3
+ */
+ @Test
+ public void
testInsertTablet_oneFailedMeasurement_withBitmap_pointsInsertedNotNegative()
+ throws IllegalPathException, WriteProcessException {
+ int rowCount = 3;
+ // bitmap for column 0: all rows marked as null
+ BitMap[] bitMaps = new BitMap[2];
+ bitMaps[0] = new BitMap(rowCount);
+ bitMaps[0].markAll();
+ bitMaps[1] = null;
+
+ // mark column 0 as failed (partial insert)
+ InsertTabletNode node =
+ buildInsertTabletNode(
+ new String[] {"s0", "s1"}, rowCount, bitMaps, 0 /* mark index 0 as
failed */);
+
+ int points = memTable.insertTablet(node, 0, rowCount);
+
+ assertEquals(3, points);
+ assertEquals(3, memTable.getTotalPointsNum());
+ }
+
+ /** All measurements fail → pointsInserted == 0. formula: (2-2)*3 - 0 = 0 */
+ @Test
+ public void testInsertTablet_allMeasurementsFailed_pointsInsertedIsZero()
+ throws IllegalPathException, WriteProcessException {
+ int rowCount = 3;
+ InsertTabletNode node = buildInsertTabletNode(new String[] {"s0", "s1"},
rowCount, null, -1);
+ node.markFailedMeasurement(0);
+ node.markFailedMeasurement(1);
+ node.setFailedMeasurementNumber(2);
+
+ int points = memTable.insertTablet(node, 0, rowCount);
+
+ assertEquals(0, points);
+ assertEquals(0, memTable.getTotalPointsNum());
+ }
+
+ /**
+ * Tablet with no failures but some null values in bitmap. Since
+ * includeNullValueInWriteThroughputMetric defaults to false, null values
are deducted.
+ *
+ * <p>formula: (dataTypes.length(2) - failedNum(0)) * rows(4) -
nullPoints(2) = 6
+ */
+ @Test
+ public void
testInsertTablet_noFailure_withNullBits_pointsInsertedNotNegative()
+ throws IllegalPathException, WriteProcessException {
+ int rowCount = 4;
+ // column 1: rows 0 and 2 are null
+ BitMap[] bitMaps = new BitMap[2];
+ bitMaps[0] = null;
+ bitMaps[1] = new BitMap(rowCount);
+ bitMaps[1].mark(0);
+ bitMaps[1].mark(2);
+
+ InsertTabletNode node =
+ buildInsertTabletNode(new String[] {"s0", "s1"}, rowCount, bitMaps, -1
/* no failure */);
+
+ int points = memTable.insertTablet(node, 0, rowCount);
+
+ assertEquals(6, points);
+ assertEquals(6, memTable.getTotalPointsNum());
+ }
+
+ // =========================================================================
+ // Helpers
+ // =========================================================================
+
+ /**
+ * Builds an aligned InsertRowNode. If {@code failedIndex >= 0} that
measurement is marked failed
+ * via {@link InsertRowNode#markFailedMeasurement(int)}.
+ */
+ private static InsertRowNode buildAlignedInsertRowNode(
+ String[] measurementNames, Object[] values, int failedIndex) throws
IllegalPathException {
+ int n = measurementNames.length;
+ TSDataType[] dataTypes = new TSDataType[n];
+ MeasurementSchema[] schemas = new MeasurementSchema[n];
+ for (int i = 0; i < n; i++) {
+ dataTypes[i] = TSDataType.INT32;
+ schemas[i] = new MeasurementSchema(measurementNames[i],
TSDataType.INT32);
+ }
+ InsertRowNode node =
+ new InsertRowNode(
+ new PlanNodeId("test"),
+ new PartialPath("root.sg.d1"),
+ true /* isAligned */,
+ measurementNames,
+ dataTypes,
+ schemas,
+ 1L,
+ values,
+ false);
+ if (failedIndex >= 0) {
+ node.markFailedMeasurement(failedIndex);
+ node.setFailedMeasurementNumber(1);
+ }
+ return node;
+ }
+
+ /**
+ * Builds a non-aligned InsertTabletNode. If {@code failedIndex >= 0} that
measurement is marked
+ * failed via {@link InsertTabletNode#markFailedMeasurement(int)}.
+ */
+ private static InsertTabletNode buildInsertTabletNode(
+ String[] measurementNames, int rowCount, BitMap[] bitMaps, int
failedIndex)
+ throws IllegalPathException {
+ int n = measurementNames.length;
+ TSDataType[] dataTypes = new TSDataType[n];
+ Object[] columns = new Object[n];
+ MeasurementSchema[] schemas = new MeasurementSchema[n];
+ for (int i = 0; i < n; i++) {
+ dataTypes[i] = TSDataType.INT32;
+ columns[i] = new int[rowCount];
+ schemas[i] = new MeasurementSchema(measurementNames[i],
TSDataType.INT32);
+ }
+ long[] times = new long[rowCount];
+ for (int i = 0; i < rowCount; i++) {
+ times[i] = i + 1L;
+ }
+ InsertTabletNode node =
+ new InsertTabletNode(
+ new PlanNodeId("test"),
+ new PartialPath("root.sg.d1"),
+ false /* isAligned */,
+ measurementNames,
+ dataTypes,
+ schemas,
+ times,
+ bitMaps,
+ columns,
+ rowCount);
+ if (failedIndex >= 0) {
+ node.markFailedMeasurement(failedIndex);
+ node.setFailedMeasurementNumber(1);
+ }
+ return node;
+ }
+}
