This is an automated email from the ASF dual-hosted git repository.
jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 77d59829e14 [Fix] Reject table inserts when all fields fail validation
(#18908)
77d59829e14 is described below
commit 77d59829e144eedea4bb568e6e8ffc4881e42f9d
Author: Hongzhi Gao <[email protected]>
AuthorDate: Mon Oct 19 14:19:19 2026 +0800
[Fix] Reject table inserts when all fields fail validation (#18908)
* [Fix] Reject table inserts when all fields fail validation
* [Test] Simplify rejected-field insert regression coverage
---
.../relational/it/db/it/IoTDBInsertTableIT.java | 120 ++++++++++++---------
.../it/session/IoTDBSessionRelationalIT.java | 37 ++-----
.../db/queryengine/plan/analyze/AnalyzeUtils.java | 2 +-
.../plan/analyze/schema/SchemaValidator.java | 22 ++++
.../queryengine/plan/analyze/AnalyzeUtilsTest.java | 25 +++++
5 files changed, 122 insertions(+), 84 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java
b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java
index 70eab82727c..210173bc4e0 100644
---
a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java
@@ -1380,65 +1380,81 @@ public class IoTDBInsertTableIT {
}
@Test
- public void testRejectedOnlyFieldStillWritesTime() throws Exception {
- try (ITableSession session =
EnvFactory.getEnv().getTableSessionConnection()) {
- session.executeNonQueryStatement("use \"test\"");
- session.executeNonQueryStatement(
- "create table rejected_only_field_time (tag1 string tag, s1 int64
field)");
-
- Tablet tablet =
- new Tablet(
- "rejected_only_field_time",
- Arrays.asList("tag1", "s1"),
- Arrays.asList(TSDataType.STRING, TSDataType.BOOLEAN),
- Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD),
- 2);
- for (int i = 0; i < 2; i++) {
- int row = tablet.getRowSize();
- tablet.addTimestamp(row, i + 1);
- tablet.addValue("tag1", row, "d" + i);
- tablet.addValue("s1", row, true);
+ public void testRejectedSqlFields() throws Exception {
+ try (Connection connection =
EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+ Statement statement = connection.createStatement()) {
+ statement.execute("use test");
+ statement.execute(
+ "create table rejected_sql_fields (tag1 string tag, s1 int64 field,
s2 int64 field)");
+ SQLException error =
+ assertThrows(
+ SQLException.class,
+ () -> statement.execute("insert into rejected_sql_fields(time,
s1) values(1, true)"));
+ assertEquals(
+ "507: No column other than Time present, please check the request",
error.getMessage());
+ try (ResultSet result = statement.executeQuery("select * from
rejected_sql_fields")) {
+ assertFalse(result.next());
}
- StatementExecutionException error =
- assertThrows(StatementExecutionException.class, () ->
session.insert(tablet));
- assertTrue(error.getMessage().contains("s1"));
-
- Tablet withoutTag =
- new Tablet(
- "rejected_only_field_time",
- Arrays.asList("s1"),
- Arrays.asList(TSDataType.BOOLEAN),
- Arrays.asList(ColumnCategory.FIELD),
- 1);
- withoutTag.addTimestamp(0, 3);
- withoutTag.addValue("s1", 0, true);
- error = assertThrows(StatementExecutionException.class, () ->
session.insert(withoutTag));
- assertTrue(error.getMessage().contains("s1"));
- assertRejectedFieldTimeRows(session);
-
- session.executeNonQueryStatement("flush");
- assertRejectedFieldTimeRows(session);
+ error =
+ assertThrows(
+ SQLException.class,
+ () ->
+ statement.execute(
+ "insert into rejected_sql_fields(time, tag1, s1, s2)
values(2, 'd1', true, 42)"));
+ assertEquals(507, error.getErrorCode());
+ assertTrue(error.getMessage().contains("Fail to insert measurements
[s1]"));
+ statement.execute("flush");
+ try (ResultSet result =
+ statement.executeQuery("select time, tag1, s1, s2 from
rejected_sql_fields")) {
+ assertTrue(result.next());
+ assertEquals(2, result.getLong(1));
+ assertEquals("d1", result.getString(2));
+ assertNull(result.getObject(3));
+ assertEquals(42, result.getLong(4));
+ assertFalse(result.next());
+ }
}
}
- private static void assertRejectedFieldTimeRows(ITableSession session)
- throws IoTDBConnectionException, StatementExecutionException {
- SessionDataSet dataSet =
- session.executeQueryStatement(
- "select time, tag1, s1 from rejected_only_field_time order by
time");
- for (int i = 0; i < 3; i++) {
- assertTrue(dataSet.hasNext());
- RowRecord row = dataSet.next();
- assertEquals(i + 1, row.getFields().get(0).getLongV());
- if (i < 2) {
- assertEquals("d" + i, row.getFields().get(1).getStringValue());
- } else {
- assertNull(row.getFields().get(1).getObjectValue(TSDataType.STRING));
+ @Test
+ public void testRejectedOnlyFieldDoesNotWriteTime() throws Exception {
+ try (ITableSession session =
EnvFactory.getEnv().getTableSessionConnection()) {
+ session.executeNonQueryStatement("use test");
+ session.executeNonQueryStatement(
+ "create table rejected_only_field_time (tag1 string tag, s1 int64
field)");
+ for (boolean withTag : new boolean[] {true, false}) {
+ Tablet tablet =
+ new Tablet(
+ "rejected_only_field_time",
+ withTag ? Arrays.asList("tag1", "s1") : Arrays.asList("s1"),
+ withTag
+ ? Arrays.asList(TSDataType.STRING, TSDataType.BOOLEAN)
+ : Arrays.asList(TSDataType.BOOLEAN),
+ withTag
+ ? Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD)
+ : Arrays.asList(ColumnCategory.FIELD),
+ 1);
+ tablet.addTimestamp(0, withTag ? 1 : 2);
+ if (withTag) {
+ tablet.addValue("tag1", 0, "d1");
+ }
+ tablet.addValue("s1", 0, true);
+ StatementExecutionException error =
+ assertThrows(StatementExecutionException.class, () ->
session.insert(tablet));
+ assertEquals(
+ "507: No column other than Time present, please check the
request", error.getMessage());
+ }
+ session.executeNonQueryStatement("flush");
+ for (String sql :
+ Arrays.asList(
+ "select * from rejected_only_field_time",
+ "show devices from rejected_only_field_time")) {
+ try (SessionDataSet dataSet = session.executeQueryStatement(sql)) {
+ assertFalse(dataSet.hasNext());
+ }
}
- assertNull(row.getFields().get(2).getObjectValue(TSDataType.INT64));
}
- assertFalse(dataSet.hasNext());
}
@Test
diff --git
a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java
b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java
index 0fae395cc2d..0f1e995055b 100644
---
a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java
@@ -439,7 +439,7 @@ public class IoTDBSessionRelationalIT {
session.executeNonQueryStatement("CREATE TABLE partial_insert (s1
boolean)");
try {
session.executeNonQueryStatement(
- "insert into partial_insert(time, s1) values
(10000,true),(20000,false),(35000,-1.5),(30000,-1),(40000,0),(50000,1),(60000,1.5),(70000,'string'),(80000,'1989-06-15'),(90000,638323200000)");
+ "insert into partial_insert(time, s1) values
(10000,true),(11000,null),(20000,false),(35000,-1.5),(30000,-1),(40000,0),(50000,1),(60000,1.5),(70000,'string'),(80000,'1989-06-15'),(90000,638323200000)");
fail("Exception expected");
} catch (StatementExecutionException e) {
assertEquals(
@@ -449,11 +449,9 @@ public class IoTDBSessionRelationalIT {
SessionDataSet dataSet =
session.executeQueryStatement("select * from partial_insert order by
time");
- // Partial insertion retains timestamps even when the only FIELD value
is rejected.
- long[] timestamps =
- new long[] {10000, 20000, 30000, 35000, 40000, 50000, 60000, 70000,
80000, 90000};
- Boolean[] values =
- new Boolean[] {true, false, null, null, false, true, null, null,
null, null};
+ // Rows without any valid FIELD are skipped during partial insertion.
+ long[] timestamps = new long[] {10000, 11000, 20000, 40000, 50000};
+ Boolean[] values = new Boolean[] {true, null, false, false, true};
int cnt = 0;
while (dataSet.hasNext()) {
RowRecord rec = dataSet.next();
@@ -805,23 +803,12 @@ public class IoTDBSessionRelationalIT {
fail("Insert should fail");
} catch (StatementExecutionException e) {
assertEquals(
- "507: Fail to insert measurements [f1, f2] "
- + "caused by [Incompatible data type of column f1:
DOUBLE/INT32, "
- + "Incompatible data type of column f2: DOUBLE/INT32]",
- e.getMessage());
+ "507: No column other than Time present, please check the
request", e.getMessage());
}
tablet.reset();
for (int phase = 0; phase < 2; phase++) {
try (SessionDataSet dataSet =
session.executeQueryStatement("select * from field_wrong_type
order by time")) {
- for (int row = 0; row < 10; row++) {
- assertTrue(dataSet.hasNext());
- RowRecord record = dataSet.next();
- assertEquals(row, record.getFields().get(0).getLongV());
- assertEquals("tag:" + row,
record.getFields().get(1).getStringValue());
- assertNull(record.getFields().get(2).getDataType());
- assertNull(record.getFields().get(3).getDataType());
- }
assertFalse(dataSet.hasNext());
}
if (phase == 0) {
@@ -1555,23 +1542,11 @@ public class IoTDBSessionRelationalIT {
fail("Exception expected: from=" + from + ", to=" + to);
} catch (StatementExecutionException e) {
assertEquals(
- "507: Fail to insert measurements [m1] caused by [Incompatible
data type of column m1: "
- + from
- + "/"
- + to
- + "]",
- e.getMessage());
+ "507: No column other than Time present, please check the
request", e.getMessage());
}
// time, tag1, m1
try (SessionDataSet dataSet =
session.executeQueryStatement("select * from table" + testNum +
" order by time")) {
- for (int row = 0; row < 2; row++) {
- assertTrue(dataSet.hasNext());
- RowRecord record = dataSet.next();
- assertEquals(row, record.getFields().get(0).getLongV());
- assertEquals("d1", record.getFields().get(1).getStringValue());
- assertNull(record.getFields().get(2).getDataType());
- }
assertFalse(dataSet.hasNext());
}
} else {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java
index 5da73066567..a1d5dc7a27f 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java
@@ -331,7 +331,7 @@ public class AnalyzeUtils {
}
boolean hasFailedMeasurement = insertStatement.hasFailedMeasurements();
String partialInsertMessage;
- if (hasFailedMeasurement) {
+ if (hasFailedMeasurement && !analysis.isFinishQueryAfterAnalyze()) {
partialInsertMessage =
String.format(
"Fail to insert measurements %s caused by %s",
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/SchemaValidator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/SchemaValidator.java
index e0a85c73cc9..02817fb4245 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/SchemaValidator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/SchemaValidator.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.calc.exception.QueryProcessException;
import org.apache.iotdb.commons.exception.SemanticException;
import org.apache.iotdb.commons.path.PartialPath;
import
org.apache.iotdb.commons.queryengine.plan.relational.metadata.QualifiedObjectName;
+import org.apache.iotdb.db.i18n.DataNodeQueryMessages;
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.common.schematree.ISchemaTree;
import org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeUtils;
@@ -46,6 +47,7 @@ import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
+import java.util.stream.Collectors;
import static org.apache.iotdb.commons.utils.PathUtils.unQualifyDatabaseName;
import static
org.apache.iotdb.db.queryengine.plan.execution.config.TableConfigTaskVisitor.DATABASE_NOT_SPECIFIED;
@@ -89,6 +91,26 @@ public class SchemaValidator {
}
insertStatement.validateTableSchema(metadata, context);
insertStatement.updateAfterSchemaValidation(context);
+ if (innerInsertStatement.hasFailedMeasurements()) {
+ final boolean hasValidFields;
+ if (innerInsertStatement instanceof InsertRowsStatement) {
+ final InsertRowsStatement rows = (InsertRowsStatement)
innerInsertStatement;
+ final List<InsertRowStatement> validRows =
+ rows.getInsertRowStatementList().stream()
+ .filter(InsertBaseStatement::hasValidMeasurements)
+ .collect(Collectors.toList());
+ hasValidFields = !validRows.isEmpty();
+ if (hasValidFields) {
+ rows.setInsertRowStatementList(validRows);
+ }
+ } else {
+ hasValidFields = innerInsertStatement.hasValidMeasurements();
+ }
+ if (!hasValidFields) {
+ throw new SemanticException(
+
DataNodeQueryMessages.NO_COLUMN_OTHER_THAN_TIME_PRESENT_PLEASE_CHECK);
+ }
+ }
insertStatement.validateDeviceSchema(metadata, context);
insertStatement.removeAttributeColumns();
} catch (final QueryProcessException e) {
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtilsTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtilsTest.java
index e5a1fb23355..648d3cc6b6e 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtilsTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtilsTest.java
@@ -38,6 +38,7 @@ import
org.apache.iotdb.commons.schema.table.column.TimeColumnSchema;
import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
import org.apache.iotdb.confignode.rpc.thrift.TGetRegionGroupsByTimeReq;
import org.apache.iotdb.confignode.rpc.thrift.TGetRegionGroupsByTimeResp;
+import org.apache.iotdb.db.i18n.DataNodeQueryMessages;
import org.apache.iotdb.db.protocol.client.ConfigNodeClient;
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.common.QueryId;
@@ -73,6 +74,30 @@ import static org.junit.Assert.assertTrue;
public class AnalyzeUtilsTest {
+ @Test
+ public void testSchemaFailureIsNotOverwrittenByPartialInsertError() {
+ final InsertRowStatement row = new InsertRowStatement();
+ row.setMeasurements(new String[] {"field"});
+ row.setDataTypes(new TSDataType[] {TSDataType.INT64});
+ row.setValues(new Object[] {1L});
+ final Analysis analysis = new Analysis();
+
+ AnalyzeUtils.validateSchema(
+ analysis,
+ row,
+ () -> {
+ row.markFailedMeasurement(0, new IllegalArgumentException("type
mismatch"));
+ throw new SemanticException(
+
DataNodeQueryMessages.NO_COLUMN_OTHER_THAN_TIME_PRESENT_PLEASE_CHECK);
+ });
+
+ assertTrue(analysis.isFinishQueryAfterAnalyze());
+ assertEquals(TSStatusCode.METADATA_ERROR.getStatusCode(),
analysis.getFailStatus().getCode());
+ assertEquals(
+ DataNodeQueryMessages.NO_COLUMN_OTHER_THAN_TIME_PRESENT_PLEASE_CHECK,
+ analysis.getFailStatus().getMessage());
+ }
+
@Test
public void testRejectedOnlyFieldStillLooksUpRowAndTabletPartition() throws
Exception {
final InsertRowStatement row = new InsertRowStatement();
