This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new db268afc910 Add datatype check for restapi insertRecords and fix query
bug (#12631)
db268afc910 is described below
commit db268afc910ac1af4ecb3abb7175e9708e47537e
Author: Haonan <[email protected]>
AuthorDate: Fri May 31 19:36:45 2024 +0800
Add datatype check for restapi insertRecords and fix query bug (#12631)
---
.../metadata/DataTypeMismatchException.java | 13 +++++++++++
.../db/protocol/rest/utils/InsertRowDataUtils.java | 20 +++++++++++++++-
.../v2/handler/StatementConstructionHandler.java | 27 +++++++++++++++++++++-
.../schemaregion/utils/ResourceByPathUtils.java | 3 ++-
4 files changed, 60 insertions(+), 3 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/DataTypeMismatchException.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/DataTypeMismatchException.java
index 8165f291765..12a04e1d282 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/DataTypeMismatchException.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/DataTypeMismatchException.java
@@ -48,6 +48,19 @@ public class DataTypeMismatchException extends
MetadataException {
value == null ? "null" : processValue(value.toString())));
}
+ public DataTypeMismatchException(
+ String deviceName, String measurementName, TSDataType insertType, long
time, Object value) {
+ super(
+ String.format(
+ "data type and value of %s.%s is not consistent, "
+ + "inserting type %s, timestamp %s, value %s",
+ deviceName,
+ measurementName,
+ insertType,
+ time,
+ value == null ? "null" : processValue(value.toString())));
+ }
+
private static String processValue(String value) {
return value.length() < 100 ? value : value.substring(0, 100);
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/utils/InsertRowDataUtils.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/utils/InsertRowDataUtils.java
index d963def3c43..fae611dcf77 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/utils/InsertRowDataUtils.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/utils/InsertRowDataUtils.java
@@ -25,6 +25,7 @@ import org.apache.tsfile.utils.Binary;
import java.nio.charset.StandardCharsets;
import java.util.List;
+import java.util.Map;
import static org.apache.iotdb.session.Session.MSG_UNSUPPORTED_DATA_TYPE;
@@ -68,7 +69,8 @@ public class InsertRowDataUtils {
return valuesList.isEmpty();
}
- public static List<Object> reGenValues(List<TSDataType> types, List<Object>
values)
+ public static List<Object> reGenValues(
+ List<TSDataType> types, List<Object> values, Map<Integer, Object>
mismatchedInfo)
throws IoTDBConnectionException {
for (int i = 0; i < values.size(); i++) {
if (values.get(i) == null) {
@@ -77,23 +79,38 @@ public class InsertRowDataUtils {
Object val = values.get(i);
switch (types.get(i)) {
case BOOLEAN:
+ if (!(val instanceof Boolean)) {
+ mismatchedInfo.put(i, val);
+ }
+ break;
case INT32:
case DATE:
+ if (val instanceof Number) {
+ values.set(i, ((Number) val).intValue());
+ } else {
+ mismatchedInfo.put(i, val);
+ }
break;
case INT64:
case TIMESTAMP:
if (val instanceof Number) {
values.set(i, ((Number) val).longValue());
+ } else {
+ mismatchedInfo.put(i, val);
}
break;
case FLOAT:
if (val instanceof Number) {
values.set(i, ((Number) val).floatValue());
+ } else {
+ mismatchedInfo.put(i, val);
}
break;
case DOUBLE:
if (val instanceof Number) {
values.set(i, ((Number) val).doubleValue());
+ } else {
+ mismatchedInfo.put(i, val);
}
break;
case TEXT:
@@ -105,6 +122,7 @@ public class InsertRowDataUtils {
throw new IoTDBConnectionException(MSG_UNSUPPORTED_DATA_TYPE +
types.get(i));
}
}
+
return values;
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/handler/StatementConstructionHandler.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/handler/StatementConstructionHandler.java
index c24089616fa..c0559b40c2d 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/handler/StatementConstructionHandler.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/handler/StatementConstructionHandler.java
@@ -20,6 +20,7 @@ package org.apache.iotdb.db.protocol.rest.v2.handler;
import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.commons.utils.PathUtils;
import org.apache.iotdb.db.exception.WriteProcessRejectException;
+import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException;
import org.apache.iotdb.db.protocol.rest.utils.InsertRowDataUtils;
import org.apache.iotdb.db.protocol.rest.v2.model.InsertRecordsRequest;
import org.apache.iotdb.db.protocol.rest.v2.model.InsertTabletRequest;
@@ -36,8 +37,10 @@ import org.apache.tsfile.utils.BitMap;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Locale;
+import java.util.Map;
public class StatementConstructionHandler {
private StatementConstructionHandler() {}
@@ -221,15 +224,37 @@ public class StatementConstructionHandler {
TimestampPrecisionUtils.checkTimestampPrecision(insertRecordsRequest.getTimestamps().get(i));
statement.setTime(insertRecordsRequest.getTimestamps().get(i));
statement.setDataTypes(dataTypesList.get(i).toArray(new TSDataType[0]));
+ Map<Integer, Object> dataTypeMismatchInfo = new HashMap<>();
List<Object> values =
InsertRowDataUtils.reGenValues(
- dataTypesList.get(i),
insertRecordsRequest.getValuesList().get(i));
+ dataTypesList.get(i),
+ insertRecordsRequest.getValuesList().get(i),
+ dataTypeMismatchInfo);
statement.setValues(values.toArray());
statement.setAligned(insertRecordsRequest.getIsAligned());
// skip empty statement
if (statement.isEmpty()) {
continue;
}
+ if (!dataTypeMismatchInfo.isEmpty()) {
+ for (Map.Entry<Integer, Object> entry :
dataTypeMismatchInfo.entrySet()) {
+ int index = entry.getKey();
+ String measurement = statement.getMeasurements()[index];
+ TSDataType dataType = statement.getDataTypes()[index];
+ statement.markFailedMeasurement(
+ index,
+ new DataTypeMismatchException(
+ insertRecordsRequest.getDevices().get(i),
+ statement.getMeasurements()[index],
+ statement.getDataTypes()[index],
+ statement.getTime(),
+ entry.getValue()));
+ // markFailedMeasurement will set datatype and measurements null
+ // setting them back in order to pass the schema validation
+ statement.getDataTypes()[index] = dataType;
+ statement.getMeasurements()[index] = measurement;
+ }
+ }
insertRowStatementList.add(statement);
}
insertStatement.setInsertRowStatementList(insertRowStatementList);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java
index d25248af44b..a58277d8d88 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java
@@ -266,7 +266,8 @@ class AlignedResourceByPathUtils extends
ResourceByPathUtils {
// if all the sub sensors doesn't exist, it will be false
boolean exits = false;
for (List<ChunkMetadata> chunkMetadata : valueChunkMetadataList) {
- boolean currentExist = i < chunkMetadata.size();
+ boolean currentExist =
+ i < chunkMetadata.size() &&
chunkMetadata.get(i).getNumOfPoints() > 0;
exits = (exits || currentExist);
valueChunkMetadata.add(currentExist ? chunkMetadata.get(i) : null);
}
