This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch fix_rest_api in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f370bc54367763ef30a4cf4f3becb980d8980e2a Author: HTHou <[email protected]> AuthorDate: Fri May 31 18:08:26 2024 +0800 Add datatype check for restapi insertRecords and fix query bug --- .../metadata/DataTypeMismatchException.java | 13 +++++++++++ .../db/protocol/rest/utils/InsertRowDataUtils.java | 20 +++++++++++++++- .../v2/handler/StatementConstructionHandler.java | 27 +++++++++++++++++++++- .../schemaregion/utils/ResourceByPathUtils.java | 3 ++- 4 files changed, 60 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/DataTypeMismatchException.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/DataTypeMismatchException.java index 8165f291765..12a04e1d282 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/DataTypeMismatchException.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/metadata/DataTypeMismatchException.java @@ -48,6 +48,19 @@ public class DataTypeMismatchException extends MetadataException { value == null ? "null" : processValue(value.toString()))); } + public DataTypeMismatchException( + String deviceName, String measurementName, TSDataType insertType, long time, Object value) { + super( + String.format( + "data type and value of %s.%s is not consistent, " + + "inserting type %s, timestamp %s, value %s", + deviceName, + measurementName, + insertType, + time, + value == null ? "null" : processValue(value.toString()))); + } + private static String processValue(String value) { return value.length() < 100 ? value : value.substring(0, 100); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/utils/InsertRowDataUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/utils/InsertRowDataUtils.java index d963def3c43..fae611dcf77 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/utils/InsertRowDataUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/utils/InsertRowDataUtils.java @@ -25,6 +25,7 @@ import org.apache.tsfile.utils.Binary; import java.nio.charset.StandardCharsets; import java.util.List; +import java.util.Map; import static org.apache.iotdb.session.Session.MSG_UNSUPPORTED_DATA_TYPE; @@ -68,7 +69,8 @@ public class InsertRowDataUtils { return valuesList.isEmpty(); } - public static List<Object> reGenValues(List<TSDataType> types, List<Object> values) + public static List<Object> reGenValues( + List<TSDataType> types, List<Object> values, Map<Integer, Object> mismatchedInfo) throws IoTDBConnectionException { for (int i = 0; i < values.size(); i++) { if (values.get(i) == null) { @@ -77,23 +79,38 @@ public class InsertRowDataUtils { Object val = values.get(i); switch (types.get(i)) { case BOOLEAN: + if (!(val instanceof Boolean)) { + mismatchedInfo.put(i, val); + } + break; case INT32: case DATE: + if (val instanceof Number) { + values.set(i, ((Number) val).intValue()); + } else { + mismatchedInfo.put(i, val); + } break; case INT64: case TIMESTAMP: if (val instanceof Number) { values.set(i, ((Number) val).longValue()); + } else { + mismatchedInfo.put(i, val); } break; case FLOAT: if (val instanceof Number) { values.set(i, ((Number) val).floatValue()); + } else { + mismatchedInfo.put(i, val); } break; case DOUBLE: if (val instanceof Number) { values.set(i, ((Number) val).doubleValue()); + } else { + mismatchedInfo.put(i, val); } break; case TEXT: @@ -105,6 +122,7 @@ public class InsertRowDataUtils { throw new IoTDBConnectionException(MSG_UNSUPPORTED_DATA_TYPE + types.get(i)); } } + return values; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/handler/StatementConstructionHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/handler/StatementConstructionHandler.java index c24089616fa..c0559b40c2d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/handler/StatementConstructionHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/handler/StatementConstructionHandler.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.protocol.rest.v2.handler; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.exception.WriteProcessRejectException; +import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException; import org.apache.iotdb.db.protocol.rest.utils.InsertRowDataUtils; import org.apache.iotdb.db.protocol.rest.v2.model.InsertRecordsRequest; import org.apache.iotdb.db.protocol.rest.v2.model.InsertTabletRequest; @@ -36,8 +37,10 @@ import org.apache.tsfile.utils.BitMap; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Locale; +import java.util.Map; public class StatementConstructionHandler { private StatementConstructionHandler() {} @@ -221,15 +224,37 @@ public class StatementConstructionHandler { TimestampPrecisionUtils.checkTimestampPrecision(insertRecordsRequest.getTimestamps().get(i)); statement.setTime(insertRecordsRequest.getTimestamps().get(i)); statement.setDataTypes(dataTypesList.get(i).toArray(new TSDataType[0])); + Map<Integer, Object> dataTypeMismatchInfo = new HashMap<>(); List<Object> values = InsertRowDataUtils.reGenValues( - dataTypesList.get(i), insertRecordsRequest.getValuesList().get(i)); + dataTypesList.get(i), + insertRecordsRequest.getValuesList().get(i), + dataTypeMismatchInfo); statement.setValues(values.toArray()); statement.setAligned(insertRecordsRequest.getIsAligned()); // skip empty statement if (statement.isEmpty()) { continue; } + if (!dataTypeMismatchInfo.isEmpty()) { + for (Map.Entry<Integer, Object> entry : dataTypeMismatchInfo.entrySet()) { + int index = entry.getKey(); + String measurement = statement.getMeasurements()[index]; + TSDataType dataType = statement.getDataTypes()[index]; + statement.markFailedMeasurement( + index, + new DataTypeMismatchException( + insertRecordsRequest.getDevices().get(i), + statement.getMeasurements()[index], + statement.getDataTypes()[index], + statement.getTime(), + entry.getValue())); + // markFailedMeasurement will set datatype and measurements null + // setting them back in order to pass the schema validation + statement.getDataTypes()[index] = dataType; + statement.getMeasurements()[index] = measurement; + } + } insertRowStatementList.add(statement); } insertStatement.setInsertRowStatementList(insertRowStatementList); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java index d25248af44b..a58277d8d88 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java @@ -266,7 +266,8 @@ class AlignedResourceByPathUtils extends ResourceByPathUtils { // if all the sub sensors doesn't exist, it will be false boolean exits = false; for (List<ChunkMetadata> chunkMetadata : valueChunkMetadataList) { - boolean currentExist = i < chunkMetadata.size(); + boolean currentExist = + i < chunkMetadata.size() && chunkMetadata.get(i).getNumOfPoints() > 0; exits = (exits || currentExist); valueChunkMetadata.add(currentExist ? chunkMetadata.get(i) : null); }
