This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch IOTDB-4913 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 6708777f06684d7c12459d8afb04e8ad903d50f3 Author: HTHou <[email protected]> AuthorDate: Fri Nov 11 17:02:18 2022 +0800 [IOTDB-4913] Fix NPE when insert multi rows with null by sql --- .../apache/iotdb/db/it/IoTDBInsertMultiRowIT.java | 19 +++++++++++ .../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 1 - .../db/mpp/plan/parser/StatementGenerator.java | 5 +-- .../plan/node/write/InsertRowsOfOneDeviceNode.java | 39 ++++++++++------------ .../crud/InsertRowsOfOneDeviceStatement.java | 25 ++++++-------- 5 files changed, 47 insertions(+), 42 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertMultiRowIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertMultiRowIT.java index 7295e9203e..fc314aa872 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertMultiRowIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertMultiRowIT.java @@ -140,4 +140,23 @@ public class IoTDBInsertMultiRowIT { e.getMessage().contains(Integer.toString(TSStatusCode.METADATA_ERROR.getStatusCode()))); } } + + @Test + public void testInsertMultiRowWithNull() { + try (Statement st1 = connection.createStatement()) { + st1.execute( + "insert into root.t1.wf01.wt01(timestamp, s1, s2) values(100, null, 1), (101, null, 2)"); + fail(); + } catch (SQLException e) { + assertTrue( + e.getMessage().contains(Integer.toString(TSStatusCode.METADATA_ERROR.getStatusCode()))); + } + try (Statement st2 = connection.createStatement()) { + st2.execute("CREATE TIMESERIES root.t1.d1.s1 WITH DATATYPE=double, ENCODING=PLAIN;"); + st2.execute( + "INSERT INTO root.t1.d1(timestamp, s1) VALUES (6, 10),(7,12),(8,14),(9,160),(10,null),(11,58)"); + } catch (SQLException e) { + fail(); + } + } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java index 37a4f76a48..3779abb660 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java @@ -1251,7 +1251,6 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> statement.setMeasurements(measurements); statement.setTime(timeArray[i]); TSDataType[] dataTypes = new TSDataType[measurementList.length]; - Arrays.fill(dataTypes, TSDataType.TEXT); statement.setDataTypes(dataTypes); Object[] values = new Object[measurementList.length]; System.arraycopy(insertStatement.getValuesList().get(i), 0, values, 0, values.length); diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java index 1ee50674ec..0189b2fb7f 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java @@ -95,7 +95,6 @@ import org.antlr.v4.runtime.tree.ParseTree; import java.nio.ByteBuffer; import java.time.ZoneId; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -346,9 +345,7 @@ public class StatementGenerator { statement.setDevicePath(insertStatement.getDevicePath()); addMeasurementAndValue( statement, req.getMeasurementsList().get(i), req.getValuesList().get(i)); - TSDataType[] dataTypes = new TSDataType[statement.getMeasurements().length]; - Arrays.fill(dataTypes, TSDataType.TEXT); - statement.setDataTypes(dataTypes); + statement.setDataTypes(new TSDataType[statement.getMeasurements().length]); statement.setTime(req.timestamps.get(i)); statement.setNeedInferType(true); statement.setAligned(req.isAligned); diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java index ea25196a26..3bc7e2803d 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java @@ -41,13 +41,13 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.stream.Collectors; +import java.util.Set; public class InsertRowsOfOneDeviceNode extends InsertNode implements BatchInsertNode { @@ -153,9 +153,6 @@ public class InsertRowsOfOneDeviceNode extends InsertNode implements BatchInsert this.failedMeasurementIndex2Info = insertRowNode.failedMeasurementIndex2Info; } } - if (!this.hasFailedMeasurements()) { - storeMeasurementsAndDataType(); - } } @Override @@ -198,24 +195,22 @@ public class InsertRowsOfOneDeviceNode extends InsertNode implements BatchInsert } private void storeMeasurementsAndDataType() { - Map<String, TSDataType> measurementsAndDataType = new HashMap<>(); - /* - Traverse from end to start. For example, consider this sql: - "insert into root.t1.wf01.wt01(timestamp, s1) values(1, 1.0), (2, 'hello')" - we want the type of 's1' to be inferred as FLOAT(1.0) rather than TEXT('hello'). - */ - for (int i = insertRowNodeList.size() - 1; i >= 0; i--) { - InsertRowNode insertRowNode = insertRowNodeList.get(i); - List<String> measurements = Arrays.asList(insertRowNode.getMeasurements()); - Map<String, TSDataType> subMap = - measurements.stream() - .collect( - Collectors.toMap( - key -> key, key -> insertRowNode.getDataTypes()[measurements.indexOf(key)])); - measurementsAndDataType.putAll(subMap); + Set<String> measurementSet = new HashSet<>(); + List<TSDataType> dataTypeList = new ArrayList<>(); + List<String> measurementList = new ArrayList<>(); + for (InsertRowNode insertRowNode : insertRowNodeList) { + String[] measurements = insertRowNode.getMeasurements(); + TSDataType[] dataTypes = insertRowNode.getDataTypes(); + for (int i = 0; i < measurements.length; i++) { + if (!measurementSet.contains(measurements[i])) { + measurementList.add(measurements[i]); + dataTypeList.add(dataTypes[i]); + measurementSet.add(measurements[i]); + } + } } - measurements = measurementsAndDataType.keySet().toArray(new String[0]); - dataTypes = measurementsAndDataType.values().toArray(new TSDataType[0]); + measurements = measurementList.toArray(new String[0]); + dataTypes = dataTypeList.toArray(new TSDataType[0]); } public static InsertRowsOfOneDeviceNode deserialize(ByteBuffer byteBuffer) { diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertRowsOfOneDeviceStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertRowsOfOneDeviceStatement.java index f9bc18c066..25049ff053 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertRowsOfOneDeviceStatement.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/crud/InsertRowsOfOneDeviceStatement.java @@ -26,16 +26,11 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.mpp.plan.constant.StatementType; import org.apache.iotdb.db.mpp.plan.statement.StatementVisitor; import org.apache.iotdb.db.utils.TimePartitionUtils; -import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; public class InsertRowsOfOneDeviceStatement extends InsertBaseStatement { @@ -65,18 +60,18 @@ public class InsertRowsOfOneDeviceStatement extends InsertBaseStatement { } devicePath = insertRowStatementList.get(0).getDevicePath(); isAligned = insertRowStatementList.get(0).isAligned; - Map<String, TSDataType> measurementsAndDataType = new HashMap<>(); + Set<String> measurementSet = new HashSet<>(); + List<String> measurementList = new ArrayList<>(); for (InsertRowStatement insertRowStatement : insertRowStatementList) { - List<String> measurements = Arrays.asList(insertRowStatement.getMeasurements()); - Map<String, TSDataType> subMap = - measurements.stream() - .collect( - Collectors.toMap( - key -> key, key -> insertRowStatement.dataTypes[measurements.indexOf(key)])); - measurementsAndDataType.putAll(subMap); + String[] measurements = insertRowStatement.getMeasurements(); + for (String measurement : measurements) { + if (!measurementSet.contains(measurement)) { + measurementList.add(measurement); + measurementSet.add(measurement); + } + } } - measurements = measurementsAndDataType.keySet().toArray(new String[0]); - dataTypes = measurementsAndDataType.values().toArray(new TSDataType[0]); + measurements = measurementList.toArray(new String[0]); } public List<TTimePartitionSlot> getTimePartitionSlots() {
