This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch do_not_cache_table_name in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a11f21cdd31ae9b9e74f6ffbd91ff65f56f2d6ec Author: HTHou <[email protected]> AuthorDate: Mon Sep 15 10:55:50 2025 +0800 Avoid using tree model device cache to cache table names --- .../it/session/IoTDBSessionRelationalIT.java | 82 +++++++++++++++++ .../db/pipe/source/mqtt/MQTTPublishHandler.java | 8 +- .../iotdb/db/protocol/mqtt/MPPPublishHandler.java | 8 +- .../v1/handler/StatementConstructionHandler.java | 3 +- .../plan/parser/StatementGenerator.java | 4 +- .../plan/planner/TableOperatorGenerator.java | 100 ++++++++++----------- .../plan/statement/crud/InsertTabletStatement.java | 6 +- .../execute/utils/CompactionPathUtils.java | 9 +- iotdb-core/metrics/interface/pom.xml | 8 +- 9 files changed, 153 insertions(+), 75 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java index 4e78947d77f..22e3e0ccd34 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java @@ -26,6 +26,7 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.io.WALReader; import org.apache.iotdb.isession.ISession; import org.apache.iotdb.isession.ITableSession; import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.isession.SessionDataSet.DataIterator; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.env.cluster.env.SimpleEnv; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; @@ -49,6 +50,7 @@ import org.apache.tsfile.write.v4.ITsFileWriter; import org.apache.tsfile.write.v4.TsFileWriterBuilder; import org.junit.After; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -1227,6 +1229,86 @@ public class IoTDBSessionRelationalIT { } } + @Test + public void autoCreateChineseCharacterTableTest() + throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + + List<IMeasurementSchema> schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("tag1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE)); + final List<ColumnCategory> columnTypes = + Arrays.asList(ColumnCategory.TAG, ColumnCategory.ATTRIBUTE, ColumnCategory.FIELD); + + long timestamp = 0; + Tablet tablet = + new Tablet( + "\"表一表一\"", + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + 15); + + for (long row = 0; row < 15; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp + row); + tablet.addValue("tag1", rowIndex, "tag:" + row); + tablet.addValue("attr1", rowIndex, "attr:" + row); + tablet.addValue("m1", rowIndex, row * 1.0); + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + session.insert(tablet); + tablet.reset(); + } + } + + SessionDataSet dataSet = session.executeQueryStatement("show tables from db1"); + DataIterator iterator = dataSet.iterator(); + while (iterator.next()) { + Assert.assertEquals("\"表一表一\"", iterator.getString(1)); + } + } + + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db2\""); + + List<IMeasurementSchema> schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("tag1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE)); + final List<ColumnCategory> columnTypes = + Arrays.asList(ColumnCategory.TAG, ColumnCategory.ATTRIBUTE, ColumnCategory.FIELD); + + long timestamp = 0; + Tablet tablet = + new Tablet( + "\"表一\"", + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + 15); + + for (long row = 0; row < 15; row++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, timestamp + row); + tablet.addValue("tag1", rowIndex, "tag:" + row); + tablet.addValue("attr1", rowIndex, "attr:" + row); + tablet.addValue("m1", rowIndex, row * 1.0); + if (tablet.getRowSize() == tablet.getMaxRowNumber()) { + session.insert(tablet); + tablet.reset(); + } + } + + SessionDataSet dataSet = session.executeQueryStatement("show tables from db2"); + DataIterator iterator = dataSet.iterator(); + while (iterator.next()) { + Assert.assertEquals("\"表一\"", iterator.getString(1)); + } + } + } + @Test public void insertNonExistTableTest() throws IoTDBConnectionException, StatementExecutionException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/mqtt/MQTTPublishHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/mqtt/MQTTPublishHandler.java index f1eb102b32e..cb113f47948 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/mqtt/MQTTPublishHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/mqtt/MQTTPublishHandler.java @@ -20,7 +20,7 @@ package org.apache.iotdb.db.pipe.source.mqtt; import org.apache.iotdb.commons.conf.IoTDBConstant.ClientVersion; -import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta; import org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskSourceRuntimeEnvironment; @@ -204,11 +204,9 @@ public class MQTTPublishHandler extends AbstractInterceptHandler { } } - private InsertTabletStatement constructInsertTabletStatement(TableMessage message) - throws IllegalPathException { + private InsertTabletStatement constructInsertTabletStatement(TableMessage message) { InsertTabletStatement statement = new InsertTabletStatement(); - statement.setDevicePath( - DataNodeDevicePathCache.getInstance().getPartialPath(message.getTable())); + statement.setDevicePath(new PartialPath(message.getTable(), false)); List<String> measurements = Stream.of(message.getFields(), message.getTagKeys(), message.getAttributeKeys()) .flatMap(List::stream) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/mqtt/MPPPublishHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/mqtt/MPPPublishHandler.java index 9f104007591..a6464a56c49 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/mqtt/MPPPublishHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/mqtt/MPPPublishHandler.java @@ -20,7 +20,7 @@ package org.apache.iotdb.db.protocol.mqtt; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.conf.IoTDBConstant.ClientVersion; -import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.conf.IoTDBConfig; @@ -204,11 +204,9 @@ public class MPPPublishHandler extends AbstractInterceptHandler { } } - private InsertTabletStatement constructInsertTabletStatement(TableMessage message) - throws IllegalPathException { + private InsertTabletStatement constructInsertTabletStatement(TableMessage message) { InsertTabletStatement insertStatement = new InsertTabletStatement(); - insertStatement.setDevicePath( - DataNodeDevicePathCache.getInstance().getPartialPath(message.getTable())); + insertStatement.setDevicePath(new PartialPath(message.getTable(), false)); List<String> measurements = Stream.of(message.getFields(), message.getTagKeys(), message.getAttributeKeys()) .flatMap(List::stream) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/table/v1/handler/StatementConstructionHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/table/v1/handler/StatementConstructionHandler.java index 70f3cfb826e..3d599cc903d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/table/v1/handler/StatementConstructionHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/table/v1/handler/StatementConstructionHandler.java @@ -18,6 +18,7 @@ package org.apache.iotdb.db.protocol.rest.table.v1.handler; import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.db.exception.WriteProcessRejectException; import org.apache.iotdb.db.protocol.rest.table.v1.model.InsertTabletRequest; @@ -45,7 +46,7 @@ public class StatementConstructionHandler { InsertTabletRequest insertTabletReq) throws IllegalPathException, WriteProcessRejectException { InsertTabletStatement insertStatement = new InsertTabletStatement(); - insertStatement.setDevicePath(DEVICE_PATH_CACHE.getPartialPath(insertTabletReq.getTable())); + insertStatement.setDevicePath(new PartialPath(insertTabletReq.getTable(), false)); insertStatement.setMeasurements(insertTabletReq.getColumnNames().toArray(new String[0])); long[] timestamps = insertTabletReq.getTimestamps().stream().mapToLong(Long::longValue).toArray(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java index 8d5fe8d48f0..5ff879cf141 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java @@ -333,7 +333,9 @@ public class StatementGenerator { // construct insert statement InsertTabletStatement insertStatement = new InsertTabletStatement(); insertStatement.setDevicePath( - DEVICE_PATH_CACHE.getPartialPath(insertTabletReq.getPrefixPath())); + insertTabletReq.isWriteToTable() + ? new PartialPath(insertTabletReq.getPrefixPath(), false) + : DEVICE_PATH_CACHE.getPartialPath(insertTabletReq.getPrefixPath())); insertStatement.setMeasurements(insertTabletReq.getMeasurements().toArray(new String[0])); TSDataType[] dataTypes = new TSDataType[insertTabletReq.types.size()]; for (int i = 0; i < insertTabletReq.types.size(); i++) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index d6c2e53c24b..88c4cc4b982 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -3695,63 +3695,59 @@ public class TableOperatorGenerator extends PlanVisitor<Operator, LocalExecution node.getPlanNodeId(), TableIntoOperator.class.getSimpleName()); - try { - PartialPath targetTable = DEVICE_PATH_CACHE.getPartialPath(node.getTable()); - - Map<String, TSDataType> tsDataTypeMap = new LinkedHashMap<>(); - Map<String, InputLocation> inputLocationMap = new LinkedHashMap<>(); - List<TSDataType> inputColumnTypes = new ArrayList<>(); - List<TsTableColumnCategory> inputColumnCategories = new ArrayList<>(); - - List<ColumnSchema> originColumns = node.getColumns(); - List<Symbol> originInputColumnNames = node.getNeededInputColumnNames(); - int size = originColumns.size(); - List<ColumnSchema> inputColumns = new ArrayList<>(size); - - List<Symbol> childOutputName = node.getChild().getOutputSymbols(); - Map<Symbol, Integer> map = new HashMap<>(childOutputName.size()); - for (int i = 0; i < size; i++) { - map.put(childOutputName.get(i), i); - inputColumns.add(null); - } - for (int i = 0; i < size; i++) { - int index = map.get(originInputColumnNames.get(i)); - inputColumns.set(index, originColumns.get(i)); - } - - for (int i = 0; i < inputColumns.size(); i++) { - String columnName = inputColumns.get(i).getName(); - inputLocationMap.put(columnName, new InputLocation(0, i)); + PartialPath targetTable = new PartialPath(node.getTable(), false); + + Map<String, TSDataType> tsDataTypeMap = new LinkedHashMap<>(); + Map<String, InputLocation> inputLocationMap = new LinkedHashMap<>(); + List<TSDataType> inputColumnTypes = new ArrayList<>(); + List<TsTableColumnCategory> inputColumnCategories = new ArrayList<>(); + + List<ColumnSchema> originColumns = node.getColumns(); + List<Symbol> originInputColumnNames = node.getNeededInputColumnNames(); + int size = originColumns.size(); + List<ColumnSchema> inputColumns = new ArrayList<>(size); + + List<Symbol> childOutputName = node.getChild().getOutputSymbols(); + Map<Symbol, Integer> map = new HashMap<>(childOutputName.size()); + for (int i = 0; i < size; i++) { + map.put(childOutputName.get(i), i); + inputColumns.add(null); + } + for (int i = 0; i < size; i++) { + int index = map.get(originInputColumnNames.get(i)); + inputColumns.set(index, originColumns.get(i)); + } - TsTableColumnCategory columnCategory = inputColumns.get(i).getColumnCategory(); - if (columnCategory == TIME) { - continue; - } + for (int i = 0; i < inputColumns.size(); i++) { + String columnName = inputColumns.get(i).getName(); + inputLocationMap.put(columnName, new InputLocation(0, i)); - TSDataType columnType = InternalTypeManager.getTSDataType(inputColumns.get(i).getType()); - tsDataTypeMap.put(columnName, columnType); - inputColumnTypes.add(columnType); - inputColumnCategories.add(columnCategory); + TsTableColumnCategory columnCategory = inputColumns.get(i).getColumnCategory(); + if (columnCategory == TIME) { + continue; } - long statementSizePerLine = - OperatorGeneratorUtil.calculateStatementSizePerLine(inputColumnTypes); - - return new TableIntoOperator( - operatorContext, - child, - node.getDatabase(), - targetTable, - inputColumnTypes, - inputColumnCategories, - inputLocationMap, - tsDataTypeMap, - true, - FragmentInstanceManager.getInstance().getIntoOperationExecutor(), - statementSizePerLine); - } catch (IllegalPathException e) { - throw new IllegalArgumentException(e); + TSDataType columnType = InternalTypeManager.getTSDataType(inputColumns.get(i).getType()); + tsDataTypeMap.put(columnName, columnType); + inputColumnTypes.add(columnType); + inputColumnCategories.add(columnCategory); } + + long statementSizePerLine = + OperatorGeneratorUtil.calculateStatementSizePerLine(inputColumnTypes); + + return new TableIntoOperator( + operatorContext, + child, + node.getDatabase(), + targetTable, + inputColumnTypes, + inputColumnCategories, + inputLocationMap, + tsDataTypeMap, + true, + FragmentInstanceManager.getInstance().getIntoOperationExecutor(), + statementSizePerLine); } private boolean[] checkStatisticAndScanOrder( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java index 4c255e0882f..4680c313ba5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/InsertTabletStatement.java @@ -106,7 +106,11 @@ public class InsertTabletStatement extends InsertBaseStatement implements ISchem .toArray(String[]::new)); setDataTypes( tablet.getSchemas().stream().map(IMeasurementSchema::getType).toArray(TSDataType[]::new)); - setDevicePath(DataNodeDevicePathCache.getInstance().getPartialPath(tablet.getDeviceId())); + if (Objects.nonNull(databaseName)) { + setDevicePath(new PartialPath(tablet.getTableName(), false)); + } else { + setDevicePath(DataNodeDevicePathCache.getInstance().getPartialPath(tablet.getDeviceId())); + } setAligned(isAligned); setTimes(tablet.getTimestamps()); setColumns(Arrays.stream(tablet.getValues()).map(this::convertTableColumn).toArray()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionPathUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionPathUtils.java index 5e452708bca..0b26202a0ac 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionPathUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionPathUtils.java @@ -38,13 +38,10 @@ public class CompactionPathUtils { public static PartialPath getPath(final IDeviceID device) throws IllegalPathException { if (device.isTableModel()) { - final String[] tableNameSegments = - DataNodeDevicePathCache.getInstance().getPartialPath(device.getTableName()).getNodes(); - final String[] nodes = new String[device.segmentNum() + tableNameSegments.length - 1]; - System.arraycopy(tableNameSegments, 0, nodes, 0, tableNameSegments.length); + final String[] nodes = new String[device.segmentNum() + 1]; + nodes[0] = device.getTableName(); for (int i = 0; i < device.segmentNum() - 1; i++) { - nodes[i + tableNameSegments.length] = - device.segment(i + 1) == null ? null : device.segment(i + 1).toString(); + nodes[i + 1] = device.segment(i + 1) == null ? null : device.segment(i + 1).toString(); } return new PartialPath(nodes); } else { diff --git a/iotdb-core/metrics/interface/pom.xml b/iotdb-core/metrics/interface/pom.xml index 1e5028f6859..f0709c62ff2 100644 --- a/iotdb-core/metrics/interface/pom.xml +++ b/iotdb-core/metrics/interface/pom.xml @@ -79,10 +79,10 @@ <groupId>io.netty</groupId> <artifactId>netty-transport</artifactId> </dependency> - <dependency> - <groupId>io.netty</groupId> - <artifactId>netty-handler</artifactId> - </dependency> + <dependency> + <groupId>io.netty</groupId> + <artifactId>netty-handler</artifactId> + </dependency> <dependency> <groupId>io.netty</groupId> <artifactId>netty-codec-http</artifactId>
