This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch object_type in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 963532caeaba63fdfec3ac3d94cce0894fdeff15 Author: shuwenwei <[email protected]> AuthorDate: Tue Jul 15 08:47:13 2025 +0800 Fixed the issue that non aligned tree view query used too much memory (cherry picked from commit d7844931a87a9478c5ada93276202e13565cae22) --- .../plan/planner/TableOperatorGenerator.java | 4 +- ...nAlignedTreeDeviceViewScanOperatorTreeTest.java | 69 ++++++++++++++-------- 2 files changed, 46 insertions(+), 27 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index 17f53216d40..e19204e2480 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -648,9 +648,7 @@ public class TableOperatorGenerator extends PlanVisitor<Operator, LocalExecution SeriesScanOptions.Builder builder = new SeriesScanOptions.Builder(); // time filter may be stateful, so we need to copy it builder.withGlobalTimeFilter(timeFilter == null ? null : timeFilter.copy()); - builder - .withIsTableViewForTreeModel(true) - .withAllSensors(new HashSet<>(measurementColumnNames)); + builder.withIsTableViewForTreeModel(true).withAllSensors(allSensors); if (pushDownPredicateForCurrentMeasurement != null) { builder.withPushDownFilter( convertPredicateToFilter( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/NonAlignedTreeDeviceViewScanOperatorTreeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/NonAlignedTreeDeviceViewScanOperatorTreeTest.java index 2188a04479b..01a00529503 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/NonAlignedTreeDeviceViewScanOperatorTreeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/NonAlignedTreeDeviceViewScanOperatorTreeTest.java @@ -126,28 +126,20 @@ public class NonAlignedTreeDeviceViewScanOperatorTreeTest { new Symbol("time"), new ColumnSchema( "time", TypeFactory.getType(TSDataType.INT64), false, TsTableColumnCategory.TIME)); - columnSchemaMap.put( - new Symbol("sensor0"), - new ColumnSchema( - "sensor0", TypeFactory.getType(TSDataType.INT32), false, TsTableColumnCategory.FIELD)); - columnSchemaMap.put( - new Symbol("sensor1"), - new ColumnSchema( - "sensor1", TypeFactory.getType(TSDataType.INT32), false, TsTableColumnCategory.FIELD)); - columnSchemaMap.put( - new Symbol("sensor2"), - new ColumnSchema( - "sensor2", TypeFactory.getType(TSDataType.INT32), false, TsTableColumnCategory.FIELD)); - columnSchemaMap.put( - new Symbol("sensor3"), - new ColumnSchema( - "sensor3", TypeFactory.getType(TSDataType.INT32), false, TsTableColumnCategory.FIELD)); + for (int i = 0; i < 10000; i++) { + columnSchemaMap.put( + new Symbol("sensor" + i), + new ColumnSchema( + "sensor" + i, + TypeFactory.getType(TSDataType.INT32), + false, + TsTableColumnCategory.FIELD)); + } Map<Symbol, Type> symbolTSDataTypeMap = new HashMap<>(); - symbolTSDataTypeMap.put(new Symbol("sensor0"), TypeFactory.getType(TSDataType.INT32)); - symbolTSDataTypeMap.put(new Symbol("sensor1"), TypeFactory.getType(TSDataType.INT32)); - symbolTSDataTypeMap.put(new Symbol("sensor2"), TypeFactory.getType(TSDataType.INT32)); - symbolTSDataTypeMap.put(new Symbol("sensor3"), TypeFactory.getType(TSDataType.INT32)); + for (int i = 0; i < 10000; i++) { + symbolTSDataTypeMap.put(new Symbol("sensor" + i), TypeFactory.getType(TSDataType.INT32)); + } symbolTSDataTypeMap.put(new Symbol("time"), TypeFactory.getType(TypeEnum.INT64)); symbolTSDataTypeMap.put(new Symbol("tag1"), TypeFactory.getType(TSDataType.TEXT)); typeProvider = new TypeProvider(symbolTSDataTypeMap); @@ -158,10 +150,9 @@ public class NonAlignedTreeDeviceViewScanOperatorTreeTest { tsTable.addColumnSchema(new TagColumnSchema("id_column", TSDataType.STRING)); tsTable.addColumnSchema(new TimeColumnSchema("time", TSDataType.INT64)); tsTable.addColumnSchema(new TagColumnSchema("tag1", TSDataType.TEXT)); - tsTable.addColumnSchema(new FieldColumnSchema("sensor0", TSDataType.INT32)); - tsTable.addColumnSchema(new FieldColumnSchema("sensor1", TSDataType.INT32)); - tsTable.addColumnSchema(new FieldColumnSchema("sensor2", TSDataType.INT32)); - tsTable.addColumnSchema(new FieldColumnSchema("sensor3", TSDataType.INT32)); + for (int i = 0; i < 10000; i++) { + tsTable.addColumnSchema(new FieldColumnSchema("sensor" + i, TSDataType.INT32)); + } tsTable.addProp(TsTable.TTL_PROPERTY, Long.MAX_VALUE + ""); tsTable.addProp( TreeViewSchema.TREE_PATH_PATTERN, @@ -176,6 +167,36 @@ public class NonAlignedTreeDeviceViewScanOperatorTreeTest { SeriesReaderTestUtil.tearDown(seqResources, unSeqResources); } + @Test + public void testQueryManyDevices() throws Exception { + List<String> outputColumnList = new ArrayList<>(10000 + 2); + TreeNonAlignedDeviceViewScanNode node = getTreeNonAlignedDeviceViewScanNode(outputColumnList); + node.setPushDownOffset(500); + node.setPushDownLimit(500); + node.setPushDownPredicate( + new ComparisonExpression( + ComparisonExpression.Operator.GREATER_THAN, + new Symbol("sensor1").toSymbolReference(), + new LongLiteral("1000"))); + for (int i = 0; i < 10000; i++) { + outputColumnList.add("sensor" + i); + } + outputColumnList.add("time"); + outputColumnList.add("tag1"); + ExecutorService instanceNotificationExecutor = + IoTDBThreadPoolFactory.newFixedThreadPool(1, "test-instance-notification"); + Operator operator = getOperator(node, instanceNotificationExecutor); + try { + assertTrue(operator instanceof DeviceIteratorScanOperator); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } finally { + operator.close(); + instanceNotificationExecutor.shutdown(); + } + } + @Test public void testScanWithPushDownPredicateAndLimitAndOffset() throws Exception { List<String> outputColumnList = Arrays.asList("sensor0", "sensor1", "sensor2", "time", "tag1");
