This is an automated email from the ASF dual-hosted git repository. zyk pushed a commit to branch table-model-debug in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 80c6b3f9f0db58ef50418ede6dbce5ba15e00292 Author: MarcosZyk <[email protected]> AuthorDate: Sun Apr 28 11:38:35 2024 +0800 refactor schemafilter to device pattern --- .../schema/source/TableDeviceSchemaSource.java | 74 +++++++------------ .../queryengine/plan/analyze/AnalyzeVisitor.java | 58 +++------------ .../ConvertSchemaPredicateToFilterVisitor.java | 13 +++- .../analyzer/schema/TableModelSchemaFetcher.java | 15 +--- .../schema/filter/impl/DeviceFilterToPathUtil.java | 85 ++++++++++++++++++++++ 5 files changed, 132 insertions(+), 113 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceSchemaSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceSchemaSource.java index 52999761142..b09e38e040b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceSchemaSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceSchemaSource.java @@ -1,11 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.iotdb.db.queryengine.execution.operator.schema.source; import org.apache.iotdb.commons.exception.runtime.SchemaExecutionException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.filter.SchemaFilter; -import org.apache.iotdb.commons.schema.filter.SchemaFilterType; -import org.apache.iotdb.commons.schema.filter.impl.DeviceIdFilter; -import org.apache.iotdb.commons.schema.filter.impl.OrFilter; +import org.apache.iotdb.commons.schema.filter.impl.DeviceFilterToPathUtil; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; @@ -21,15 +38,9 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.read.common.block.TsBlockBuilder; import org.apache.tsfile.utils.Binary; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.NoSuchElementException; -import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; - public class TableDeviceSchemaSource implements ISchemaSource<IDeviceSchemaInfo> { private String database; @@ -137,46 +148,11 @@ public class TableDeviceSchemaSource implements ISchemaSource<IDeviceSchemaInfo> } private List<PartialPath> getDevicePatternList() { - int length = DataNodeTableCache.getInstance().getTable(database, tableName).getIdNums() + 3; - String[] nodes = new String[length]; - Arrays.fill(nodes, "*"); - nodes[0] = PATH_ROOT; - nodes[1] = database; - nodes[2] = tableName; - Map<Integer, List<String>> orValueMap = new HashMap<>(); - for (SchemaFilter schemaFilter : idDeterminedFilterList) { - if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID)) { - DeviceIdFilter deviceIdFilter = (DeviceIdFilter) schemaFilter; - nodes[deviceIdFilter.getIndex() + 3] = deviceIdFilter.getValue(); - } else if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.OR)) { - OrFilter orFilter = (OrFilter) schemaFilter; - if (orFilter.getLeft().getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID) - && orFilter.getRight().getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID)) { - DeviceIdFilter deviceIdFilter = (DeviceIdFilter) orFilter.getLeft(); - nodes[deviceIdFilter.getIndex() + 3] = deviceIdFilter.getValue(); - deviceIdFilter = (DeviceIdFilter) orFilter.getLeft(); - orValueMap - .computeIfAbsent(deviceIdFilter.getIndex(), k -> new ArrayList<>()) - .add(deviceIdFilter.getValue()); - } - } - } - - PartialPath path = new PartialPath(nodes); - List<PartialPath> pathList = new ArrayList<>(); - pathList.add(path); - for (Map.Entry<Integer, List<String>> entry : orValueMap.entrySet()) { - for (int i = 0, size = pathList.size(); i < size; i++) { - for (String value : entry.getValue()) { - nodes = Arrays.copyOf(pathList.get(i).getNodes(), length); - nodes[entry.getKey() + 3] = value; - path = new PartialPath(nodes); - pathList.add(path); - } - } - } - - return pathList; + return DeviceFilterToPathUtil.convertToDevicePattern( + database, + tableName, + DataNodeTableCache.getInstance().getTable(database, tableName), + idDeterminedFilterList); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index bd94bf2b0fd..b685882230c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -33,11 +33,7 @@ import org.apache.iotdb.commons.partition.SchemaPartition; import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; -import org.apache.iotdb.commons.schema.filter.SchemaFilter; -import org.apache.iotdb.commons.schema.filter.SchemaFilterType; -import org.apache.iotdb.commons.schema.filter.impl.DeviceIdFilter; -import org.apache.iotdb.commons.schema.filter.impl.OrFilter; -import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.commons.schema.filter.impl.DeviceFilterToPathUtil; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics; @@ -186,7 +182,6 @@ import static org.apache.iotdb.commons.conf.IoTDBConstant.ALLOWED_SCHEMA_PROPS; import static org.apache.iotdb.commons.conf.IoTDBConstant.DEADBAND; import static org.apache.iotdb.commons.conf.IoTDBConstant.LOSS; import static org.apache.iotdb.commons.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD; -import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_MATCH_PATTERN; import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.DEVICE; import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.ENDTIME; @@ -3717,49 +3712,16 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> String database = statement.getDatabase(); String tableName = statement.getTableName(); - List<TsTableColumnSchema> columnSchemaList = - DataNodeTableCache.getInstance().getTable(database, tableName).getColumnList(); - - int length = DataNodeTableCache.getInstance().getTable(database, tableName).getIdNums() + 3 + 1; - String[] nodes = new String[length]; - Arrays.fill(nodes, "*"); - nodes[0] = PATH_ROOT; - nodes[1] = database; - nodes[2] = tableName; - nodes[nodes.length - 1] = ONE_LEVEL_PATH_WILDCARD; - Map<Integer, List<String>> orValueMap = new HashMap<>(); - for (SchemaFilter schemaFilter : statement.getIdDeterminedFilterList()) { - if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID)) { - DeviceIdFilter deviceIdFilter = (DeviceIdFilter) schemaFilter; - nodes[deviceIdFilter.getIndex() + 3] = deviceIdFilter.getValue(); - } else if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.OR)) { - OrFilter orFilter = (OrFilter) schemaFilter; - if (orFilter.getLeft().getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID) - && orFilter.getRight().getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID)) { - DeviceIdFilter deviceIdFilter = (DeviceIdFilter) orFilter.getLeft(); - nodes[deviceIdFilter.getIndex() + 3] = deviceIdFilter.getValue(); - deviceIdFilter = (DeviceIdFilter) orFilter.getLeft(); - orValueMap - .computeIfAbsent(deviceIdFilter.getIndex(), k -> new ArrayList<>()) - .add(deviceIdFilter.getValue()); - } - } - } + + List<PartialPath> devicePatternList = + DeviceFilterToPathUtil.convertToDevicePattern( + database, + tableName, + DataNodeTableCache.getInstance().getTable(database, tableName), + statement.getIdDeterminedFilterList()); PathPatternTree patternTree = new PathPatternTree(); - PartialPath path = new PartialPath(nodes); - patternTree.appendFullPath(path); - List<PartialPath> pathList = new ArrayList<>(); - pathList.add(path); - for (Map.Entry<Integer, List<String>> entry : orValueMap.entrySet()) { - for (int i = 0, size = pathList.size(); i < size; i++) { - for (String value : entry.getValue()) { - nodes = Arrays.copyOf(pathList.get(i).getNodes(), length); - nodes[entry.getKey() + 3] = value; - path = new PartialPath(nodes); - pathList.add(path); - patternTree.appendFullPath(path); - } - } + for (PartialPath devicePattern : devicePatternList) { + patternTree.appendFullPath(devicePattern, ONE_LEVEL_PATH_WILDCARD); } SchemaPartition partition = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertSchemaPredicateToFilterVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertSchemaPredicateToFilterVisitor.java index 1e9f116d3e2..5c8fa1e6bb2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertSchemaPredicateToFilterVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertSchemaPredicateToFilterVisitor.java @@ -43,7 +43,6 @@ import org.apache.iotdb.db.relational.sql.tree.SimpleCaseExpression; import org.apache.iotdb.db.relational.sql.tree.StringLiteral; import org.apache.iotdb.db.relational.sql.tree.SymbolReference; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -109,6 +108,7 @@ public class ConvertSchemaPredicateToFilterVisitor .equals(TsTableColumnCategory.ID)) { return new DeviceIdFilter(context.idColumeIndexMap.get(columnName), value); } else { + context.hasAttribute = true; return new DeviceAttributeFilter(columnName, value); } } @@ -143,8 +143,7 @@ public class ConvertSchemaPredicateToFilterVisitor private final TsTable table; private final Map<String, Integer> idColumeIndexMap; - private final List<SchemaFilter> idDeterminedFilters = new ArrayList<>(); - private final List<SchemaFilter> idFuzzyFilters = new ArrayList<>(); + private boolean hasAttribute = false; public Context(TsTable table) { this.table = table; @@ -163,5 +162,13 @@ public class ConvertSchemaPredicateToFilterVisitor } return map; } + + public boolean hasAttribute() { + return hasAttribute; + } + + public void reset() { + hasAttribute = false; + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java index 62557581979..239212f461c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/schema/TableModelSchemaFetcher.java @@ -21,8 +21,6 @@ package org.apache.iotdb.db.queryengine.plan.relational.analyzer.schema; import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.commons.schema.filter.SchemaFilter; -import org.apache.iotdb.commons.schema.filter.SchemaFilterType; -import org.apache.iotdb.commons.schema.filter.impl.OrFilter; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; @@ -184,7 +182,6 @@ public class TableModelSchemaFetcher { } finally { coordinator.cleanupQueryExecution(queryId, null, t); } - System.out.println(deviceEntryList); return deviceEntryList; } @@ -199,8 +196,9 @@ public class TableModelSchemaFetcher { if (expression == null) { continue; } + context.reset(); SchemaFilter schemaFilter = expression.accept(visitor, context); - if (hasAttribute(schemaFilter)) { + if (context.hasAttribute()) { idFuzzyFilters.add(schemaFilter); } else { idDeterminedFilters.add(schemaFilter); @@ -208,13 +206,4 @@ public class TableModelSchemaFetcher { } return new Pair<>(idDeterminedFilters, idFuzzyFilters); } - - private boolean hasAttribute(SchemaFilter schemaFilter) { - if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.OR)) { - return hasAttribute(((OrFilter) schemaFilter).getLeft()) - || hasAttribute(((OrFilter) schemaFilter).getRight()); - } - - return schemaFilter.getSchemaFilterType().equals(SchemaFilterType.DEVICE_ATTRIBUTE); - } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceFilterToPathUtil.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceFilterToPathUtil.java new file mode 100644 index 00000000000..ff674942055 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/DeviceFilterToPathUtil.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.schema.filter.impl; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.commons.schema.filter.SchemaFilterType; +import org.apache.iotdb.commons.schema.table.TsTable; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.iotdb.commons.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD; +import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; + +public class DeviceFilterToPathUtil { + + private DeviceFilterToPathUtil() { + // do nothing + } + + public static List<PartialPath> convertToDevicePattern( + String database, String tableName, TsTable table, List<SchemaFilter> idDeterminedFilterList) { + int length = table.getIdNums() + 3; + String[] nodes = new String[length]; + Arrays.fill(nodes, ONE_LEVEL_PATH_WILDCARD); + nodes[0] = PATH_ROOT; + nodes[1] = database; + nodes[2] = tableName; + Map<Integer, List<String>> orValueMap = new HashMap<>(); + for (SchemaFilter schemaFilter : idDeterminedFilterList) { + if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID)) { + DeviceIdFilter deviceIdFilter = (DeviceIdFilter) schemaFilter; + nodes[deviceIdFilter.getIndex() + 3] = deviceIdFilter.getValue(); + } else if (schemaFilter.getSchemaFilterType().equals(SchemaFilterType.OR)) { + OrFilter orFilter = (OrFilter) schemaFilter; + if (orFilter.getLeft().getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID) + && orFilter.getRight().getSchemaFilterType().equals(SchemaFilterType.DEVICE_ID)) { + DeviceIdFilter deviceIdFilter = (DeviceIdFilter) orFilter.getLeft(); + nodes[deviceIdFilter.getIndex() + 3] = deviceIdFilter.getValue(); + deviceIdFilter = (DeviceIdFilter) orFilter.getLeft(); + orValueMap + .computeIfAbsent(deviceIdFilter.getIndex(), k -> new ArrayList<>()) + .add(deviceIdFilter.getValue()); + } + } + } + + PartialPath path = new PartialPath(nodes); + List<PartialPath> pathList = new ArrayList<>(); + pathList.add(path); + for (Map.Entry<Integer, List<String>> entry : orValueMap.entrySet()) { + for (int i = 0, size = pathList.size(); i < size; i++) { + for (String value : entry.getValue()) { + nodes = Arrays.copyOf(pathList.get(i).getNodes(), length); + nodes[entry.getKey() + 3] = value; + path = new PartialPath(nodes); + pathList.add(path); + } + } + } + + return pathList; + } +}
