Beyyes commented on code in PR #11522: URL: https://github.com/apache/iotdb/pull/11522#discussion_r1396067355
########## iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/TemplatedAnalyze.java: ########## @@ -0,0 +1,436 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.analyze; + +import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.partition.DataPartition; +import org.apache.iotdb.commons.partition.DataPartitionQueryParam; +import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.exception.sql.MeasurementNotExistException; +import org.apache.iotdb.db.exception.sql.SemanticException; +import org.apache.iotdb.db.queryengine.common.schematree.DeviceSchemaInfo; +import org.apache.iotdb.db.queryengine.common.schematree.ISchemaTree; +import org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet; +import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaFetcher; +import org.apache.iotdb.db.queryengine.plan.expression.Expression; +import org.apache.iotdb.db.queryengine.plan.expression.leaf.TimeSeriesOperand; +import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; +import org.apache.iotdb.db.queryengine.plan.statement.component.ResultColumn; +import org.apache.iotdb.db.queryengine.plan.statement.component.SortItem; +import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; +import org.apache.iotdb.db.schemaengine.template.Template; +import org.apache.iotdb.tsfile.enums.TSDataType; +import org.apache.iotdb.tsfile.read.filter.basic.Filter; +import org.apache.iotdb.tsfile.utils.Pair; +import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet.PARTITION_FETCHER; +import static org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.CONFIG; +import static org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.DEVICE_EXPRESSION; +import static org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.END_TIME_EXPRESSION; +import static org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.WHERE_WRONG_TYPE_ERROR_MSG; +import static org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.analyzeDeviceViewSpecialProcess; +import static org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.analyzeExpressionType; +import static org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.analyzeFill; +import static org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.analyzeOutput; +import static org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.getTimePartitionSlotList; +import static org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer.concatDeviceAndBindSchemaForExpression; +import static org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer.getMeasurementExpression; +import static org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer.normalizeExpression; + +/** + * This class provides accelerated implementation for multiple devices align by device query. This + * optimization is only used for devices with same template, using template can avoid many + * unnecessary judgements. + * + * <p>e.g. for query `SELECT * FROM root.xx.** order by device/time/expression align by device`, the + * device list of `root.xx.**` must use same template. + */ +public class TemplatedAnalyze { Review Comment: Some methods may can just reuse AnalyzeVisitor. ########## iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java: ########## @@ -2536,11 +2543,37 @@ private Map<String, List<InputLocation>> makeLayout(PlanNode node) { } private List<TSDataType> getInputColumnTypes(PlanNode node, TypeProvider typeProvider) { - return node.getChildren().stream() - .map(PlanNode::getOutputColumnNames) - .flatMap(List::stream) - .map(typeProvider::getType) - .collect(Collectors.toList()); + if (typeProvider.getMeasurementList() == null) { + return node.getChildren().stream() + .map(PlanNode::getOutputColumnNames) + .flatMap(List::stream) + .map(typeProvider::getType) + .collect(Collectors.toList()); + } else { + return getInputColumnTypesUseTemplate(node, typeProvider); + } + } + + private List<TSDataType> getInputColumnTypesUseTemplate( + PlanNode node, TypeProvider typeProvider) { + // in template situation, the children of FilterNode/TransformNode can be TimeJoinNode, Review Comment: right? ########## iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SourceRewriter.java: ########## @@ -162,17 +162,21 @@ public List<PlanNode> visitDeviceView(DeviceViewNode node, DistributionPlanConte Set<TRegionReplicaSet> relatedDataRegions = new HashSet<>(); List<DeviceViewSplit> deviceViewSplits = new ArrayList<>(); + // Step 1: constructs DeviceViewSplit - Map<String, List<String>> outputDeviceToQueriedDevicesMap = + Map<String, String> outputDeviceToQueriedDevicesMap = analysis.getOutputDeviceToQueriedDevicesMap(); for (int i = 0; i < node.getDevices().size(); i++) { String outputDevice = node.getDevices().get(i); PlanNode child = node.getChildren().get(i); - List<TRegionReplicaSet> regionReplicaSets = new ArrayList<>(); - for (String queriedDevice : outputDeviceToQueriedDevicesMap.get(outputDevice)) { - regionReplicaSets.addAll( - analysis.getPartitionInfo(queriedDevice, analysis.getGlobalTimeFilter())); - } + List<TRegionReplicaSet> regionReplicaSets = + analysis.isAllDevicesInOneTemplate() Review Comment: if AllDevicesInOneTemplate, no view, no need using outputDeviceToQueriedDevicesMap ########## iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TemplatedLogicalPlan.java: ########## @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.queryengine.plan.planner; + +import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.expression.Expression; +import org.apache.iotdb.db.queryengine.plan.expression.leaf.TimeSeriesOperand; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.TopKNode; +import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; +import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.DEVICE; +import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.ENDTIME; +import static org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer.searchSourceExpressions; +import static org.apache.iotdb.db.queryengine.plan.planner.LogicalPlanVisitor.pushDownLimitToScanNode; + +public class TemplatedLogicalPlan { + + private final Analysis analysis; + private final QueryStatement queryStatement; + private final MPPQueryContext context; + private final List<String> measurementList; + private final List<IMeasurementSchema> schemaList; + + public TemplatedLogicalPlan( + Analysis analysis, QueryStatement queryStatement, MPPQueryContext context) { + this.analysis = analysis; + this.queryStatement = queryStatement; + this.context = context; + + measurementList = new ArrayList<>(analysis.getMeasurementList()); + schemaList = new ArrayList<>(analysis.getMeasurementSchemaList()); + + context.getTypeProvider().setMeasurementList(measurementList); + context.getTypeProvider().setSchemaList(schemaList); + } + + public PlanNode visitQuery() { + LogicalPlanBuilder planBuilder = new LogicalPlanBuilder(analysis, context); + + Map<String, PlanNode> deviceToSubPlanMap = new LinkedHashMap<>(); + for (PartialPath devicePath : analysis.getDeviceList()) { + String deviceName = devicePath.getFullPath(); + PlanNode rootNode = + visitQueryBody( + devicePath, + analysis, + queryStatement, + analysis.getDeviceToWhereExpression() != null + ? analysis.getDeviceToWhereExpression().get(deviceName) + : null, + context); + + LogicalPlanBuilder subPlanBuilder = + new TemplatedLogicalPlanBuilder(analysis, context, measurementList, schemaList) + .withNewRoot(rootNode); + + // sortOperator push down + if (queryStatement.needPushDownSort()) { Review Comment: test queryStatement.needPushDownSort() and fill -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
