This is an automated email from the ASF dual-hosted git repository. hui pushed a commit to branch lmh/SupportQueryWithView in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d42d4789ad21e136de0f2ae573995d40e9fbeda2 Author: liuminghui233 <[email protected]> AuthorDate: Sat May 27 23:27:16 2023 +0800 finish analyzer --- .../apache/iotdb/db/mpp/plan/analyze/Analysis.java | 15 ++- .../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 129 ++++++++++++--------- 2 files changed, 85 insertions(+), 59 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/Analysis.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/Analysis.java index db2ae86e9d6..42e25ccd21f 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/Analysis.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/Analysis.java @@ -147,13 +147,15 @@ public class Analysis { // expression of order by that need to be calculated private Map<String, Set<Expression>> deviceToOrderByExpressions; - // the sortItems used in order by push down of align by device + // the sortItems used in order by push down of align by device private Map<String, List<SortItem>> deviceToSortItems; // e.g. [s1,s2,s3] is query, but [s1, s3] exists in device1, then device1 -> [1, 3], s1 is 1 but // not 0 because device is the first column private Map<String, List<Integer>> deviceViewInputIndexesMap; + private Map<String, List<String>> outputDeviceToQueriedDeviceMap; + private Set<Expression> deviceViewOutputExpressions; // indicates whether DeviceView need special process when rewriteSource in DistributionPlan, @@ -435,6 +437,15 @@ public class Analysis { return deviceViewInputIndexesMap; } + public Map<String, List<String>> getOutputDeviceToQueriedDeviceMap() { + return outputDeviceToQueriedDeviceMap; + } + + public void setOutputDeviceToQueriedDeviceMap( + Map<String, List<String>> outputDeviceToQueriedDeviceMap) { + this.outputDeviceToQueriedDeviceMap = outputDeviceToQueriedDeviceMap; + } + public Set<Expression> getSourceExpressions() { return sourceExpressions; } @@ -681,7 +692,7 @@ public class Analysis { ///////////////////////////////////////////////////////////////////////////////////////////////// public void setHasViewsInQuery(boolean hasViewsInQuery) { - this.hasViewsInQuery = hasViewsInQuery; + this.hasViewsInQuery = this.hasViewsInQuery || hasViewsInQuery; } public boolean hasViewsInQuery() { diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java index 909b3e24685..98a59db9107 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java @@ -50,7 +50,6 @@ import org.apache.iotdb.db.exception.sql.MeasurementNotExistException; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.exception.sql.StatementAnalyzeException; import org.apache.iotdb.db.metadata.template.Template; -import org.apache.iotdb.db.metadata.view.viewExpression.visitor.GetSourcePathsVisitor; import org.apache.iotdb.db.mpp.common.MPPQueryContext; import org.apache.iotdb.db.mpp.common.header.ColumnHeader; import org.apache.iotdb.db.mpp.common.header.ColumnHeaderConstant; @@ -189,6 +188,7 @@ import static org.apache.iotdb.commons.conf.IoTDBConstant.ALLOWED_SCHEMA_PROPS; import static org.apache.iotdb.commons.conf.IoTDBConstant.DEADBAND; import static org.apache.iotdb.commons.conf.IoTDBConstant.LOSS; import static org.apache.iotdb.commons.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD; +import static org.apache.iotdb.db.metadata.view.viewExpression.visitor.GetSourcePathsVisitor.getSourcePaths; import static org.apache.iotdb.db.mpp.common.header.ColumnHeaderConstant.DEVICE; import static org.apache.iotdb.db.mpp.common.header.ColumnHeaderConstant.ENDTIME; import static org.apache.iotdb.db.mpp.metric.QueryPlanCostMetricSet.PARTITION_FETCHER; @@ -322,6 +322,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> outputExpressions = new ArrayList<>(); outputExpressionMap.values().forEach(outputExpressions::addAll); + analysis.setOutputExpressions(outputExpressions); if (outputExpressions.isEmpty()) { return finishQuery(queryStatement, analysis); } @@ -351,7 +352,6 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> analyzeInto(analysis, queryStatement, outputExpressions); } - analysis.setOutputExpressions(outputExpressions); analyzeGroupByTime(analysis, queryStatement); @@ -446,45 +446,43 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> for (Expression selectExpression : selectExpressions) { sourceExpressions.addAll( - ExpressionAnalyzer.removeWildcardAndViewInExpression( - selectExpression, analysis, schemaTree)); + ExpressionAnalyzer.bindSchemaForExpression(selectExpression, analysis, schemaTree)); } analysis.setSourceExpressions(sourceExpressions); } - private ISchemaTree findAllViewsInTreeThenReFetchAndMerge(ISchemaTree originSchemaTree) { - if (originSchemaTree.hasLogicalViewMeasurement()) { - PathPatternTree patternTree = new PathPatternTree(); - GetSourcePathsVisitor getSourcePathsVisitor = new GetSourcePathsVisitor(); - boolean needToReFetch = false; - try { - Pair<List<MeasurementPath>, Integer> tempPair = - originSchemaTree.searchMeasurementPaths(new PartialPath("root.**")); - for (MeasurementPath measurementPath : tempPair.left) { - if (measurementPath.getMeasurementSchema().isLogicalView()) { - LogicalViewSchema logicalViewSchema = - (LogicalViewSchema) measurementPath.getMeasurementSchema(); - ViewExpression viewExpression = logicalViewSchema.getExpression(); - List<PartialPath> pathsNeedToReFetch = - getSourcePathsVisitor.process(viewExpression, null); - for (PartialPath path : pathsNeedToReFetch) { - patternTree.appendFullPath(path); - needToReFetch = true; - } + private void findAllViewsInTreeThenReFetchAndMerge(ISchemaTree originSchemaTree) { + if (!originSchemaTree.hasLogicalViewMeasurement()) { + return; + } + + PathPatternTree patternTree = new PathPatternTree(); + boolean needToReFetch = false; + try { + Pair<List<MeasurementPath>, Integer> tempPair = + originSchemaTree.searchMeasurementPaths(new PartialPath("root.**")); + for (MeasurementPath measurementPath : tempPair.left) { + if (measurementPath.getMeasurementSchema().isLogicalView()) { + LogicalViewSchema logicalViewSchema = + (LogicalViewSchema) measurementPath.getMeasurementSchema(); + ViewExpression viewExpression = logicalViewSchema.getExpression(); + List<PartialPath> pathsNeedToReFetch = getSourcePaths(viewExpression); + for (PartialPath path : pathsNeedToReFetch) { + patternTree.appendFullPath(path); + needToReFetch = true; } } - } catch (Exception e) { - throw new RuntimeException(e); - } - if (needToReFetch) { - ISchemaTree viewSchemaTree = this.schemaFetcher.fetchSchema(patternTree, null); - originSchemaTree.mergeSchemaTree(viewSchemaTree); - Set<String> allDatabases = viewSchemaTree.getDatabases(); - allDatabases.addAll(originSchemaTree.getDatabases()); - originSchemaTree.setDatabases(allDatabases); } + } catch (Exception e) { + throw new RuntimeException(e); + } + if (needToReFetch) { + ISchemaTree viewSchemaTree = this.schemaFetcher.fetchSchema(patternTree, null); + originSchemaTree.mergeSchemaTree(viewSchemaTree); + Set<String> allDatabases = viewSchemaTree.getDatabases(); + allDatabases.addAll(originSchemaTree.getDatabases()); + originSchemaTree.setDatabases(allDatabases); } - return originSchemaTree; } private Map<Integer, List<Pair<Expression, String>>> analyzeSelect( @@ -507,7 +505,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> boolean hasAlias = resultColumn.hasAlias(); List<Expression> resultExpressions = - ExpressionAnalyzer.removeWildcardAndViewInExpression( + ExpressionAnalyzer.bindSchemaForExpression( resultColumn.getExpression(), analysis, schemaTree); for (Expression expression : resultExpressions) { @@ -525,17 +523,18 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> } else { Expression expressionWithoutAlias = ExpressionAnalyzer.removeAliasFromExpression(expression); - String alias = expression.getStringWithViewOfThisExpression(); - alias = hasAlias ? resultColumn.getAlias() : alias; if (hasAlias) { + String alias = resultColumn.getAlias(); if (aliasSet.contains(alias)) { throw new SemanticException( String.format("alias '%s' can only be matched with one time series", alias)); } aliasSet.add(alias); } + String outputColumnName = + hasAlias ? resultColumn.getAlias() : expression.getOutputSymbol(); analyzeExpression(analysis, expressionWithoutAlias); - outputExpressions.add(new Pair<>(expressionWithoutAlias, alias)); + outputExpressions.add(new Pair<>(expressionWithoutAlias, outputColumnName)); } paginationController.consumeLimit(); } else { @@ -585,7 +584,8 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> new LinkedHashMap<>(); for (PartialPath device : deviceSet) { List<Expression> selectExpressionsOfOneDevice = - ExpressionAnalyzer.concatDeviceAndRemoveWildcard(selectExpression, device, schemaTree); + ExpressionAnalyzer.concatDeviceAndBindSchemaForExpression( + selectExpression, device, schemaTree); if (selectExpressionsOfOneDevice.isEmpty()) { continue; } @@ -625,13 +625,10 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> // add outputExpressions Expression measurementExpressionWithoutAlias = ExpressionAnalyzer.removeAliasFromExpression(measurementExpression); - String alias = - !Objects.equals(measurementExpressionWithoutAlias, measurementExpression) - ? measurementExpression.getExpressionString() - : null; - alias = hasAlias ? resultColumn.getAlias() : alias; + String outputColumnName = + hasAlias ? resultColumn.getAlias() : measurementExpression.getOutputSymbol(); analyzeExpression(analysis, measurementExpressionWithoutAlias); - outputExpressions.add(new Pair<>(measurementExpressionWithoutAlias, alias)); + outputExpressions.add(new Pair<>(measurementExpressionWithoutAlias, outputColumnName)); // add deviceToSelectExpressions for (String deviceName : deviceToSelectExpressionsOfOneMeasurement.keySet()) { @@ -665,7 +662,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> // get removeWildcard Expressions in Having List<Expression> conJunctions = - ExpressionAnalyzer.removeWildcardAndViewInFilter( + ExpressionAnalyzer.bindSchemaForPredicate( queryStatement.getHavingCondition().getPredicate(), queryStatement.getFromComponent().getPrefixPaths(), schemaTree, @@ -698,7 +695,8 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> for (PartialPath device : deviceSet) { List<Expression> expressionsInHaving = - ExpressionAnalyzer.concatDeviceAndRemoveWildcard(havingExpression, device, schemaTree); + ExpressionAnalyzer.concatDeviceAndBindSchemaForExpression( + havingExpression, device, schemaTree); conJunctions.addAll( expressionsInHaving.stream() @@ -1117,7 +1115,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> return; } List<Expression> conJunctions = - ExpressionAnalyzer.removeWildcardAndViewInFilter( + ExpressionAnalyzer.bindSchemaForPredicate( queryStatement.getWhereCondition().getPredicate(), queryStatement.getFromComponent().getPrefixPaths(), schemaTree, @@ -1138,7 +1136,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> private Expression analyzeWhereSplitByDevice( QueryStatement queryStatement, PartialPath devicePath, ISchemaTree schemaTree) { List<Expression> conJunctions = - ExpressionAnalyzer.removeWildcardInFilterByDevice( + ExpressionAnalyzer.concatDeviceAndBindSchemaForPredicate( queryStatement.getWhereCondition().getPredicate(), devicePath, schemaTree, true); return ExpressionUtils.constructQueryFilter( conJunctions.stream().distinct().collect(Collectors.toList())); @@ -1190,7 +1188,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> List<String> deviceViewOutputColumns = deviceViewOutputExpressions.stream() - .map(Expression::getExpressionString) + .map(Expression::getOutputSymbol) .collect(Collectors.toList()); Map<String, Set<String>> deviceToOutputColumnsMap = new LinkedHashMap<>(); @@ -1205,11 +1203,25 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> outputColumns.add(ENDTIME); } for (Expression expression : outputExpressionsUnderDevice) { - outputColumns.add(ExpressionAnalyzer.getMeasurementExpression(expression).toString()); + outputColumns.add( + ExpressionAnalyzer.getMeasurementExpression(expression).getOutputSymbol()); } deviceToOutputColumnsMap.put(deviceName, outputColumns); } + Map<String, Set<Expression>> deviceToSourceExpressions = + analysis.getDeviceToSourceExpressions(); + Map<String, List<String>> outputDeviceToQueriedDeviceMap = new LinkedHashMap<>(); + for (String deviceName : deviceToSourceExpressions.keySet()) { + Set<Expression> sourceExpressionsUnderDevice = deviceToSourceExpressions.get(deviceName); + Set<String> queriedDevices = new HashSet<>(); + for (Expression expression : sourceExpressionsUnderDevice) { + queriedDevices.add(ExpressionAnalyzer.getDeviceNameInSourceExpression(expression)); + } + outputDeviceToQueriedDeviceMap.put(deviceName, new ArrayList<>(queriedDevices)); + } + analysis.setOutputDeviceToQueriedDeviceMap(outputDeviceToQueriedDeviceMap); + Map<String, List<Integer>> deviceViewInputIndexesMap = new HashMap<>(); for (String deviceName : deviceToOutputColumnsMap.keySet()) { List<String> outputsUnderDevice = new ArrayList<>(deviceToOutputColumnsMap.get(deviceName)); @@ -1286,8 +1298,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> for (Expression expressionForItem : queryStatement.getExpressionSortItemList()) { // Expression in a sortItem only indicates one column List<Expression> expressions = - ExpressionAnalyzer.removeWildcardAndViewInExpression( - expressionForItem, analysis, schemaTree); + ExpressionAnalyzer.bindSchemaForExpression(expressionForItem, analysis, schemaTree); if (expressions.size() != 1) { throw new SemanticException("One sort item in order by should only indicate one value"); } @@ -1324,7 +1335,8 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> Expression expression = groupByComponent.getControlColumnExpression(); for (PartialPath device : deviceSet) { List<Expression> groupByExpressionsOfOneDevice = - ExpressionAnalyzer.concatDeviceAndRemoveWildcard(expression, device, schemaTree); + ExpressionAnalyzer.concatDeviceAndBindSchemaForExpression( + expression, device, schemaTree); if (groupByExpressionsOfOneDevice.size() != 1) { throw new SemanticException("Expression in group by should indicate one value"); @@ -1395,7 +1407,8 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> Set<Expression> orderByExpressionsForOneDevice = new LinkedHashSet<>(); for (Expression expressionForItem : queryStatement.getExpressionSortItemList()) { List<Expression> expressions = - ExpressionAnalyzer.concatDeviceAndRemoveWildcard(expressionForItem, device, schemaTree); + ExpressionAnalyzer.concatDeviceAndBindSchemaForExpression( + expressionForItem, device, schemaTree); if (expressions.size() != 1) { throw new SemanticException( String.format( @@ -1441,8 +1454,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> groupByExpression = groupByComponent.getControlColumnExpression(); // Expression in group by variation clause only indicates one column List<Expression> expressions = - ExpressionAnalyzer.removeWildcardAndViewInExpression( - groupByExpression, analysis, schemaTree); + ExpressionAnalyzer.bindSchemaForExpression(groupByExpression, analysis, schemaTree); if (expressions.size() != 1) { throw new SemanticException("Expression in group by should indicate one value"); } @@ -1558,7 +1570,10 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> Analysis analysis, QueryStatement queryStatement, ISchemaTree schemaTree) { Set<String> deviceSet = new HashSet<>(); if (queryStatement.isAlignByDevice()) { - deviceSet = analysis.getDeviceToSourceExpressions().keySet(); + deviceSet = + analysis.getOutputDeviceToQueriedDeviceMap().values().stream() + .flatMap(List::stream) + .collect(Collectors.toSet()); } else { for (Expression expression : analysis.getSourceExpressions()) { deviceSet.add(ExpressionAnalyzer.getDeviceNameInSourceExpression(expression));
