This is an automated email from the ASF dual-hosted git repository. caogaofei pushed a commit to branch TableModelGrammar0606 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ed474142d3e4edef52f6237f8915e71ee6cfeed3 Author: Beyyes <[email protected]> AuthorDate: Thu Jun 6 22:44:44 2024 +0800 cp from new-table-model-debug --- .../db/queryengine/common/MPPQueryContext.java | 4 + .../relational/ColumnTransformerBuilder.java | 2 +- ....java => PredicatePushIntoMetadataChecker.java} | 8 +- .../planner/optimizations/IndexScan.java | 49 ++++++++-- .../planner/optimizations/PredicatePushDown.java | 105 ++++++++++++++++++++- .../plan/optimization/PredicatePushDownTest.java | 2 +- 6 files changed, 154 insertions(+), 16 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java index 988224da729..bafd57aa3db 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; import org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider; import org.apache.iotdb.db.queryengine.plan.analyze.lock.SchemaLockType; import org.apache.iotdb.db.queryengine.plan.planner.LocalExecutionPlanner; +import org.apache.iotdb.db.queryengine.plan.relational.sql.tree.Expression; import org.apache.iotdb.db.queryengine.statistics.QueryPlanStatistics; import org.apache.tsfile.read.filter.basic.Filter; @@ -89,6 +90,9 @@ public class MPPQueryContext { private final LocalExecutionPlanner LOCAL_EXECUTION_PLANNER = LocalExecutionPlanner.getInstance(); + // 0 : metadataExpressions, 1 : expressionsPushDownToOperator, 2 : expressionsCannotPushDown + public List<List<Expression>> splitPredicateExpression; + public MPPQueryContext(QueryId queryId) { this.queryId = queryId; this.endPointBlackList = new LinkedList<>(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java index fb405d1c62e..b911e2f0da4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java @@ -113,7 +113,7 @@ import java.util.Optional; import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; -import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoIndexScanChecker.isStringLiteral; +import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoMetadataChecker.isStringLiteral; import static org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager.getTSDataType; import static org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignatureTranslator.toTypeSignature; import static org.apache.tsfile.read.common.type.BinaryType.TEXT; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicatePushIntoIndexScanChecker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicatePushIntoMetadataChecker.java similarity index 93% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicatePushIntoIndexScanChecker.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicatePushIntoMetadataChecker.java index f7d2c1764a3..4561b6a838f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicatePushIntoIndexScanChecker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicatePushIntoMetadataChecker.java @@ -40,11 +40,15 @@ import java.util.Set; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoScanChecker.isSymbolReference; -public class PredicatePushIntoIndexScanChecker extends PredicateVisitor<Boolean, Void> { +public class PredicatePushIntoMetadataChecker extends PredicateVisitor<Boolean, Void> { private final Set<String> idOrAttributeColumnNames; - public PredicatePushIntoIndexScanChecker(Set<String> idOrAttributeColumnNames) { + public static boolean check(Set<String> idOrAttributeColumnNames, Expression expression) { + return new PredicatePushIntoMetadataChecker(idOrAttributeColumnNames).process(expression); + } + + public PredicatePushIntoMetadataChecker(Set<String> idOrAttributeColumnNames) { this.idOrAttributeColumnNames = idOrAttributeColumnNames; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/IndexScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/IndexScan.java index fd3fe00bd9f..db728f9ae52 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/IndexScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/IndexScan.java @@ -27,7 +27,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.IPartitionFetcher; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis; -import org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoIndexScanChecker; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoMetadataChecker; import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName; @@ -85,20 +85,28 @@ public class IndexScan implements RelationalPlanOptimizer { @Override public PlanNode visitFilter(FilterNode node, RewriterContext context) { context.setPredicate(node.getPredicate()); + context.setFilterNode(node); node.getChild().accept(this, context); return node; } @Override public PlanNode visitTableScan(TableScanNode node, RewriterContext context) { - List<String> attributeColumns = - node.getAssignments().entrySet().stream() - .filter(e -> e.getValue().getColumnCategory().equals(ATTRIBUTE)) - .map(e -> e.getKey().getName()) - .collect(Collectors.toList()); - List<Expression> conjExpressions = getConjunctionExpressions(context.getPredicate(), node); + // only when exist diff predicate in FilterNode, context.predicate will not equal null + if (context.predicate == null) { + context.predicate = node.getPushDownPredicate(); + } + + List<Expression> conjExpressions = getConjunctionExpressions(context, node); + node.setPushDownPredicate(context.getPredicate()); String dbName = context.getSessionInfo().getDatabaseName().get(); + List<String> attributeColumns = + node.getOutputSymbols().stream() + .filter( + symbol -> ATTRIBUTE.equals(node.getAssignments().get(symbol).getColumnCategory())) + .map(Symbol::getName) + .collect(Collectors.toList()); List<DeviceEntry> deviceEntries = context .getMetadata() @@ -152,7 +160,8 @@ public class IndexScan implements RelationalPlanOptimizer { } private static List<Expression> getConjunctionExpressions( - Expression predicate, TableScanNode node) { + RewriterContext context, TableScanNode node) { + Expression predicate = context.predicate; if (predicate == null) { return Collections.emptyList(); } @@ -164,20 +173,31 @@ public class IndexScan implements RelationalPlanOptimizer { if (predicate instanceof LogicalExpression && ((LogicalExpression) predicate).getOperator() == LogicalExpression.Operator.AND) { List<Expression> resultExpressions = new ArrayList<>(); + List<Expression> beExpressions = new ArrayList<>(); for (Expression subExpression : ((LogicalExpression) predicate).getTerms()) { if (Boolean.TRUE.equals( - new PredicatePushIntoIndexScanChecker(idOrAttributeColumnNames) + new PredicatePushIntoMetadataChecker(idOrAttributeColumnNames) .process(subExpression))) { resultExpressions.add(subExpression); + } else { + beExpressions.add(subExpression); } } + if (beExpressions.isEmpty()) { + context.predicate = null; + } else if (beExpressions.size() == 1) { + context.predicate = beExpressions.get(0); + } else { + context.predicate = new LogicalExpression(LogicalExpression.Operator.AND, beExpressions); + } return resultExpressions; } if (Boolean.FALSE.equals( - new PredicatePushIntoIndexScanChecker(idOrAttributeColumnNames).process(predicate))) { + new PredicatePushIntoMetadataChecker(idOrAttributeColumnNames).process(predicate))) { return Collections.emptyList(); } else { + context.predicate = null; return Collections.singletonList(predicate); } } @@ -219,6 +239,7 @@ public class IndexScan implements RelationalPlanOptimizer { private final Analysis analysis; private final IPartitionFetcher partitionFetcher; private final MPPQueryContext queryContext; + private FilterNode filterNode; RewriterContext( Expression predicate, @@ -266,5 +287,13 @@ public class IndexScan implements RelationalPlanOptimizer { public MPPQueryContext getQueryContext() { return queryContext; } + + public FilterNode getFilterNode() { + return filterNode; + } + + public void setFilterNode(FilterNode filterNode) { + this.filterNode = filterNode; + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PredicatePushDown.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PredicatePushDown.java index ae05611ce08..842448be624 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PredicatePushDown.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PredicatePushDown.java @@ -22,15 +22,23 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.MultiChildProcessNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.SingleChildProcessNode; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoMetadataChecker; import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.tree.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.tree.FunctionCall; +import org.apache.iotdb.db.queryengine.plan.relational.sql.tree.IsNullPredicate; +import org.apache.iotdb.db.queryengine.plan.relational.sql.tree.LogicalExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.tree.Node; +import org.apache.iotdb.db.queryengine.plan.relational.sql.tree.NotExpression; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; /** Push down predicate to TableScanNode as possible. */ public class PredicatePushDown implements RelationalPlanOptimizer { @@ -78,9 +86,12 @@ public class PredicatePushDown implements RelationalPlanOptimizer { @Override public PlanNode visitFilter(FilterNode node, RewriterContext context) { + context.filterNode = node; + if (node.getPredicate() != null) { // when exist diff function, predicate can not be pushed down if (containsDiffFunction(node.getPredicate())) { + context.pushDownPredicate = null; return node; } @@ -91,17 +102,85 @@ public class PredicatePushDown implements RelationalPlanOptimizer { return node.getChild(); } + // when reach this case? node.getChild().accept(this, context); - return node; + return node.getChild(); } @Override public PlanNode visitTableScan(TableScanNode node, RewriterContext context) { - node.setPushDownPredicate(context.pushDownPredicate); + // has diff in FilterNode + if (context.pushDownPredicate == null) { + node.setPushDownPredicate(null); + return node; + } + + context.queryContext.splitPredicateExpression = splitConjunctionExpressions(context, node); + + if (context.queryContext.splitPredicateExpression.get(1).size() > 0) { + List<Expression> expressions = context.queryContext.splitPredicateExpression.get(1); + node.setPushDownPredicate( + expressions.size() == 1 + ? expressions.get(0) + : new LogicalExpression(LogicalExpression.Operator.AND, expressions)); + } else { + node.setPushDownPredicate(null); + } + + // exist expressions can not push down + if (context.queryContext.splitPredicateExpression.get(2).size() > 0) { + List<Expression> expressions = context.queryContext.splitPredicateExpression.get(2); + return new FilterNode( + context.queryContext.getQueryId().genPlanNodeId(), + node, + expressions.size() == 1 + ? expressions.get(0) + : new LogicalExpression(LogicalExpression.Operator.AND, expressions)); + } + return node; } } + private static List<List<Expression>> splitConjunctionExpressions( + RewriterContext context, TableScanNode node) { + Expression predicate = context.pushDownPredicate; + + Set<String> idOrAttributeColumnNames = + node.getIdAndAttributeIndexMap().keySet().stream() + .map(Symbol::getName) + .collect(Collectors.toSet()); + + List<Expression> metadataExpressions = new ArrayList<>(); + List<Expression> expressionsCanPushDownToOperator = new ArrayList<>(); + List<Expression> expressionsCannotPushDown = new ArrayList<>(); + + if (predicate instanceof LogicalExpression + && ((LogicalExpression) predicate).getOperator() == LogicalExpression.Operator.AND) { + + for (Expression expression : ((LogicalExpression) predicate).getTerms()) { + if (PredicatePushIntoMetadataChecker.check(idOrAttributeColumnNames, expression)) { + metadataExpressions.add(expression); + } else if (cannotPushDownToOperator(expression)) { + expressionsCannotPushDown.add(expression); + } else { + expressionsCanPushDownToOperator.add(expression); + } + } + } + + if (PredicatePushIntoMetadataChecker.check(idOrAttributeColumnNames, predicate)) { + metadataExpressions.add(predicate); + } else if (cannotPushDownToOperator(predicate)) { + expressionsCannotPushDown.add(predicate); + } else { + expressionsCanPushDownToOperator.add(predicate); + } + + return Arrays.asList( + metadataExpressions, expressionsCanPushDownToOperator, expressionsCannotPushDown); + } + static boolean containsDiffFunction(Expression expression) { if (expression instanceof FunctionCall && "diff".equalsIgnoreCase(((FunctionCall) expression).getName().toString())) { @@ -119,7 +198,29 @@ public class PredicatePushDown implements RelationalPlanOptimizer { return false; } + /** + * When filter exists NotExpression or IsNullPredicate, this filter can not be pushed down into + * Operator + */ + static boolean cannotPushDownToOperator(Expression expression) { + if (expression instanceof NotExpression || expression instanceof IsNullPredicate) { + return true; + } + + if (!expression.getChildren().isEmpty()) { + for (Node node : expression.getChildren()) { + if (containsDiffFunction((Expression) node)) { + return true; + } + } + } + + return false; + } + private static class RewriterContext { Expression pushDownPredicate; + MPPQueryContext queryContext; + FilterNode filterNode; } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/optimization/PredicatePushDownTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/optimization/PredicatePushDownTest.java index 9d67070dc47..dc6cc08d7dd 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/optimization/PredicatePushDownTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/optimization/PredicatePushDownTest.java @@ -295,7 +295,7 @@ public class PredicatePushDownTest { } @Test - public void testCannotPushDownAlignByTime() { + public void testCannotPushDownToOperatorAlignByTime() { checkCannotPushDown( "select s1, s2 from root.sg.d1 where time > 100 and (s1 > 10 or s2 > 10)", new TestPlanBuilder()
