This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch ty/TryOpt in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d008c227648f9e170ab6d56011ae770a6632e7ae Author: JackieTien97 <[email protected]> AuthorDate: Tue Sep 3 16:06:10 2024 +0800 PruneColumn firstly in RelationPlanner --- .../plan/relational/analyzer/Analysis.java | 24 ++++++ .../relational/analyzer/StatementAnalyzer.java | 23 ++++-- .../plan/relational/planner/RelationPlanner.java | 29 ++++--- .../optimizations/LogicalOptimizeFactory.java | 89 ++++++++++++++++++++-- 4 files changed, 140 insertions(+), 25 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java index 87e7fe15548..6250f394cf4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java @@ -70,6 +70,7 @@ import javax.annotation.Nullable; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -479,6 +480,9 @@ public class Analysis implements IAnalysis { } private <T extends Node> List<T> dereference(Collection<NodeRef<T>> nodeRefs) { + if (nodeRefs.isEmpty()) { + return Collections.emptyList(); + } return nodeRefs.stream().map(NodeRef::getNode).collect(toImmutableList()); } @@ -517,6 +521,26 @@ public class Analysis implements IAnalysis { return columnReferences.containsKey(NodeRef.of(expression)); } + public boolean hasFieldReference(QualifiedObjectName tableName, String fieldName) { + for (Map<QualifiedObjectName, Set<String>> map : tableColumnReferences.values()) { + Set<String> fields = map.get(tableName); + if (fields != null && fields.contains(fieldName)) { + return true; + } + } + return false; + } + + public Set<String> getUsedColumns(QualifiedObjectName tableName) { + for (Map<QualifiedObjectName, Set<String>> map : tableColumnReferences.values()) { + Set<String> fields = map.get(tableName); + if (fields != null) { + return fields; + } + } + return Collections.emptySet(); + } + public void addTableColumnReferences( AccessControl accessControl, Identity identity, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index a2b91ac794d..df2ee265edf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -1381,10 +1381,18 @@ public class StatementAnalyzer { } if (name != null) { - List<Field> matchingFields = sourceScope.getRelationType().resolveFields(name); - if (!matchingFields.isEmpty()) { - originTable = matchingFields.get(0).getOriginTable(); - originColumn = matchingFields.get(0).getOriginColumnName(); + Field matchingField = null; + try { + matchingField = analysis.getResolvedField(expression).getField(); + } catch (IllegalArgumentException e) { + List<Field> matchingFields = sourceScope.getRelationType().resolveFields(name); + if (!matchingFields.isEmpty()) { + matchingField = matchingFields.get(0); + } + } + if (matchingField != null) { + originTable = matchingField.getOriginTable(); + originColumn = matchingField.getOriginColumnName(); } } @@ -1582,15 +1590,14 @@ public class StatementAnalyzer { List<Field> outputFields = fields.build(); + RelationType relationType = new RelationType(outputFields); Scope accessControlScope = - Scope.builder() - .withRelationType(RelationId.anonymous(), new RelationType(outputFields)) - .build(); + Scope.builder().withRelationType(RelationId.anonymous(), relationType).build(); // analyzeFiltersAndMasks(table, name, new RelationType(outputFields), // accessControlScope); analysis.registerTable(table, tableSchema, name); - Scope tableScope = createAndAssignScope(table, scope, outputFields); + Scope tableScope = createAndAssignScope(table, scope, relationType); // if (addRowIdColumn) { // FieldReference reference = new FieldReference(outputFields.size() - 1); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index cdfb2023ae9..5a112f3c911 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -60,6 +60,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import static java.util.Objects.requireNonNull; @@ -128,14 +129,32 @@ public class RelationPlanner extends AstVisitor<RelationPlan, Void> { ImmutableList.Builder<Symbol> outputSymbolsBuilder = ImmutableList.builder(); ImmutableMap.Builder<Symbol, ColumnSchema> symbolToColumnSchema = ImmutableMap.builder(); Collection<Field> fields = scope.getRelationType().getAllFields(); + QualifiedName qualifiedName = analysis.getRelationName(table); + if (!qualifiedName.getPrefix().isPresent()) { + throw new IllegalStateException("Table " + table.getName() + " has no prefix!"); + } + + QualifiedObjectName qualifiedObjectName = + new QualifiedObjectName( + qualifiedName.getPrefix().map(QualifiedName::toString).orElse(null), + qualifiedName.getSuffix()); + + Set<String> usedColumns = analysis.getUsedColumns(qualifiedObjectName); + // on the basis of that the order of fields is same with the column category order of segments // in DeviceEntry Map<Symbol, Integer> idAndAttributeIndexMap = new HashMap<>(); int idIndex = 0; for (Field field : fields) { + TsTableColumnCategory category = field.getColumnCategory(); + // only keep used columns and all ID columns + if (category != TsTableColumnCategory.ID + && field.getOriginColumnName().isPresent() + && usedColumns.contains(field.getOriginColumnName().get())) { + continue; + } Symbol symbol = symbolAllocator.newSymbol(field); outputSymbolsBuilder.add(symbol); - TsTableColumnCategory category = field.getColumnCategory(); symbolToColumnSchema.put( symbol, new ColumnSchema( @@ -146,15 +165,7 @@ public class RelationPlanner extends AstVisitor<RelationPlan, Void> { } List<Symbol> outputSymbols = outputSymbolsBuilder.build(); - QualifiedName qualifiedName = analysis.getRelationName(table); - if (!qualifiedName.getPrefix().isPresent()) { - throw new IllegalStateException("Table " + table.getName() + " has no prefix!"); - } - QualifiedObjectName qualifiedObjectName = - new QualifiedObjectName( - qualifiedName.getPrefix().map(QualifiedName::toString).orElse(null), - qualifiedName.getSuffix()); Map<Symbol, ColumnSchema> tableColumnSchema = symbolToColumnSchema.build(); analysis.addTableSchema(qualifiedObjectName, tableColumnSchema); TableScanNode tableScanNode = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java index 8acede85ba5..415fa40d6a6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java @@ -53,6 +53,7 @@ public class LogicalOptimizeFactory { public LogicalOptimizeFactory(PlannerContext plannerContext) { IrTypeAnalyzer typeAnalyzer = new IrTypeAnalyzer(plannerContext); Metadata metadata = plannerContext.getMetadata(); + RuleStatsRecorder ruleStats = new RuleStatsRecorder(); PlanOptimizer pushPredicateIntoTableScanOptimizer = new PushPredicateIntoTableScan(); PlanOptimizer transformSortToStreamSortOptimizer = new TransformSortToStreamSort(); @@ -68,12 +69,31 @@ public class LogicalOptimizeFactory { new PruneTableScanColumns(plannerContext.getMetadata()), new PruneTopKColumns()); IterativeOptimizer columnPruningOptimizer = - new IterativeOptimizer(plannerContext, new RuleStatsRecorder(), columnPruningRules); + new IterativeOptimizer(plannerContext, ruleStats, columnPruningRules); + + // Set<Rule<?>> projectionPushdownRules = ImmutableSet.of( + // new PushProjectionThroughUnion(), + // new PushProjectionThroughExchange(), + // // Dereference pushdown rules + // new PushDownDereferencesThroughMarkDistinct(typeAnalyzer), + // new PushDownDereferenceThroughProject(typeAnalyzer), + // new PushDownDereferenceThroughUnnest(typeAnalyzer), + // new PushDownDereferenceThroughSemiJoin(typeAnalyzer), + // new PushDownDereferenceThroughJoin(typeAnalyzer), + // new PushDownDereferenceThroughFilter(typeAnalyzer), + // new ExtractDereferencesFromFilterAboveScan(typeAnalyzer), + // new PushDownDereferencesThroughLimit(typeAnalyzer), + // new PushDownDereferencesThroughSort(typeAnalyzer), + // new PushDownDereferencesThroughAssignUniqueId(typeAnalyzer), + // new PushDownDereferencesThroughWindow(typeAnalyzer), + // new PushDownDereferencesThroughTopN(typeAnalyzer), + // new PushDownDereferencesThroughRowNumber(typeAnalyzer), + // new PushDownDereferencesThroughTopNRanking(typeAnalyzer)); IterativeOptimizer inlineProjectionLimitFiltersOptimizer = new IterativeOptimizer( plannerContext, - new RuleStatsRecorder(), + ruleStats, ImmutableSet.of( new InlineProjections(plannerContext), new RemoveRedundantIdentityProjections(), @@ -88,24 +108,77 @@ public class LogicalOptimizeFactory { .add(new RemoveTrivialFilters()) .build(); IterativeOptimizer simplifyOptimizer = - new IterativeOptimizer(plannerContext, new RuleStatsRecorder(), simplifyOptimizerRules); + new IterativeOptimizer(plannerContext, ruleStats, simplifyOptimizerRules); + Set<Rule<?>> limitPushdownRules = + ImmutableSet.of(new PushLimitThroughOffset(), new PushLimitThroughProject()); IterativeOptimizer limitPushdownOptimizer = - new IterativeOptimizer( - plannerContext, - new RuleStatsRecorder(), - ImmutableSet.of(new PushLimitThroughOffset(), new PushLimitThroughProject())); + new IterativeOptimizer(plannerContext, ruleStats, limitPushdownRules); PlanOptimizer pushLimitOffsetIntoTableScanOptimizer = new PushLimitOffsetIntoTableScan(); IterativeOptimizer topKOptimizer = new IterativeOptimizer( plannerContext, - new RuleStatsRecorder(), + ruleStats, ImmutableSet.of(new MergeLimitWithSort(), new MergeLimitOverProjectWithSort())); this.planOptimizers = ImmutableList.of( + new IterativeOptimizer( + plannerContext, + ruleStats, + ImmutableSet.<Rule<?>>builder() + .addAll(columnPruningRules) + // .addAll(projectionPushdownRules) + // .addAll(new UnwrapRowSubscript().rules()) + // .addAll(new PushCastIntoRow().rules()) + .addAll( + ImmutableSet.of( + new MergeFilters(), + new InlineProjections(plannerContext), + new RemoveRedundantIdentityProjections(), + new MergeLimits(), + new RemoveTrivialFilters() + // new RemoveRedundantLimit(), + // new RemoveRedundantOffset(), + // new RemoveRedundantSort(), + // new RemoveRedundantSortBelowLimitWithTies(), + // new RemoveRedundantTopN(), + // new RemoveRedundantDistinctLimit(), + // new ReplaceRedundantJoinWithSource(), + // new RemoveRedundantJoin(), + // new ReplaceRedundantJoinWithProject(), + // new RemoveRedundantExists(), + // new RemoveRedundantWindow(), + // new SingleDistinctAggregationToGroupBy(), + // new MergeLimitWithDistinct(), + // new PruneCountAggregationOverScalar(metadata), + // new SimplifyCountOverConstant(plannerContext), + // new + // PreAggregateCaseAggregations(plannerContext, typeAnalyzer))) + )) + .build()), + // MergeUnion and related projection pruning rules must run before limit pushdown rules, + // otherwise + // an intermediate limit node will prevent unions from being merged later on + new IterativeOptimizer( + plannerContext, + ruleStats, + ImmutableSet.<Rule<?>>builder() + // .addAll(projectionPushdownRules) + .addAll(columnPruningRules) + .addAll(limitPushdownRules) + .addAll( + ImmutableSet.of( + // new MergeUnion(), + // new RemoveEmptyUnionBranches(), + new MergeFilters(), + new RemoveTrivialFilters(), + new MergeLimits(), + new InlineProjections(plannerContext), + new RemoveRedundantIdentityProjections())) + .build()), simplifyOptimizer, columnPruningOptimizer, inlineProjectionLimitFiltersOptimizer,
