This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 0f6e0d6c7a2 PruneColumn firstly in RelationPlanner
0f6e0d6c7a2 is described below
commit 0f6e0d6c7a2f0973fa2ed7e3c726bd5cda09e900
Author: Jackie Tien <[email protected]>
AuthorDate: Wed Sep 4 11:47:31 2024 +0800
PruneColumn firstly in RelationPlanner
---
.../plan/relational/analyzer/Analysis.java | 14 ++++
.../relational/analyzer/StatementAnalyzer.java | 23 ++++--
.../plan/relational/planner/RelationPlanner.java | 29 ++++---
.../plan/relational/planner/TranslationMap.java | 17 +++--
.../optimizations/LogicalOptimizeFactory.java | 89 ++++++++++++++++++++--
.../plan/relational/security/Identity.java | 19 +++++
.../plan/relational/analyzer/AnalyzerTest.java | 21 ++---
7 files changed, 171 insertions(+), 41 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java
index 87e7fe15548..4fca93c199e 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java
@@ -70,6 +70,7 @@ import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
@@ -479,6 +480,9 @@ public class Analysis implements IAnalysis {
}
private <T extends Node> List<T> dereference(Collection<NodeRef<T>>
nodeRefs) {
+ if (nodeRefs.isEmpty()) {
+ return Collections.emptyList();
+ }
return nodeRefs.stream().map(NodeRef::getNode).collect(toImmutableList());
}
@@ -517,6 +521,16 @@ public class Analysis implements IAnalysis {
return columnReferences.containsKey(NodeRef.of(expression));
}
+ public Set<String> getUsedColumns(QualifiedObjectName tableName) {
+ for (Map<QualifiedObjectName, Set<String>> map :
tableColumnReferences.values()) {
+ Set<String> fields = map.get(tableName);
+ if (fields != null) {
+ return fields;
+ }
+ }
+ return Collections.emptySet();
+ }
+
public void addTableColumnReferences(
AccessControl accessControl,
Identity identity,
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java
index a2b91ac794d..df2ee265edf 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java
@@ -1381,10 +1381,18 @@ public class StatementAnalyzer {
}
if (name != null) {
- List<Field> matchingFields =
sourceScope.getRelationType().resolveFields(name);
- if (!matchingFields.isEmpty()) {
- originTable = matchingFields.get(0).getOriginTable();
- originColumn = matchingFields.get(0).getOriginColumnName();
+ Field matchingField = null;
+ try {
+ matchingField = analysis.getResolvedField(expression).getField();
+ } catch (IllegalArgumentException e) {
+ List<Field> matchingFields =
sourceScope.getRelationType().resolveFields(name);
+ if (!matchingFields.isEmpty()) {
+ matchingField = matchingFields.get(0);
+ }
+ }
+ if (matchingField != null) {
+ originTable = matchingField.getOriginTable();
+ originColumn = matchingField.getOriginColumnName();
}
}
@@ -1582,15 +1590,14 @@ public class StatementAnalyzer {
List<Field> outputFields = fields.build();
+ RelationType relationType = new RelationType(outputFields);
Scope accessControlScope =
- Scope.builder()
- .withRelationType(RelationId.anonymous(), new
RelationType(outputFields))
- .build();
+ Scope.builder().withRelationType(RelationId.anonymous(),
relationType).build();
// analyzeFiltersAndMasks(table, name, new
RelationType(outputFields),
// accessControlScope);
analysis.registerTable(table, tableSchema, name);
- Scope tableScope = createAndAssignScope(table, scope, outputFields);
+ Scope tableScope = createAndAssignScope(table, scope, relationType);
// if (addRowIdColumn) {
// FieldReference reference = new
FieldReference(outputFields.size() - 1);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java
index cdfb2023ae9..d04e1bd7533 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java
@@ -60,6 +60,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.Set;
import static java.util.Objects.requireNonNull;
@@ -128,14 +129,32 @@ public class RelationPlanner extends
AstVisitor<RelationPlan, Void> {
ImmutableList.Builder<Symbol> outputSymbolsBuilder =
ImmutableList.builder();
ImmutableMap.Builder<Symbol, ColumnSchema> symbolToColumnSchema =
ImmutableMap.builder();
Collection<Field> fields = scope.getRelationType().getAllFields();
+ QualifiedName qualifiedName = analysis.getRelationName(table);
+ if (!qualifiedName.getPrefix().isPresent()) {
+ throw new IllegalStateException("Table " + table.getName() + " has no
prefix!");
+ }
+
+ QualifiedObjectName qualifiedObjectName =
+ new QualifiedObjectName(
+
qualifiedName.getPrefix().map(QualifiedName::toString).orElse(null),
+ qualifiedName.getSuffix());
+
+ Set<String> usedColumns = analysis.getUsedColumns(qualifiedObjectName);
+
// on the basis of that the order of fields is same with the column
category order of segments
// in DeviceEntry
Map<Symbol, Integer> idAndAttributeIndexMap = new HashMap<>();
int idIndex = 0;
for (Field field : fields) {
+ TsTableColumnCategory category = field.getColumnCategory();
+ // only keep used columns and all ID columns
+ if (category != TsTableColumnCategory.ID
+ && field.getOriginColumnName().isPresent()
+ && !usedColumns.contains(field.getOriginColumnName().get())) {
+ continue;
+ }
Symbol symbol = symbolAllocator.newSymbol(field);
outputSymbolsBuilder.add(symbol);
- TsTableColumnCategory category = field.getColumnCategory();
symbolToColumnSchema.put(
symbol,
new ColumnSchema(
@@ -146,15 +165,7 @@ public class RelationPlanner extends
AstVisitor<RelationPlan, Void> {
}
List<Symbol> outputSymbols = outputSymbolsBuilder.build();
- QualifiedName qualifiedName = analysis.getRelationName(table);
- if (!qualifiedName.getPrefix().isPresent()) {
- throw new IllegalStateException("Table " + table.getName() + " has no
prefix!");
- }
- QualifiedObjectName qualifiedObjectName =
- new QualifiedObjectName(
-
qualifiedName.getPrefix().map(QualifiedName::toString).orElse(null),
- qualifiedName.getSuffix());
Map<Symbol, ColumnSchema> tableColumnSchema = symbolToColumnSchema.build();
analysis.addTableSchema(qualifiedObjectName, tableColumnSchema);
TableScanNode tableScanNode =
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TranslationMap.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TranslationMap.java
index 0e2af0b70fe..715111907f6 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TranslationMap.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TranslationMap.java
@@ -43,7 +43,6 @@ import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
-import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.base.Verify.verify;
import static java.lang.String.format;
@@ -131,11 +130,11 @@ public class TranslationMap {
requireNonNull(astToSymbols, "astToSymbols is null");
this.astToSymbols = ImmutableMap.copyOf(astToSymbols);
- checkArgument(
- scope.getLocalScopeFieldCount() == fieldSymbols.length,
- "scope: %s, fields mappings: %s",
- scope.getRelationType().getAllFieldCount(),
- fieldSymbols.length);
+ // checkArgument(
+ // scope.getLocalScopeFieldCount() == fieldSymbols.length,
+ // "scope: %s, fields mappings: %s",
+ // scope.getRelationType().getAllFieldCount(),
+ // fieldSymbols.length);
astToSymbols.keySet().stream()
.map(ScopeAware::getNode)
@@ -383,7 +382,11 @@ public class TranslationMap {
ResolvedField field =
analysis.getColumnReferenceFields().get(NodeRef.of(expression));
if (scope.isLocalScope(field.getScope())) {
- return Optional.of(fieldSymbols[field.getHierarchyFieldIndex()]);
+ if (field.getField().getOriginColumnName().isPresent()) {
+ return
Optional.of(Symbol.of(field.getField().getOriginColumnName().get()));
+ } else {
+ return Optional.of(fieldSymbols[field.getHierarchyFieldIndex()]);
+ }
}
if (outerContext.isPresent()) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java
index 8acede85ba5..415fa40d6a6 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/LogicalOptimizeFactory.java
@@ -53,6 +53,7 @@ public class LogicalOptimizeFactory {
public LogicalOptimizeFactory(PlannerContext plannerContext) {
IrTypeAnalyzer typeAnalyzer = new IrTypeAnalyzer(plannerContext);
Metadata metadata = plannerContext.getMetadata();
+ RuleStatsRecorder ruleStats = new RuleStatsRecorder();
PlanOptimizer pushPredicateIntoTableScanOptimizer = new
PushPredicateIntoTableScan();
PlanOptimizer transformSortToStreamSortOptimizer = new
TransformSortToStreamSort();
@@ -68,12 +69,31 @@ public class LogicalOptimizeFactory {
new PruneTableScanColumns(plannerContext.getMetadata()),
new PruneTopKColumns());
IterativeOptimizer columnPruningOptimizer =
- new IterativeOptimizer(plannerContext, new RuleStatsRecorder(),
columnPruningRules);
+ new IterativeOptimizer(plannerContext, ruleStats, columnPruningRules);
+
+ // Set<Rule<?>> projectionPushdownRules = ImmutableSet.of(
+ // new PushProjectionThroughUnion(),
+ // new PushProjectionThroughExchange(),
+ // // Dereference pushdown rules
+ // new PushDownDereferencesThroughMarkDistinct(typeAnalyzer),
+ // new PushDownDereferenceThroughProject(typeAnalyzer),
+ // new PushDownDereferenceThroughUnnest(typeAnalyzer),
+ // new PushDownDereferenceThroughSemiJoin(typeAnalyzer),
+ // new PushDownDereferenceThroughJoin(typeAnalyzer),
+ // new PushDownDereferenceThroughFilter(typeAnalyzer),
+ // new ExtractDereferencesFromFilterAboveScan(typeAnalyzer),
+ // new PushDownDereferencesThroughLimit(typeAnalyzer),
+ // new PushDownDereferencesThroughSort(typeAnalyzer),
+ // new PushDownDereferencesThroughAssignUniqueId(typeAnalyzer),
+ // new PushDownDereferencesThroughWindow(typeAnalyzer),
+ // new PushDownDereferencesThroughTopN(typeAnalyzer),
+ // new PushDownDereferencesThroughRowNumber(typeAnalyzer),
+ // new PushDownDereferencesThroughTopNRanking(typeAnalyzer));
IterativeOptimizer inlineProjectionLimitFiltersOptimizer =
new IterativeOptimizer(
plannerContext,
- new RuleStatsRecorder(),
+ ruleStats,
ImmutableSet.of(
new InlineProjections(plannerContext),
new RemoveRedundantIdentityProjections(),
@@ -88,24 +108,77 @@ public class LogicalOptimizeFactory {
.add(new RemoveTrivialFilters())
.build();
IterativeOptimizer simplifyOptimizer =
- new IterativeOptimizer(plannerContext, new RuleStatsRecorder(),
simplifyOptimizerRules);
+ new IterativeOptimizer(plannerContext, ruleStats,
simplifyOptimizerRules);
+ Set<Rule<?>> limitPushdownRules =
+ ImmutableSet.of(new PushLimitThroughOffset(), new
PushLimitThroughProject());
IterativeOptimizer limitPushdownOptimizer =
- new IterativeOptimizer(
- plannerContext,
- new RuleStatsRecorder(),
- ImmutableSet.of(new PushLimitThroughOffset(), new
PushLimitThroughProject()));
+ new IterativeOptimizer(plannerContext, ruleStats, limitPushdownRules);
PlanOptimizer pushLimitOffsetIntoTableScanOptimizer = new
PushLimitOffsetIntoTableScan();
IterativeOptimizer topKOptimizer =
new IterativeOptimizer(
plannerContext,
- new RuleStatsRecorder(),
+ ruleStats,
ImmutableSet.of(new MergeLimitWithSort(), new
MergeLimitOverProjectWithSort()));
this.planOptimizers =
ImmutableList.of(
+ new IterativeOptimizer(
+ plannerContext,
+ ruleStats,
+ ImmutableSet.<Rule<?>>builder()
+ .addAll(columnPruningRules)
+ // .addAll(projectionPushdownRules)
+ // .addAll(new
UnwrapRowSubscript().rules())
+ // .addAll(new
PushCastIntoRow().rules())
+ .addAll(
+ ImmutableSet.of(
+ new MergeFilters(),
+ new InlineProjections(plannerContext),
+ new RemoveRedundantIdentityProjections(),
+ new MergeLimits(),
+ new RemoveTrivialFilters()
+ // new
RemoveRedundantLimit(),
+ // new
RemoveRedundantOffset(),
+ // new
RemoveRedundantSort(),
+ // new
RemoveRedundantSortBelowLimitWithTies(),
+ // new
RemoveRedundantTopN(),
+ // new
RemoveRedundantDistinctLimit(),
+ // new
ReplaceRedundantJoinWithSource(),
+ // new
RemoveRedundantJoin(),
+ // new
ReplaceRedundantJoinWithProject(),
+ // new
RemoveRedundantExists(),
+ // new
RemoveRedundantWindow(),
+ // new
SingleDistinctAggregationToGroupBy(),
+ // new
MergeLimitWithDistinct(),
+ // new
PruneCountAggregationOverScalar(metadata),
+ // new
SimplifyCountOverConstant(plannerContext),
+ // new
+ // PreAggregateCaseAggregations(plannerContext,
typeAnalyzer)))
+ ))
+ .build()),
+ // MergeUnion and related projection pruning rules must run before
limit pushdown rules,
+ // otherwise
+ // an intermediate limit node will prevent unions from being
merged later on
+ new IterativeOptimizer(
+ plannerContext,
+ ruleStats,
+ ImmutableSet.<Rule<?>>builder()
+ // .addAll(projectionPushdownRules)
+ .addAll(columnPruningRules)
+ .addAll(limitPushdownRules)
+ .addAll(
+ ImmutableSet.of(
+ // new MergeUnion(),
+ // new
RemoveEmptyUnionBranches(),
+ new MergeFilters(),
+ new RemoveTrivialFilters(),
+ new MergeLimits(),
+ new InlineProjections(plannerContext),
+ new RemoveRedundantIdentityProjections()))
+ .build()),
simplifyOptimizer,
columnPruningOptimizer,
inlineProjectionLimitFiltersOptimizer,
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/Identity.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/Identity.java
index fad9562a916..1f9d13716df 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/Identity.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/Identity.java
@@ -19,6 +19,8 @@
package org.apache.iotdb.db.queryengine.plan.relational.security;
+import java.util.Objects;
+
public class Identity {
private final String user;
@@ -29,4 +31,21 @@ public class Identity {
public String getUser() {
return user;
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Identity identity = (Identity) o;
+ return Objects.equals(user, identity.user);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(user);
+ }
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java
index 9024de680b3..ef47e225ba1 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java
@@ -939,18 +939,21 @@ public class AnalyzerTest {
rootNode = logicalQueryPlan.getRootNode();
distributionPlanner = new TableDistributedPlanner(actualAnalysis,
logicalQueryPlan, context);
distributedQueryPlan = distributionPlanner.plan();
+
assertTrue(actualAnalysis.getRespDatasetHeader().getColumnNameIndexMap().containsKey("time"));
+
assertTrue(actualAnalysis.getRespDatasetHeader().getColumnNameIndexMap().containsKey("Time"));
+
assertTrue(actualAnalysis.getRespDatasetHeader().getColumnNameIndexMap().containsKey("_col3"));
+
assertTrue(actualAnalysis.getRespDatasetHeader().getColumnNameIndexMap().containsKey("_col2"));
+
assertTrue(actualAnalysis.getRespDatasetHeader().getColumnNameIndexMap().containsKey("tag1"));
+
assertTrue(actualAnalysis.getRespDatasetHeader().getColumnNameIndexMap().containsKey("TAG1"));
assertEquals(
- 0,
actualAnalysis.getRespDatasetHeader().getColumnNameIndexMap().get("Time").intValue());
+
actualAnalysis.getRespDatasetHeader().getColumnNameIndexMap().get("time").intValue(),
+
actualAnalysis.getRespDatasetHeader().getColumnNameIndexMap().get("Time").intValue());
assertEquals(
- 0,
actualAnalysis.getRespDatasetHeader().getColumnNameIndexMap().get("time").intValue());
+
actualAnalysis.getRespDatasetHeader().getColumnNameIndexMap().get("_col3").intValue(),
+
actualAnalysis.getRespDatasetHeader().getColumnNameIndexMap().get("_col2").intValue());
assertEquals(
- 2,
actualAnalysis.getRespDatasetHeader().getColumnNameIndexMap().get("_col2").intValue());
- assertEquals(
- 2,
actualAnalysis.getRespDatasetHeader().getColumnNameIndexMap().get("_col3").intValue());
- assertEquals(
- 1,
actualAnalysis.getRespDatasetHeader().getColumnNameIndexMap().get("tag1").intValue());
- assertEquals(
- 1,
actualAnalysis.getRespDatasetHeader().getColumnNameIndexMap().get("TAG1").intValue());
+
actualAnalysis.getRespDatasetHeader().getColumnNameIndexMap().get("tag1").intValue(),
+
actualAnalysis.getRespDatasetHeader().getColumnNameIndexMap().get("TAG1").intValue());
}
private Metadata mockMetadataForInsertion() {
