This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch ty/TableModelGrammar
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/ty/TableModelGrammar by this
push:
new 5f41dfbd5e6 Make globalTableTimePredicate valid, add
PruneUnUsedColumns and PredicatePushDown optimize rules
5f41dfbd5e6 is described below
commit 5f41dfbd5e60c9b8c7a99ae9e9d098889ecfe2b6
Author: Beyyes <[email protected]>
AuthorDate: Wed Jun 5 17:40:28 2024 +0800
Make globalTableTimePredicate valid, add PruneUnUsedColumns and
PredicatePushDown optimize rules
---
.../db/queryengine/common/MPPQueryContext.java | 6 +
.../plan/planner/plan/TableModelTimePredicate.java | 5 +-
.../plan/planner/plan/TimePredicate.java | 12 +-
.../plan/planner/plan/TreeModelTimePredicate.java | 2 +
.../plan/planner/plan/node/PlanVisitor.java | 4 +-
.../plan/relational/analyzer/Analysis.java | 11 +
.../relational/metadata/TableMetadataImpl.java | 4 +
.../plan/relational/planner/Assignments.java | 19 +-
.../plan/relational/planner/LogicalPlanner.java | 6 +-
.../plan/relational/planner/QueryPlanner.java | 5 +
.../relational/planner/node/TableScanNode.java | 32 ++-
.../planner/optimizations/IndexScan.java | 29 ++-
...ableScanColumns.java => PredicatePushDown.java} | 85 +++++--
...bleScanColumns.java => PruneUnUsedColumns.java} | 57 ++++-
.../RemoveRedundantIdentityProjections.java | 1 +
.../plan/relational/sql/tree/DataType.java | 1 -
.../plan/relational/analyzer/AnalyzerTest.java | 278 ++++++++++++++++++---
.../plan/relational/analyzer/TestMatadata.java | 118 +--------
18 files changed, 469 insertions(+), 206 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java
index 3a42ee805b0..988224da729 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java
@@ -218,11 +218,17 @@ public class MPPQueryContext {
}
}
+ // used for tree model
public void generateGlobalTimeFilter(Analysis analysis) {
this.globalTimeFilter =
PredicateUtils.convertPredicateToTimeFilter(analysis.getGlobalTimePredicate());
}
+ // used for table model
+ public void setGlobalTimeFilter(Filter globalTimeFilter) {
+ this.globalTimeFilter = globalTimeFilter;
+ }
+
public Filter getGlobalTimeFilter() {
// time filter may be stateful, so we need to copy it
return globalTimeFilter != null ? globalTimeFilter.copy() : null;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TableModelTimePredicate.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TableModelTimePredicate.java
index b4d1337b16e..fa7a82f484e 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TableModelTimePredicate.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TableModelTimePredicate.java
@@ -23,10 +23,10 @@ import
org.apache.iotdb.db.queryengine.plan.analyze.PredicateUtils;
import org.apache.iotdb.db.queryengine.plan.relational.sql.tree.Expression;
import org.apache.tsfile.read.filter.basic.Filter;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
import java.io.DataOutputStream;
import java.io.IOException;
-import java.io.UnsupportedEncodingException;
import java.util.Objects;
public class TableModelTimePredicate implements TimePredicate {
@@ -39,7 +39,8 @@ public class TableModelTimePredicate implements TimePredicate
{
@Override
public void serialize(DataOutputStream stream) throws IOException {
- throw new UnsupportedEncodingException();
+ ReadWriteIOUtils.write((byte) 1, stream);
+ Expression.serialize(timePredicate, stream);
}
@Override
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TimePredicate.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TimePredicate.java
index 04cf67ed3aa..348fa1bfe5e 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TimePredicate.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TimePredicate.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.queryengine.plan.planner.plan;
import org.apache.iotdb.db.queryengine.plan.expression.Expression;
import org.apache.tsfile.read.filter.basic.Filter;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
import java.io.DataOutputStream;
import java.io.IOException;
@@ -34,7 +35,14 @@ public interface TimePredicate {
Filter convertPredicateToTimeFilter();
static TimePredicate deserialize(ByteBuffer byteBuffer) {
- // TODO will return another kind of TimePredicate like
TableModelTimePredicate in the future
- return new TreeModelTimePredicate(Expression.deserialize(byteBuffer));
+ // 0 for tree model, 1 for table model
+ byte type = ReadWriteIOUtils.readByte(byteBuffer);
+ if (type == 0) {
+ return new TreeModelTimePredicate(Expression.deserialize(byteBuffer));
+ } else {
+ return new TableModelTimePredicate(
+
org.apache.iotdb.db.queryengine.plan.relational.sql.tree.Expression.deserialize(
+ byteBuffer));
+ }
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TreeModelTimePredicate.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TreeModelTimePredicate.java
index 5b9e90cd8d0..5cc00125aa7 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TreeModelTimePredicate.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/TreeModelTimePredicate.java
@@ -23,6 +23,7 @@ import
org.apache.iotdb.db.queryengine.plan.analyze.PredicateUtils;
import org.apache.iotdb.db.queryengine.plan.expression.Expression;
import org.apache.tsfile.read.filter.basic.Filter;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
import java.io.DataOutputStream;
import java.io.IOException;
@@ -38,6 +39,7 @@ public class TreeModelTimePredicate implements TimePredicate {
@Override
public void serialize(DataOutputStream stream) throws IOException {
+ ReadWriteIOUtils.write((byte) 0, stream);
Expression.serialize(timePredicate, stream);
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java
index 85cc1a83625..2e5a61b7a83 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java
@@ -548,7 +548,7 @@ public abstract class PlanVisitor<R, C> {
public R visitProject(
org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode
node, C context) {
- return visitPlan(node, context);
+ return visitSingleChildProcess(node, context);
}
public R visitLimit(
@@ -567,7 +567,7 @@ public abstract class PlanVisitor<R, C> {
public R visitOutput(
org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode
node, C context) {
- return visitPlan(node, context);
+ return visitSingleChildProcess(node, context);
}
public R visitSort(
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java
index adcc56e464b..a74f35526b5 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java
@@ -158,6 +158,9 @@ public class Analysis implements IAnalysis {
private boolean finishQueryAfterAnalyze;
+ // indicate is there a value filter
+ private boolean hasValueFilter = false;
+
public Expression getGlobalTableModelTimePredicate() {
return this.globalTableModelTimePredicate;
}
@@ -567,6 +570,14 @@ public class Analysis implements IAnalysis {
return aliasedRelations.contains(NodeRef.of(relation));
}
+ public boolean hasValueFilter() {
+ return hasValueFilter;
+ }
+
+ public void setHasValueFilter(boolean hasValueFilter) {
+ this.hasValueFilter = hasValueFilter;
+ }
+
@Override
public boolean isFailed() {
return false;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java
index 8d622c8dc02..7859fc64186 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java
@@ -106,6 +106,10 @@ public class TableMetadataImpl implements Metadata {
@Override
public Type getFunctionReturnType(String functionName, List<? extends Type>
argumentTypes) {
+ return getFunctionType(functionName, argumentTypes);
+ }
+
+ public static Type getFunctionType(String functionName, List<? extends Type>
argumentTypes) {
// builtin scalar function
if
(BuiltinScalarFunction.DIFF.getFunctionName().equalsIgnoreCase(functionName)) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/Assignments.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/Assignments.java
index ffb2b62c521..5bc8a178c11 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/Assignments.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/Assignments.java
@@ -16,7 +16,6 @@ package
org.apache.iotdb.db.queryengine.plan.relational.planner;
import org.apache.iotdb.db.queryengine.plan.relational.sql.tree.Expression;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import java.util.Collection;
@@ -68,7 +67,7 @@ public class Assignments {
}
public Assignments(Map<Symbol, Expression> assignments) {
- this.assignments = ImmutableMap.copyOf(requireNonNull(assignments,
"assignments is null"));
+ this.assignments = requireNonNull(assignments, "assignments is null");
}
public List<Symbol> getOutputs() {
@@ -95,6 +94,18 @@ public class Assignments {
.collect(toAssignments());
}
+ public boolean contains(Symbol symbol) {
+ return assignments.containsKey(symbol);
+ }
+
+ public Expression get(Symbol symbol) {
+ return assignments.get(symbol);
+ }
+
+ public void remove(Symbol symbol) {
+ this.assignments.remove(symbol);
+ }
+
private Collector<Entry<Symbol, Expression>, Builder, Assignments>
toAssignments() {
return Collector.of(
Assignments::builder,
@@ -118,10 +129,6 @@ public class Assignments {
return assignments.entrySet();
}
- public Expression get(Symbol symbol) {
- return assignments.get(symbol);
- }
-
public int size() {
return assignments.size();
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/LogicalPlanner.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/LogicalPlanner.java
index fb593d4c852..dfc2bfae5a2 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/LogicalPlanner.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/LogicalPlanner.java
@@ -29,7 +29,8 @@ import
org.apache.iotdb.db.queryengine.plan.relational.analyzer.RelationType;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode;
import
org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.IndexScan;
-import
org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.PruneTableScanColumns;
+import
org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.PredicatePushDown;
+import
org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.PruneUnUsedColumns;
import
org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.RelationalPlanOptimizer;
import
org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.RemoveRedundantIdentityProjections;
import
org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.SimplifyExpressions;
@@ -77,8 +78,9 @@ public class LogicalPlanner {
this.relationalPlanOptimizers =
Arrays.asList(
new SimplifyExpressions(),
+ new PruneUnUsedColumns(),
+ new PredicatePushDown(),
new RemoveRedundantIdentityProjections(),
- new PruneTableScanColumns(),
new IndexScan());
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java
index a3e1dec9819..0130b13af47 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java
@@ -19,6 +19,7 @@ import org.apache.iotdb.db.queryengine.common.SessionInfo;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis;
import org.apache.iotdb.db.queryengine.plan.relational.analyzer.NodeRef;
+import
org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.ConvertPredicateToTimeFilterVisitor;
import
org.apache.iotdb.db.queryengine.plan.relational.planner.ir.ExpressionTranslateVisitor;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode;
@@ -250,12 +251,16 @@ public class QueryPlanner {
if (resultPair.left != null) {
globalTimePredicate =
ExpressionTranslateVisitor.translateToSymbolReference(resultPair.left,
planBuilder);
+
+ queryContext.setGlobalTimeFilter(
+ globalTimePredicate.accept(new
ConvertPredicateToTimeFilterVisitor(), null));
}
analysis.setGlobalTableModelTimePredicate(globalTimePredicate);
boolean hasValueFilter = resultPair.right;
if (!hasValueFilter) {
return planBuilder;
}
+ analysis.setHasValueFilter(true);
// TODO if predicate equals TrueConstant, no need filter
return planBuilder.withNewRoot(
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java
index 33dfa3e04bc..6fd46d63678 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java
@@ -45,8 +45,8 @@ public class TableScanNode extends SourceNode {
// db.tablename
private final String qualifiedTableName;
- private final List<Symbol> outputSymbols;
- private final Map<Symbol, ColumnSchema> assignments;
+ private List<Symbol> outputSymbols;
+ private Map<Symbol, ColumnSchema> assignments;
private List<DeviceEntry> deviceEntries;
private Map<Symbol, Integer> idAndAttributeIndexMap;
@@ -276,6 +276,18 @@ public class TableScanNode extends SourceNode {
return this.qualifiedTableName;
}
+ public void setOutputSymbols(List<Symbol> outputSymbols) {
+ this.outputSymbols = outputSymbols;
+ }
+
+ public void setAssignments(Map<Symbol, ColumnSchema> assignments) {
+ this.assignments = assignments;
+ }
+
+ public Map<Symbol, ColumnSchema> getAssignments() {
+ return this.assignments;
+ }
+
public void setDeviceEntries(List<DeviceEntry> deviceEntries) {
this.deviceEntries = deviceEntries;
}
@@ -288,10 +300,6 @@ public class TableScanNode extends SourceNode {
this.idAndAttributeIndexMap = idAndAttributeIndexMap;
}
- public Map<Symbol, ColumnSchema> getAssignments() {
- return this.assignments;
- }
-
public Ordering getScanOrder() {
return this.scanOrder;
}
@@ -300,10 +308,6 @@ public class TableScanNode extends SourceNode {
return deviceEntries;
}
- public Expression getPushDownPredicate() {
- return this.pushDownPredicate;
- }
-
public long getPushDownLimit() {
return this.pushDownLimit;
}
@@ -312,6 +316,14 @@ public class TableScanNode extends SourceNode {
return this.pushDownOffset;
}
+ public Expression getPushDownPredicate() {
+ return this.pushDownPredicate;
+ }
+
+ public void setPushDownPredicate(@Nullable Expression pushDownPredicate) {
+ this.pushDownPredicate = pushDownPredicate;
+ }
+
public boolean isPushLimitToEachDevice() {
return pushLimitToEachDevice;
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/IndexScan.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/IndexScan.java
index 56eb8ea082d..fd3fe00bd9f 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/IndexScan.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/IndexScan.java
@@ -58,8 +58,6 @@ public class IndexScan implements RelationalPlanOptimizer {
static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
- private MPPQueryContext mppQueryContext;
-
@Override
public PlanNode optimize(
PlanNode planNode,
@@ -67,14 +65,15 @@ public class IndexScan implements RelationalPlanOptimizer {
Metadata metadata,
IPartitionFetcher partitionFetcher,
SessionInfo sessionInfo,
- MPPQueryContext context) {
+ MPPQueryContext queryContext) {
return planNode.accept(
new Rewriter(),
- new RewriterContext(null, metadata, sessionInfo, analysis,
partitionFetcher));
+ new RewriterContext(null, metadata, sessionInfo, analysis,
partitionFetcher, queryContext));
}
private static class Rewriter extends PlanVisitor<PlanNode, RewriterContext>
{
+
@Override
public PlanNode visitPlan(PlanNode node, RewriterContext context) {
for (PlanNode child : node.getChildren()) {
@@ -109,9 +108,7 @@ public class IndexScan implements RelationalPlanOptimizer {
attributeColumns);
node.setDeviceEntries(deviceEntries);
- // TODO getDataPartition, Change globalTimeFilter to Filter
- String treeDatabase = "root." + dbName;
- Filter globalTimeFilter = null;
+ String treeModelDatabase = "root." + dbName;
Set<String> deviceSet = new HashSet<>();
for (DeviceEntry deviceEntry : deviceEntries) {
StringArrayDeviceID arrayDeviceID = (StringArrayDeviceID)
deviceEntry.getDeviceID();
@@ -121,7 +118,10 @@ public class IndexScan implements RelationalPlanOptimizer {
DataPartition dataPartition =
fetchDataPartitionByDevices(
- deviceSet, treeDatabase, globalTimeFilter,
context.partitionFetcher);
+ deviceSet,
+ treeModelDatabase,
+ context.getQueryContext().getGlobalTimeFilter(),
+ context.getPartitionFetcher());
context.getAnalysis().setDataPartition(dataPartition);
if (dataPartition.getDataPartitionMap().size() > 1) {
@@ -218,18 +218,21 @@ public class IndexScan implements RelationalPlanOptimizer
{
private final SessionInfo sessionInfo;
private final Analysis analysis;
private final IPartitionFetcher partitionFetcher;
+ private final MPPQueryContext queryContext;
RewriterContext(
Expression predicate,
Metadata metadata,
SessionInfo sessionInfo,
Analysis analysis,
- IPartitionFetcher partitionFetcher) {
+ IPartitionFetcher partitionFetcher,
+ MPPQueryContext queryContext) {
this.predicate = predicate;
this.metadata = metadata;
this.sessionInfo = sessionInfo;
this.analysis = analysis;
this.partitionFetcher = partitionFetcher;
+ this.queryContext = queryContext;
}
public Expression getPredicate() {
@@ -255,5 +258,13 @@ public class IndexScan implements RelationalPlanOptimizer {
public Analysis getAnalysis() {
return this.analysis;
}
+
+ public IPartitionFetcher getPartitionFetcher() {
+ return partitionFetcher;
+ }
+
+ public MPPQueryContext getQueryContext() {
+ return queryContext;
+ }
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PruneTableScanColumns.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PredicatePushDown.java
similarity index 54%
copy from
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PruneTableScanColumns.java
copy to
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PredicatePushDown.java
index cbc18a51465..ae05611ce08 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PruneTableScanColumns.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PredicatePushDown.java
@@ -19,22 +19,22 @@ import org.apache.iotdb.db.queryengine.common.SessionInfo;
import org.apache.iotdb.db.queryengine.plan.analyze.IPartitionFetcher;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor;
+import
org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.MultiChildProcessNode;
+import
org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.SingleChildProcessNode;
import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata;
-import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode;
-import
org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode;
import
org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode;
-import
org.apache.iotdb.db.queryengine.plan.relational.sql.tree.DefaultTraversalVisitor;
-import
org.apache.iotdb.db.queryengine.plan.relational.sql.tree.SymbolReference;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.tree.Expression;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.tree.FunctionCall;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.tree.Node;
-import com.google.common.collect.ImmutableList;
-
-import java.util.HashSet;
+import java.util.ArrayList;
import java.util.List;
-import java.util.Set;
-public class PruneTableScanColumns implements RelationalPlanOptimizer {
+/** Push down predicate to TableScanNode as possible. */
+public class PredicatePushDown implements RelationalPlanOptimizer {
+
@Override
public PlanNode optimize(
PlanNode planNode,
@@ -43,52 +43,83 @@ public class PruneTableScanColumns implements
RelationalPlanOptimizer {
IPartitionFetcher partitionFetcher,
SessionInfo sessionInfo,
MPPQueryContext context) {
+
+ if (!analysis.hasValueFilter()) {
+ return planNode;
+ }
+
return planNode.accept(new Rewriter(), new RewriterContext());
}
private static class Rewriter extends PlanVisitor<PlanNode, RewriterContext>
{
+
@Override
public PlanNode visitPlan(PlanNode node, RewriterContext context) {
- for (PlanNode child : node.getChildren()) {
- child.accept(this, context);
- }
+ throw new IllegalArgumentException(
+ String.format("Unexpected plan node: %s in TableModel
PredicatePushDown", node));
+ }
+
+ @Override
+ public PlanNode visitSingleChildProcess(SingleChildProcessNode node,
RewriterContext context) {
+ PlanNode rewrittenChild = node.getChild().accept(this, context);
+ node.setChild(rewrittenChild);
return node;
}
@Override
- public PlanNode visitProject(ProjectNode node, RewriterContext context) {
- context.symbolHashSet.addAll(node.getOutputSymbols());
- node.getChild().accept(this, context);
+ public PlanNode visitMultiChildProcess(MultiChildProcessNode node,
RewriterContext context) {
+ List<PlanNode> rewrittenChildren = new ArrayList<>();
+ for (PlanNode child : node.getChildren()) {
+ rewrittenChildren.add(child.accept(this, context));
+ }
+ node.setChildren(rewrittenChildren);
return node;
}
@Override
public PlanNode visitFilter(FilterNode node, RewriterContext context) {
- ImmutableList.Builder<Symbol> symbolBuilder = ImmutableList.builder();
- new SymbolBuilderVisitor().process(node.getPredicate(),
ImmutableList.builder());
- List<Symbol> ret = symbolBuilder.build();
- context.symbolHashSet.addAll(ret);
+ if (node.getPredicate() != null) {
+ // when exist diff function, predicate can not be pushed down
+ if (containsDiffFunction(node.getPredicate())) {
+ return node;
+ }
+
+ context.pushDownPredicate = node.getPredicate();
+ node.getChild().accept(this, context);
+
+ // remove FilterNode after push down
+ return node.getChild();
+ }
+
node.getChild().accept(this, context);
return node;
}
@Override
public PlanNode visitTableScan(TableScanNode node, RewriterContext
context) {
+ node.setPushDownPredicate(context.pushDownPredicate);
return node;
}
}
- private static class SymbolBuilderVisitor
- extends DefaultTraversalVisitor<ImmutableList.Builder<Symbol>> {
- @Override
- protected Void visitSymbolReference(
- SymbolReference node, ImmutableList.Builder<Symbol> builder) {
- builder.add(Symbol.from(node));
- return null;
+ static boolean containsDiffFunction(Expression expression) {
+ if (expression instanceof FunctionCall
+ && "diff".equalsIgnoreCase(((FunctionCall)
expression).getName().toString())) {
+ return true;
}
+
+ if (!expression.getChildren().isEmpty()) {
+ for (Node node : expression.getChildren()) {
+ if (containsDiffFunction((Expression) node)) {
+ return true;
+ }
+ }
+ }
+
+ return false;
}
private static class RewriterContext {
- Set<Symbol> symbolHashSet = new HashSet<>();
+ Expression pushDownPredicate;
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PruneTableScanColumns.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PruneUnUsedColumns.java
similarity index 58%
rename from
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PruneTableScanColumns.java
rename to
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PruneUnUsedColumns.java
index cbc18a51465..d6310c37dc0 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PruneTableScanColumns.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PruneUnUsedColumns.java
@@ -20,21 +20,37 @@ import
org.apache.iotdb.db.queryengine.plan.analyze.IPartitionFetcher;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor;
import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis;
+import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata;
import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode;
+import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode;
import
org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode;
import
org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode;
import
org.apache.iotdb.db.queryengine.plan.relational.sql.tree.DefaultTraversalVisitor;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.tree.Expression;
import
org.apache.iotdb.db.queryengine.plan.relational.sql.tree.SymbolReference;
import com.google.common.collect.ImmutableList;
+import java.util.ArrayList;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
-public class PruneTableScanColumns implements RelationalPlanOptimizer {
+import static org.apache.iotdb.commons.conf.IoTDBConstant.TIME;
+
+/**
+ * Remove unused columns in TableScanNode.
+ *
+ * <p>For example, The output columns of TableScanNode in `select * from
table1` query are `tag1,
+ * attr1, s1`, but the output columns of TableScanNode in `select s1 from
table1` query can only be
+ * `s1`.
+ */
+public class PruneUnUsedColumns implements RelationalPlanOptimizer {
+
@Override
public PlanNode optimize(
PlanNode planNode,
@@ -47,6 +63,7 @@ public class PruneTableScanColumns implements
RelationalPlanOptimizer {
}
private static class Rewriter extends PlanVisitor<PlanNode, RewriterContext>
{
+
@Override
public PlanNode visitPlan(PlanNode node, RewriterContext context) {
for (PlanNode child : node.getChildren()) {
@@ -55,9 +72,28 @@ public class PruneTableScanColumns implements
RelationalPlanOptimizer {
return node;
}
+ @Override
+ public PlanNode visitOutput(OutputNode node, RewriterContext context) {
+ context.allUsedSymbolSet.addAll(node.getOutputSymbols());
+ node.getChild().accept(this, context);
+ return node;
+ }
+
@Override
public PlanNode visitProject(ProjectNode node, RewriterContext context) {
- context.symbolHashSet.addAll(node.getOutputSymbols());
+ // There must exist OutputNode above ProjectNode
+ node.getAssignments()
+ .getMap()
+ .entrySet()
+ .removeIf(entry ->
!context.allUsedSymbolSet.contains(entry.getKey()));
+ Set<Symbol> usedSymbolSet = new HashSet<>();
+ for (Map.Entry<Symbol, Expression> entry :
node.getAssignments().getMap().entrySet()) {
+ ImmutableList.Builder<Symbol> symbolBuilder = ImmutableList.builder();
+ new SymbolBuilderVisitor().process(entry.getValue(), symbolBuilder);
+ usedSymbolSet.addAll(symbolBuilder.build());
+ }
+
+ context.allUsedSymbolSet.addAll(usedSymbolSet);
node.getChild().accept(this, context);
return node;
}
@@ -65,15 +101,24 @@ public class PruneTableScanColumns implements
RelationalPlanOptimizer {
@Override
public PlanNode visitFilter(FilterNode node, RewriterContext context) {
ImmutableList.Builder<Symbol> symbolBuilder = ImmutableList.builder();
- new SymbolBuilderVisitor().process(node.getPredicate(),
ImmutableList.builder());
- List<Symbol> ret = symbolBuilder.build();
- context.symbolHashSet.addAll(ret);
+ new SymbolBuilderVisitor().process(node.getPredicate(), symbolBuilder);
+ context.allUsedSymbolSet.addAll(symbolBuilder.build());
node.getChild().accept(this, context);
return node;
}
@Override
public PlanNode visitTableScan(TableScanNode node, RewriterContext
context) {
+ List<Symbol> newOutputSymbols = new ArrayList<>();
+ Map<Symbol, ColumnSchema> newAssignments = new HashMap<>();
+ for (Symbol symbol : node.getOutputSymbols()) {
+ if (TIME.equalsIgnoreCase(symbol.getName()) ||
context.allUsedSymbolSet.contains(symbol)) {
+ newOutputSymbols.add(symbol);
+ newAssignments.put(symbol, node.getAssignments().get(symbol));
+ }
+ }
+ node.setOutputSymbols(newOutputSymbols);
+ node.setAssignments(newAssignments);
return node;
}
}
@@ -89,6 +134,6 @@ public class PruneTableScanColumns implements
RelationalPlanOptimizer {
}
private static class RewriterContext {
- Set<Symbol> symbolHashSet = new HashSet<>();
+ Set<Symbol> allUsedSymbolSet = new HashSet<>();
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/RemoveRedundantIdentityProjections.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/RemoveRedundantIdentityProjections.java
index 28b5ab8d1d7..474a327292f 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/RemoveRedundantIdentityProjections.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/RemoveRedundantIdentityProjections.java
@@ -42,6 +42,7 @@ public class RemoveRedundantIdentityProjections implements
RelationalPlanOptimiz
}
private static class Rewriter extends PlanVisitor<PlanNode, RewriterContext>
{
+
@Override
public PlanNode visitPlan(PlanNode node, RewriterContext context) {
PlanNode newNode = node.clone();
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/tree/DataType.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/tree/DataType.java
index 7123c457ae7..d6abeb4f5d4 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/tree/DataType.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/tree/DataType.java
@@ -22,7 +22,6 @@ package
org.apache.iotdb.db.queryengine.plan.relational.sql.tree;
import javax.annotation.Nullable;
public abstract class DataType extends Expression {
-
DataType(@Nullable NodeLocation location) {
super(location);
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java
index 104cdc9ed48..a2a3b56fc8d 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AnalyzerTest.java
@@ -45,7 +45,9 @@ import
org.apache.iotdb.db.queryengine.plan.relational.metadata.TableHandle;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableSchema;
import org.apache.iotdb.db.queryengine.plan.relational.planner.LogicalPlanner;
import
org.apache.iotdb.db.queryengine.plan.relational.planner.distribute.TableDistributionPlanner;
+import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode;
+import
org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode;
import
org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode;
import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl;
import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser;
@@ -79,6 +81,26 @@ public class AnalyzerTest {
private static final NopAccessControl nopAccessControl = new
NopAccessControl();
+ QueryId queryId = new QueryId("tmp_query");
+ SessionInfo sessionInfo =
+ new SessionInfo(
+ 1L,
+ "iotdb-user",
+ ZoneId.systemDefault(),
+ IoTDBConstant.ClientVersion.V_1_0,
+ "db",
+ IClientSession.SqlDialect.TABLE);
+ Metadata metadata = new TestMatadata();
+ String sql;
+ Analysis actualAnalysis;
+ MPPQueryContext context;
+ LogicalPlanner logicalPlanner;
+ LogicalQueryPlan logicalQueryPlan;
+ PlanNode rootNode;
+ TableDistributionPlanner distributionPlanner;
+ DistributedQueryPlan distributedQueryPlan;
+ TableScanNode tableScanNode;
+
@Test
public void testMockQuery() throws OperatorNotFoundException {
String sql =
@@ -145,52 +167,45 @@ public class AnalyzerTest {
System.out.println(actualAnalysis.getTypes());
}
- QueryId queryId = new QueryId("tmp_query");
- SessionInfo sessionInfo =
- new SessionInfo(
- 1L,
- "iotdb-user",
- ZoneId.systemDefault(),
- IoTDBConstant.ClientVersion.V_1_0,
- "db",
- IClientSession.SqlDialect.TABLE);
- Metadata metadata = new TestMatadata();
-
@Test
public void singleTableNoFilterTest() throws IoTDBException {
- // 1. wildcard
- String sql = "SELECT * FROM table1";
- Analysis actualAnalysis = analyzeSQL(sql, metadata);
+ // wildcard
+ sql = "SELECT * FROM table1";
+ actualAnalysis = analyzeSQL(sql, metadata);
assertNotNull(actualAnalysis);
assertEquals(1, actualAnalysis.getTables().size());
- MPPQueryContext context = new MPPQueryContext(sql, queryId, sessionInfo,
null, null);
- LogicalPlanner logicalPlanner =
+ context = new MPPQueryContext(sql, queryId, sessionInfo, null, null);
+ logicalPlanner =
new LogicalPlanner(
context, metadata, sessionInfo, getFakePartitionFetcher(),
WarningCollector.NOOP);
- LogicalQueryPlan logicalQueryPlan = logicalPlanner.plan(actualAnalysis);
- PlanNode rootNode = logicalQueryPlan.getRootNode();
+ logicalQueryPlan = logicalPlanner.plan(actualAnalysis);
+ rootNode = logicalQueryPlan.getRootNode();
assertTrue(rootNode instanceof OutputNode);
assertTrue(((OutputNode) rootNode).getChild() instanceof TableScanNode);
- TableScanNode tableScanNode = (TableScanNode) ((OutputNode)
rootNode).getChild();
+ tableScanNode = (TableScanNode) ((OutputNode) rootNode).getChild();
assertEquals("table1", tableScanNode.getQualifiedTableName());
- assertEquals(8, tableScanNode.getOutputSymbols().size());
- assertEquals(8, tableScanNode.getAssignments().size());
+ assertEquals(
+ Arrays.asList("time", "tag1", "tag2", "tag3", "attr1", "attr2", "s1",
"s2", "s3"),
+ tableScanNode.getOutputColumnNames());
+ assertEquals(9, tableScanNode.getOutputSymbols().size());
+ assertEquals(9, tableScanNode.getAssignments().size());
assertEquals(1, tableScanNode.getDeviceEntries().size());
assertEquals(5, tableScanNode.getIdAndAttributeIndexMap().size());
assertEquals(ASC, tableScanNode.getScanOrder());
- TableDistributionPlanner distributionPlanner =
- new TableDistributionPlanner(actualAnalysis, logicalQueryPlan,
context);
+ distributionPlanner = new TableDistributionPlanner(actualAnalysis,
logicalQueryPlan, context);
DistributedQueryPlan distributedQueryPlan = distributionPlanner.plan();
assertEquals(4, distributedQueryPlan.getInstances().size());
+ }
- // 2. global time filter
+ @Test
+ public void singleTableWithFilterTest() throws IoTDBException {
+ // global time filter
sql = "SELECT * FROM table1 where time > 1";
actualAnalysis = analyzeSQL(sql, metadata);
assertNotNull(actualAnalysis);
assertEquals(1, actualAnalysis.getTables().size());
-
context = new MPPQueryContext(sql, queryId, sessionInfo, null, null);
logicalPlanner =
new LogicalPlanner(
@@ -201,15 +216,222 @@ public class AnalyzerTest {
assertTrue(((OutputNode) rootNode).getChild() instanceof TableScanNode);
tableScanNode = (TableScanNode) ((OutputNode) rootNode).getChild();
assertEquals("table1", tableScanNode.getQualifiedTableName());
- assertEquals(8, tableScanNode.getOutputSymbols().size());
- assertEquals(8, tableScanNode.getAssignments().size());
+ assertEquals(
+ Arrays.asList("time", "tag1", "tag2", "tag3", "attr1", "attr2", "s1",
"s2", "s3"),
+ tableScanNode.getOutputColumnNames());
+ assertEquals(9, tableScanNode.getAssignments().size());
assertEquals(1, tableScanNode.getDeviceEntries().size());
assertEquals(5, tableScanNode.getIdAndAttributeIndexMap().size());
assertEquals(ASC, tableScanNode.getScanOrder());
-
distributionPlanner = new TableDistributionPlanner(actualAnalysis,
logicalQueryPlan, context);
distributedQueryPlan = distributionPlanner.plan();
assertEquals(4, distributedQueryPlan.getInstances().size());
+
+ // value filter which can be pushed down
+ sql = "SELECT tag1, attr1, s2 FROM table1 where s1 > 1";
+ actualAnalysis = analyzeSQL(sql, metadata);
+ assertNotNull(actualAnalysis);
+ assertEquals(1, actualAnalysis.getTables().size());
+ context = new MPPQueryContext(sql, queryId, sessionInfo, null, null);
+ logicalPlanner =
+ new LogicalPlanner(
+ context, metadata, sessionInfo, getFakePartitionFetcher(),
WarningCollector.NOOP);
+ logicalQueryPlan = logicalPlanner.plan(actualAnalysis);
+ rootNode = logicalQueryPlan.getRootNode();
+ assertTrue(rootNode instanceof OutputNode);
+ assertTrue(rootNode.getChildren().get(0) instanceof ProjectNode);
+ assertTrue(rootNode.getChildren().get(0).getChildren().get(0) instanceof
TableScanNode);
+ tableScanNode = (TableScanNode)
rootNode.getChildren().get(0).getChildren().get(0);
+ assertEquals(
+ Arrays.asList("time", "tag1", "attr1", "s1", "s2"),
tableScanNode.getOutputColumnNames());
+
+ // value filter which can not be pushed down
+ sql = "SELECT tag1, attr1, s2 FROM table1 where diff(s1) > 1";
+ actualAnalysis = analyzeSQL(sql, metadata);
+ assertEquals(1, actualAnalysis.getTables().size());
+ context = new MPPQueryContext(sql, queryId, sessionInfo, null, null);
+ logicalPlanner =
+ new LogicalPlanner(
+ context, metadata, sessionInfo, getFakePartitionFetcher(),
WarningCollector.NOOP);
+ logicalQueryPlan = logicalPlanner.plan(actualAnalysis);
+ rootNode = logicalQueryPlan.getRootNode();
+ assertTrue(rootNode instanceof OutputNode);
+ assertTrue(rootNode.getChildren().get(0) instanceof ProjectNode);
+ assertTrue(rootNode.getChildren().get(0).getChildren().get(0) instanceof
FilterNode);
+ assertTrue(
+ rootNode.getChildren().get(0).getChildren().get(0).getChildren().get(0)
+ instanceof TableScanNode);
+ tableScanNode =
+ (TableScanNode)
rootNode.getChildren().get(0).getChildren().get(0).getChildren().get(0);
+ assertEquals(
+ Arrays.asList("time", "tag1", "attr1", "s1", "s2"),
tableScanNode.getOutputColumnNames());
+
+ sql = "SELECT tag1, attr1, s2 FROM table1 where diff(s1) + 1 > 1";
+ actualAnalysis = analyzeSQL(sql, metadata);
+ assertEquals(1, actualAnalysis.getTables().size());
+ context = new MPPQueryContext(sql, queryId, sessionInfo, null, null);
+ logicalPlanner =
+ new LogicalPlanner(
+ context, metadata, sessionInfo, getFakePartitionFetcher(),
WarningCollector.NOOP);
+ logicalQueryPlan = logicalPlanner.plan(actualAnalysis);
+ rootNode = logicalQueryPlan.getRootNode();
+ assertTrue(rootNode instanceof OutputNode);
+ assertTrue(rootNode.getChildren().get(0) instanceof ProjectNode);
+ assertTrue(rootNode.getChildren().get(0).getChildren().get(0) instanceof
FilterNode);
+ assertTrue(
+ rootNode.getChildren().get(0).getChildren().get(0).getChildren().get(0)
+ instanceof TableScanNode);
+ tableScanNode =
+ (TableScanNode)
rootNode.getChildren().get(0).getChildren().get(0).getChildren().get(0);
+ assertEquals(
+ Arrays.asList("time", "tag1", "attr1", "s1", "s2"),
tableScanNode.getOutputColumnNames());
+ }
+
+ @Test
+ public void singleTableProjectTest() throws IoTDBException {
+ // 1. project without filter
+ sql = "SELECT tag1, attr1, s1 FROM table1";
+ actualAnalysis = analyzeSQL(sql, metadata);
+ assertNotNull(actualAnalysis);
+ assertEquals(1, actualAnalysis.getTables().size());
+ context = new MPPQueryContext(sql, queryId, sessionInfo, null, null);
+ logicalPlanner =
+ new LogicalPlanner(
+ context, metadata, sessionInfo, getFakePartitionFetcher(),
WarningCollector.NOOP);
+ logicalQueryPlan = logicalPlanner.plan(actualAnalysis);
+ rootNode = logicalQueryPlan.getRootNode();
+ PlanNode tableScanNode =
rootNode.getChildren().get(0).getChildren().get(0);
+ assertEquals(
+ Arrays.asList("time", "tag1", "attr1", "s1"),
tableScanNode.getOutputColumnNames());
+
+ // 2. project with filter
+ sql = "SELECT tag1, attr1, s1 FROM table1 WHERE tag2='A' and s2=8";
+ actualAnalysis = analyzeSQL(sql, metadata);
+ assertNotNull(actualAnalysis);
+ assertEquals(1, actualAnalysis.getTables().size());
+ context = new MPPQueryContext(sql, queryId, sessionInfo, null, null);
+ logicalPlanner =
+ new LogicalPlanner(
+ context, metadata, sessionInfo, getFakePartitionFetcher(),
WarningCollector.NOOP);
+ logicalQueryPlan = logicalPlanner.plan(actualAnalysis);
+ rootNode = logicalQueryPlan.getRootNode();
+ tableScanNode = rootNode.getChildren().get(0).getChildren().get(0);
+ assertEquals(
+ Arrays.asList("time", "tag1", "tag2", "attr1", "s1", "s2"),
+ tableScanNode.getOutputColumnNames());
+
+ // 3. project with filter and function
+ sql =
+ "SELECT s1+s3, CAST(s2 AS DOUBLE) FROM table1 WHERE REPLACE(tag1,
'low', '!')='!' AND attr2='B'";
+ actualAnalysis = analyzeSQL(sql, metadata);
+ context = new MPPQueryContext(sql, queryId, sessionInfo, null, null);
+ logicalQueryPlan =
+ new LogicalPlanner(
+ context, metadata, sessionInfo, getFakePartitionFetcher(),
WarningCollector.NOOP)
+ .plan(actualAnalysis);
+ rootNode = logicalQueryPlan.getRootNode();
+ tableScanNode = rootNode.getChildren().get(0).getChildren().get(0);
+ assertEquals(
+ Arrays.asList("time", "tag1", "attr2", "s1", "s2", "s3"),
+ tableScanNode.getOutputColumnNames());
+ }
+
+ @Test
+ public void expressionTest() throws IoTDBException {
+ // 1. is null / is not null
+ sql = "SELECT * FROM table1 WHERE tag1 is not null and s1 is null";
+ context = new MPPQueryContext(sql, queryId, sessionInfo, null, null);
+ actualAnalysis = analyzeSQL(sql, metadata);
+ logicalQueryPlan =
+ new LogicalPlanner(
+ context, metadata, sessionInfo, getFakePartitionFetcher(),
WarningCollector.NOOP)
+ .plan(actualAnalysis);
+ rootNode = logicalQueryPlan.getRootNode();
+
+ // 2. like
+ sql = "SELECT * FROM table1 WHERE tag1 like '%m'";
+ context = new MPPQueryContext(sql, queryId, sessionInfo, null, null);
+ actualAnalysis = analyzeSQL(sql, metadata);
+ logicalQueryPlan =
+ new LogicalPlanner(
+ context, metadata, sessionInfo, getFakePartitionFetcher(),
WarningCollector.NOOP)
+ .plan(actualAnalysis);
+ rootNode = logicalQueryPlan.getRootNode();
+
+ // 3. in / not in
+ sql =
+ "SELECT *, s1/2, s2+1, s2*3, s1+s2, s2%1 FROM table1 WHERE tag1 in
('A', 'B') and tag2 not in ('A', 'C')";
+ context = new MPPQueryContext(sql, queryId, sessionInfo, null, null);
+ actualAnalysis = analyzeSQL(sql, metadata);
+ logicalQueryPlan =
+ new LogicalPlanner(
+ context, metadata, sessionInfo, getFakePartitionFetcher(),
WarningCollector.NOOP)
+ .plan(actualAnalysis);
+ rootNode = logicalQueryPlan.getRootNode();
+
+ // 4. not
+ sql = "SELECT * FROM table1 WHERE tag1 not like '%m'";
+ context = new MPPQueryContext(sql, queryId, sessionInfo, null, null);
+ actualAnalysis = analyzeSQL(sql, metadata);
+ logicalQueryPlan =
+ new LogicalPlanner(
+ context, metadata, sessionInfo, getFakePartitionFetcher(),
WarningCollector.NOOP)
+ .plan(actualAnalysis);
+ rootNode = logicalQueryPlan.getRootNode();
+ }
+
+ @Test
+ public void functionTest() throws IoTDBException {
+ // 1. cast
+ sql = "SELECT CAST(s2 AS DOUBLE) FROM table1 WHERE CAST(s1 AS DOUBLE) >
1.0";
+ context = new MPPQueryContext(sql, queryId, sessionInfo, null, null);
+ actualAnalysis = analyzeSQL(sql, metadata);
+ logicalPlanner =
+ new LogicalPlanner(
+ context, metadata, sessionInfo, getFakePartitionFetcher(),
WarningCollector.NOOP);
+ logicalQueryPlan = logicalPlanner.plan(actualAnalysis);
+ rootNode = logicalQueryPlan.getRootNode();
+
+ // 2. substring
+ sql =
+ "SELECT SUBSTRING(tag1, 2), SUBSTRING(tag2, s1) FROM table1 WHERE
SUBSTRING(tag2, 1) = 'A'";
+ context = new MPPQueryContext(sql, queryId, sessionInfo, null, null);
+ actualAnalysis = analyzeSQL(sql, metadata);
+ logicalPlanner =
+ new LogicalPlanner(
+ context, metadata, sessionInfo, getFakePartitionFetcher(),
WarningCollector.NOOP);
+ logicalQueryPlan = logicalPlanner.plan(actualAnalysis);
+ rootNode = logicalQueryPlan.getRootNode();
+
+ // 3. round
+ sql = "SELECT ROUND(s1, 1) FROM table1 WHERE ROUND(s2, 2) > 1.0";
+ context = new MPPQueryContext(sql, queryId, sessionInfo, null, null);
+ actualAnalysis = analyzeSQL(sql, metadata);
+ logicalPlanner =
+ new LogicalPlanner(
+ context, metadata, sessionInfo, getFakePartitionFetcher(),
WarningCollector.NOOP);
+ logicalQueryPlan = logicalPlanner.plan(actualAnalysis);
+ rootNode = logicalQueryPlan.getRootNode();
+
+ // 4. replace
+ sql = "SELECT REPLACE(tag1, 'A', 'B') FROM table1 WHERE REPLACE(attr1,
'C', 'D') = 'D'";
+ context = new MPPQueryContext(sql, queryId, sessionInfo, null, null);
+ actualAnalysis = analyzeSQL(sql, metadata);
+ logicalPlanner =
+ new LogicalPlanner(
+ context, metadata, sessionInfo, getFakePartitionFetcher(),
WarningCollector.NOOP);
+ logicalQueryPlan = logicalPlanner.plan(actualAnalysis);
+ rootNode = logicalQueryPlan.getRootNode();
+
+ // 5. diff
+ sql = "SELECT DIFF(s1) FROM table1 WHERE DIFF(s2) > 0";
+ context = new MPPQueryContext(sql, queryId, sessionInfo, null, null);
+ actualAnalysis = analyzeSQL(sql, metadata);
+ logicalPlanner =
+ new LogicalPlanner(
+ context, metadata, sessionInfo, getFakePartitionFetcher(),
WarningCollector.NOOP);
+ logicalQueryPlan = logicalPlanner.plan(actualAnalysis);
+ rootNode = logicalQueryPlan.getRootNode();
}
public static Analysis analyzeSQL(String sql, Metadata metadata) {
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMatadata.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMatadata.java
index bbb6ef02d3c..02e01b29503 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMatadata.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMatadata.java
@@ -16,8 +16,6 @@ package
org.apache.iotdb.db.queryengine.plan.relational.analyzer;
import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
import org.apache.iotdb.commons.udf.builtin.BuiltinAggregationFunction;
-import org.apache.iotdb.commons.udf.builtin.BuiltinScalarFunction;
-import org.apache.iotdb.db.exception.sql.SemanticException;
import org.apache.iotdb.db.queryengine.common.SessionInfo;
import org.apache.iotdb.db.queryengine.plan.relational.function.OperatorType;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnMetadata;
@@ -33,7 +31,6 @@ import
org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager;
import org.apache.iotdb.db.queryengine.plan.relational.type.TypeManager;
import
org.apache.iotdb.db.queryengine.plan.relational.type.TypeNotFoundException;
import org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignature;
-import org.apache.iotdb.db.utils.constant.SqlConstant;
import org.apache.tsfile.file.metadata.StringArrayDeviceID;
import org.apache.tsfile.read.common.type.BinaryType;
@@ -44,6 +41,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Optional;
+import static
org.apache.iotdb.db.queryengine.plan.relational.metadata.TableMetadataImpl.getFunctionType;
import static org.apache.tsfile.read.common.type.BinaryType.TEXT;
import static org.apache.tsfile.read.common.type.BooleanType.BOOLEAN;
import static org.apache.tsfile.read.common.type.DoubleType.DOUBLE;
@@ -65,6 +63,7 @@ public class TestMatadata implements Metadata {
private static final String ATTR2 = "attr2";
private static final String S1 = "s1";
private static final String S2 = "s2";
+ private static final String S3 = "s3";
private static final ColumnMetadata TIME_CM = new ColumnMetadata(TIME,
INT64);
private static final ColumnMetadata TAG1_CM = new ColumnMetadata(TAG1,
BinaryType.TEXT);
private static final ColumnMetadata TAG2_CM = new ColumnMetadata(TAG2,
BinaryType.TEXT);
@@ -73,6 +72,7 @@ public class TestMatadata implements Metadata {
private static final ColumnMetadata ATTR2_CM = new ColumnMetadata(ATTR2,
BinaryType.TEXT);
private static final ColumnMetadata S1_CM = new ColumnMetadata(S1, INT64);
private static final ColumnMetadata S2_CM = new ColumnMetadata(S2, INT64);
+ private static final ColumnMetadata S3_CM = new ColumnMetadata(S3, DOUBLE);
public static final String DB2 = "db2";
public static final String TABLE2 = "table2";
@@ -101,6 +101,9 @@ public class TestMatadata implements Metadata {
.setColumnCategory(TsTableColumnCategory.MEASUREMENT)
.build(),
ColumnSchema.builder(S2_CM)
+ .setColumnCategory(TsTableColumnCategory.MEASUREMENT)
+ .build(),
+ ColumnSchema.builder(S3_CM)
.setColumnCategory(TsTableColumnCategory.MEASUREMENT)
.build());
@@ -150,114 +153,7 @@ public class TestMatadata implements Metadata {
@Override
public Type getFunctionReturnType(String functionName, List<? extends Type>
argumentTypes) {
-
- // builtin scalar function
- if
(BuiltinScalarFunction.DIFF.getFunctionName().equalsIgnoreCase(functionName)
- ||
BuiltinScalarFunction.ROUND.getFunctionName().equalsIgnoreCase(functionName)) {
- if (!isOneNumericType(argumentTypes)) {
- throw new SemanticException(
- "Scalar function"
- + functionName.toLowerCase(Locale.ENGLISH)
- + " only supports numeric data types [INT32, INT64, FLOAT,
DOUBLE]");
- }
- return DOUBLE;
- } else if
(BuiltinScalarFunction.REPLACE.getFunctionName().equalsIgnoreCase(functionName)
- ||
BuiltinScalarFunction.SUBSTRING.getFunctionName().equalsIgnoreCase(functionName))
{
- if (!isOneTextType(argumentTypes)) {
- throw new SemanticException(
- "Scalar function"
- + functionName.toLowerCase(Locale.ENGLISH)
- + " only supports text data type.");
- }
- return TEXT;
- }
-
- // builtin aggregation function
- // check argument type
- switch (functionName.toLowerCase()) {
- case SqlConstant.AVG:
- case SqlConstant.SUM:
- case SqlConstant.EXTREME:
- case SqlConstant.MIN_VALUE:
- case SqlConstant.MAX_VALUE:
- case SqlConstant.STDDEV:
- case SqlConstant.STDDEV_POP:
- case SqlConstant.STDDEV_SAMP:
- case SqlConstant.VARIANCE:
- case SqlConstant.VAR_POP:
- case SqlConstant.VAR_SAMP:
- if (!isOneNumericType(argumentTypes)) {
- throw new SemanticException(
- String.format(
- "Aggregate functions [%s] only support numeric data types
[INT32, INT64, FLOAT, DOUBLE]",
- functionName));
- }
- break;
- case SqlConstant.MIN_TIME:
- case SqlConstant.MAX_TIME:
- case SqlConstant.FIRST_VALUE:
- case SqlConstant.LAST_VALUE:
- case SqlConstant.TIME_DURATION:
- case SqlConstant.MODE:
- if (argumentTypes.size() != 1) {
- throw new SemanticException(
- String.format(
- "Aggregate functions [%s] should only have one argument",
functionName));
- }
- break;
- case SqlConstant.MAX_BY:
- case SqlConstant.MIN_BY:
- if (argumentTypes.size() != 2) {
- throw new SemanticException(
- String.format(
- "Aggregate functions [%s] should only have two arguments",
functionName));
- } else if (!argumentTypes.get(1).isOrderable()) {
- throw new SemanticException(
- String.format(
- "Second argument of Aggregate functions [%s] should be
orderable", functionName));
- }
-
- break;
- case SqlConstant.COUNT:
- break;
- default:
- // ignore
- }
-
- // get return type
- switch (functionName.toLowerCase()) {
- case SqlConstant.MIN_TIME:
- case SqlConstant.MAX_TIME:
- case SqlConstant.COUNT:
- case SqlConstant.TIME_DURATION:
- return INT64;
- case SqlConstant.MIN_VALUE:
- case SqlConstant.LAST_VALUE:
- case SqlConstant.FIRST_VALUE:
- case SqlConstant.MAX_VALUE:
- case SqlConstant.EXTREME:
- case SqlConstant.MODE:
- case SqlConstant.MAX_BY:
- case SqlConstant.MIN_BY:
- return argumentTypes.get(0);
- case SqlConstant.AVG:
- case SqlConstant.SUM:
- case SqlConstant.STDDEV:
- case SqlConstant.STDDEV_POP:
- case SqlConstant.STDDEV_SAMP:
- case SqlConstant.VARIANCE:
- case SqlConstant.VAR_POP:
- case SqlConstant.VAR_SAMP:
- return DOUBLE;
- default:
- // ignore
- }
-
- // TODO scalar UDF function
-
- // TODO UDAF
-
- throw new SemanticException("Unknown function: " + functionName);
+ return getFunctionType(functionName, argumentTypes);
}
@Override
