This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch ty/TableModelGrammar in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 64bd103f54e1a9996e4810dea9c3030ddc0f8baa Author: JackieTien97 <[email protected]> AuthorDate: Tue Apr 16 21:32:24 2024 +0800 add some filter converter --- .../source/relational/TableScanOperator.java | 16 +- .../queryengine/plan/analyze/PredicateUtils.java | 20 ++ .../plan/planner/TableOperatorGenerator.java | 162 ++++++++++++++- .../predicate/ConvertPredicateToFilterVisitor.java | 22 +++ .../ConvertPredicateToTimeFilterVisitor.java | 218 +++++++++++++++++++++ .../predicate/PredicatePushIntoScanChecker.java | 131 +++++++++++++ .../analyzer/predicate/PredicateVisitor.java | 87 ++++++++ .../relational/planner/node/TableScanNode.java | 8 + .../tsfile/read/filter/factory/FilterFactory.java | 18 ++ 9 files changed, 671 insertions(+), 11 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableScanOperator.java index 97ffe2b2282..1e26fa5d636 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableScanOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableScanOperator.java @@ -28,7 +28,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions; import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema; import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry; -import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; import org.apache.iotdb.tsfile.common.conf.TSFileConfig; @@ -56,8 +55,6 @@ import static org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeM public class TableScanOperator extends AbstractDataSourceOperator { - private final List<Symbol> outputColumnNames; - private final List<ColumnSchema> columnSchemas; private final int[] columnsIndexArray; @@ -78,18 +75,18 @@ public class TableScanOperator extends AbstractDataSourceOperator { private final List<TSDataType> measurementColumnTSDataTypes; private final TsBlockBuilder measurementDataBuilder; + + private final int maxTsBlockLineNum; + private TsBlock measurementDataBlock; private QueryDataSource queryDataSource; - private int currentDeviceIndex = 0; - - private int maxTsBlockLineNum = -1; + private int currentDeviceIndex; public TableScanOperator( OperatorContext context, PlanNodeId sourceId, - List<Symbol> outputColumnNames, List<ColumnSchema> columnSchemas, int[] columnsIndexArray, int measurementColumnCount, @@ -101,7 +98,6 @@ public class TableScanOperator extends AbstractDataSourceOperator { int maxTsBlockLineNum) { this.sourceId = sourceId; this.operatorContext = context; - this.outputColumnNames = outputColumnNames; this.columnSchemas = columnSchemas; this.columnsIndexArray = columnsIndexArray; this.measurementColumnCount = measurementColumnCount; @@ -118,7 +114,7 @@ public class TableScanOperator extends AbstractDataSourceOperator { this.maxReturnSize = Math.min( maxReturnSize, - (1L + outputColumnNames.size()) + (1L + columnsIndexArray.length) * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte()); this.maxTsBlockLineNum = maxTsBlockLineNum; @@ -303,7 +299,7 @@ public class TableScanOperator extends AbstractDataSourceOperator { @Override public long calculateMaxPeekMemory() { - return (1L + outputColumnNames.size()) + return (1L + columnsIndexArray.length) * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte() * 3L; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/PredicateUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/PredicateUtils.java index ce41726768b..d635f074789 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/PredicateUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/PredicateUtils.java @@ -290,6 +290,19 @@ public class PredicateUtils { allMeasurements, isBuildPlanUseTemplate, typeProvider)); } + public static Filter convertPredicateToFilter( + org.apache.iotdb.db.relational.sql.tree.Expression predicate, + List<String> allMeasurements, + TypeProvider typeProvider) { + if (predicate == null) { + return null; + } + return predicate.accept( + new ConvertPredicateToFilterVisitor(), + new ConvertPredicateToFilterVisitor.Context( + allMeasurements, isBuildPlanUseTemplate, typeProvider)); + } + /** * Combine the given conjuncts into a single expression using "and". * @@ -396,4 +409,11 @@ public class PredicateUtils { public static boolean predicateCanPushIntoScan(Expression predicate) { return new PredicatePushIntoScanChecker().process(predicate, null); } + + public static boolean predicateCanPushIntoScan( + org.apache.iotdb.db.relational.sql.tree.Expression predicate) { + return new org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate + .PredicatePushIntoScanChecker() + .process(predicate, null); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index 9899ce6922e..f5e22768ce5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -19,9 +19,18 @@ package org.apache.iotdb.db.queryengine.plan.planner; +import org.apache.iotdb.commons.path.AlignedPath; +import org.apache.iotdb.db.queryengine.execution.driver.DataDriverContext; import org.apache.iotdb.db.queryengine.execution.operator.Operator; +import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; +import org.apache.iotdb.db.queryengine.execution.operator.source.AlignedSeriesScanOperator; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.TableScanOperator; +import org.apache.iotdb.db.queryengine.plan.analyze.PredicateUtils; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FilterNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.MergeSortNode; @@ -31,6 +40,22 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode; +import org.apache.iotdb.db.relational.sql.tree.Expression; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.read.filter.basic.Filter; +import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +import static java.util.Objects.requireNonNull; +import static org.apache.iotdb.db.queryengine.plan.analyze.PredicateUtils.convertPredicateToFilter; +import static org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager.getTSDataType; /** This Visitor is responsible for transferring Table PlanNode Tree to Table Operator Tree. */ public class TableOperatorGenerator extends PlanVisitor<Operator, LocalExecutionPlanContext> { @@ -42,7 +67,142 @@ public class TableOperatorGenerator extends PlanVisitor<Operator, LocalExecution @Override public Operator visitTableScan(TableScanNode node, LocalExecutionPlanContext context) { - return super.visitTableScan(node, context); + + List<Symbol> outputColumnNames = node.getOutputSymbols(); + int outputColumnCount = outputColumnNames.size(); + List<ColumnSchema> columnSchemas = new ArrayList<>(outputColumnCount); + int[] columnsIndexArray = new int[outputColumnCount]; + Map<Symbol, ColumnSchema> columnSchemaMap = node.getAssignments(); + Map<Symbol, Integer> idAndAttributeColumnsIndexMap = node.getAttributesMap(); + List<String> measurementColumnNames = new ArrayList<>(); + List<IMeasurementSchema> measurementSchemas = new ArrayList<>(); + int measurementColumnCount = 0; + for (int i = 0; i < outputColumnCount; i++) { + Symbol columnName = outputColumnNames.get(i); + ColumnSchema schema = + requireNonNull(columnSchemaMap.get(columnName), columnName + " is null"); + columnSchemas.add(schema); + switch (schema.getColumnCategory()) { + case ID: + case ATTRIBUTE: + columnsIndexArray[i] = + requireNonNull( + idAndAttributeColumnsIndexMap.get(columnName), columnName + " is null"); + break; + case MEASUREMENT: + columnsIndexArray[i] = measurementColumnCount; + measurementColumnCount++; + measurementColumnNames.add(columnName.getName()); + measurementSchemas.add( + new MeasurementSchema(schema.getName(), getTSDataType(schema.getType()))); + break; + default: + throw new IllegalArgumentException( + "Unexpected column category: " + schema.getColumnCategory()); + } + } + + SeriesScanOptions.Builder scanOptionsBuilder = getSeriesScanOptionsBuilder(node, context); + scanOptionsBuilder.withPushDownLimit(node.getPushDownLimit()); + scanOptionsBuilder.withPushDownOffset(node.getPushDownOffset()); + scanOptionsBuilder.withAllSensors(new HashSet<>(measurementColumnNames)); + + Expression pushDownPredicate = node.getPushDownPredicate(); + boolean predicateCanPushIntoScan = canPushIntoScan(pushDownPredicate); + if (pushDownPredicate != null && predicateCanPushIntoScan) { + scanOptionsBuilder.withPushDownFilter( + convertPredicateToFilter( + pushDownPredicate, + node.getAlignedPath().getMeasurementList(), + context.getTypeProvider().getTemplatedInfo() != null, + context.getTypeProvider())); + } + + OperatorContext operatorContext = + context + .getDriverContext() + .addOperatorContext( + context.getNextOperatorId(), + node.getPlanNodeId(), + AlignedSeriesScanOperator.class.getSimpleName()); + + int maxTsBlockLineNum = TSFileDescriptor.getInstance().getConfig().getMaxTsBlockLineNumber(); + if (context.getTypeProvider().getTemplatedInfo() != null) { + maxTsBlockLineNum = + (int) + Math.min( + context.getTypeProvider().getTemplatedInfo().getLimitValue(), maxTsBlockLineNum); + } + + TableScanOperator tableScanOperator = + new TableScanOperator( + operatorContext, + node.getPlanNodeId(), + seriesPath, + node.getScanOrder(), + scanOptionsBuilder.build(), + node.isQueryAllSensors(), + context.getTypeProvider().getTemplatedInfo() != null + ? context.getTypeProvider().getTemplatedInfo().getDataTypes() + : null, + maxTsBlockLineNum); + + ((DataDriverContext) context.getDriverContext()).addSourceOperator(seriesScanOperator); + ((DataDriverContext) context.getDriverContext()).addPath(seriesPath); + context.getDriverContext().setInputDriver(true); + + if (!predicateCanPushIntoScan) { + if (context.isBuildPlanUseTemplate()) { + TemplatedInfo templatedInfo = context.getTemplatedInfo(); + return constructFilterOperator( + pushDownPredicate, + seriesScanOperator, + templatedInfo.getProjectExpressions(), + templatedInfo.getDataTypes(), + templatedInfo.getLayoutMap(), + templatedInfo.isKeepNull(), + node.getPlanNodeId(), + templatedInfo.getScanOrder(), + context); + } + + AlignedPath alignedPath = node.getAlignedPath(); + List<Expression> expressions = new ArrayList<>(); + List<TSDataType> dataTypes = new ArrayList<>(); + for (int i = 0; i < alignedPath.getMeasurementList().size(); i++) { + expressions.add(ExpressionFactory.timeSeries(alignedPath.getSubMeasurementPath(i))); + dataTypes.add(alignedPath.getSubMeasurementDataType(i)); + } + + return constructFilterOperator( + pushDownPredicate, + seriesScanOperator, + expressions.toArray(new Expression[0]), + dataTypes, + makeLayout(Collections.singletonList(node)), + false, + node.getPlanNodeId(), + node.getScanOrder(), + context); + } + return seriesScanOperator; + } + + private SeriesScanOptions.Builder getSeriesScanOptionsBuilder( + TableScanNode node, LocalExecutionPlanContext context) { + SeriesScanOptions.Builder scanOptionsBuilder = new SeriesScanOptions.Builder(); + + Filter globalTimeFilter = context.getGlobalTimeFilter(); + if (globalTimeFilter != null) { + // time filter may be stateful, so we need to copy it + scanOptionsBuilder.withGlobalTimeFilter(globalTimeFilter.copy()); + } + + return scanOptionsBuilder; + } + + private boolean canPushIntoScan(Expression pushDownPredicate) { + return pushDownPredicate == null || PredicateUtils.predicateCanPushIntoScan(pushDownPredicate); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java new file mode 100644 index 00000000000..7ab2448ded1 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate; + +public class ConvertPredicateToFilterVisitor {} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToTimeFilterVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToTimeFilterVisitor.java new file mode 100644 index 00000000000..f48a78151df --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToTimeFilterVisitor.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate; + +import org.apache.iotdb.db.relational.sql.tree.BetweenPredicate; +import org.apache.iotdb.db.relational.sql.tree.ComparisonExpression; +import org.apache.iotdb.db.relational.sql.tree.Expression; +import org.apache.iotdb.db.relational.sql.tree.IfExpression; +import org.apache.iotdb.db.relational.sql.tree.InListExpression; +import org.apache.iotdb.db.relational.sql.tree.InPredicate; +import org.apache.iotdb.db.relational.sql.tree.IsNotNullPredicate; +import org.apache.iotdb.db.relational.sql.tree.IsNullPredicate; +import org.apache.iotdb.db.relational.sql.tree.LikePredicate; +import org.apache.iotdb.db.relational.sql.tree.LogicalExpression; +import org.apache.iotdb.db.relational.sql.tree.LongLiteral; +import org.apache.iotdb.db.relational.sql.tree.NotExpression; +import org.apache.iotdb.db.relational.sql.tree.NullIfExpression; +import org.apache.iotdb.db.relational.sql.tree.SearchedCaseExpression; +import org.apache.iotdb.db.relational.sql.tree.SimpleCaseExpression; +import org.apache.iotdb.db.relational.sql.tree.SymbolReference; +import org.apache.iotdb.tsfile.read.filter.basic.Filter; +import org.apache.iotdb.tsfile.read.filter.factory.FilterFactory; +import org.apache.iotdb.tsfile.read.filter.factory.TimeFilterApi; + +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.google.common.base.Preconditions.checkArgument; + +public class ConvertPredicateToTimeFilterVisitor extends PredicateVisitor<Filter, Void> { + + @Override + protected Filter visitInPredicate(InPredicate node, Void context) { + checkArgument(isTimeColumn(node.getValue())); + Expression valueList = node.getValueList(); + checkArgument(valueList instanceof InListExpression); + List<Expression> values = ((InListExpression) valueList).getValues(); + for (Expression value : values) { + checkArgument(value instanceof LongLiteral); + } + if (values.size() == 1) { + TimeFilterApi.eq(((LongLiteral) values.get(0)).getParsedValue()); + } + Set<Long> longValues = new HashSet<>(); + for (Expression value : values) { + longValues.add(((LongLiteral) value).getParsedValue()); + } + return TimeFilterApi.in(longValues); + } + + @Override + protected Filter visitIsNullPredicate(IsNullPredicate node, Void context) { + throw new UnsupportedOperationException("TIMESTAMP does not support IS NULL"); + } + + @Override + protected Filter visitIsNotNullPredicate(IsNotNullPredicate node, Void context) { + throw new UnsupportedOperationException("TIMESTAMP does not support IS NOT NULL"); + } + + @Override + protected Filter visitLikePredicate(LikePredicate node, Void context) { + throw new UnsupportedOperationException("TIMESTAMP does not support LIKE"); + } + + @Override + protected Filter visitLogicalExpression(LogicalExpression node, Void context) { + List<Filter> filterList = + node.getTerms().stream() + .map(n -> n.accept(this, context)) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + switch (node.getOperator()) { + case OR: + return FilterFactory.or(filterList); + case AND: + return FilterFactory.and(filterList); + default: + throw new IllegalArgumentException("Unsupported operator: " + node.getOperator()); + } + } + + @Override + protected Filter visitNotExpression(NotExpression node, Void context) { + return FilterFactory.not(node.getValue().accept(this, context)); + } + + @Override + protected Filter visitComparisonExpression(ComparisonExpression node, Void context) { + long value; + if (node.getLeft() instanceof LongLiteral) { + value = getLongValue(node.getLeft()); + } else if (node.getRight() instanceof LongLiteral) { + value = getLongValue(node.getRight()); + } else { + throw new IllegalStateException( + "Either left or right operand of Time ComparisonExpression should be LongLiteral"); + } + switch (node.getOperator()) { + case EQUAL: + return TimeFilterApi.eq(value); + case NOT_EQUAL: + return TimeFilterApi.notEq(value); + case GREATER_THAN: + return TimeFilterApi.gt(value); + case GREATER_THAN_OR_EQUAL: + return TimeFilterApi.gtEq(value); + case LESS_THAN: + return TimeFilterApi.lt(value); + case LESS_THAN_OR_EQUAL: + return TimeFilterApi.ltEq(value); + default: + throw new IllegalArgumentException("Unsupported operator: " + node.getOperator()); + } + } + + @Override + protected Filter visitSimpleCaseExpression(SimpleCaseExpression node, Void context) { + throw new UnsupportedOperationException("TIMESTAMP does not CASE WHEN"); + } + + @Override + protected Filter visitSearchedCaseExpression(SearchedCaseExpression node, Void context) { + throw new UnsupportedOperationException("TIMESTAMP does not CASE WHEN"); + } + + @Override + protected Filter visitIfExpression(IfExpression node, Void context) { + throw new UnsupportedOperationException("TIMESTAMP does not IF"); + } + + @Override + protected Filter visitNullIfExpression(NullIfExpression node, Void context) { + throw new UnsupportedOperationException("TIMESTAMP does not NULLIF"); + } + + @Override + protected Filter visitBetweenPredicate(BetweenPredicate node, Void context) { + Expression firstExpression = node.getValue(); + Expression secondExpression = node.getMin(); + Expression thirdExpression = node.getMax(); + + if (isTimeColumn(firstExpression)) { + // firstExpression is TIMESTAMP + long minValue = getLongValue(secondExpression); + long maxValue = getLongValue(thirdExpression); + + if (minValue == maxValue) { + return TimeFilterApi.eq(minValue); + } + return TimeFilterApi.between(minValue, maxValue); + } else if (isTimeColumn(secondExpression)) { + // secondExpression is TIMESTAMP + long value = getLongValue(firstExpression); + long maxValue = getLongValue(thirdExpression); + + // cases: + // 1 BETWEEN time AND 2 => time <= 1 + // 1 BETWEEN time AND 1 => time <= 1 + // 1 BETWEEN time AND 0 => FALSE + // 1 NOT BETWEEN time AND 2 => time > 1 + // 1 NOT BETWEEN time AND 1 => time > 1 + // 1 NOT BETWEEN time AND 0 => TRUE + checkArgument( + value <= maxValue, + String.format("Predicate [%s] should be simplified in previous step", node)); + return TimeFilterApi.ltEq(value); + } else if (isTimeColumn(thirdExpression)) { + // thirdExpression is TIMESTAMP + long value = getLongValue(firstExpression); + long minValue = getLongValue(secondExpression); + + // cases: + // 1 BETWEEN 2 AND time => FALSE + // 1 BETWEEN 1 AND time => time >= 1 + // 1 BETWEEN 0 AND time => time >= 1 + // 1 NOT BETWEEN 2 AND time => TRUE + // 1 NOT BETWEEN 1 AND time => time < 1 + // 1 NOT BETWEEN 0 AND time => time < 1 + checkArgument( + value >= minValue, + String.format("Predicate [%s] should be simplified in previous step", node)); + return TimeFilterApi.gtEq(value); + } else { + throw new IllegalStateException( + "Three operand of between expression should have time column."); + } + } + + public static boolean isTimeColumn(Expression expression) { + return expression instanceof SymbolReference + && "time".equalsIgnoreCase(((SymbolReference) expression).getName()); + } + + public static long getLongValue(Expression expression) { + return ((LongLiteral) expression).getParsedValue(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicatePushIntoScanChecker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicatePushIntoScanChecker.java new file mode 100644 index 00000000000..519027bf832 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicatePushIntoScanChecker.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate; + +import org.apache.iotdb.db.relational.sql.tree.BetweenPredicate; +import org.apache.iotdb.db.relational.sql.tree.ComparisonExpression; +import org.apache.iotdb.db.relational.sql.tree.Expression; +import org.apache.iotdb.db.relational.sql.tree.IfExpression; +import org.apache.iotdb.db.relational.sql.tree.InPredicate; +import org.apache.iotdb.db.relational.sql.tree.IsNotNullPredicate; +import org.apache.iotdb.db.relational.sql.tree.IsNullPredicate; +import org.apache.iotdb.db.relational.sql.tree.LikePredicate; +import org.apache.iotdb.db.relational.sql.tree.Literal; +import org.apache.iotdb.db.relational.sql.tree.LogicalExpression; +import org.apache.iotdb.db.relational.sql.tree.NotExpression; +import org.apache.iotdb.db.relational.sql.tree.NullIfExpression; +import org.apache.iotdb.db.relational.sql.tree.SearchedCaseExpression; +import org.apache.iotdb.db.relational.sql.tree.SimpleCaseExpression; +import org.apache.iotdb.db.relational.sql.tree.SymbolReference; + +import java.util.List; + +public class PredicatePushIntoScanChecker extends PredicateVisitor<Boolean, Void> { + + @Override + public Boolean visitExpression(Expression expression, Void context) { + return Boolean.FALSE; + } + + @Override + protected Boolean visitInPredicate(InPredicate node, Void context) { + return isSymbolReference(node.getValue()); + } + + @Override + protected Boolean visitIsNullPredicate(IsNullPredicate node, Void context) { + throw new IllegalArgumentException("IS NULL Expression can't be pushed down"); + } + + @Override + protected Boolean visitIsNotNullPredicate(IsNotNullPredicate node, Void context) { + return isSymbolReference(node.getValue()); + } + + @Override + protected Boolean visitLikePredicate(LikePredicate node, Void context) { + return isSymbolReference(node.getValue()) + && isLiteral(node.getPattern()) + && node.getEscape().map(this::isLiteral).orElse(true); + } + + @Override + protected Boolean visitLogicalExpression(LogicalExpression node, Void context) { + List<Expression> children = node.getTerms(); + for (Expression child : children) { + if (!process(child, context)) { + return Boolean.FALSE; + } + } + return Boolean.TRUE; + } + + @Override + protected Boolean visitNotExpression(NotExpression node, Void context) { + throw new IllegalArgumentException("Not Expression can't be pushed down"); + } + + @Override + protected Boolean visitComparisonExpression(ComparisonExpression node, Void context) { + return (isSymbolReference(node.getLeft()) && isLiteral(node.getRight())) + || (isSymbolReference(node.getRight()) && isLiteral(node.getLeft())); + } + + @Override + protected Boolean visitSimpleCaseExpression(SimpleCaseExpression node, Void context) { + return Boolean.FALSE; + } + + @Override + protected Boolean visitSearchedCaseExpression(SearchedCaseExpression node, Void context) { + return Boolean.FALSE; + } + + @Override + protected Boolean visitIfExpression(IfExpression node, Void context) { + return Boolean.FALSE; + } + + @Override + protected Boolean visitNullIfExpression(NullIfExpression node, Void context) { + return Boolean.FALSE; + } + + @Override + protected Boolean visitBetweenPredicate(BetweenPredicate node, Void context) { + return (isSymbolReference(node.getValue()) + && isLiteral(node.getMin()) + && isLiteral(node.getMax())) + || (isLiteral(node.getValue()) + && isSymbolReference(node.getMin()) + && isLiteral(node.getMax())) + || (isLiteral(node.getValue()) + && isLiteral(node.getMin()) + && isSymbolReference(node.getMax())); + } + + public static boolean isLiteral(Expression expression) { + return expression instanceof Literal; + } + + public static boolean isSymbolReference(Expression expression) { + return expression instanceof SymbolReference; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicateVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicateVisitor.java new file mode 100644 index 00000000000..70d1ae11d0b --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicateVisitor.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate; + +import org.apache.iotdb.db.relational.sql.tree.AstVisitor; +import org.apache.iotdb.db.relational.sql.tree.BetweenPredicate; +import org.apache.iotdb.db.relational.sql.tree.ComparisonExpression; +import org.apache.iotdb.db.relational.sql.tree.Expression; +import org.apache.iotdb.db.relational.sql.tree.IfExpression; +import org.apache.iotdb.db.relational.sql.tree.InPredicate; +import org.apache.iotdb.db.relational.sql.tree.IsNotNullPredicate; +import org.apache.iotdb.db.relational.sql.tree.IsNullPredicate; +import org.apache.iotdb.db.relational.sql.tree.LikePredicate; +import org.apache.iotdb.db.relational.sql.tree.LogicalExpression; +import org.apache.iotdb.db.relational.sql.tree.NotExpression; +import org.apache.iotdb.db.relational.sql.tree.NullIfExpression; +import org.apache.iotdb.db.relational.sql.tree.SearchedCaseExpression; +import org.apache.iotdb.db.relational.sql.tree.SimpleCaseExpression; + +/** + * This class provides a visitor of {@link Expression}, which can be extended to create a visitor + * which only needs to handle a subset of the available methods. + * + * @param <R> The return type of the visit operation. + * @param <C> The context information during visiting. + */ +public abstract class PredicateVisitor<R, C> extends AstVisitor<R, C> { + + @Override + public R visitExpression(Expression expression, C context) { + throw new IllegalArgumentException( + "Unsupported expression: " + expression.getClass().getSimpleName()); + } + + @Override + protected abstract R visitInPredicate(InPredicate node, C context); + + @Override + protected abstract R visitIsNullPredicate(IsNullPredicate node, C context); + + @Override + protected abstract R visitIsNotNullPredicate(IsNotNullPredicate node, C context); + + @Override + protected abstract R visitLikePredicate(LikePredicate node, C context); + + @Override + protected abstract R visitLogicalExpression(LogicalExpression node, C context); + + @Override + protected abstract R visitNotExpression(NotExpression node, C context); + + @Override + protected abstract R visitComparisonExpression(ComparisonExpression node, C context); + + @Override + protected abstract R visitSimpleCaseExpression(SimpleCaseExpression node, C context); + + @Override + protected abstract R visitSearchedCaseExpression(SearchedCaseExpression node, C context); + + @Override + protected abstract R visitIfExpression(IfExpression node, C context); + + @Override + protected abstract R visitNullIfExpression(NullIfExpression node, C context); + + @Override + protected abstract R visitBetweenPredicate(BetweenPredicate node, C context); +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java index 8849b3ddb18..521c4749b13 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/TableScanNode.java @@ -131,6 +131,14 @@ public class TableScanNode extends PlanNode { return this.scanOrder; } + public List<DeviceEntry> getDeviceEntries() { + return deviceEntries; + } + + public Map<Symbol, Integer> getAttributesMap() { + return attributesMap; + } + public Expression getPushDownPredicate() { return this.pushDownPredicate; } diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/factory/FilterFactory.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/factory/FilterFactory.java index 23546767987..14f8d43bac4 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/factory/FilterFactory.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/factory/FilterFactory.java @@ -24,6 +24,8 @@ import org.apache.iotdb.tsfile.read.filter.operator.And; import org.apache.iotdb.tsfile.read.filter.operator.Not; import org.apache.iotdb.tsfile.read.filter.operator.Or; +import java.util.List; + import static org.apache.iotdb.tsfile.utils.Preconditions.checkArgument; public class FilterFactory { @@ -43,6 +45,14 @@ public class FilterFactory { return new And(left, right); } + public static Filter and(List<Filter> filterList) { + And result = new And(filterList.get(0), filterList.get(1)); + for (int i = 2, size = filterList.size(); i < size; i++) { + result = new And(result, filterList.get(i)); + } + return result; + } + public static Filter or(Filter left, Filter right) { if (left == null && right == null) { return null; @@ -54,6 +64,14 @@ public class FilterFactory { return new Or(left, right); } + public static Filter or(List<Filter> filterList) { + Or result = new Or(filterList.get(0), filterList.get(1)); + for (int i = 2, size = filterList.size(); i < size; i++) { + result = new Or(result, filterList.get(i)); + } + return result; + } + public static Not not(Filter filter) { checkArgument(filter != null, "filter cannot be null"); return new Not(filter);
