This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch rc/2.0.5 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f12055153ac349ce2f9c531a37c2c6a9beb9ba93 Author: Weihao Li <[email protected]> AuthorDate: Wed Jul 23 09:10:59 2025 +0800 Support extract value filter push down (cherry picked from commit 2c6e00f1f1e89a311f44090e191a61fd7e32a9fc) --- .../query/recent/extract/IoTDBExtractTableIT.java | 107 +++++++++++++++++++- .../predicate/ConvertPredicateToFilterVisitor.java | 108 ++++++++++++++++++++- .../PredicateCombineIntoTableScanChecker.java | 50 +++++----- .../optimizations/PushPredicateIntoTableScan.java | 21 ++-- .../relational/analyzer/ExtractExpressionTest.java | 10 ++ .../plan/relational/analyzer/TestMetadata.java | 57 +++++++---- pom.xml | 2 +- 7 files changed, 296 insertions(+), 59 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/extract/IoTDBExtractTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/extract/IoTDBExtractTableIT.java index 8b065b1571f..f0b27007126 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/extract/IoTDBExtractTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/extract/IoTDBExtractTableIT.java @@ -118,7 +118,7 @@ public class IoTDBExtractTableIT { } @Test - public void extractTimeFilterPushDownTest() { + public void extractFilterPushDownTest() { String[] expectedHeader = new String[] {"time", "s1"}; String[] retArray = new String[] { @@ -129,23 +129,34 @@ public class IoTDBExtractTableIT { expectedHeader, retArray, DATABASE_NAME); - // verify the pushdown result is same with non-pushdown tableResultSetEqualTest( "SELECT time, s1 FROM table1 where extract(hour from ts) > 1", expectedHeader, retArray, DATABASE_NAME); + // verify the pushdown result is same with non-pushdown + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from ts) + 1 > 2", + expectedHeader, + retArray, + DATABASE_NAME); tableResultSetEqualTest( "SELECT time, s1 FROM table1 where extract(hour from time) >= 2", expectedHeader, retArray, DATABASE_NAME); - // verify the pushdown result is same with non-pushdown tableResultSetEqualTest( "SELECT time, s1 FROM table1 where extract(hour from ts) >= 2", expectedHeader, retArray, DATABASE_NAME); + // verify the pushdown result is same with non-pushdown + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from ts) + 1>= 3", + expectedHeader, + retArray, + DATABASE_NAME); + retArray = new String[] { getTimeStrUTC8("2025-07-08T10:18:51") + ",2,", @@ -162,6 +173,12 @@ public class IoTDBExtractTableIT { expectedHeader, retArray, DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from ts) + 1 > 10", + "+08:00", + expectedHeader, + retArray, + DATABASE_NAME); tableResultSetEqualTest( "SELECT time, s1 FROM table1 where extract(hour from time) >= 10", "+08:00", @@ -174,6 +191,12 @@ public class IoTDBExtractTableIT { expectedHeader, retArray, DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from ts)+1>= 11", + "+08:00", + expectedHeader, + retArray, + DATABASE_NAME); expectedHeader = new String[] {"time", "s1"}; retArray = @@ -190,6 +213,11 @@ public class IoTDBExtractTableIT { expectedHeader, retArray, DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from ts) + 1< 2", + expectedHeader, + retArray, + DATABASE_NAME); tableResultSetEqualTest( "SELECT time, s1 FROM table1 where extract(hour from time) <= 0", expectedHeader, @@ -200,6 +228,11 @@ public class IoTDBExtractTableIT { expectedHeader, retArray, DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from ts) +1<= 1", + expectedHeader, + retArray, + DATABASE_NAME); tableResultSetEqualTest( "SELECT time, s1 FROM table1 where extract(hour from time) = 0", expectedHeader, @@ -210,6 +243,11 @@ public class IoTDBExtractTableIT { expectedHeader, retArray, DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from ts) +1= 1", + expectedHeader, + retArray, + DATABASE_NAME); retArray = new String[] { getTimeStrUTC8("2025-07-09T08:17:50") + ",3,", @@ -226,6 +264,12 @@ public class IoTDBExtractTableIT { expectedHeader, retArray, DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from ts) + 1 < 10", + "+08:00", + expectedHeader, + retArray, + DATABASE_NAME); tableResultSetEqualTest( "SELECT time, s1 FROM table1 where extract(hour from time) <= 8", "+08:00", @@ -238,6 +282,12 @@ public class IoTDBExtractTableIT { expectedHeader, retArray, DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from ts) +1<= 9", + "+08:00", + expectedHeader, + retArray, + DATABASE_NAME); tableResultSetEqualTest( "SELECT time, s1 FROM table1 where extract(hour from time) = 8", "+08:00", @@ -250,6 +300,12 @@ public class IoTDBExtractTableIT { expectedHeader, retArray, DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from ts) +1= 9", + "+08:00", + expectedHeader, + retArray, + DATABASE_NAME); retArray = new String[] { @@ -266,6 +322,27 @@ public class IoTDBExtractTableIT { expectedHeader, retArray, DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from ts) +1!= 1", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from time) between 1 and 2", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from ts) between 1 and 2", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from ts) +1 between 2 and 3", + expectedHeader, + retArray, + DATABASE_NAME); + retArray = new String[] { getTimeStrUTC8("2025-07-08T09:18:51") + ",1,", @@ -283,6 +360,30 @@ public class IoTDBExtractTableIT { expectedHeader, retArray, DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from ts) +1 != 9", + "+08:00", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from time) between 9 and 10", + "+08:00", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from ts) between 9 and 10", + "+08:00", + expectedHeader, + retArray, + DATABASE_NAME); + tableResultSetEqualTest( + "SELECT time, s1 FROM table1 where extract(hour from ts) +1 between 10 and 11", + "+08:00", + expectedHeader, + retArray, + DATABASE_NAME); } @Test diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java index 0c9f42b3e63..be2218ba9d3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToFilterVisitor.java @@ -29,6 +29,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DoubleLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GenericLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IfExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InListExpression; @@ -47,6 +48,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StringLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager; +import com.google.common.collect.ImmutableList; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.common.regexp.LikePattern; import org.apache.tsfile.enums.TSDataType; @@ -54,6 +56,7 @@ import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.filter.factory.FilterFactory; import org.apache.tsfile.read.filter.factory.ValueFilterApi; +import org.apache.tsfile.read.filter.operator.ExtractTimeFilterOperators; import org.apache.tsfile.utils.Binary; import javax.annotation.Nullable; @@ -72,18 +75,24 @@ import static org.apache.iotdb.db.queryengine.plan.expression.unary.LikeExpressi import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.ConvertPredicateToTimeFilterVisitor.getLongValue; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoScanChecker.isLiteral; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoScanChecker.isSymbolReference; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.ir.GlobalTimePredicateExtractVisitor.isExtractTimeColumn; import static org.apache.iotdb.db.queryengine.plan.relational.planner.ir.GlobalTimePredicateExtractVisitor.isTimeColumn; +import static org.apache.tsfile.read.common.type.TimestampType.TIMESTAMP; public class ConvertPredicateToFilterVisitor extends PredicateVisitor<Filter, ConvertPredicateToFilterVisitor.Context> { @Nullable private final String timeColumnName; private final ConvertPredicateToTimeFilterVisitor timeFilterVisitor; + private final ZoneId zoneId; + private final TimeUnit currPrecision; public ConvertPredicateToFilterVisitor( @Nullable String timeColumnName, ZoneId zoneId, TimeUnit currPrecision) { this.timeColumnName = timeColumnName; this.timeFilterVisitor = new ConvertPredicateToTimeFilterVisitor(zoneId, currPrecision); + this.zoneId = zoneId; + this.currPrecision = currPrecision; } @Override @@ -168,6 +177,48 @@ public class ConvertPredicateToFilterVisitor } } + private Filter constructExtractCompareFilter( + ComparisonExpression.Operator operator, + SymbolReference symbolReference, + Extract.Field field, + Literal literal, + Context context) { + + if (!context.isMeasurementColumn(symbolReference)) { + throw new IllegalStateException( + String.format("Only support measurement column in filter: %s", symbolReference)); + } + + int measurementIndex = context.getMeasurementIndex(symbolReference.getName()); + long value = getValue(literal, TIMESTAMP); + ExtractTimeFilterOperators.Field field1 = + ExtractTimeFilterOperators.Field.values()[field.ordinal()]; + + switch (operator) { + case EQUAL: + return ValueFilterApi.extractValueEq( + measurementIndex, value, field1, zoneId, currPrecision); + case NOT_EQUAL: + return ValueFilterApi.extractValueNotEq( + measurementIndex, value, field1, zoneId, currPrecision); + case GREATER_THAN: + return ValueFilterApi.extractValueGt( + measurementIndex, value, field1, zoneId, currPrecision); + case GREATER_THAN_OR_EQUAL: + return ValueFilterApi.extractValueGtEq( + measurementIndex, value, field1, zoneId, currPrecision); + case LESS_THAN: + return ValueFilterApi.extractValueLt( + measurementIndex, value, field1, zoneId, currPrecision); + case LESS_THAN_OR_EQUAL: + return ValueFilterApi.extractValueLtEq( + measurementIndex, value, field1, zoneId, currPrecision); + default: + throw new IllegalArgumentException( + String.format("Unsupported extract comparison operator %s", operator)); + } + } + @SuppressWarnings("unchecked") public static <T extends Comparable<T>> T getValue(Literal value, Type dataType) { try { @@ -273,6 +324,22 @@ public class ConvertPredicateToFilterVisitor && context.isMeasurementColumn((SymbolReference) right)) { return constructCompareFilter( node.getOperator().flip(), (SymbolReference) right, (Literal) left, context); + } else if (context.isExtractMeasurementColumn(left) && isLiteral(right)) { + Extract extract = (Extract) left; + return constructExtractCompareFilter( + node.getOperator(), + (SymbolReference) extract.getExpression(), + extract.getField(), + (Literal) right, + context); + } else if (isLiteral(left) && context.isExtractMeasurementColumn(right)) { + Extract extract = (Extract) right; + return constructExtractCompareFilter( + node.getOperator().flip(), + (SymbolReference) extract.getExpression(), + extract.getField(), + (Literal) left, + context); } else { throw new IllegalStateException( String.format("%s is not supported in value push down", node)); @@ -307,7 +374,10 @@ public class ConvertPredicateToFilterVisitor if (isTimeColumn(firstExpression, timeColumnName) || isTimeColumn(secondExpression, timeColumnName) - || isTimeColumn(thirdExpression, timeColumnName)) { + || isTimeColumn(thirdExpression, timeColumnName) + || isExtractTimeColumn(firstExpression, timeColumnName) + || isExtractTimeColumn(secondExpression, timeColumnName) + || isExtractTimeColumn(thirdExpression, timeColumnName)) { return timeFilterVisitor.process(node, null); } @@ -331,6 +401,33 @@ public class ConvertPredicateToFilterVisitor (SymbolReference) thirdExpression, (Literal) firstExpression, context); + } else if (context.isExtractMeasurementColumn(firstExpression)) { + checkArgument(isLiteral(secondExpression)); + checkArgument(isLiteral(thirdExpression)); + long minValue = getLongValue(secondExpression); + long maxValue = getLongValue(thirdExpression); + Extract extract = (Extract) firstExpression; + int measurementIndex = + context.getMeasurementIndex(((SymbolReference) extract.getExpression()).getName()); + ExtractTimeFilterOperators.Field field = + ExtractTimeFilterOperators.Field.values()[extract.getField().ordinal()]; + + if (minValue == maxValue) { + return ValueFilterApi.extractValueEq( + measurementIndex, minValue, field, zoneId, currPrecision); + } + return FilterFactory.and( + ImmutableList.of( + ValueFilterApi.extractValueGtEq( + measurementIndex, minValue, field, zoneId, currPrecision), + ValueFilterApi.extractValueLtEq( + measurementIndex, maxValue, field, zoneId, currPrecision))); + } else if (context.isExtractMeasurementColumn(secondExpression)) { + throw new IllegalStateException( + "Should not reach here before PredicateCombineIntoTableScanChecker support Extract push-down in third child"); + } else if (context.isExtractMeasurementColumn(thirdExpression)) { + throw new IllegalStateException( + "Should not reach here before PredicateCombineIntoTableScanChecker support Extract push-down in third child"); } else { throw new IllegalStateException( String.format("%s is not supported in value push down", node)); @@ -429,5 +526,14 @@ public class ConvertPredicateToFilterVisitor ColumnSchema schema = schemaMap.get(Symbol.from(symbolReference)); return schema != null && schema.getColumnCategory() == TsTableColumnCategory.FIELD; } + + public boolean isExtractMeasurementColumn(Expression expression) { + if (expression instanceof Extract + && ((Extract) expression).getExpression() instanceof SymbolReference) { + ColumnSchema schema = schemaMap.get(Symbol.from(((Extract) expression).getExpression())); + return schema != null && schema.getColumnCategory() == TsTableColumnCategory.FIELD; + } + return false; + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicateCombineIntoTableScanChecker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicateCombineIntoTableScanChecker.java index b0764213dfe..e77f08da585 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicateCombineIntoTableScanChecker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/PredicateCombineIntoTableScanChecker.java @@ -22,6 +22,7 @@ package org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BetweenPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Extract; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.IfExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InListExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InPredicate; @@ -41,23 +42,17 @@ import java.util.Set; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoScanChecker.isLiteral; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoScanChecker.isSymbolReference; -import static org.apache.iotdb.db.queryengine.plan.relational.planner.ir.GlobalTimePredicateExtractVisitor.isExtractTimeColumn; public class PredicateCombineIntoTableScanChecker extends PredicateVisitor<Boolean, Void> { - private final Set<String> measurementColumns; - private final String timeColumnName; + private final Set<String> timeOrMeasurementColumns; - public static boolean check( - Set<String> measurementColumns, String timeColumnName, Expression expression) { - return new PredicateCombineIntoTableScanChecker(measurementColumns, timeColumnName) - .process(expression); + public static boolean check(Set<String> measurementColumns, Expression expression) { + return new PredicateCombineIntoTableScanChecker(measurementColumns).process(expression); } - public PredicateCombineIntoTableScanChecker( - Set<String> measurementColumns, String timeColumnName) { - this.measurementColumns = measurementColumns; - this.timeColumnName = timeColumnName; + public PredicateCombineIntoTableScanChecker(Set<String> timeOrMeasurementColumns) { + this.timeOrMeasurementColumns = timeOrMeasurementColumns; } @Override @@ -77,7 +72,7 @@ public class PredicateCombineIntoTableScanChecker extends PredicateVisitor<Boole @Override protected Boolean visitInPredicate(InPredicate node, Void context) { - return isMeasurementColumn(node.getValue()) && isInListAllLiteral(node); + return isTimeOrMeasurementColumn(node.getValue()) && isInListAllLiteral(node); } public static Boolean isInListAllLiteral(InPredicate node) { @@ -96,7 +91,7 @@ public class PredicateCombineIntoTableScanChecker extends PredicateVisitor<Boole @Override protected Boolean visitLikePredicate(LikePredicate node, Void context) { - return isMeasurementColumn(node.getValue()) + return isTimeOrMeasurementColumn(node.getValue()) && isLiteral(node.getPattern()) && node.getEscape().map(PredicatePushIntoScanChecker::isLiteral).orElse(true); } @@ -120,32 +115,32 @@ public class PredicateCombineIntoTableScanChecker extends PredicateVisitor<Boole @Override protected Boolean visitComparisonExpression(ComparisonExpression node, Void context) { - return (isMeasurementColumn(node.getLeft()) && isLiteral(node.getRight())) - || (isMeasurementColumn(node.getRight()) && isLiteral(node.getLeft())) - || (isExtractTimeColumn(node.getLeft(), timeColumnName) && isLiteral(node.getRight())) - || (isExtractTimeColumn(node.getRight(), timeColumnName) && isLiteral(node.getLeft())); + return (isTimeOrMeasurementColumn(node.getLeft()) && isLiteral(node.getRight())) + || (isTimeOrMeasurementColumn(node.getRight()) && isLiteral(node.getLeft())) + || (isExtractTimeOrMeasurementColumn(node.getLeft()) && isLiteral(node.getRight())) + || (isExtractTimeOrMeasurementColumn(node.getRight()) && isLiteral(node.getLeft())); } @Override protected Boolean visitBetweenPredicate(BetweenPredicate node, Void context) { - return (isMeasurementColumn(node.getValue()) + return (isTimeOrMeasurementColumn(node.getValue()) && isLiteral(node.getMin()) && isLiteral(node.getMax())) - || (isExtractTimeColumn(node.getValue(), timeColumnName) + || (isExtractTimeOrMeasurementColumn(node.getValue()) && isLiteral(node.getMin()) && isLiteral(node.getMax())); // TODO After Constant-Folding introduced /*|| (isLiteral(node.getValue()) - && isMeasurementColumn(node.getMin()) + && isTimeOrMeasurementColumn(node.getMin()) && isLiteral(node.getMax())) || (isLiteral(node.getValue()) && isLiteral(node.getMin()) - && isMeasurementColumn(node.getMax()));*/ + && isTimeOrMeasurementColumn(node.getMax()));*/ } @Override protected Boolean visitIsNotNullPredicate(IsNotNullPredicate node, Void context) { - return isMeasurementColumn(node.getValue()); + return isTimeOrMeasurementColumn(node.getValue()); } // expression below will be supported later @@ -169,8 +164,15 @@ public class PredicateCombineIntoTableScanChecker extends PredicateVisitor<Boole return Boolean.FALSE; } - private boolean isMeasurementColumn(Expression expression) { + private boolean isTimeOrMeasurementColumn(Expression expression) { return isSymbolReference(expression) - && measurementColumns.contains(((SymbolReference) expression).getName()); + && timeOrMeasurementColumns.contains(((SymbolReference) expression).getName()); + } + + private boolean isExtractTimeOrMeasurementColumn(Expression expression) { + return expression instanceof Extract + && ((Extract) expression).getExpression() instanceof SymbolReference + && timeOrMeasurementColumns.contains( + ((SymbolReference) ((Extract) expression).getExpression()).getName()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index 4284c37a03a..cfaa4a99f84 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -445,7 +445,7 @@ public class PushPredicateIntoTableScan implements PlanOptimizer { // no predicate, just scan all matched deviceEntries if (TRUE_LITERAL.equals(context.inheritedPredicate)) { - getDeviceEntriesWithDataPartitions(tableScanNode, Collections.emptyList(), null); + getDeviceEntriesWithDataPartitions(tableScanNode, Collections.emptyList()); return tableScanNode; } @@ -484,10 +484,7 @@ public class PushPredicateIntoTableScan implements PlanOptimizer { } // do index scan after expressionCanPushDown is processed - getDeviceEntriesWithDataPartitions( - tableScanNode, - splitExpression.getMetadataExpressions(), - splitExpression.getTimeColumnName()); + getDeviceEntriesWithDataPartitions(tableScanNode, splitExpression.getMetadataExpressions()); // exist expressions can not push down to scan operator if (!splitExpression.getExpressionsCannotPushDown().isEmpty()) { @@ -505,16 +502,16 @@ public class PushPredicateIntoTableScan implements PlanOptimizer { private SplitExpression splitPredicate(DeviceTableScanNode node, Expression predicate) { Set<String> idOrAttributeColumnNames = new HashSet<>(node.getAssignments().size()); - Set<String> measurementColumnNames = new HashSet<>(node.getAssignments().size()); + Set<String> timeOrMeasurementColumnNames = new HashSet<>(node.getAssignments().size()); String timeColumnName = null; for (Map.Entry<Symbol, ColumnSchema> entry : node.getAssignments().entrySet()) { Symbol columnSymbol = entry.getKey(); ColumnSchema columnSchema = entry.getValue(); if (TIME.equals(columnSchema.getColumnCategory())) { - measurementColumnNames.add(columnSymbol.getName()); + timeOrMeasurementColumnNames.add(columnSymbol.getName()); timeColumnName = columnSymbol.getName(); } else if (FIELD.equals(columnSchema.getColumnCategory())) { - measurementColumnNames.add(columnSymbol.getName()); + timeOrMeasurementColumnNames.add(columnSymbol.getName()); } else { idOrAttributeColumnNames.add(columnSymbol.getName()); } @@ -531,7 +528,7 @@ public class PushPredicateIntoTableScan implements PlanOptimizer { if (PredicatePushIntoMetadataChecker.check(idOrAttributeColumnNames, expression)) { metadataExpressions.add(expression); } else if (PredicateCombineIntoTableScanChecker.check( - measurementColumnNames, timeColumnName, expression)) { + timeOrMeasurementColumnNames, expression)) { expressionsCanPushDown.add(expression); } else { expressionsCannotPushDown.add(expression); @@ -545,7 +542,7 @@ public class PushPredicateIntoTableScan implements PlanOptimizer { if (PredicatePushIntoMetadataChecker.check(idOrAttributeColumnNames, predicate)) { metadataExpressions.add(predicate); } else if (PredicateCombineIntoTableScanChecker.check( - measurementColumnNames, timeColumnName, predicate)) { + timeOrMeasurementColumnNames, predicate)) { expressionsCanPushDown.add(predicate); } else { expressionsCannotPushDown.add(predicate); @@ -556,9 +553,7 @@ public class PushPredicateIntoTableScan implements PlanOptimizer { } private void getDeviceEntriesWithDataPartitions( - final DeviceTableScanNode tableScanNode, - final List<Expression> metadataExpressions, - String timeColumnName) { + final DeviceTableScanNode tableScanNode, final List<Expression> metadataExpressions) { final List<String> attributeColumns = new ArrayList<>(); int attributeIndex = 0; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExtractExpressionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExtractExpressionTest.java index ce432bf42c4..463384e2935 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExtractExpressionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExtractExpressionTest.java @@ -40,6 +40,16 @@ public class ExtractExpressionTest { assertPlan( planTester.createPlan("select * from table1 where extract(ms from time) > 5"), output(tableScan("testdb.table1"))); + assertPlan( + planTester.createPlan("select * from table1 where extract(ms from time) between 1 and 2"), + output(tableScan("testdb.table1"))); + + assertPlan( + planTester.createPlan("select * from table2 where extract(ms from s4) > 5"), + output(tableScan("testdb.table2"))); + assertPlan( + planTester.createPlan("select * from table2 where extract(ms from s4) between 1 and 2"), + output(tableScan("testdb.table2"))); } @Test diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMetadata.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMetadata.java index 2fc2e7989b8..4dce72b2cef 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMetadata.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMetadata.java @@ -133,6 +133,8 @@ public class TestMetadata implements Metadata { public static final String DB2 = "db2"; public static final String TABLE2 = "table2"; + private static final String S4 = "s4"; + private static final ColumnMetadata S4_CM = new ColumnMetadata(S4, TIMESTAMP); public static final String TREE_VIEW_DB = "tree_view"; public static final String DEVICE_VIEW_TEST_TABLE = "root.test.device_view"; @@ -191,23 +193,44 @@ public class TestMetadata implements Metadata { return Optional.of(new TableSchema(table.getTableName(), columnSchemaList)); } - final List<ColumnSchema> columnSchemas = - Arrays.asList( - ColumnSchema.builder(TIME_CM).setColumnCategory(TsTableColumnCategory.TIME).build(), - ColumnSchema.builder(TAG1_CM).setColumnCategory(TsTableColumnCategory.TAG).build(), - ColumnSchema.builder(TAG2_CM).setColumnCategory(TsTableColumnCategory.TAG).build(), - ColumnSchema.builder(TAG3_CM).setColumnCategory(TsTableColumnCategory.TAG).build(), - ColumnSchema.builder(ATTR1_CM) - .setColumnCategory(TsTableColumnCategory.ATTRIBUTE) - .build(), - ColumnSchema.builder(ATTR2_CM) - .setColumnCategory(TsTableColumnCategory.ATTRIBUTE) - .build(), - ColumnSchema.builder(S1_CM).setColumnCategory(TsTableColumnCategory.FIELD).build(), - ColumnSchema.builder(S2_CM).setColumnCategory(TsTableColumnCategory.FIELD).build(), - ColumnSchema.builder(S3_CM).setColumnCategory(TsTableColumnCategory.FIELD).build()); - - return Optional.of(new TableSchema(TABLE1, columnSchemas)); + if (name.getObjectName().equalsIgnoreCase(TABLE1)) { + final List<ColumnSchema> columnSchemas = + Arrays.asList( + ColumnSchema.builder(TIME_CM).setColumnCategory(TsTableColumnCategory.TIME).build(), + ColumnSchema.builder(TAG1_CM).setColumnCategory(TsTableColumnCategory.TAG).build(), + ColumnSchema.builder(TAG2_CM).setColumnCategory(TsTableColumnCategory.TAG).build(), + ColumnSchema.builder(TAG3_CM).setColumnCategory(TsTableColumnCategory.TAG).build(), + ColumnSchema.builder(ATTR1_CM) + .setColumnCategory(TsTableColumnCategory.ATTRIBUTE) + .build(), + ColumnSchema.builder(ATTR2_CM) + .setColumnCategory(TsTableColumnCategory.ATTRIBUTE) + .build(), + ColumnSchema.builder(S1_CM).setColumnCategory(TsTableColumnCategory.FIELD).build(), + ColumnSchema.builder(S2_CM).setColumnCategory(TsTableColumnCategory.FIELD).build(), + ColumnSchema.builder(S3_CM).setColumnCategory(TsTableColumnCategory.FIELD).build()); + + return Optional.of(new TableSchema(TABLE1, columnSchemas)); + } else { + List<ColumnSchema> columnSchemas = + Arrays.asList( + ColumnSchema.builder(TIME_CM).setColumnCategory(TsTableColumnCategory.TIME).build(), + ColumnSchema.builder(TAG1_CM).setColumnCategory(TsTableColumnCategory.TAG).build(), + ColumnSchema.builder(TAG2_CM).setColumnCategory(TsTableColumnCategory.TAG).build(), + ColumnSchema.builder(TAG3_CM).setColumnCategory(TsTableColumnCategory.TAG).build(), + ColumnSchema.builder(ATTR1_CM) + .setColumnCategory(TsTableColumnCategory.ATTRIBUTE) + .build(), + ColumnSchema.builder(ATTR2_CM) + .setColumnCategory(TsTableColumnCategory.ATTRIBUTE) + .build(), + ColumnSchema.builder(S1_CM).setColumnCategory(TsTableColumnCategory.FIELD).build(), + ColumnSchema.builder(S2_CM).setColumnCategory(TsTableColumnCategory.FIELD).build(), + ColumnSchema.builder(S3_CM).setColumnCategory(TsTableColumnCategory.FIELD).build(), + ColumnSchema.builder(S4_CM).setColumnCategory(TsTableColumnCategory.FIELD).build()); + + return Optional.of(new TableSchema(TABLE2, columnSchemas)); + } } @Override diff --git a/pom.xml b/pom.xml index 21a7316c1ec..9399fef2d4b 100644 --- a/pom.xml +++ b/pom.xml @@ -175,7 +175,7 @@ <thrift.version>0.14.1</thrift.version> <xz.version>1.9</xz.version> <zstd-jni.version>1.5.6-3</zstd-jni.version> - <tsfile.version>2.2.0-250717-SNAPSHOT</tsfile.version> + <tsfile.version>2.2.0-250722-SNAPSHOT</tsfile.version> </properties> <!-- if we claim dependencies in dependencyManagement, then we do not claim
