This is an automated email from the ASF dual-hosted git repository.
JackieTien97 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new d9122d83761 Support timestamp cast literals in table-model time
planning (#19223)
d9122d83761 is described below
commit d9122d83761003343f9aa2551883a092bd60dc9c
Author: shuwenwei <[email protected]>
AuthorDate: Thu Nov 12 10:28:15 2026 +0800
Support timestamp cast literals in table-model time planning (#19223)
---
.../it/query/recent/IoTDBComplexQueryIT.java | 109 +++++++++++++++++++++
.../it/query/recent/IoTDBGapFillTableIT.java | 40 ++++++++
.../ir/GlobalTimePredicateExtractVisitor.java | 18 +++-
.../ConvertPredicateToTimeFilterVisitor.java | 12 ++-
.../plan/relational/planner/QueryPlanner.java | 12 ++-
.../plan/relational/planner/TranslationMap.java | 5 +-
.../ir/GapFillStartAndEndTimeExtractVisitor.java | 46 ++++++++-
7 files changed, 229 insertions(+), 13 deletions(-)
diff --git
a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBComplexQueryIT.java
b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBComplexQueryIT.java
index e3ca746ecb7..f491c868f56 100644
---
a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBComplexQueryIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBComplexQueryIT.java
@@ -19,19 +19,29 @@
package org.apache.iotdb.relational.it.query.recent;
+import org.apache.iotdb.isession.SessionConfig;
import org.apache.iotdb.it.env.EnvFactory;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
import org.apache.iotdb.itbase.category.TableClusterIT;
import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
+import org.apache.iotdb.itbase.env.BaseEnv;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+
import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData;
import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest;
+import static org.junit.Assert.assertTrue;
@RunWith(IoTDBTestRunner.class)
@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
@@ -91,6 +101,105 @@ public class IoTDBComplexQueryIT {
DATABASE_NAME);
}
+ @Test
+ public void testTimeFilterWithTimestampCast() throws Exception {
+ String[] expectedHeader = new String[] {"time", "name"};
+ String rangeQuery =
+ "SELECT time, name FROM employees"
+ + " WHERE time >= CAST(2 AS TIMESTAMP) AND time < CAST(5 AS
TIMESTAMP)"
+ + " ORDER BY time";
+ tableResultSetEqualTest(
+ rangeQuery,
+ expectedHeader,
+ new String[] {
+ "1970-01-01T00:00:00.002Z,John,",
+ "1970-01-01T00:00:00.003Z,Nancy,",
+ "1970-01-01T00:00:00.004Z,Jack,"
+ },
+ DATABASE_NAME);
+ // Time predicates must reach the table scan so partition pruning remains
effective.
+ assertHasTimePredicate(rangeQuery);
+
+ String betweenQuery =
+ "SELECT time, name FROM employees"
+ + " WHERE time BETWEEN CAST(2 AS TIMESTAMP) AND CAST(4 AS
TIMESTAMP)"
+ + " ORDER BY time";
+ tableResultSetEqualTest(
+ betweenQuery,
+ expectedHeader,
+ new String[] {
+ "1970-01-01T00:00:00.002Z,John,",
+ "1970-01-01T00:00:00.003Z,Nancy,",
+ "1970-01-01T00:00:00.004Z,Jack,"
+ },
+ DATABASE_NAME);
+ assertHasTimePredicate(betweenQuery);
+
+ tableResultSetEqualTest(
+ "SELECT time, name FROM employees"
+ + " WHERE time IN (CAST(2 AS TIMESTAMP), CAST(4 AS TIMESTAMP))
ORDER BY time",
+ expectedHeader,
+ new String[] {"1970-01-01T00:00:00.002Z,John,",
"1970-01-01T00:00:00.004Z,Jack,"},
+ DATABASE_NAME);
+
+ String singleValueInQuery =
+ "SELECT time, name FROM employees WHERE time IN (CAST(2 AS TIMESTAMP))
ORDER BY time";
+ tableResultSetEqualTest(
+ singleValueInQuery,
+ expectedHeader,
+ new String[] {"1970-01-01T00:00:00.002Z,John,"},
+ DATABASE_NAME);
+ assertHasTimePredicate(singleValueInQuery);
+ }
+
+ private static void assertHasTimePredicate(String query) throws Exception {
+ // EXPLAIN coordinates and node IDs may differ by DataNode, so verify the
plan on one node
+ // instead of comparing all cluster plans as regular query results.
+ try (Connection connection =
+ EnvFactory.getEnv()
+ .getConnection(
+ EnvFactory.getEnv().getDataNodeWrapper(0),
+ SessionConfig.DEFAULT_USER,
+ SessionConfig.DEFAULT_PASSWORD,
+ BaseEnv.TABLE_SQL_DIALECT);
+ Statement statement = connection.createStatement()) {
+ statement.execute("USE " + DATABASE_NAME);
+ try (ResultSet resultSet = statement.executeQuery("EXPLAIN (FORMAT JSON)
" + query)) {
+ assertTrue(resultSet.next());
+ assertTrue(
+ "Expected DeviceTableScanNode to contain a TimePredicate",
+
containsTimePredicate(JsonParser.parseString(resultSet.getString(1))));
+ }
+ }
+ }
+
+ private static boolean containsTimePredicate(JsonElement element) {
+ if (element == null || element.isJsonNull()) {
+ return false;
+ }
+ if (element.isJsonObject()) {
+ JsonObject object = element.getAsJsonObject();
+ JsonElement properties = object.get("properties");
+ if (properties != null
+ && properties.isJsonObject()
+ && properties.getAsJsonObject().has("TimePredicate")) {
+ return true;
+ }
+ for (String key : object.keySet()) {
+ if (containsTimePredicate(object.get(key))) {
+ return true;
+ }
+ }
+ } else if (element.isJsonArray()) {
+ for (JsonElement child : element.getAsJsonArray()) {
+ if (containsTimePredicate(child)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
@Test
public void testTableLessQuery() {
String[] expectedHeader;
diff --git
a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBGapFillTableIT.java
b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBGapFillTableIT.java
index 1bc7382ea76..b67f34cb2d1 100644
---
a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBGapFillTableIT.java
+++
b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBGapFillTableIT.java
@@ -31,6 +31,8 @@ import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
+import java.time.Instant;
+
import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData;
import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail;
import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest;
@@ -264,6 +266,44 @@ public class IoTDBGapFillTableIT {
DATABASE_NAME);
}
+ @Test
+ public void gapFillWithTimestampCastBoundsTest() {
+ long startTime = Instant.parse("2024-09-24T04:00:00Z").toEpochMilli();
+ long endTime = Instant.parse("2024-09-24T12:00:00Z").toEpochMilli();
+ String[] expectedHeader = new String[] {"hour_time", "avg_s1"};
+ String[] retArray =
+ new String[] {
+ "2024-09-24T04:00:00.000Z,null,",
+ "2024-09-24T05:00:00.000Z,null,",
+ "2024-09-24T06:00:00.000Z,null,",
+ "2024-09-24T07:00:00.000Z,27.2,",
+ "2024-09-24T08:00:00.000Z,27.3,",
+ "2024-09-24T09:00:00.000Z,null,",
+ "2024-09-24T10:00:00.000Z,null,",
+ "2024-09-24T11:00:00.000Z,29.3,"
+ };
+ tableResultSetEqualTest(
+ "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1
from table1"
+ + " where time >= CAST("
+ + startTime
+ + " AS TIMESTAMP) and time < CAST("
+ + endTime
+ + " AS TIMESTAMP) and device_id = 'd1' group by 1 order by 1",
+ expectedHeader,
+ retArray,
+ DATABASE_NAME);
+ tableResultSetEqualTest(
+ "select date_bin_gapfill(1h, time) as hour_time, avg(s1) as avg_s1
from table1"
+ + " where time between CAST("
+ + startTime
+ + " AS TIMESTAMP) and CAST("
+ + (endTime - 1)
+ + " AS TIMESTAMP) and device_id = 'd1' group by 1 order by 1",
+ expectedHeader,
+ retArray,
+ DATABASE_NAME);
+ }
+
@Test
public void gapFillWithFillClauseTest() {
diff --git
a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/relational/planner/ir/GlobalTimePredicateExtractVisitor.java
b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/relational/planner/ir/GlobalTimePredicateExtractVisitor.java
index b109912e037..029230cc85e 100644
---
a/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/relational/planner/ir/GlobalTimePredicateExtractVisitor.java
+++
b/iotdb-core/calc-commons/src/main/java/org/apache/iotdb/calc/plan/relational/planner/ir/GlobalTimePredicateExtractVisitor.java
@@ -27,6 +27,7 @@ import
org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.ComparisonEx
import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression;
import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Extract;
import
org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.FunctionCall;
+import
org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.GenericLiteral;
import
org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.IfExpression;
import
org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InPredicate;
import
org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.IsNotNullPredicate;
@@ -286,13 +287,22 @@ public class GlobalTimePredicateExtractVisitor
Expression timeExpression, String timeColumnName, Expression
valueExpression) {
return (isTimeColumn(timeExpression, timeColumnName)
|| isExtractTimeColumn(timeExpression, timeColumnName))
- && valueExpression instanceof LongLiteral;
+ && isTimeValueLiteral(valueExpression);
}
private static boolean checkBetweenConstantSatisfy(Expression e1, Expression
e2) {
- return e1 instanceof LongLiteral
- && e2 instanceof LongLiteral
- && ((LongLiteral) e1).getParsedValue() <= ((LongLiteral)
e2).getParsedValue();
+ return isTimeValueLiteral(e1) && isTimeValueLiteral(e2) &&
getLongValue(e1) <= getLongValue(e2);
+ }
+
+ private static boolean isTimeValueLiteral(Expression expression) {
+ return expression instanceof LongLiteral || expression instanceof
GenericLiteral;
+ }
+
+ private static long getLongValue(Expression expression) {
+ if (expression instanceof GenericLiteral) {
+ return Long.parseLong(((GenericLiteral) expression).getValue());
+ }
+ return ((LongLiteral) expression).getParsedValue();
}
public static class Context {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToTimeFilterVisitor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToTimeFilterVisitor.java
index 18a1aa33f50..517068317ed 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToTimeFilterVisitor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/predicate/ConvertPredicateToTimeFilterVisitor.java
@@ -23,6 +23,7 @@ import
org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BetweenPredi
import
org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.ComparisonExpression;
import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression;
import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Extract;
+import
org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.GenericLiteral;
import
org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.IfExpression;
import
org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InListExpression;
import
org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.InPredicate;
@@ -70,14 +71,14 @@ public class ConvertPredicateToTimeFilterVisitor extends
PredicateVisitor<Filter
checkArgument(valueList instanceof InListExpression);
List<Expression> values = ((InListExpression) valueList).getValues();
for (Expression value : values) {
- checkArgument(value instanceof LongLiteral);
+ checkArgument(isLongValue(value));
}
if (values.size() == 1) {
return TimeFilterApi.eq(getLongValue(values.get(0)));
}
Set<Long> longValues = new HashSet<>();
for (Expression value : values) {
- longValues.add(((LongLiteral) value).getParsedValue());
+ longValues.add(getLongValue(value));
}
return TimeFilterApi.in(longValues);
}
@@ -317,6 +318,13 @@ public class ConvertPredicateToTimeFilterVisitor extends
PredicateVisitor<Filter
}
public static long getLongValue(Expression expression) {
+ if (expression instanceof GenericLiteral) {
+ return Long.parseLong(((GenericLiteral) expression).getValue());
+ }
return ((LongLiteral) expression).getParsedValue();
}
+
+ private static boolean isLongValue(Expression expression) {
+ return expression instanceof LongLiteral || expression instanceof
GenericLiteral;
+ }
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java
index f8dd5548595..e1207243ac1 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java
@@ -1356,7 +1356,12 @@ public class QueryPlanner {
long origin = ((LongLiteral)
gapFillColumn.getChildren().get(3)).getParsedValue();
long[] startAndEndTime =
getStartTimeAndEndTimeOfGapFill(
- timeColumn, wherePredicate, origin, monthDuration,
nonMonthDuration);
+ timeColumn,
+ wherePredicate,
+ origin,
+ monthDuration,
+ nonMonthDuration,
+ subPlan.getTranslations().getPlannerContext());
return subPlan.withNewRoot(
new GapFillNode(
queryIdAllocator.genPlanNodeId(),
@@ -1375,11 +1380,12 @@ public class QueryPlanner {
Expression wherePredicate,
long origin,
int monthDuration,
- long nonMonthDuration) {
+ long nonMonthDuration,
+ PlannerContext plannerContext) {
GapFillStartAndEndTimeExtractVisitor.Context context =
new GapFillStartAndEndTimeExtractVisitor.Context();
GapFillStartAndEndTimeExtractVisitor visitor =
- new GapFillStartAndEndTimeExtractVisitor(timeColumn);
+ new GapFillStartAndEndTimeExtractVisitor(timeColumn, plannerContext,
session);
if (!Boolean.TRUE.equals(wherePredicate.accept(visitor, context))) {
throw new SemanticException(CAN_NOT_INFER_TIME_RANGE);
} else {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TranslationMap.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TranslationMap.java
index 29675036828..90f05101ed7 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TranslationMap.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TranslationMap.java
@@ -73,7 +73,6 @@ public class TranslationMap {
private final Scope scope;
private final Analysis analysis;
private final Optional<TranslationMap> outerContext;
- // TODO unused now
private final PlannerContext plannerContext;
// current mappings of underlying field -> symbol for translating direct
field references
@@ -202,6 +201,10 @@ public class TranslationMap {
return analysis;
}
+ public PlannerContext getPlannerContext() {
+ return plannerContext;
+ }
+
public boolean canTranslate(Expression expression) {
verifyAstExpression(expression);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/GapFillStartAndEndTimeExtractVisitor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/GapFillStartAndEndTimeExtractVisitor.java
index 8dfd49f97d0..a04c691e854 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/GapFillStartAndEndTimeExtractVisitor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/GapFillStartAndEndTimeExtractVisitor.java
@@ -20,20 +20,28 @@
package org.apache.iotdb.db.queryengine.plan.relational.planner.ir;
import org.apache.iotdb.commons.exception.SemanticException;
+import org.apache.iotdb.commons.queryengine.common.SessionInfo;
import org.apache.iotdb.commons.queryengine.plan.relational.planner.Symbol;
import
org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.BetweenPredicate;
import
org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.ComparisonExpression;
import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Expression;
+import
org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.GenericLiteral;
import
org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LogicalExpression;
import
org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.LongLiteral;
import org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.Node;
import
org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.SymbolReference;
import org.apache.iotdb.db.i18n.DataNodeQueryMessages;
+import
org.apache.iotdb.db.queryengine.plan.relational.planner.IrExpressionInterpreter;
+import org.apache.iotdb.db.queryengine.plan.relational.planner.PlannerContext;
+import
org.apache.iotdb.db.queryengine.plan.relational.planner.SymbolsExtractor;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AstVisitor;
+import com.google.common.base.VerifyException;
+
import javax.annotation.Nullable;
import java.time.ZoneId;
+import java.util.OptionalLong;
import static
org.apache.iotdb.calc.transformation.dag.column.unary.scalar.DateBinFunctionColumnTransformer.dateBin;
import static
org.apache.iotdb.commons.queryengine.plan.relational.sql.ast.ComparisonExpression.Operator.GREATER_THAN;
@@ -55,9 +63,14 @@ public class GapFillStartAndEndTimeExtractVisitor
.EXCEPTION_OPERATOR_OF_UPDATEENDTIME_SHOULD_ONLY_BE_LESS_THAN_AND_LESS_THAN_OR_EQUAL_NOW_IS_ARG_BF20DCC9;
private final Symbol timeColumn;
+ private final PlannerContext plannerContext;
+ private final SessionInfo session;
- public GapFillStartAndEndTimeExtractVisitor(Symbol timeColumn) {
+ public GapFillStartAndEndTimeExtractVisitor(
+ Symbol timeColumn, PlannerContext plannerContext, SessionInfo session) {
this.timeColumn = timeColumn;
+ this.plannerContext = plannerContext;
+ this.session = session;
}
public static final String CAN_NOT_INFER_TIME_RANGE =
@@ -156,8 +169,10 @@ public class GapFillStartAndEndTimeExtractVisitor
Expression valueExpression,
ComparisonExpression.Operator operator,
GapFillStartAndEndTimeExtractVisitor.Context context) {
- if (isTimeIdentifier(timeExpression) && valueExpression instanceof
LongLiteral) {
- long value = ((LongLiteral) valueExpression).getParsedValue();
+ OptionalLong timeValue =
+ isTimeIdentifier(timeExpression) ? extractTimeValue(valueExpression) :
OptionalLong.empty();
+ if (timeValue.isPresent()) {
+ long value = timeValue.getAsLong();
switch (operator) {
case EQUAL:
case NOT_EQUAL:
@@ -176,6 +191,31 @@ public class GapFillStartAndEndTimeExtractVisitor
return false;
}
+ private OptionalLong extractTimeValue(Expression expression) {
+ if (expression instanceof LongLiteral) {
+ return OptionalLong.of(((LongLiteral) expression).getParsedValue());
+ }
+ if (expression instanceof GenericLiteral) {
+ return OptionalLong.of(Long.parseLong(((GenericLiteral)
expression).getValue()));
+ }
+ if (!SymbolsExtractor.extractUnique(expression).isEmpty()) {
+ return OptionalLong.empty();
+ }
+ try {
+ Object value =
+ IrExpressionInterpreter.evaluateConstantExpression(expression,
plannerContext, session);
+ if (value instanceof Byte
+ || value instanceof Short
+ || value instanceof Integer
+ || value instanceof Long) {
+ return OptionalLong.of(((Number) value).longValue());
+ }
+ } catch (VerifyException ignored) {
+ // A non-constant value cannot define the gap-fill range.
+ }
+ return OptionalLong.empty();
+ }
+
public static class Context {
@Nullable public ComparisonExpression.Operator leftOperator;
long startTime;