This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch ty/SupportFill in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 133d41b77abda0ac8e3e2cd5e238b1c59c12993e Author: JackieTien97 <[email protected]> AuthorDate: Fri Sep 27 21:18:48 2024 +0800 Support Fill in Table Model --- .../it/query/recent/IoTDBFillTableIT.java | 583 +++++++++++++++++++++ .../relational/analyzer/StatementAnalyzer.java | 59 ++- .../planner/CastToStringLiteralVisitor.java | 6 +- .../planner/CastToTimestampLiteralVisitor.java | 2 +- .../plan/relational/planner/QueryPlanner.java | 22 +- .../distribute/TableDistributedPlanGenerator.java | 29 +- .../relational/planner/node/PreviousFillNode.java | 8 +- .../planner/optimizations/SortElimination.java | 46 +- .../optimizations/TransformSortToStreamSort.java | 17 + .../plan/relational/sql/parser/AstBuilder.java | 10 +- .../db/relational/grammar/sql/RelationalSql.g4 | 2 +- 11 files changed, 741 insertions(+), 43 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBFillTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBFillTableIT.java new file mode 100644 index 00000000000..512dc0958a2 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBFillTableIT.java @@ -0,0 +1,583 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBFillTableIT { + private static final String DATABASE_NAME = "test"; + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE table1(device_id STRING ID, s1 INT32 MEASUREMENT, s2 INT64 MEASUREMENT, s3 FLOAT MEASUREMENT, s4 DOUBLE MEASUREMENT, s5 BOOLEAN MEASUREMENT, s6 TEXT MEASUREMENT, s7 STRING MEASUREMENT, s8 BLOB MEASUREMENT, s9 TIMESTAMP MEASUREMENT, s10 DATE MEASUREMENT)", + "INSERT INTO table1(time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) " + + " values(1, 'd1', 1, 11, 1.1, 11.1, true, 'text1', 'string1', X'cafebabe01', 1, '2024-10-01')", + "INSERT INTO table1(time,device_id,s1,s2,s3,s4,s5) " + + " values(2, 'd1', 2, 22, 2.2, 22.2, false)", + "INSERT INTO table1(time,device_id,s6,s7,s8,s9,s10) " + + " values(3, 'd1', 'text3', 'string3', X'cafebabe03', 3, '2024-10-03')", + "INSERT INTO table1(time,device_id,s6,s7,s8,s9,s10) " + + " values(4, 'd1', 'text4', 'string4', X'cafebabe04', 4, '2024-10-04')", + "INSERT INTO table1(time,device_id,s1,s2,s3,s4,s5) " + + " values(5, 'd1', 5, 55, 5.5, 55.5, false)", + "INSERT INTO table1(time,device_id,s1,s2,s3,s4,s5) " + + " values(7, 'd1', 7, 77, 7.7, 77.7, true)", + "INSERT INTO table1(time,device_id,s6,s7,s8,s9,s10) " + + " values(8, 'd1', 'text8', 'string8', X'cafebabe08', 8, '2024-10-08')", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setEnableCrossSpaceCompaction(false); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void normalFillTest() { + + // --------------------------------- PREVIOUS FILL --------------------------------- + + // case 1: all without time filter using previous fill without timeDuration + String[] expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.003Z,d1,2,22,2.2,22.2,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,2,22,2.2,22.2,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.008Z,d1,7,77,7.7,77.7,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL(PREVIOUS)", expectedHeader, retArray, DATABASE_NAME); + + // case 2: all with time filter using previous fill without timeDuration + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,null,null,null,null,null,", + "1970-01-01T00:00:00.003Z,d1,2,22,2.2,22.2,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,2,22,2.2,22.2,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.008Z,d1,7,77,7.7,77.7,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 WHERE time > 1 FILL(PREVIOUS)", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 3: all with time filter and value filter using previous fill without timeDuration + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,null,null,null,null,null,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,null,null,null,null,null,", + }; + tableResultSetEqualTest( + "select * from table1 WHERE time > 1 and time < 8 and s2 > 22 FILL(PREVIOUS)", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 4: all without time filter using previous fill with timeDuration + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.003Z,d1,2,22,2.2,22.2,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,2,22,2.2,22.2,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,null,null,null,null,null,", + "1970-01-01T00:00:00.008Z,d1,7,77,7.7,77.7,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL(PREVIOUS, 2ms)", expectedHeader, retArray, DATABASE_NAME); + + // case 5: all without time filter using previous fill with timeDuration with helper column + // index + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.003Z,d1,2,22,2.2,22.2,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,2,22,2.2,22.2,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,null,null,null,null,null,", + "1970-01-01T00:00:00.008Z,d1,7,77,7.7,77.7,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL(PREVIOUS(1), 2ms)", expectedHeader, retArray, DATABASE_NAME); + + // case 6: all without time filter using previous fill with timeDuration with helper column + // index + // TODO need fix + // expectedHeader = new String[] {"time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", + // "s7", "s8", "s9", "s10"}; + // retArray = + // new String[] { + // + // "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + // + // "1970-01-01T00:00:00.002Z,d1,2,22,2.2,11.1,false,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + // + // "1970-01-01T00:00:00.003Z,d1,2,22,2.2,11.1,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + // + // "1970-01-01T00:00:00.004Z,d1,2,22,2.2,11.1,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + // + // "1970-01-01T00:00:00.005Z,d1,5,55,5.5,11.1,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + // "1970-01-01T00:00:00.007Z,d1,7,77,7.7,11.1,true,null,null,null,null,null,", + // + // "1970-01-01T00:00:00.008Z,d1,7,77,7.7,11.1,true,text8,string1,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + // }; + // tableResultSetEqualTest( + // "select * from table1 FILL(PREVIOUS(11), 2ms)", expectedHeader, retArray, + // DATABASE_NAME); + + // case7: all without time filter using previous fill with order by time desc + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.008Z,d1,7,77,7.7,77.7,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,null,null,null,null,null,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.004Z,d1,2,22,2.2,22.2,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.003Z,d1,2,22,2.2,22.2,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + }; + tableResultSetEqualTest( + "select * from table1 FILL(PREVIOUS(1), 2ms) order by time desc", + expectedHeader, + retArray, + DATABASE_NAME); + + // case8: all without time filter using previous fill with order by value + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.008Z,d1,7,77,7.7,77.7,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.004Z,d1,2,22,2.2,22.2,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.003Z,d1,2,22,2.2,22.2,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,null,null,null,null,null,", + }; + tableResultSetEqualTest( + "select * from table1 FILL(PREVIOUS(1), 2ms) order by s9 desc, time desc", + expectedHeader, + retArray, + DATABASE_NAME); + + // case9: all without time filter using previous fill with subQuery + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.003Z,d1,5,55,5.5,55.5,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,5,55,5.5,55.5,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,null,null,null,null,null,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from (select * from table1 order by time desc) FILL(previous, 2ms) order by time", + expectedHeader, + retArray, + DATABASE_NAME); + + // --------------------------------- LINEAR FILL --------------------------------- + // case 1: all without time filter using linear fill + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,false,null,null,null,1970-01-01T00:00:00.002Z,2024-10-02,", + "1970-01-01T00:00:00.003Z,d1,3,33,3.3,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,4,44,4.4,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,1970-01-01T00:00:00.005Z,2024-10-05,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,1970-01-01T00:00:00.007Z,2024-10-07,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 FILL(LINEAR)", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 2: all with time filter using linear fill + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,false,null,null,null,null,null,", + "1970-01-01T00:00:00.003Z,d1,3,33,3.3,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,4,44,4.4,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,1970-01-01T00:00:00.005Z,2024-10-05,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,1970-01-01T00:00:00.007Z,2024-10-07,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 WHERE time > 1 FILL(LINEAR)", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 3: all with time filter and value filter using linear fill + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,null,null,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,null,null,", + }; + tableResultSetEqualTest( + "select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 WHERE time > 1 and time < 8 and s2 > 22 FILL(LINEAR)", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 5: all without time filter using linear fill with helper column + // index + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,false,null,null,null,1970-01-01T00:00:00.002Z,2024-10-02,", + "1970-01-01T00:00:00.003Z,d1,3,33,3.3,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,4,44,4.4,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,1970-01-01T00:00:00.005Z,2024-10-05,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,1970-01-01T00:00:00.007Z,2024-10-07,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 FILL(LINEAR(1))", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 6: all without time filter using linear fill with helper column + // index + // TODO need fix + // expectedHeader = new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", + // "s7", "s8", "s9", "s10"}; + // retArray = + // new String[] { + // + // "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + // + // "1970-01-01T00:00:00.002Z,d1,2,22,2.2,11.1,false,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + // + // "1970-01-01T00:00:00.003Z,d1,2,22,2.2,11.1,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + // + // "1970-01-01T00:00:00.004Z,d1,2,22,2.2,11.1,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + // + // "1970-01-01T00:00:00.005Z,d1,5,55,5.5,11.1,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + // "1970-01-01T00:00:00.007Z,d1,7,77,7.7,11.1,true,null,null,null,null,null,", + // + // "1970-01-01T00:00:00.008Z,d1,7,77,7.7,11.1,true,text8,string1,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + // }; + // tableResultSetEqualTest( + // "select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 FILL(LINEAR(10))", + // expectedHeader, retArray, + // DATABASE_NAME); + + // case7: all without time filter using linear fill with order by time desc + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,1970-01-01T00:00:00.007Z,2024-10-07,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,1970-01-01T00:00:00.005Z,2024-10-05,", + "1970-01-01T00:00:00.004Z,d1,4,44,4.4,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.003Z,d1,3,33,3.3,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,false,null,null,null,1970-01-01T00:00:00.002Z,2024-10-02,", + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + }; + tableResultSetEqualTest( + "select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 FILL(LINEAR) order by time desc", + expectedHeader, + retArray, + DATABASE_NAME); + + // case8: all without time filter using linear fill with order by value + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,1970-01-01T00:00:00.007Z,2024-10-07,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,1970-01-01T00:00:00.005Z,2024-10-05,", + "1970-01-01T00:00:00.004Z,d1,4,44,4.4,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.003Z,d1,3,33,3.3,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,false,null,null,null,1970-01-01T00:00:00.002Z,2024-10-02,", + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + }; + tableResultSetEqualTest( + "select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 FILL(LINEAR(1)) order by s9 desc, time desc", + expectedHeader, + retArray, + DATABASE_NAME); + + // case9: all without time filter using linear fill with subQuery + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,false,null,null,null,1970-01-01T00:00:00.002Z,2024-10-02,", + "1970-01-01T00:00:00.003Z,d1,3,33,3.3,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,4,44,4.4,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,1970-01-01T00:00:00.005Z,2024-10-05,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,1970-01-01T00:00:00.007Z,2024-10-07,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from (select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 order by time desc) FILL(LINEAR) order by time", + expectedHeader, + retArray, + DATABASE_NAME); + + // --------------------------------- VALUE FILL --------------------------------- + // case 1: fill with integer value + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,100,100,null,1970-01-01T00:00:00.100Z,0-01-00,", + "1970-01-01T00:00:00.003Z,d1,100,100,100.0,100.0,true,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,100,100,100.0,100.0,true,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,100,100,null,1970-01-01T00:00:00.100Z,0-01-00,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,100,100,null,1970-01-01T00:00:00.100Z,0-01-00,", + "1970-01-01T00:00:00.008Z,d1,100,100,100.0,100.0,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL(100)", expectedHeader, retArray, DATABASE_NAME); + + // case 2: fill with float value + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,110.2,110.2,null,1970-01-01T00:00:00.110Z,0-01-10,", + "1970-01-01T00:00:00.003Z,d1,110,110,110.2,110.2,true,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,110,110,110.2,110.2,true,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,110.2,110.2,null,1970-01-01T00:00:00.110Z,0-01-10,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,110.2,110.2,null,1970-01-01T00:00:00.110Z,0-01-10,", + "1970-01-01T00:00:00.008Z,d1,110,110,110.2,110.2,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL(110.2)", expectedHeader, retArray, DATABASE_NAME); + + // case 3: fill with boolean value + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,false,false,null,null,null,", + "1970-01-01T00:00:00.003Z,d1,0,0,0.0,0.0,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,0,0,0.0,0.0,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,false,false,null,null,null,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,false,false,null,null,null,", + "1970-01-01T00:00:00.008Z,d1,0,0,0.0,0.0,false,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL(false)", expectedHeader, retArray, DATABASE_NAME); + + // case 3: fill with string value + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,iotdb,iotdb,null,null,null,", + "1970-01-01T00:00:00.003Z,d1,null,null,null,null,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,null,null,null,null,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,iotdb,iotdb,null,null,null,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,iotdb,iotdb,null,null,null,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,false,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL('iotdb')", expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,2018-05-06,2018-05-06,null,2018-05-06T00:00:00.000Z,2018-05-06,", + "1970-01-01T00:00:00.003Z,d1,null,null,null,null,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,null,null,null,null,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,2018-05-06,2018-05-06,null,2018-05-06T00:00:00.000Z,2018-05-06,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,2018-05-06,2018-05-06,null,2018-05-06T00:00:00.000Z,2018-05-06,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,false,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL('2018-05-06')", expectedHeader, retArray, DATABASE_NAME); + + // case 4: fill with blob value + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,0xcafebabe99,0xcafebabe99,0xcafebabe99,null,null,", + "1970-01-01T00:00:00.003Z,d1,null,null,null,null,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,null,null,null,null,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,0xcafebabe99,0xcafebabe99,0xcafebabe99,null,null,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,0xcafebabe99,0xcafebabe99,0xcafebabe99,null,null,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL(X'cafebabe99')", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void abNormalFillTest() { + + // --------------------------------- PREVIOUS FILL --------------------------------- + tableAssertTestFail( + "select s1 from table1 FILL(PREVIOUS(1))", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Don't need to specify helper column index while timeDuration parameter is not specified", + DATABASE_NAME); + + tableAssertTestFail( + "select s1 from table1 FILL(PREVIOUS, 2ms)", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Cannot infer the helper column for PREVIOUS FILL, there exists no column whose type is TIMESTAMP", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL(PREVIOUS(1), 2ms)", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Type of helper column for PREVIOUS FILL should only be TIMESTAMP, but type of the column you specify is INT32", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL(PREVIOUS(0), 2ms)", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": PREVIOUS FILL position 0 is not in select list", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL(PREVIOUS(3), 2ms)", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": PREVIOUS FILL position 3 is not in select list", + DATABASE_NAME); + + // --------------------------------- LINEAR FILL --------------------------------- + + tableAssertTestFail( + "select s1 from table1 FILL(LINEAR)", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Cannot infer the helper column for LINEAR FILL, there exists no column whose type is TIMESTAMP", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL(LINEAR(1))", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Type of helper column for LINEAR FILL should only be TIMESTAMP, but type of the column you specify is INT32", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL(LINEAR(0))", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": LINEAR FILL position 0 is not in select list", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL(LINEAR(3))", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": LINEAR FILL position 3 is not in select list", + DATABASE_NAME); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index d56cec28114..4c992bff86f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -195,6 +195,8 @@ public class StatementAnalyzer { private final StatementAnalyzerFactory statementAnalyzerFactory; private Analysis analysis; + + private boolean hasFillInParentScope = false; private final MPPQueryContext queryContext; private final AccessControl accessControl; @@ -508,6 +510,7 @@ public class StatementAnalyzer { @Override protected Scope visitQuery(Query node, Optional<Scope> context) { Scope withScope = analyzeWith(node, context); + hasFillInParentScope = node.getFill().isPresent() || hasFillInParentScope; Scope queryBodyScope = process(node.getQueryBody(), withScope); if (node.getFill().isPresent()) { @@ -521,7 +524,8 @@ public class StatementAnalyzer { if ((queryBodyScope.getOuterQueryParent().isPresent() || !isTopLevel) && !node.getLimit().isPresent() - && !node.getOffset().isPresent()) { + && !node.getOffset().isPresent() + && hasFillInParentScope) { // not the root scope and ORDER BY is ineffective analysis.markRedundantOrderBy(node.getOrderBy().get()); warningCollector.add( @@ -696,6 +700,14 @@ public class StatementAnalyzer { throw new SemanticException( "immediate WITH clause in recursive query is not supported"); }); + withQuery + .getQuery() + .getFill() + .ifPresent( + orderBy -> { + throw new SemanticException( + "immediate FILL clause in recursive query is not supported"); + }); withQuery .getQuery() .getOrderBy() @@ -790,6 +802,7 @@ public class StatementAnalyzer { protected Scope visitQuerySpecification(QuerySpecification node, Optional<Scope> scope) { // TODO: extract candidate names from SELECT, WHERE, HAVING, GROUP BY and ORDER BY expressions // to pass down to analyzeFrom + hasFillInParentScope = node.getFill().isPresent() || hasFillInParentScope; Scope sourceScope = analyzeFrom(node, scope); @@ -802,9 +815,12 @@ public class StatementAnalyzer { Scope outputScope = computeAndAssignOutputScope(node, scope, sourceScope); - if (node.getFill().isPresent()) { - analyzeFill(node.getFill().get(), outputScope); - } + node.getFill() + .ifPresent( + fill -> { + Scope fillScope = computeAndAssignFillScope(fill, sourceScope, outputScope); + analyzeFill(fill, fillScope); + }); List<Expression> orderByExpressions = emptyList(); Optional<Scope> orderByScope = Optional.empty(); @@ -816,7 +832,8 @@ public class StatementAnalyzer { if ((sourceScope.getOuterQueryParent().isPresent() || !isTopLevel) && !node.getLimit().isPresent() - && !node.getOffset().isPresent()) { + && !node.getOffset().isPresent() + && hasFillInParentScope) { // not the root scope and ORDER BY is ineffective analysis.markRedundantOrderBy(orderBy); warningCollector.add( @@ -1471,6 +1488,18 @@ public class StatementAnalyzer { return orderByScope; } + private Scope computeAndAssignFillScope(Fill node, Scope sourceScope, Scope outputScope) { + // Fill should "see" both output and FROM fields during initial analysis and + // non-aggregation query planning + Scope fillScope = + Scope.builder() + .withParent(sourceScope) + .withRelationType(outputScope.getRelationId(), outputScope.getRelationType()) + .build(); + analysis.setScope(node, fillScope); + return fillScope; + } + @Override protected Scope visitSubqueryExpression(SubqueryExpression node, Optional<Scope> context) { return process(node.getQuery(), context); @@ -2098,7 +2127,7 @@ public class StatementAnalyzer { Analysis.FillAnalysis fillAnalysis; if (node.getFillMethod() == FillPolicy.PREVIOUS) { if (node.getTimeDurationThreshold().isPresent()) { - FieldReference helperColumn = getHelperColumn(node, scope); + FieldReference helperColumn = getHelperColumn(node, scope, FillPolicy.PREVIOUS); ExpressionAnalyzer.analyzeExpression( metadata, queryContext, @@ -2131,7 +2160,7 @@ public class StatementAnalyzer { correlationSupport); fillAnalysis = new Analysis.ValueFillAnalysis(literal); } else if (node.getFillMethod() == FillPolicy.LINEAR) { - FieldReference helperColumn = getHelperColumn(node, scope); + FieldReference helperColumn = getHelperColumn(node, scope, FillPolicy.LINEAR); ExpressionAnalyzer.analyzeExpression( metadata, queryContext, @@ -2151,19 +2180,21 @@ public class StatementAnalyzer { analysis.setFill(node, fillAnalysis); } - private FieldReference getHelperColumn(Fill node, Scope scope) { + private FieldReference getHelperColumn(Fill node, Scope scope, FillPolicy fillMethod) { FieldReference helperColumn; if (node.getIndex().isPresent()) { long ordinal = node.getIndex().get().getParsedValue(); if (ordinal < 1 || ordinal > scope.getRelationType().getVisibleFieldCount()) { throw new SemanticException( - String.format("LINEAR FILL position %s is not in select list", ordinal)); + String.format( + "%s FILL position %s is not in select list", fillMethod.name(), ordinal)); } else if (!isTimestampType( - scope.getRelationType().getFieldByIndex((int) ordinal).getType())) { + scope.getRelationType().getFieldByIndex((int) ordinal - 1).getType())) { throw new SemanticException( String.format( - "Type of helper column for LINEAR FILL should only be TIMESTAMP, but type of the column you specify is %s", - scope.getRelationType().getFieldByIndex((int) ordinal).getType())); + "Type of helper column for %s FILL should only be TIMESTAMP, but type of the column you specify is %s", + fillMethod.name(), + scope.getRelationType().getFieldByIndex((int) ordinal - 1).getType())); } else { helperColumn = new FieldReference(toIntExact(ordinal - 1)); } @@ -2179,7 +2210,9 @@ public class StatementAnalyzer { } if (index == -1) { throw new SemanticException( - "Cannot infer the helper column for LINEAR FILL, there exists no column whose type is TIMESTAMP"); + String.format( + "Cannot infer the helper column for %s FILL, there exists no column whose type is TIMESTAMP", + fillMethod.name())); } helperColumn = new FieldReference(index); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CastToStringLiteralVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CastToStringLiteralVisitor.java index a64a909ec5f..e14a58b30bc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CastToStringLiteralVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CastToStringLiteralVisitor.java @@ -50,7 +50,7 @@ public class CastToStringLiteralVisitor extends AstVisitor<Binary, Void> { @Override protected Binary visitBooleanLiteral(BooleanLiteral node, Void context) { - return new Binary(BytesUtils.boolToBytes(node.getValue())); + return new Binary(String.valueOf(node.getValue()), charset); } @Override @@ -60,7 +60,7 @@ public class CastToStringLiteralVisitor extends AstVisitor<Binary, Void> { @Override protected Binary visitDoubleLiteral(DoubleLiteral node, Void context) { - return new Binary(BytesUtils.doubleToBytes(node.getValue())); + return new Binary(String.valueOf(node.getValue()), charset); } @Override @@ -70,7 +70,7 @@ public class CastToStringLiteralVisitor extends AstVisitor<Binary, Void> { @Override protected Binary visitBinaryLiteral(BinaryLiteral node, Void context) { - return new Binary(node.toHexString(), charset); + return new Binary(BytesUtils.parseBlobByteArrayToString(node.getValue()), charset); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CastToTimestampLiteralVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CastToTimestampLiteralVisitor.java index 9187437ecd3..2a6dead013b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CastToTimestampLiteralVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CastToTimestampLiteralVisitor.java @@ -58,7 +58,7 @@ public class CastToTimestampLiteralVisitor extends AstVisitor<Long, Void> { @Override protected Long visitDoubleLiteral(DoubleLiteral node, Void context) { - return null; + return (long) node.getValue(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java index 099e2aeacf0..4f8c59b5e45 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java @@ -119,6 +119,8 @@ public class QueryPlanner { public RelationPlan plan(Query query) { PlanBuilder builder = planQueryBody(query.getQueryBody()); + builder = fill(builder, query.getFill()); + // TODO result is :input[0], :input[1], :input[2] List<Analysis.SelectExpression> selectExpressions = analysis.getSelectExpressions(query); List<Expression> outputs = @@ -132,8 +134,6 @@ public class QueryPlanner { builder.appendProjections( Iterables.concat(orderBy, outputs), symbolAllocator, queryContext); } - - builder = fill(builder, query.getFill()); Optional<OrderingScheme> orderingScheme = orderingScheme(builder, query.getOrderBy(), analysis.getOrderByExpressions(query)); builder = sort(builder, orderingScheme); @@ -166,6 +166,23 @@ public class QueryPlanner { } List<Expression> outputs = outputExpressions(selectExpressions); + + if (node.getFill().isPresent()) { + // Add projections for the outputs of SELECT, but stack them on top of the ones from the FROM + // clause so both are visible + // when resolving the ORDER BY clause. + builder = builder.appendProjections(outputs, symbolAllocator, queryContext); + // The new scope is the composite of the fields from the FROM and SELECT clause (local nested + // scopes). Fields from the bottom of + // the scope stack need to be placed first to match the expected layout for nested scopes. + List<Symbol> newFields = new ArrayList<>(builder.getTranslations().getFieldSymbolsList()); + + outputs.stream().map(builder::translate).forEach(newFields::add); + + builder = builder.withScope(analysis.getScope(node.getFill().get()), newFields); + builder = fill(builder, node.getFill()); + } + if (node.getOrderBy().isPresent()) { // ORDER BY requires outputs of SELECT to be visible. // For queries with aggregation, it also requires grouping keys and translated aggregations. @@ -201,7 +218,6 @@ public class QueryPlanner { Iterables.concat(orderBy, outputs), symbolAllocator, queryContext); } - builder = fill(builder, node.getFill()); Optional<OrderingScheme> orderingScheme = orderingScheme(builder, node.getOrderBy(), analysis.getOrderByExpressions(node)); builder = sort(builder, orderingScheme); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index c7e3883fc45..cef978762ea 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -50,6 +50,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.StreamSortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; @@ -142,6 +143,9 @@ public class TableDistributedPlanGenerator @Override public List<PlanNode> visitFill(FillNode node, PlanContext context) { + if (!(node instanceof ValueFillNode)) { + context.clearExpectedOrderingScheme(); + } List<PlanNode> childrenNodes = node.getChild().accept(this, context); OrderingScheme childOrdering = nodeOrderingMap.get(childrenNodes.get(0).getPlanNodeId()); if (childOrdering != null) { @@ -154,6 +158,7 @@ public class TableDistributedPlanGenerator } node.setChild(mergeChildrenViaCollectOrMergeSort(childOrdering, childrenNodes)); + context.setHasSeenFill(true); return Collections.singletonList(node); } @@ -227,8 +232,7 @@ public class TableDistributedPlanGenerator @Override public List<PlanNode> visitTopK(TopKNode node, PlanContext context) { - context.expectedOrderingScheme = node.getOrderingScheme(); - context.hasSortProperty = true; + context.setExpectedOrderingScheme(node.getOrderingScheme()); nodeOrderingMap.put(node.getPlanNodeId(), node.getOrderingScheme()); checkArgument( @@ -259,8 +263,7 @@ public class TableDistributedPlanGenerator @Override public List<PlanNode> visitSort(SortNode node, PlanContext context) { - context.expectedOrderingScheme = node.getOrderingScheme(); - context.hasSortProperty = true; + context.setExpectedOrderingScheme(node.getOrderingScheme()); nodeOrderingMap.put(node.getPlanNodeId(), node.getOrderingScheme()); List<PlanNode> childrenNodes = node.getChild().accept(this, context); @@ -285,8 +288,7 @@ public class TableDistributedPlanGenerator @Override public List<PlanNode> visitStreamSort(StreamSortNode node, PlanContext context) { - context.expectedOrderingScheme = node.getOrderingScheme(); - context.hasSortProperty = true; + context.setExpectedOrderingScheme(node.getOrderingScheme()); nodeOrderingMap.put(node.getPlanNodeId(), node.getOrderingScheme()); List<PlanNode> childrenNodes = node.getChild().accept(this, context); @@ -825,6 +827,7 @@ public class TableDistributedPlanGenerator public static class PlanContext { final Map<PlanNodeId, NodeDistribution> nodeDistributionMap; + boolean hasSeenFill = false; boolean hasExchangeNode = false; boolean hasSortProperty = false; OrderingScheme expectedOrderingScheme; @@ -837,5 +840,19 @@ public class TableDistributedPlanGenerator public NodeDistribution getNodeDistribution(PlanNodeId nodeId) { return this.nodeDistributionMap.get(nodeId); } + + public void clearExpectedOrderingScheme() { + expectedOrderingScheme = null; + hasSortProperty = false; + } + + public void setExpectedOrderingScheme(OrderingScheme expectedOrderingScheme) { + this.expectedOrderingScheme = expectedOrderingScheme; + hasSortProperty = true; + } + + public void setHasSeenFill(boolean hasSeenFill) { + this.hasSeenFill = hasSeenFill; + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/PreviousFillNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/PreviousFillNode.java index c6c58dd5324..2fb89422094 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/PreviousFillNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/PreviousFillNode.java @@ -104,14 +104,14 @@ public class PreviousFillNode extends FillNode { } public static PreviousFillNode deserialize(ByteBuffer byteBuffer) { - boolean isNull = ReadWriteIOUtils.readBool(byteBuffer); + boolean hasValue = ReadWriteIOUtils.readBool(byteBuffer); TimeDuration timeDuration = null; - if (!isNull) { + if (hasValue) { timeDuration = TimeDuration.deserialize(byteBuffer); } - isNull = ReadWriteIOUtils.readBool(byteBuffer); + hasValue = ReadWriteIOUtils.readBool(byteBuffer); Symbol helperColumn = null; - if (!isNull) { + if (hasValue) { helperColumn = Symbol.deserialize(byteBuffer); } PlanNodeId planNodeId = PlanNodeId.deserialize(byteBuffer); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SortElimination.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SortElimination.java index da83b773124..d28901690e1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SortElimination.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SortElimination.java @@ -22,9 +22,13 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.relational.planner.OrderingScheme; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.StreamSortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; + +import java.util.Collections; import static org.apache.iotdb.db.utils.constant.TestConstant.TIMESTAMP_STR; @@ -62,23 +66,29 @@ public class SortElimination implements PlanOptimizer { public PlanNode visitSort(SortNode node, Context context) { Context newContext = new Context(); PlanNode child = node.getChild().accept(this, newContext); + context.setHasSeenFill(newContext.hasSeenFill); OrderingScheme orderingScheme = node.getOrderingScheme(); - if (newContext.getTotalDeviceEntrySize() == 1 + if (!context.hasSeenFill() + && newContext.getTotalDeviceEntrySize() == 1 && TIMESTAMP_STR.equalsIgnoreCase(orderingScheme.getOrderBy().get(0).getName())) { return child; } - - return node.isOrderByAllIdsAndTime() ? child : node; + return !context.hasSeenFill() && node.isOrderByAllIdsAndTime() + ? child + : node.replaceChildren(Collections.singletonList(child)); } @Override public PlanNode visitStreamSort(StreamSortNode node, Context context) { - PlanNode child = node.getChild().accept(this, context); - return node.isOrderByAllIdsAndTime() - || node.getStreamCompareKeyEndIndex() - == node.getOrderingScheme().getOrderBy().size() - 1 + Context newContext = new Context(); + PlanNode child = node.getChild().accept(this, newContext); + context.setHasSeenFill(newContext.hasSeenFill); + return !context.hasSeenFill() + && (node.isOrderByAllIdsAndTime() + || node.getStreamCompareKeyEndIndex() + == node.getOrderingScheme().getOrderBy().size() - 1) ? child - : node; + : node.replaceChildren(Collections.singletonList(child)); } @Override @@ -86,11 +96,23 @@ public class SortElimination implements PlanOptimizer { context.addDeviceEntrySize(node.getDeviceEntries().size()); return node; } + + @Override + public PlanNode visitFill(FillNode node, Context context) { + PlanNode newNode = node.clone(); + for (PlanNode child : node.getChildren()) { + newNode.addChild(child.accept(this, context)); + } + context.setHasSeenFill(!(node instanceof ValueFillNode)); + return newNode; + } } private static class Context { private int totalDeviceEntrySize = 0; + private boolean hasSeenFill = false; + Context() {} public void addDeviceEntrySize(int deviceEntrySize) { @@ -100,5 +122,13 @@ public class SortElimination implements PlanOptimizer { public int getTotalDeviceEntrySize() { return totalDeviceEntrySize; } + + public boolean hasSeenFill() { + return hasSeenFill; + } + + public void setHasSeenFill(boolean hasSeenFill) { + this.hasSeenFill = hasSeenFill; + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/TransformSortToStreamSort.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/TransformSortToStreamSort.java index 16bc3b4eeba..c9b3cc071d0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/TransformSortToStreamSort.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/TransformSortToStreamSort.java @@ -29,6 +29,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.OrderingScheme; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationTableScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.StreamSortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; @@ -135,17 +136,28 @@ public class TransformSortToStreamSort implements PlanOptimizer { orderingScheme.getOrderBy().get(streamSortIndex + 1).getName()); } + @Override + public PlanNode visitFill(FillNode node, Context context) { + PlanNode newNode = node.clone(); + for (PlanNode child : node.getChildren()) { + newNode.addChild(child.accept(this, context)); + } + return newNode; + } + @Override public PlanNode visitTableScan(TableScanNode node, Context context) { context.setTableScanNode(node); return node; } + @Override public PlanNode visitAggregation(AggregationNode node, Context context) { context.setCanTransform(false); return visitSingleChildProcess(node, context); } + @Override public PlanNode visitAggregationTableScan(AggregationTableScanNode node, Context context) { context.setCanTransform(false); return visitTableScan(node, context); @@ -154,6 +166,7 @@ public class TransformSortToStreamSort implements PlanOptimizer { private static class Context { private TableScanNode tableScanNode; + private boolean canTransform = true; public TableScanNode getTableScanNode() { @@ -171,5 +184,9 @@ public class TransformSortToStreamSort implements PlanOptimizer { public void setCanTransform(boolean canTransform) { this.canTransform = canTransform; } + + public boolean isCanTransform() { + return canTransform; + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index 73e59ec1cb7..1aba94853c8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -994,7 +994,7 @@ public class AstBuilder extends RelationalSqlBaseVisitor<Node> { Optional<Fill> fill = Optional.empty(); if (ctx.fillClause() != null) { - fill = visitIfPresent(ctx.fillClause(), Fill.class); + fill = visitIfPresent(ctx.fillClause().fillMethod(), Fill.class); } Optional<OrderBy> orderBy = Optional.empty(); @@ -1061,12 +1061,13 @@ public class AstBuilder extends RelationalSqlBaseVisitor<Node> { } if (ctx.INTEGER_VALUE() != null) { - helperColumnIndex = new LongLiteral(getLocation(ctx.INTEGER_VALUE()), ctx.getText()); + helperColumnIndex = + new LongLiteral(getLocation(ctx.INTEGER_VALUE()), ctx.INTEGER_VALUE().getText()); } } else { if (ctx.INTEGER_VALUE() != null) { throw new SemanticException( - "Don't need to specify helper column index while timeDuration parameter is not specified."); + "Don't need to specify helper column index while timeDuration parameter is not specified"); } } return new Fill(getLocation(ctx), timeDuration, helperColumnIndex); @@ -1076,7 +1077,8 @@ public class AstBuilder extends RelationalSqlBaseVisitor<Node> { public Node visitLinearFill(RelationalSqlParser.LinearFillContext ctx) { if (ctx.INTEGER_VALUE() != null) { return new Fill( - getLocation(ctx), new LongLiteral(getLocation(ctx.INTEGER_VALUE()), ctx.getText())); + getLocation(ctx), + new LongLiteral(getLocation(ctx.INTEGER_VALUE()), ctx.INTEGER_VALUE().getText())); } else { return new Fill(getLocation(ctx)); } diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index 99584724417..464f21a0fcc 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -845,7 +845,7 @@ nonReserved | CACHE | CALL | CALLED | CASCADE | CATALOG | CATALOGS | CHAR | CHARACTER | CHARSET | CLEAR | CLUSTER | CLUSTERID | COLUMN | COLUMNS | COMMENT | COMMIT | COMMITTED | CONDITION | CONDITIONAL | CONFIGNODES | CONFIGURATION | CONNECTOR | COPARTITION | COUNT | CURRENT | DATA | DATABASE | DATABASES | DATANODES | DATE | DAY | DECLARE | DEFAULT | DEFINE | DEFINER | DENY | DESC | DESCRIPTOR | DETAILS| DETERMINISTIC | DEVICES | DISTRIBUTED | DO | DOUBLE | ELSEIF | EMPTY | ENCODING | ERROR | EXCLUDING | EXPLAIN | EXTRACTOR - | FETCH | FILL | FILTER | FINAL | FIRST | FLUSH | FOLLOWING | FORMAT | FUNCTION | FUNCTIONS + | FETCH | FILTER | FINAL | FIRST | FLUSH | FOLLOWING | FORMAT | FUNCTION | FUNCTIONS | GRACE | GRANT | GRANTED | GRANTS | GRAPHVIZ | GROUPS | HOUR | ID | INDEX | INDEXES | IF | IGNORE | IMMEDIATE | INCLUDING | INITIAL | INPUT | INTERVAL | INVOKER | IO | ITERATE | ISOLATION
