This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch ty/NewDataTypeSelectInto in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 2abe5417af9eaf9088bfc23a09eb45b9c806952e Author: JackieTien97 <[email protected]> AuthorDate: Mon Jul 15 16:34:16 2024 +0800 Support new dataType in select into clause and fill clause --- .../db/it/fill/IoTDBFillWithNewDataTypeIT.java | 145 +++++++++++++++++++++ .../iotdb/db/it/selectinto/IoTDBSelectIntoIT.java | 53 +++++++- .../plan/planner/OperatorTreeGenerator.java | 10 +- .../plan/statement/literal/BooleanLiteral.java | 4 +- .../plan/statement/literal/DoubleLiteral.java | 3 +- .../plan/statement/literal/Literal.java | 4 + .../plan/statement/literal/LongLiteral.java | 26 +++- .../plan/statement/literal/StringLiteral.java | 21 ++- .../org/apache/iotdb/db/utils/CommonUtils.java | 39 +++--- 9 files changed, 277 insertions(+), 28 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/fill/IoTDBFillWithNewDataTypeIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/fill/IoTDBFillWithNewDataTypeIT.java new file mode 100644 index 00000000000..64f246d316b --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/fill/IoTDBFillWithNewDataTypeIT.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.it.fill; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.Statement; +import java.util.Locale; + +import static org.apache.iotdb.db.it.utils.TestUtils.resultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({LocalStandaloneIT.class, ClusterIT.class}) +public class IoTDBFillWithNewDataTypeIT { + + private static String[] creationSqls = + new String[] { + "create timeseries root.db.d1.s1 BOOLEAN encoding=PLAIN", + "create timeseries root.db.d1.s2 FLOAT encoding=RLE", + "create timeseries root.db.d1.s3 TEXT encoding=PLAIN", + "create timeseries root.db.d1.s4 INT32 encoding=PLAIN", + "create timeseries root.db.d1.s5 INT64 encoding=PLAIN", + "create timeseries root.db.d1.s6 DOUBLE encoding=PLAIN", + "create timeseries root.db.d1.s7 timestamp encoding=PLAIN", + "create timeseries root.db.d1.s8 string encoding=PLAIN", + "create timeseries root.db.d1.s9 blob encoding=PLAIN", + "create timeseries root.db.d1.s10 date encoding=PLAIN", + "insert into root.db.d1(time,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1,true,1.1,'hello1',1,1,1.9,1997-01-01T08:00:00.001+08:00,'Hong Kong',X'486f6e67204b6f6e6720426c6f6221','1997-07-01')", + "insert into root.db.d1(time,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(3,false,1.3,'hello3',3,3,2.1,1997-01-03T08:00:00.001+08:00,'Hong Kong-3',X'486f6e67204b6f6e6720426c6f6224','1997-07-03')", + "insert into root.db.d1(time,s2) values(2,1.2)" + }; + + @Before + public void setUp() throws Exception { + Locale.setDefault(Locale.ENGLISH); + EnvFactory.getEnv().initClusterEnvironment(); + prepareData(); + } + + private void prepareData() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + + for (String sql : creationSqls) { + statement.addBatch(sql); + } + statement.executeBatch(); + statement.clearBatch(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + @After + public void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testFill() { + + String[] resultSet = + new String[] { + "1,1.1,852076800001,Hong Kong,0x486f6e67204b6f6e6720426c6f6221,1997-07-01,", + "2,1.2,2,2,null,null,", + "3,1.3,852249600001,Hong Kong-3,0x486f6e67204b6f6e6720426c6f6224,1997-07-03,", + }; + + String expectedQueryHeader = + "Time,root.db.d1.s2,root.db.d1.s7,root.db.d1.s8,root.db.d1.s9,root.db.d1.s10,"; + resultSetEqualTest( + "select s2,s7,s8,s9,s10 from root.db.d1 fill(2);", expectedQueryHeader, resultSet); + + resultSet = + new String[] { + "1,1.1,852076800001,Hong Kong,0x486f6e67204b6f6e6720426c6f6221,1997-07-01,", + "2,1.2,null,1997-07-02,null,1997-07-02,", + "3,1.3,852249600001,Hong Kong-3,0x486f6e67204b6f6e6720426c6f6224,1997-07-03,", + }; + + resultSetEqualTest( + "select s2,s7,s8,s9,s10 from root.db.d1 fill('1997-07-02');", + expectedQueryHeader, + resultSet); + + resultSet = + new String[] { + "1,1.1,852076800001,Hong Kong,0x486f6e67204b6f6e6720426c6f6221,1997-07-01,", + "2,1.2,null,Hong Kong-2,null,null,", + "3,1.3,852249600001,Hong Kong-3,0x486f6e67204b6f6e6720426c6f6224,1997-07-03,", + }; + + resultSetEqualTest( + "select s2,s7,s8,s9,s10 from root.db.d1 fill('Hong Kong-2');", + expectedQueryHeader, + resultSet); + + resultSet = + new String[] { + "1,1.1,852076800001,Hong Kong,0x486f6e67204b6f6e6720426c6f6221,1997-07-01,", + "2,1.2,852076800001,Hong Kong,0x486f6e67204b6f6e6720426c6f6221,1997-07-01,", + "3,1.3,852249600001,Hong Kong-3,0x486f6e67204b6f6e6720426c6f6224,1997-07-03,", + }; + + resultSetEqualTest( + "select s2,s7,s8,s9,s10 from root.db.d1 fill(previous);", expectedQueryHeader, resultSet); + + resultSet = + new String[] { + "1,1.1,852076800001,Hong Kong,0x486f6e67204b6f6e6720426c6f6221,1997-07-01,", + "2,1.2,852163200001,null,null,1997-07-02,", + "3,1.3,852249600001,Hong Kong-3,0x486f6e67204b6f6e6720426c6f6224,1997-07-03,", + }; + + resultSetEqualTest( + "select s2,s7,s8,s9,s10 from root.db.d1 fill(linear);", expectedQueryHeader, resultSet); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/selectinto/IoTDBSelectIntoIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/selectinto/IoTDBSelectIntoIT.java index fc3f71df0ae..6c97888b32b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/selectinto/IoTDBSelectIntoIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/selectinto/IoTDBSelectIntoIT.java @@ -85,7 +85,21 @@ public class IoTDBSelectIntoIT { "flush", "CREATE DATABASE root.sg1", "CREATE TIMESERIES root.sg1.d1.s1 WITH DATATYPE=INT32, ENCODING=RLE", - "CREATE TIMESERIES root.sg1.d1.s2 WITH DATATYPE=FLOAT, ENCODING=RLE")); + "CREATE TIMESERIES root.sg1.d1.s2 WITH DATATYPE=FLOAT, ENCODING=RLE", + "create timeseries root.db.d1.s1 BOOLEAN encoding=PLAIN", + "create timeseries root.db.d1.s2 FLOAT encoding=RLE", + "create timeseries root.db.d1.s3 TEXT encoding=PLAIN", + "create timeseries root.db.d1.s4 INT32 encoding=PLAIN", + "create timeseries root.db.d1.s5 INT64 encoding=PLAIN", + "create timeseries root.db.d1.s6 DOUBLE encoding=PLAIN", + "create timeseries root.db.d1.s7 timestamp encoding=PLAIN", + "create timeseries root.db.d1.s8 string encoding=PLAIN", + "create timeseries root.db.d1.s9 blob encoding=PLAIN", + "create timeseries root.db.d1.s10 date encoding=PLAIN", + "CREATE ALIGNED TIMESERIES root.db.d2(s1 BOOLEAN encoding=PLAIN, s2 FLOAT encoding=RLE,s3 TEXT encoding=PLAIN,s4 INT32 encoding=PLAIN,s5 INT64 encoding=PLAIN,s6 DOUBLE encoding=PLAIN,s7 timestamp,s8 string,s9 blob,s10 date) ", + "insert into root.db.d1(time,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(1,true,1.1,'hello1',1,1,1.9,1997-01-01T08:00:00.001+08:00,'Hong Kong',X'486f6e67204b6f6e6720426c6f6221','1997-07-01')", + "insert into root.db.d1(time,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) values(3,false,1.3,'hello3',3,3,2.1,1997-01-03T08:00:00.001+08:00,'Hong Kong-3',X'486f6e67204b6f6e6720426c6f6224','1997-07-03')", + "insert into root.db.d1(time,s2) values(2,1.2)")); static { SELECT_INTO_SQL_LIST.add("CREATE DATABASE root.sg_type"); @@ -731,4 +745,41 @@ public class IoTDBSelectIntoIT { expectedQueryHeader, queryRetArray); } + + @Test + public void testNewDataType() { + String[] intoRetArray = + new String[] { + "root.db.d1.s7,root.db.d2.s7,2,", + "root.db.d1.s8,root.db.d2.s8,2,", + "root.db.d1.s9,root.db.d2.s9,2,", + "root.db.d1.s10,root.db.d2.s10,2,", + }; + + resultSetEqualTest( + "select s7 into root.db.d2(s7) from root.db.d1;", + selectIntoHeader, + new String[] {intoRetArray[0]}); + resultSetEqualTest( + "select s8 into root.db.d2(s8) from root.db.d1;", + selectIntoHeader, + new String[] {intoRetArray[1]}); + resultSetEqualTest( + "select s9 into root.db.d2(s9) from root.db.d1;", + selectIntoHeader, + new String[] {intoRetArray[2]}); + resultSetEqualTest( + "select s10 into root.db.d2(s10) from root.db.d1;", + selectIntoHeader, + new String[] {intoRetArray[3]}); + + String[] resultSet = + new String[] { + "1,852076800001,Hong Kong,0x486f6e67204b6f6e6720426c6f6221,1997-07-01,", + "3,852249600001,Hong Kong-3,0x486f6e67204b6f6e6720426c6f6224,1997-07-03,", + }; + + String expectedQueryHeader = "Time,root.db.d2.s7,root.db.d2.s8,root.db.d2.s9,root.db.d2.s10,"; + resultSetEqualTest("select s7,s8,s9,s10 from root.db.d2;", expectedQueryHeader, resultSet); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java index 03a1245576c..05425c23b2b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java @@ -1344,9 +1344,11 @@ public class OperatorTreeGenerator extends PlanVisitor<Operator, LocalExecutionP constantFill[i] = new BinaryConstantFill(literal.getBinary()); break; case INT32: - case DATE: constantFill[i] = new IntConstantFill(literal.getInt()); break; + case DATE: + constantFill[i] = new IntConstantFill(literal.getDate()); + break; case INT64: case TIMESTAMP: constantFill[i] = new LongConstantFill(literal.getLong()); @@ -2359,8 +2361,10 @@ public class OperatorTreeGenerator extends PlanVisitor<Operator, LocalExecutionP private static long getValueSizePerLine(TSDataType tsDataType) { switch (tsDataType) { case INT32: + case DATE: return Integer.BYTES; case INT64: + case TIMESTAMP: return Long.BYTES; case FLOAT: return Float.BYTES; @@ -2369,9 +2373,11 @@ public class OperatorTreeGenerator extends PlanVisitor<Operator, LocalExecutionP case BOOLEAN: return Byte.BYTES; case TEXT: + case BLOB: + case STRING: return StatisticsManager.getInstance().getMaxBinarySizeInBytes(); default: - throw new UnsupportedOperationException("Unknown data type " + tsDataType); + throw new UnsupportedOperationException(UNKNOWN_DATATYPE + tsDataType); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/literal/BooleanLiteral.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/literal/BooleanLiteral.java index e9bace0bd6d..3bc87664a95 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/literal/BooleanLiteral.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/literal/BooleanLiteral.java @@ -55,7 +55,9 @@ public class BooleanLiteral extends Literal { @Override public boolean isDataTypeConsistency(TSDataType dataType) { - return dataType == TSDataType.BOOLEAN || dataType == TSDataType.TEXT; + return dataType == TSDataType.BOOLEAN + || dataType == TSDataType.TEXT + || dataType == TSDataType.STRING; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/literal/DoubleLiteral.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/literal/DoubleLiteral.java index 4f95e960b79..3bfe49370cd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/literal/DoubleLiteral.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/literal/DoubleLiteral.java @@ -56,7 +56,8 @@ public class DoubleLiteral extends Literal { public boolean isDataTypeConsistency(TSDataType dataType) { return dataType == TSDataType.FLOAT || dataType == TSDataType.DOUBLE - || dataType == TSDataType.TEXT; + || dataType == TSDataType.TEXT + || dataType == TSDataType.STRING; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/literal/Literal.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/literal/Literal.java index 3e82cafb66f..a894666ffca 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/literal/Literal.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/literal/Literal.java @@ -76,6 +76,10 @@ public abstract class Literal extends StatementNode { throw new UnsupportedOperationException(getClass().getName()); } + public int getDate() { + throw new UnsupportedOperationException(getClass().getName()); + } + public long getLong() { throw new UnsupportedOperationException(getClass().getName()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/literal/LongLiteral.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/literal/LongLiteral.java index caace0f8ad5..2fa30fe9b07 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/literal/LongLiteral.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/literal/LongLiteral.java @@ -27,8 +27,11 @@ import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.time.format.DateTimeParseException; import java.util.Objects; +import static org.apache.tsfile.utils.DateUtils.parseIntToLocalDate; + public class LongLiteral extends Literal { private final long value; @@ -58,18 +61,30 @@ public class LongLiteral extends Literal { @Override public boolean isDataTypeConsistency(TSDataType dataType) { - if (dataType == TSDataType.INT32) { + if (dataType == TSDataType.INT32 || dataType == TSDataType.DATE) { try { Math.toIntExact(value); - return true; + if (dataType == TSDataType.INT32) { + return true; + } } catch (ArithmeticException e) { return false; } + + // check date type + try { + parseIntToLocalDate((int) value); + return true; + } catch (DateTimeParseException e) { + return false; + } } return dataType == TSDataType.INT64 || dataType == TSDataType.FLOAT || dataType == TSDataType.DOUBLE - || dataType == TSDataType.TEXT; + || dataType == TSDataType.TEXT + || dataType == TSDataType.STRING + || dataType == TSDataType.TIMESTAMP; } @Override @@ -82,6 +97,11 @@ public class LongLiteral extends Literal { return Math.toIntExact(value); } + @Override + public int getDate() { + return Math.toIntExact(value); + } + @Override public long getLong() { return getValue(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/literal/StringLiteral.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/literal/StringLiteral.java index 7dea9fabb9d..284003bbb72 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/literal/StringLiteral.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/literal/StringLiteral.java @@ -29,6 +29,8 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.Objects; +import static org.apache.iotdb.db.utils.CommonUtils.parseIntFromString; + public class StringLiteral extends Literal { private final String value; @@ -54,12 +56,22 @@ public class StringLiteral extends Literal { @Override public boolean isDataTypeConsistency(TSDataType dataType) { - return dataType == TSDataType.TEXT; + if (dataType == TSDataType.TEXT || dataType == TSDataType.STRING) { + return true; + } else if (dataType == TSDataType.DATE) { + try { + parseIntFromString(value); + return true; + } catch (NumberFormatException e) { + return false; + } + } + return false; } @Override public String getDataTypeString() { - return TSDataType.TEXT.toString(); + return TSDataType.STRING.toString(); } @Override @@ -67,6 +79,11 @@ public class StringLiteral extends Literal { return new Binary(value, TSFileConfig.STRING_CHARSET); } + @Override + public int getDate() { + return parseIntFromString(value); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java index d7489730613..cfca21019a8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java @@ -102,24 +102,7 @@ public class CommonUtils { + e.getMessage()); } case DATE: - try { - if (value.length() == 12 - && ((value.startsWith(SqlConstant.QUOTE) && value.endsWith(SqlConstant.QUOTE)) - || (value.startsWith(SqlConstant.DQUOTE) - && value.endsWith(SqlConstant.DQUOTE)))) { - return DateTimeUtils.parseDateExpressionToInt(value.substring(1, value.length() - 1)); - } else { - return DateTimeUtils.parseDateExpressionToInt(StringUtils.trim(value)); - } - } catch (Throwable e) { - throw new NumberFormatException( - "data type is not consistent, input " - + value - + ", registered " - + dataType - + " because " - + e.getMessage()); - } + return parseIntFromString(value); case FLOAT: float f; try { @@ -174,6 +157,26 @@ public class CommonUtils { } } + public static Integer parseIntFromString(String value) { + try { + if (value.length() == 12 + && ((value.startsWith(SqlConstant.QUOTE) && value.endsWith(SqlConstant.QUOTE)) + || (value.startsWith(SqlConstant.DQUOTE) && value.endsWith(SqlConstant.DQUOTE)))) { + return DateTimeUtils.parseDateExpressionToInt(value.substring(1, value.length() - 1)); + } else { + return DateTimeUtils.parseDateExpressionToInt(StringUtils.trim(value)); + } + } catch (Throwable e) { + throw new NumberFormatException( + "data type is not consistent, input " + + value + + ", registered " + + TSDataType.DATE + + " because " + + e.getMessage()); + } + } + public static boolean checkCanCastType(TSDataType src, TSDataType dest) { if (Objects.isNull(src)) { return true;
