This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch force_ci/object_type in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 53c730ca4801fddd1a59efbfa6897b7abc95a97d Author: alpass163 <[email protected]> AuthorDate: Fri Nov 28 16:44:31 2025 +0800 added support for Blob objects to the length() function, allowing for calculating the byte size for Blob inputs (#16170) (cherry picked from commit 85281fac2a3aa779042739706e08aa0177554477) --- .../scalar/IoTDBScalarFunctionTableIT.java | 23 ++-- .../it/query/recent/IoTDBLengthFunctionIT.java | 119 +++++++++++++++++ .../relational/ColumnTransformerBuilder.java | 8 +- .../relational/metadata/TableMetadataImpl.java | 5 +- .../unary/scalar/BlobLengthColumnTransformer.java | 56 ++++++++ .../scalar/BlobLengthColumnTransformerTest.java | 148 +++++++++++++++++++++ 6 files changed, 341 insertions(+), 18 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBScalarFunctionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBScalarFunctionTableIT.java index f6ca8918b1a..01c2d591918 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBScalarFunctionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBScalarFunctionTableIT.java @@ -1370,63 +1370,56 @@ public class IoTDBScalarFunctionTableIT { tableAssertTestFail( "select s1,Length(s1,1) from lengthTable", TSStatusCode.SEMANTIC_ERROR.getStatusCode() - + ": Scalar function length only accepts one argument and it must be text or string data type.", + + ": Scalar function length only accepts one argument and it must be text, string, or blob data type.", DATABASE_NAME); // case 2: wrong data type tableAssertTestFail( "select s1,Length(s2) from lengthTable", TSStatusCode.SEMANTIC_ERROR.getStatusCode() - + ": Scalar function length only accepts one argument and it must be text or string data type.", + + ": Scalar function length only accepts one argument and it must be text, string, or blob data type.", DATABASE_NAME); // case 3: wrong data type tableAssertTestFail( "select s1,Length(s3) from lengthTable", TSStatusCode.SEMANTIC_ERROR.getStatusCode() - + ": Scalar function length only accepts one argument and it must be text or string data type.", + + ": Scalar function length only accepts one argument and it must be text, string, or blob data type.", DATABASE_NAME); // case 4: wrong data type tableAssertTestFail( "select s1,Length(s4) from lengthTable", TSStatusCode.SEMANTIC_ERROR.getStatusCode() - + ": Scalar function length only accepts one argument and it must be text or string data type.", + + ": Scalar function length only accepts one argument and it must be text, string, or blob data type.", DATABASE_NAME); // case 5: wrong data type tableAssertTestFail( "select s1,Length(s5) from lengthTable", TSStatusCode.SEMANTIC_ERROR.getStatusCode() - + ": Scalar function length only accepts one argument and it must be text or string data type.", + + ": Scalar function length only accepts one argument and it must be text, string, or blob data type.", DATABASE_NAME); // case 6: wrong data type tableAssertTestFail( "select s1,Length(s6) from lengthTable", TSStatusCode.SEMANTIC_ERROR.getStatusCode() - + ": Scalar function length only accepts one argument and it must be text or string data type.", + + ": Scalar function length only accepts one argument and it must be text, string, or blob data type.", DATABASE_NAME); // case 7: wrong data type tableAssertTestFail( "select s1,Length(s7) from lengthTable", TSStatusCode.SEMANTIC_ERROR.getStatusCode() - + ": Scalar function length only accepts one argument and it must be text or string data type.", + + ": Scalar function length only accepts one argument and it must be text, string, or blob data type.", DATABASE_NAME); // case 8: wrong data type tableAssertTestFail( "select s1,Length(s8) from lengthTable", TSStatusCode.SEMANTIC_ERROR.getStatusCode() - + ": Scalar function length only accepts one argument and it must be text or string data type.", - DATABASE_NAME); - - // case 9: wrong data type - tableAssertTestFail( - "select s1,Length(s10) from lengthTable", - TSStatusCode.SEMANTIC_ERROR.getStatusCode() - + ": Scalar function length only accepts one argument and it must be text or string data type.", + + ": Scalar function length only accepts one argument and it must be text, string, or blob data type.", DATABASE_NAME); } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBLengthFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBLengthFunctionIT.java new file mode 100644 index 00000000000..57bf23918d5 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBLengthFunctionIT.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBLengthFunctionIT { + + private static final String DATABASE_NAME = "test_length_function"; + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE table1(c_text TEXT FIELD, c_string STRING FIELD, c_blob BLOB FIELD, c_int INT32 FIELD)", + "INSERT INTO table1(time, c_text, c_string, c_blob) VALUES (1, 'hello', 'hello', X'68656C6C6F')", + "INSERT INTO table1(time, c_text, c_string, c_blob) VALUES (2, '你好', '你好', X'e4bda0e5a5bd')", + "INSERT INTO table1(time, c_text, c_string, c_blob) VALUES (3, '', '', X'')", + "INSERT INTO table1(time, c_int) VALUES (4, 404)" + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + /** validate LENGTH() for TEXT and STRING types, correctly calculate the character count */ + @Test + public void testLengthOnTextAndString() { + String[] expectedHeader = new String[] {"time", "length(c_text)", "length(c_string)"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,5,5,", + "1970-01-01T00:00:00.002Z,2,2,", + "1970-01-01T00:00:00.003Z,0,0,", + "1970-01-01T00:00:00.004Z,null,null," + }; + + tableResultSetEqualTest( + "SELECT time, length(c_text) as \"length(c_text)\", length(c_string) as \"length(c_string)\" FROM table1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + /** validate LENGTH() for BLOB type, correctly calculate the number of bytes */ + @Test + public void testLengthOnBlob() { + String[] expectedHeader = new String[] {"time", "length(c_blob)"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,5,", + "1970-01-01T00:00:00.002Z,6,", + "1970-01-01T00:00:00.003Z,0,", + "1970-01-01T00:00:00.004Z,null," + }; + + tableResultSetEqualTest( + "SELECT time, length(c_blob) as \"length(c_blob)\" FROM table1", + expectedHeader, + retArray, + DATABASE_NAME); + } + + /** + * Test the error handling behavior of the LENGTH() function when it receives invalid parameters. + */ + @Test + public void testLengthFunctionOnInvalidInputs() { + String expectedErrorMessage = + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Scalar function length only accepts one argument and it must be text, string, or blob data type."; + + // Exception 1: Using LENGTH() on non-TEXT/BLOB/STRING types + tableAssertTestFail("SELECT length(c_int) FROM table1", expectedErrorMessage, DATABASE_NAME); + + // Exception 2: Incorrect number of arguments passed to the LENGTH() function + tableAssertTestFail( + "SELECT length(c_text, 1) FROM table1", expectedErrorMessage, DATABASE_NAME); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java index 8f2c82d6f1c..a17a824248c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java @@ -129,6 +129,7 @@ import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.Bi import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.BitwiseRightShiftColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.BitwiseXor2ColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.BitwiseXorColumnTransformer; +import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.BlobLengthColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.BytesToDoubleColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.BytesToFloatColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.BytesToIntColumnTransformer; @@ -779,7 +780,12 @@ public class ColumnTransformerBuilder } else if (TableBuiltinScalarFunction.LENGTH.getFunctionName().equalsIgnoreCase(functionName)) { ColumnTransformer first = this.process(children.get(0), context); if (children.size() == 1) { - return new LengthColumnTransformer(INT32, first); + Type argumentType = first.getType(); + if (isCharType(argumentType)) { + return new LengthColumnTransformer(INT32, first); + } else { + return new BlobLengthColumnTransformer(INT32, first); + } } } else if (TableBuiltinScalarFunction.UPPER.getFunctionName().equalsIgnoreCase(functionName)) { ColumnTransformer first = this.process(children.get(0), context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index 3b7c3c049a4..672168b4d19 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -266,11 +266,12 @@ public class TableMetadataImpl implements Metadata { } return STRING; } else if (TableBuiltinScalarFunction.LENGTH.getFunctionName().equalsIgnoreCase(functionName)) { - if (!(argumentTypes.size() == 1 && isCharType(argumentTypes.get(0)))) { + if (!(argumentTypes.size() == 1 + && (isCharType(argumentTypes.get(0)) || isBlobType(argumentTypes.get(0))))) { throw new SemanticException( "Scalar function " + functionName.toLowerCase(Locale.ENGLISH) - + " only accepts one argument and it must be text or string data type."); + + " only accepts one argument and it must be text, string, or blob data type."); } return INT32; } else if (TableBuiltinScalarFunction.UPPER.getFunctionName().equalsIgnoreCase(functionName)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/BlobLengthColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/BlobLengthColumnTransformer.java new file mode 100644 index 00000000000..e94d9ad3907 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/BlobLengthColumnTransformer.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar; + +import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; +import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.UnaryColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.utils.Binary; + +public class BlobLengthColumnTransformer extends UnaryColumnTransformer { + + public BlobLengthColumnTransformer(Type returnType, ColumnTransformer childColumnTransformer) { + super(returnType, childColumnTransformer); + } + + @Override + protected void doTransform(Column column, ColumnBuilder columnBuilder) { + doTransform(column, columnBuilder, null); + } + + @Override + protected void doTransform(Column column, ColumnBuilder columnBuilder, boolean[] selection) { + + int positionCount = column.getPositionCount(); + for (int i = 0; i < positionCount; i++) { + if ((selection != null && !selection[i]) || column.isNull(i)) { + columnBuilder.appendNull(); + continue; + } + + Binary value = column.getBinary(i); + int length = value.getValues().length; + columnBuilder.writeInt(length); + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/BlobLengthColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/BlobLengthColumnTransformerTest.java new file mode 100644 index 00000000000..ffb00505d14 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/BlobLengthColumnTransformerTest.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar; + +import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.read.common.block.column.BinaryColumn; +import org.apache.tsfile.utils.Binary; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import java.nio.charset.StandardCharsets; +import java.util.Optional; + +import static org.apache.tsfile.read.common.type.IntType.INT32; + +public class BlobLengthColumnTransformerTest { + + private ColumnTransformer mockChildColumnTransformer(Column column) { + ColumnTransformer mockColumnTransformer = Mockito.mock(ColumnTransformer.class); + Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); + Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); + Mockito.doNothing().when(mockColumnTransformer).clearCache(); + Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + return mockColumnTransformer; + } + + private static byte[] hexStringToByteArray(String s) { + int len = s.length(); + if (len % 2 != 0) { + throw new IllegalArgumentException("Hex string must have an even number of characters"); + } + byte[] data = new byte[len / 2]; + for (int i = 0; i < len; i += 2) { + data[i / 2] = + (byte) ((Character.digit(s.charAt(i), 16) << 4) + Character.digit(s.charAt(i + 1), 16)); + } + return data; + } + + /** + * Test blob length from string input, the length should be the byte length of the string in UTF-8 + * encoding. + */ + @Test + public void testBlobLengthFromString() { + String input = "hello"; + Binary[] values = new Binary[] {new Binary(input, StandardCharsets.UTF_8)}; + Column binaryColumn = new BinaryColumn(values.length, Optional.empty(), values); + + ColumnTransformer childColumnTransformer = mockChildColumnTransformer(binaryColumn); + BlobLengthColumnTransformer blobLengthColumnTransformer = + new BlobLengthColumnTransformer(INT32, childColumnTransformer); + blobLengthColumnTransformer.addReferenceCount(); + blobLengthColumnTransformer.evaluate(); + Column result = blobLengthColumnTransformer.getColumn(); + + int expectedLength = input.getBytes(StandardCharsets.UTF_8).length; + Assert.assertEquals(expectedLength, result.getInt(0)); + } + + @Test + public void testBlobLengthFromHex() { + String input = "68656C6C6F"; + byte[] inputBytes = hexStringToByteArray(input); + Binary[] values = new Binary[] {new Binary(inputBytes)}; + Column binaryColumn = new BinaryColumn(values.length, Optional.empty(), values); + + ColumnTransformer childColumnTransformer = mockChildColumnTransformer(binaryColumn); + BlobLengthColumnTransformer blobLengthColumnTransformer = + new BlobLengthColumnTransformer(INT32, childColumnTransformer); + blobLengthColumnTransformer.addReferenceCount(); + blobLengthColumnTransformer.evaluate(); + Column result = blobLengthColumnTransformer.getColumn(); + + int expectedLength = inputBytes.length; + Assert.assertEquals(expectedLength, result.getInt(0)); + } + + @Test + public void testBlobLengthMultiRowsWithNull() { + String input1 = "68656C6C6F"; + String input2 = "1F3C"; + byte[] inputBytes1 = hexStringToByteArray(input1); + byte[] inputBytes2 = hexStringToByteArray(input2); + + Binary[] values = new Binary[] {new Binary(inputBytes1), null, new Binary(inputBytes2)}; + boolean[] valueIsNull = new boolean[] {false, true, false}; + Column binaryColumn = new BinaryColumn(values.length, Optional.of(valueIsNull), values); + ColumnTransformer childColumnTransformer = mockChildColumnTransformer(binaryColumn); + + BlobLengthColumnTransformer blobLengthColumnTransformer = + new BlobLengthColumnTransformer(INT32, childColumnTransformer); + blobLengthColumnTransformer.addReferenceCount(); + blobLengthColumnTransformer.evaluate(); + Column result = blobLengthColumnTransformer.getColumn(); + Assert.assertEquals(inputBytes1.length, result.getInt(0)); + Assert.assertTrue(result.isNull(1)); + Assert.assertEquals(inputBytes2.length, result.getInt(2)); + } + + @Test + public void testBlobLengthWithSelection() { + String input1 = "68656C6C6F"; + String input2 = "1F3C"; + String input3 = ""; + + byte[] bytes1 = hexStringToByteArray(input1); + byte[] bytes2 = hexStringToByteArray(input2); + byte[] bytes3 = hexStringToByteArray(input3); + + Binary[] values = {new Binary(bytes1), new Binary(bytes2), new Binary(bytes3)}; + boolean[] booleans = {false, true, true}; + ColumnTransformer child = + mockChildColumnTransformer(new BinaryColumn(values.length, Optional.empty(), values)); + BlobLengthColumnTransformer blobLengthColumnTransformer = + new BlobLengthColumnTransformer(INT32, child); + blobLengthColumnTransformer.addReferenceCount(); + blobLengthColumnTransformer.evaluateWithSelection(booleans); + Column result = blobLengthColumnTransformer.getColumn(); + + int expectedValue2 = bytes2.length; + int expectedValue3 = bytes3.length; + + Assert.assertTrue(result.isNull(0)); + Assert.assertEquals(expectedValue2, result.getInt(1)); + Assert.assertEquals(expectedValue3, result.getInt(2)); + } +}
