This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch ty-object-fix in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 0d7adf655760050b4a06bb13c8c93db832de34d2 Author: JackieTien97 <[email protected]> AuthorDate: Mon Dec 29 20:51:36 2025 +0800 Support second and third parameter of read_object are not constant --- ...bjectQueryIT2.java => IoTDBObjectQuery2IT.java} | 2 +- .../it/query/object/IoTDBObjectQueryIT.java | 95 +++++++++++++++++++-- .../relational/ColumnTransformerBuilder.java | 40 ++++++--- .../plan/planner/OperatorTreeGenerator.java | 1 + .../binary/ReadObject2ColumnTransformer.java | 78 +++++++++++++++++ .../ternary/ReadObject3ColumnTransformer.java | 99 ++++++++++++++++++++++ .../unary/scalar/ReadObjectColumnTransformer.java | 22 +++-- 7 files changed, 308 insertions(+), 29 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/object/IoTDBObjectQueryIT2.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/object/IoTDBObjectQuery2IT.java similarity index 99% rename from integration-test/src/test/java/org/apache/iotdb/relational/it/query/object/IoTDBObjectQueryIT2.java rename to integration-test/src/test/java/org/apache/iotdb/relational/it/query/object/IoTDBObjectQuery2IT.java index 9a0dd60907e..a1fbed0c337 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/object/IoTDBObjectQueryIT2.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/object/IoTDBObjectQuery2IT.java @@ -43,7 +43,7 @@ import java.time.LocalDate; @RunWith(IoTDBTestRunner.class) @Category({TableLocalStandaloneIT.class, TableClusterIT.class}) -public class IoTDBObjectQueryIT2 { +public class IoTDBObjectQuery2IT { private static final String DATABASE_NAME = "test"; diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/object/IoTDBObjectQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/object/IoTDBObjectQueryIT.java index 65c0f09e413..db520e7d9a5 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/object/IoTDBObjectQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/object/IoTDBObjectQueryIT.java @@ -62,15 +62,15 @@ public class IoTDBObjectQueryIT { new String[] { "CREATE DATABASE " + DATABASE_NAME, "USE " + DATABASE_NAME, - "CREATE TABLE t1(device_id STRING TAG, o1 OBJECT, b1 BLOB, s1 STRING)", - "INSERT INTO t1(time, device_id, b1, o1, s1) VALUES(1, 'd1', X'cafebabe01', to_object(true, 0, X'cafebabe01'), 'cafebabe01')", - "INSERT INTO t1(time, device_id, b1, o1, s1) VALUES(2, 'd1', X'cafebabe0202', to_object(true, 0, X'cafebabe02'), 'cafebabe02')", - "INSERT INTO t1(time, device_id, b1, o1, s1) VALUES(3, 'd1', X'cafebabe0303', to_object(true, 0, X'cafebabe03'), 'cafebabe03')", - "INSERT INTO t1(time, device_id, b1, o1, s1) VALUES(4, 'd1', X'cafebabe04', to_object(true, 0, X'cafebabe04'), 'cafebabe04')", - "INSERT INTO t1(time, device_id, b1, o1, s1) VALUES(1, 'd2', X'cafebade01', to_object(true, 0, X'cafebade01'), 'cafebade01')", - "INSERT INTO t1(time, device_id, b1, o1, s1) VALUES(2, 'd2', X'cafebade0202', to_object(true, 0, X'cafebade02'), 'cafebade02')", - "INSERT INTO t1(time, device_id, b1, o1, s1) VALUES(3, 'd2', X'cafebade0302', to_object(true, 0, X'cafebade03'), 'cafebade03')", - "INSERT INTO t1(time, device_id, b1, o1, s1) VALUES(4, 'd2', X'cafebade04', to_object(true, 0, X'cafebade04'), 'cafebade04')", + "CREATE TABLE t1(device_id STRING TAG, o1 OBJECT, b1 BLOB, s1 STRING, l1 INT64, l2 INT64)", + "INSERT INTO t1(time, device_id, b1, o1, s1, l1, l2) VALUES(1, 'd1', X'cafebabe01', to_object(true, 0, X'cafebabe01'), 'cafebabe01', 0, 100)", + "INSERT INTO t1(time, device_id, b1, o1, s1, l1, l2) VALUES(2, 'd1', X'cafebabe0202', to_object(true, 0, X'cafebabe02'), 'cafebabe02', 0, 100)", + "INSERT INTO t1(time, device_id, b1, o1, s1, l1, l2) VALUES(3, 'd1', X'cafebabe0303', to_object(true, 0, X'cafebabe03'), 'cafebabe03', 0, 100)", + "INSERT INTO t1(time, device_id, b1, o1, s1, l1, l2) VALUES(4, 'd1', X'cafebabe04', to_object(true, 0, X'cafebabe04'), 'cafebabe04', 0, 100)", + "INSERT INTO t1(time, device_id, b1, o1, s1, l1, l2) VALUES(1, 'd2', X'cafebade01', to_object(true, 0, X'cafebade01'), 'cafebade01', 0, 100)", + "INSERT INTO t1(time, device_id, b1, o1, s1, l1, l2) VALUES(2, 'd2', X'cafebade0202', to_object(true, 0, X'cafebade02'), 'cafebade02', 0, 100)", + "INSERT INTO t1(time, device_id, b1, o1, s1, l1, l2) VALUES(3, 'd2', X'cafebade0302', to_object(true, 0, X'cafebade03'), 'cafebade03', 0, 100)", + "INSERT INTO t1(time, device_id, b1, o1, s1, l1, l2) VALUES(4, 'd2', X'cafebade04', to_object(true, 0, X'cafebade04'), 'cafebade04', 0, 100)", "FLUSH", }; @@ -134,6 +134,83 @@ public class IoTDBObjectQueryIT { } assertEquals(2, cnt); } + + try (ResultSet resultSet = + statement.executeQuery( + "SELECT time, b1, READ_OBJECT(o1, 0, -1), s1 FROM t1 WHERE device_id = 'd2' AND READ_OBJECT(o1)=b1 ORDER BY time")) { + int cnt = 0; + String[] ans = {"0xcafebade01", "0xcafebade04"}; + while (resultSet.next()) { + String s = resultSet.getString(3); + assertEquals(ans[cnt], s); + cnt++; + } + assertEquals(2, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "SELECT time, b1, READ_OBJECT(o1, l1), s1 FROM t1 WHERE device_id = 'd2' AND READ_OBJECT(o1)=b1 ORDER BY time")) { + int cnt = 0; + String[] ans = {"0xcafebade01", "0xcafebade04"}; + while (resultSet.next()) { + String s = resultSet.getString(3); + assertEquals(ans[cnt], s); + cnt++; + } + assertEquals(2, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "SELECT time, b1, READ_OBJECT(o1, l1, l2), s1 FROM t1 WHERE device_id = 'd2' AND READ_OBJECT(o1)=b1 ORDER BY time")) { + int cnt = 0; + String[] ans = {"0xcafebade01", "0xcafebade04"}; + while (resultSet.next()) { + String s = resultSet.getString(3); + assertEquals(ans[cnt], s); + cnt++; + } + assertEquals(2, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "SELECT time, b1, READ_OBJECT(o1, l1, -1), s1 FROM t1 WHERE device_id = 'd2' AND READ_OBJECT(o1)=b1 ORDER BY time")) { + int cnt = 0; + String[] ans = {"0xcafebade01", "0xcafebade04"}; + while (resultSet.next()) { + String s = resultSet.getString(3); + assertEquals(ans[cnt], s); + cnt++; + } + assertEquals(2, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "SELECT time, b1, READ_OBJECT(o1, 0, l2), s1 FROM t1 WHERE device_id = 'd2' AND READ_OBJECT(o1)=b1 ORDER BY time")) { + int cnt = 0; + String[] ans = {"0xcafebade01", "0xcafebade04"}; + while (resultSet.next()) { + String s = resultSet.getString(3); + assertEquals(ans[cnt], s); + cnt++; + } + assertEquals(2, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "SELECT time, b1, o1, s1 FROM t1 WHERE device_id = 'd1' FILL METHOD LINEAR")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + String s = resultSet.getString(3); + assertEquals("(Object) 5 B", s); + } + assertEquals(4, cnt); + } } } catch (SQLException e) { e.printStackTrace(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java index 41279c9b3eb..f34070c5e9a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java @@ -85,6 +85,7 @@ import org.apache.iotdb.db.queryengine.transformation.dag.column.binary.CompareL import org.apache.iotdb.db.queryengine.transformation.dag.column.binary.CompareNonEqualColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.binary.HmacColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.binary.Like2ColumnTransformer; +import org.apache.iotdb.db.queryengine.transformation.dag.column.binary.ReadObject2ColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.binary.factory.HmacStrategiesFactory; import org.apache.iotdb.db.queryengine.transformation.dag.column.leaf.ConstantColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.leaf.IdentityColumnTransformer; @@ -105,6 +106,7 @@ import org.apache.iotdb.db.queryengine.transformation.dag.column.multi.LogicalOr import org.apache.iotdb.db.queryengine.transformation.dag.column.ternary.BetweenColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.ternary.Like3ColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.ternary.LpadColumnTransformer; +import org.apache.iotdb.db.queryengine.transformation.dag.column.ternary.ReadObject3ColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.ternary.RpadColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.udf.UserDefineScalarFunctionTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.IsNullColumnTransformer; @@ -1467,21 +1469,31 @@ public class ColumnTransformerBuilder if (children.size() == 1) { return new ReadObjectColumnTransformer(BLOB, first, context.fragmentInstanceContext); } else if (children.size() == 2) { - return new ReadObjectColumnTransformer( - BLOB, - ((LongLiteral) children.get(1)).getParsedValue(), - first, - context.fragmentInstanceContext); + Expression offset = children.get(1); + if (isLongLiteral(offset)) { + return new ReadObjectColumnTransformer( + BLOB, + ((LongLiteral) children.get(1)).getParsedValue(), + first, + context.fragmentInstanceContext); + } else { + return new ReadObject2ColumnTransformer( + BLOB, first, this.process(offset, context), context.fragmentInstanceContext); + } } else { - long offset = ((LongLiteral) children.get(1)).getParsedValue(); - long length = ((LongLiteral) children.get(2)).getParsedValue(); - checkArgument(offset >= 0 && length >= 0); - return new ReadObjectColumnTransformer( - BLOB, - ((LongLiteral) children.get(1)).getParsedValue(), - ((LongLiteral) children.get(2)).getParsedValue(), - first, - context.fragmentInstanceContext); + if (isLongLiteral(children.get(1)) && isLongLiteral(children.get(2))) { + long offset = ((LongLiteral) children.get(1)).getParsedValue(); + long length = ((LongLiteral) children.get(2)).getParsedValue(); + return new ReadObjectColumnTransformer( + BLOB, offset, length, first, context.fragmentInstanceContext); + } else { + return new ReadObject3ColumnTransformer( + BLOB, + first, + this.process(children.get(1), context), + this.process(children.get(2), context), + context.fragmentInstanceContext); + } } } else { // user defined function diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java index 44be3e69afb..f869ce59d47 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java @@ -1494,6 +1494,7 @@ public class OperatorTreeGenerator extends PlanVisitor<Operator, LocalExecutionP case TEXT: case STRING: case BLOB: + case OBJECT: linearFill[i] = IDENTITY_LINEAR_FILL; break; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/ReadObject2ColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/ReadObject2ColumnTransformer.java new file mode 100644 index 00000000000..e382f0f8f65 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/binary/ReadObject2ColumnTransformer.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.column.binary; + +import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; +import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.type.Type; + +import java.util.Optional; + +import static org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.ReadObjectColumnTransformer.transform; + +public class ReadObject2ColumnTransformer extends BinaryColumnTransformer { + + private final Optional<FragmentInstanceContext> fragmentInstanceContext; + + public ReadObject2ColumnTransformer( + Type returnType, + ColumnTransformer leftTransformer, + ColumnTransformer rightTransformer, + Optional<FragmentInstanceContext> fragmentInstanceContext) { + super(returnType, leftTransformer, rightTransformer); + this.fragmentInstanceContext = fragmentInstanceContext; + } + + @Override + protected void doTransform( + Column leftColumn, Column rightColumn, ColumnBuilder builder, int positionCount) { + for (int i = 0; i < positionCount; i++) { + if (!leftColumn.isNull(i) && !rightColumn.isNull(i)) { + transform(leftColumn, builder, i, rightColumn.getLong(i), -1, this.fragmentInstanceContext); + } else { + builder.appendNull(); + } + } + } + + @Override + protected void doTransform( + Column leftColumn, + Column rightColumn, + ColumnBuilder builder, + int positionCount, + boolean[] selection) { + for (int i = 0; i < positionCount; i++) { + if (selection[i] && !leftColumn.isNull(i) && !rightColumn.isNull(i)) { + transform(leftColumn, builder, i, rightColumn.getLong(i), -1, this.fragmentInstanceContext); + } else { + builder.appendNull(); + } + } + } + + @Override + protected void checkType() { + // do nothing + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/ternary/ReadObject3ColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/ternary/ReadObject3ColumnTransformer.java new file mode 100644 index 00000000000..e3b4fad8b49 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/ternary/ReadObject3ColumnTransformer.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.column.ternary; + +import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; +import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.type.Type; + +import java.util.Optional; + +import static org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.ReadObjectColumnTransformer.transform; + +public class ReadObject3ColumnTransformer extends TernaryColumnTransformer { + + private final Optional<FragmentInstanceContext> fragmentInstanceContext; + + public ReadObject3ColumnTransformer( + Type retuenType, + ColumnTransformer firstColumnTransformer, + ColumnTransformer secondColumnTransformer, + ColumnTransformer thirdColumnTransformer, + Optional<FragmentInstanceContext> fragmentInstanceContext) { + super(retuenType, firstColumnTransformer, secondColumnTransformer, thirdColumnTransformer); + this.fragmentInstanceContext = fragmentInstanceContext; + } + + @Override + protected void checkType() { + // do nothing + } + + @Override + protected void doTransform( + Column firstColumn, + Column secondColumn, + Column thirdColumn, + ColumnBuilder builder, + int positionCount) { + for (int i = 0; i < positionCount; i++) { + if (!firstColumn.isNull(i) && !secondColumn.isNull(i) && !thirdColumn.isNull(i)) { + transform( + firstColumn, + builder, + i, + secondColumn.getLong(i), + thirdColumn.getLong(i), + this.fragmentInstanceContext); + } else { + builder.appendNull(); + } + } + } + + @Override + protected void doTransform( + Column firstColumn, + Column secondColumn, + Column thirdColumn, + ColumnBuilder builder, + int positionCount, + boolean[] selection) { + for (int i = 0; i < positionCount; i++) { + if (selection[i] + && !firstColumn.isNull(i) + && !secondColumn.isNull(i) + && !thirdColumn.isNull(i)) { + transform( + firstColumn, + builder, + i, + secondColumn.getLong(i), + thirdColumn.getLong(i), + this.fragmentInstanceContext); + } else { + builder.appendNull(); + } + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ReadObjectColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ReadObjectColumnTransformer.java index a4ad4e25756..33b32de2327 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ReadObjectColumnTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ReadObjectColumnTransformer.java @@ -73,7 +73,7 @@ public class ReadObjectColumnTransformer extends UnaryColumnTransformer { protected void doTransform(Column column, ColumnBuilder columnBuilder) { for (int i = 0, n = column.getPositionCount(); i < n; i++) { if (!column.isNull(i)) { - transform(column, columnBuilder, i); + transform(column, columnBuilder, i, this.offset, this.length, this.fragmentInstanceContext); } else { columnBuilder.appendNull(); } @@ -84,22 +84,34 @@ public class ReadObjectColumnTransformer extends UnaryColumnTransformer { protected void doTransform(Column column, ColumnBuilder columnBuilder, boolean[] selection) { for (int i = 0, n = column.getPositionCount(); i < n; i++) { if (selection[i] && !column.isNull(i)) { - transform(column, columnBuilder, i); + transform(column, columnBuilder, i, this.offset, this.length, this.fragmentInstanceContext); } else { columnBuilder.appendNull(); } } } - private void transform(Column column, ColumnBuilder columnBuilder, int i) { + public static void transform( + Column column, + ColumnBuilder columnBuilder, + int i, + long offset, + long length, + Optional<FragmentInstanceContext> fragmentInstanceContext) { // BinaryColumn.getDataType() returns TSDataType.TEXT if (TSDataType.TEXT == column.getDataType()) { Binary binary = column.getBinary(i); - columnBuilder.writeBinary(readObject(binary)); + columnBuilder.writeBinary(readObject(binary, offset, length, fragmentInstanceContext)); + } else { + throw new IllegalStateException("read_object function only accept a BinaryColumn."); } } - private Binary readObject(Binary binary) { + public static Binary readObject( + Binary binary, + long offset, + long length, + Optional<FragmentInstanceContext> fragmentInstanceContext) { Pair<Long, String> objectLengthPathPair = ObjectTypeUtils.parseObjectBinaryToSizeStringPathPair(binary); long fileLength = objectLengthPathPair.getLeft();
